diff --git a/build/lib/pyseidon_dvt/__init__.py b/build/lib/pyseidon_dvt/__init__.py new file mode 100644 index 0000000..32305a7 --- /dev/null +++ b/build/lib/pyseidon_dvt/__init__.py @@ -0,0 +1,38 @@ +#!/usr/bin/python2.7 +# encoding: utf-8 + +from __future__ import division +import os +import sys + +local = os.path.dirname(__file__) +sys.path.append(os.path.join(local,'fvcomClass')) +sys.path.append(os.path.join(local,'adcpClass')) +sys.path.append(os.path.join(local,'drifterClass')) +sys.path.append(os.path.join(local,'stationClass')) +sys.path.append(os.path.join(local,'tidegaugeClass')) +sys.path.append(os.path.join(local,'validationClass')) +sys.path.append(os.path.join(local,'utilities')) + +#Local import +from utilities import * +from adcpClass import * +from drifterClass import * +from tidegaugeClass import * +from stationClass import * +from fvcomClass import * +from validationClass import * + +# Custom error +from pyseidon_dvt.utilities.pyseidon_error import PyseidonError + +#Permission info for OpenDap server +#print "OpenDap server connexion info:" + +__version__ = '2.0' +__all__ = ["FVCOM", "ADCP", "Drifter", "TideGauge",\ + "Validation", "Station", "utilities", "PyseidonError"] +__authors__ = ['Wesley Bowman, Thomas Roc, Jonathan Smith'] +__licence__ = 'GNU Affero GPL v3.0' +__copyright__ = 'Copyright (c) 2014 EcoEnergyII' + diff --git a/build/lib/pyseidon_dvt/adcpClass/__init__.py b/build/lib/pyseidon_dvt/adcpClass/__init__.py new file mode 100644 index 0000000..f29eb7b --- /dev/null +++ b/build/lib/pyseidon_dvt/adcpClass/__init__.py @@ -0,0 +1,10 @@ +#!/usr/bin/python2.7 +# encoding: utf-8 + +#Local import +from adcpClass import ADCP + +__authors__ = ['Wesley Bowman, Thomas Roc, Jonathan Smith'] +__licence__ = 'GNU Affero GPL v3.0' +__copyright__ = 'Copyright (c) 2014 EcoEnergyII' + diff --git a/build/lib/pyseidon_dvt/adcpClass/adcpClass.py b/build/lib/pyseidon_dvt/adcpClass/adcpClass.py new file mode 100644 index 0000000..d77af6b --- /dev/null +++ b/build/lib/pyseidon_dvt/adcpClass/adcpClass.py @@ -0,0 +1,67 @@ +#!/usr/bin/python2.7 +# encoding: utf-8 +from __future__ import division +import numpy as np +import scipy.io as sio +import h5py + +#Local import +from variablesAdcp import _load_adcp +from functionsAdcp import * +from plotsAdcp import * + + +class ADCP: + """ + **A class/structure for ADCP data** + + Functionality structured as follows: :: + + _Data. = raw matlab file data + |_Variables. = useable adcp variables and quantities + |_History = Quality Control metadata + testAdcp._|_Utils. = set of useful functions + |_Plots. = plotting functions + |_method_1 + | ... = methods and analysis techniques intrinsic to ADCPs + |_method_n + + Inputs: + - Only takes a file name as input, ex: testAdcp=ADCP('./path_to_matlab_file/filename') + + *Notes* + Only handle fully processed ADCP matlab data previously quality-controlled as well + as formatted through "EnsembleData_FlowFile" matlab script at the mo. + + Throughout the package, the following conventions apply: + - Coordinates = decimal degrees East and North + - Directions = in degrees, between -180 and 180 deg., i.e. 0=East, 90=North, + +/-180=West, -90=South + - Depth = 0m is the free surface and depth is negative + """ + + def __init__(self, filename, debug=False): + """ Initialize ADCP class.""" + self._debug = debug + self._origin_file = filename + if debug: + print '-Debug mode on-' + #TR_comments: find a way to dissociate raw and processed data + self.History = ['Created from ' + filename] + #TR_comments: *_Raw and *_10minavg open with h5py whereas *_davgBS + try: + self.Data = sio.loadmat(filename, struct_as_record=False, squeeze_me=True) + except NotImplementedError: + self.Data = h5py.File(filename, 'r') + #TR_comments: Initialize class structure + self.Variables = _load_adcp(self, self.History, debug=self._debug) + self.Plots = PlotsAdcp(self.Variables, debug=self._debug) + self.Utils = FunctionsAdcp(self.Variables, + self.Plots, + self.History, + debug=self._debug) + + ##Re-assignement of utility functions as methods + self.dump_profile_data = self.Plots._dump_profile_data_as_csv + + return \ No newline at end of file diff --git a/build/lib/pyseidon_dvt/adcpClass/functionsAdcp.py b/build/lib/pyseidon_dvt/adcpClass/functionsAdcp.py new file mode 100644 index 0000000..427d3b3 --- /dev/null +++ b/build/lib/pyseidon_dvt/adcpClass/functionsAdcp.py @@ -0,0 +1,622 @@ +#!/usr/bin/python2.7 +# encoding: utf-8 + +from __future__ import division + +import numpy as np +import numexpr as ne +import datetime +from pyseidon_dvt.utilities.miscellaneous import * +from pyseidon_dvt.utilities.BP_tools import * +from utide import solve, reconstruct +import time + +# Custom error +from pyseidon_error import PyseidonError + +class FunctionsAdcp: + """ **'Utils' subset of FVCOM class gathers useful functions** """ + def __init__(self, variable, plot, History, debug=False): + self._debug = debug + self._plot = plot + #Create pointer to FVCOM class + setattr(self, '_var', variable) + setattr(self, '_History', History) + + return + + def flow_dir(self, t_start=[], t_end=[], time_ind=[], + exceedance=False, debug=False): + """ + Flow directions and associated norm + + Outputs: + - flowDir = flowDir at station, 1D array + - norm = velocity norm at station, 1D array + + Options: + - t_start = start time, as string ('yyyy-mm-ddThh:mm:ss'), + or time index as an integer + - t_end = end time, as a string ('yyyy-mm-ddThh:mm:ss'), + or time index as an integer + - time_ind = time indices to work in, list of integers + - excedance = True, compute associated exceedance curve + + *Notes* + - directions between -180 and 180 deg., i.e. 0=East, 90=North, + +/-180=West, -90=South + """ + debug = debug or self._debug + if debug: + print 'Computing flow directions at point...' + + # Find time interval to work in + argtime = [] + if not time_ind==[]: + argtime = time_ind + elif not t_start==[]: + if type(t_start)==str: + start = datetime.datetime.strptime(t_start, '%Y-%m-%d %H:%M:%S') + end = datetime.datetime.strptime(t_end, '%Y-%m-%d %H:%M:%S') + argtime = time_to_index(start, end, self._var.julianTime[:], debug=debug) + else: + argtime = np.arange(t_start, t_end) + + #Choose the right pair of velocity components + if not argtime==[]: + U = self._var.ua[argtime] + V = self._var.va[argtime] + else: + U = self._var.ua[:] + V = self._var.va[:] + + #Compute directions + if debug: + print 'Computing arctan2 and norm...' + dirFlow = np.rad2deg(np.arctan2(V,U)) + + #Compute velocity norm + norm = ne.evaluate('sqrt(U**2 + V**2)').squeeze() + if debug: + print '...Passed' + #Rose diagram + self._plot.rose_diagram(dirFlow, norm) + if exceedance: + self.exceedance(norm) + + return dirFlow, norm + + def ebb_flood_split(self, t_start=[], t_end=[], time_ind=[], debug=False): + """ + Compute time indices for ebb and flood but also the + principal flow directions and associated variances for (lon, lat) point + + Outputs: + - floodIndex = flood time index, 1D array of integers + - ebbIndex = ebb time index, 1D array of integers + - pr_axis = principal flow ax1s, float number in degrees + - pr_ax_var = associated variance, float number + + Options: + - t_start = start time, as a string ('yyyy-mm-ddThh:mm:ss'), + or time index as an integer + - t_end = end time, as a string ('yyyy-mm-ddThh:mm:ss'), + or time index as an integer + - time_ind = time indices to work in, 1D array of integers + + *Notes* + - may take time to compute if time period too long + - directions between -180 and 180 deg., i.e. 0=East, 90=North, + +/-180=West, -90=South + - use time_ind or t_start and t_end, not both + - assume that flood is aligned with principal direction + """ + debug = debug or self._debug + if debug: + start = time.time() + print 'Computing principal flow directions...' + + # Find time interval to work in + argtime = [] + if not time_ind==[]: + argtime = time_ind + elif not t_start==[]: + if type(t_start)==str: + start = datetime.datetime.strptime(t_start, '%Y-%m-%d %H:%M:%S') + end = datetime.datetime.strptime(t_end, '%Y-%m-%d %H:%M:%S') + argtime = time_to_index(start, end, self._var.julianTime[:], debug=debug) + else: + argtime = np.arange(t_start, t_end) + + #Choose the right pair of velocity components + if not argtime==[]: + U = self._var.ua[argtime] + V = self._var.va[argtime] + else: + U = self._var.ua[:] + V = self._var.va[:] + + #WB version of BP's principal axis + #Assuming principal axis = flood heading + #determine principal axes - potentially a problem if axes are very kinked + # since this would misclassify part of ebb and flood + if debug: print 'Computin principal axis at point...' + pr_axis, pr_ax_var = principal_axis(U, V) + + #ebb/flood split + if debug: print 'Splitting ebb and flood at point...' + ###TR: version 1 + # reverse 0-360 deg convention + #ra = (-pr_axis - 90.0) * np.pi /180.0 + #if ra>np.pi: + # ra = ra - (2.0*np.pi) + #elif ra<-np.pi: + # ra = ra + (2.0*np.pi) + #dirFlow = np.arctan2(V,U) + ##Define bins of angles + #if ra == 0.0: + # binP = [0.0, np.pi/2.0] + # binP = [0.0, -np.pi/2.0] + #elif ra > 0.0: + # if ra == np.pi: + # binP = [np.pi/2.0 , np.pi] + # binM = [-np.pi, -np.pi/2.0 ] + # elif ra < (np.pi/2.0): + # binP = [0.0, ra + (np.pi/2.0)] + # binM = [-((np.pi/2.0)-ra), 0.0] + # else: + # binP = [ra - (np.pi/2.0), np.pi] + # binM = [-np.pi, -np.pi + (ra-(np.pi/2.0))] + #else: + # if ra == -np.pi: + # binP = [np.pi/2.0 , np.pi] + # binM = [-np.pi, -np.pi/2.0] + # elif ra > -(np.pi/2.0): + # binP = [0.0, ra + (np.pi/2.0)] + # binM = [ ((-np.pi/2.0)+ra), 0.0] + # else: + # binP = [np.pi - (ra+(np.pi/2.0)) , np.pi] + # binM = [-np.pi, ra + (np.pi/2.0)] + # + #test = (((dirFlow > binP[0]) * (dirFlow < binP[1])) + + # ((dirFlow > binM[0]) * (dirFlow < binM[1]))) + #floodIndex = np.where(test == True)[0] + #ebbIndex = np.where(test == False)[0] + ###TR: version 2 + flood_heading = np.array([-90, 90]) + pr_axis + dir_all = np.rad2deg(np.arctan2(V,U)) + ind = np.where(dir_all<0) + dir_all[ind] = dir_all[ind] + 360 + # sign speed - eliminating wrap-around + dir_PA = dir_all - pr_axis + dir_PA[dir_PA < -90] += 360 + dir_PA[dir_PA > 270] -= 360 + #general direction of flood passed as input argument + floodIndex = np.where((dir_PA >= -90) & (dir_PA<90)) + ebbIndex = np.arange(dir_PA.shape[0]) + ebbIndex = np.delete(ebbIndex,floodIndex[:]) + #TR: quick fix + if type(floodIndex).__name__=='tuple': + floodIndex = floodIndex[0] + + if debug: + end = time.time() + print "...processing time: ", (end - start) + + return floodIndex, ebbIndex, pr_axis, pr_ax_var + + def exceedance(self, var, graph=True, dump=False, debug=False, **kwargs): + """ + This function calculate the excedence curve of a var(time). + + Inputs: + - var = given quantity, 1 array of n elements + + Options: + - graph: True->plots curve; False->does not + - dump = boolean, dump profile data in csv file + - kwargs = keyword options associated with pandas.DataFrame.to_csv, such as: + sep, header, na_rep, index,...etc + Check doc. of "to_csv" for complete list of options + + Outputs: + - Exceedance = list of % of occurences, 1D array + - Ranges = list of signal amplitude bins, 1D array + + *Notes* + - This method is not suitable for SSE + """ + debug = (debug or self._debug) + if debug: + print 'Computing exceedance...' + + signal=var + + Max = max(signal) + dy = (Max/30.0) + Ranges = np.arange(0,(Max + dy), dy) + Exceedance = np.zeros(Ranges.shape[0]) + dt = self._var.matlabTime[1] - self._var.matlabTime[0] + Period = var.shape[0] * dt + time = np.arange(0.0, Period, dt) + + N = len(signal) + M = len(Ranges) + + for i in range(M): + r = Ranges[i] + for j in range(N-1): + if signal[j] > r: + Exceedance[i] = Exceedance[i] + (time[j+1] - time[j]) + + Exceedance = (Exceedance * 100) / Period + #Plot + if graph: + error=np.ones(Exceedance.shape) * np.std(var)/2.0 + self._plot.plot_xy(Exceedance, Ranges, yerror=error, + yLabel='Amplitudes', + xLabel='Exceedance probability in %', dump=dump, **kwargs) + + if debug: + print '...Passed' + + return Exceedance, Ranges + + def speed_histogram(self, t_start=[], t_end=[], time_ind=[], + debug=False, dump=False, **kwargs): + """ + This function plots the histogram of occurrences for the signed + flow speed. + + Options: + - t_start = start time, as a string ('yyyy-mm-ddThh:mm:ss'), + or time index as an integer + - t_end = end time, as a string ('yyyy-mm-ddThh:mm:ss'), + or time index as an integer + - time_ind = time indices to work in, 1D array of integers + - dump = boolean, dump profile data in csv file + - kwargs = keyword options associated with pandas.DataFrame.to_csv, such as: + sep, header, na_rep, index,...etc + Check doc. of "to_csv" for complete list of options + + *Notes* + - use time_ind or t_start and t_end, not both + """ + debug = debug or self._debug + if debug: + start = time.time() + print 'Computing speed histogram...' + + pI, nI, pa, pav = self.ebb_flood_split(t_start=t_start, t_end=t_end, + time_ind=time_ind, debug=debug) + dirFlow, norm = self.flow_dir(t_start=t_start, t_end=t_end, + time_ind=time_ind, exceedance=False, debug=debug) + norm[nI] = -1.0 * norm[nI] + + #compute bins + #minBound = norm.min() + #maxBound = norm.max() + #step = round((maxBound-minBound/51.0),1) + #bins = np.arange(minBound,maxBound,step) + + #plot histogram + self._plot.Histogram(norm, + title='Flow speed histogram', + xLabel='Signed flow speed (m/s)', + yLabel='Occurrences (%)', dump=dump, **kwargs) + + if debug: + end = time.time() + print "...processing time: ", (end - start) + + + def Harmonic_analysis(self, + time_ind=[], t_start=[], t_end=[], + elevation=True, velocity=False, + threeD=False, debug=False, **kwargs): + ''' + This function performs a harmonic analysis on the sea surface elevation + time series or the velocity components timeseries. + + Outputs: + - harmo = harmonic coefficients, dictionary + + Options: + - time_ind = time indices to work in, list of integers + - t_start = start time, as a string ('yyyy-mm-ddThh:mm:ss'), + or time index as an integer + - t_end = end time, as a string ('yyyy-mm-ddThh:mm:ss'), + or time index as an integer + - elevation = True means that `solve' will be done for elevation. + - velocity = True means that `solve' will be done for velocity. + - threeD = True means that 'solve' will be done for each layer + of the velocity data instead of on depth-averaged data. + + Options: + Options are the same as for 'solve', which are shown below with + their default values: + conf_int=True; cnstit='auto'; notrend=0; prefilt=[]; nodsatlint=0; + nodsatnone=0; gwchlint=0; gwchnone=0; infer=[]; inferaprx=0; + rmin=1; method='cauchy'; tunrdn=1; linci=0; white=0; nrlzn=200; + lsfrqosmp=1; nodiagn=0; diagnplots=0; diagnminsnr=2; + ordercnstit=[]; runtimedisp='yyy' + + *Notes* + For more detailed information about 'solve', please see + https://github.com/wesleybowman/UTide + + ''' + debug = (debug or self._debug) + + argtime = [] + if not time_ind==[]: + argtime = time_ind + elif not t_start==[]: + if type(t_start)==str: + start = datetime.datetime.strptime(t_start, '%Y-%m-%d %H:%M:%S') + end = datetime.datetime.strptime(t_end, '%Y-%m-%d %H:%M:%S') + argtime = time_to_index(start, end, self._var.julianTime[:], debug=debug) + else: + argtime = np.arange(t_start, t_end) + + if velocity == elevation: + raise PyseidonError("---Can only process either velocities or elevation---") + + if velocity: + time = self._var.matlabTime[:] + lat = self._var.lat + if not threeD: + u = self._var.ua[:] + v = self._var.va[:] + + if not argtime==[]: + time = time[argtime[:]] + u = u[argtime[:]] + v = v[argtime[:]] + + harmo = solve(time, u, v, lat, **kwargs) + else: + harmo = {} + for layerIndex in range(self._var.u.data.shape[1]): + u = self._var.u[:,layerIndex] + v = self._var.v[:,layerIndex] + + if not argtime==[]: + time = time[argtime[:]] + u = u[argtime[:]] + v = v[argtime[:]] + + harmo['Layer_'+str(layerIndex)] = solve(time, u, v, lat, **kwargs) + + if elevation: + time = self._var.matlabTime[:] + el = self._var.el[:] + + if not argtime==[]: + time = time[argtime[:]] + el = el[argtime[:]] + + lat = self._var.lat + harmo = solve(time, el, None, lat, **kwargs) + #Write meta-data only if computed over all the elements + + return harmo + + def Harmonic_reconstruction(self, harmo, time_ind=slice(None), debug=False, **kwargs): + """ + This function reconstructs the velocity components or the surface elevation + from harmonic coefficients. + Harmonic_reconstruction calls 'reconstruct'. This function assumes harmonics + ('solve') has already been executed. + + Inputs: + - Harmo = harmonic coefficient from harmo_analysis + - elevation =True means that 'reconstruct' will be done for elevation. + - velocity =True means that 'reconstruct' will be done for velocity. + - time_ind = time indices to process, list of integers + + Output: + - Reconstruct = reconstructed signal, dictionary + + Utide's options: + Options are the same as for 'reconstruct', which are shown below with + their default values: + cnstit = [], minsnr = 2, minpe = 0 + + *Notes* + For more detailed information about 'reconstruct', please see + https://github.com/wesleybowman/UTide + """ + debug = (debug or self._debug) + time = self._var.matlabTime[time_ind] + #TR_comments: Add debug flag in Utide: debug=self._debug + if not 'Layer_0' in harmo: + Reconstruct = reconstruct(time, harmo) + else: + Reconstruct = {} + for layer in harmo: + Reconstruct[layer] = reconstruct(time, harmo[layer]) + + return Reconstruct + + def velo_stack(self, Reconstruct, debug=False): + """ + This function seperates and stacks u & v into respective matrices + from a 3D harmonically reconstructed dictionary + + Inputs: + - Reconstruct = reconstructed signal, dictionary from Harmonic_reconstruction() + + Outputs: + - u = stacked matrix of u velocity + - v = stacked matrix of v velocity + + """ + debug = (debug or self._debug) + u = Reconstruct['Layer_0']['u'] + v = Reconstruct['Layer_0']['v'] + for i in range(len(Reconstruct))[1:]: + u = np.vstack((u, Reconstruct['Layer_'+str(i)]['u'])) + v = np.vstack((v, Reconstruct['Layer_'+str(i)]['v'])) + + return u, v + + def verti_shear(self, t_start=[], t_end=[], time_ind=[], + graph=True, dump=False, debug=False, **kwargs): + """ + Compute vertical shear + + Outputs: + - dveldz = vertical shear (1/s), 2D array (time, nlevel - 1) + + Utide's options: + - t_start = start time, as a string ('yyyy-mm-ddThh:mm:ss'), + or time index as an integer + - t_end = end time, as a string ('yyyy-mm-ddThh:mm:ss'), + or time index as an integer + - time_ind = time indices to work in, list of integers + - graph = plots graph if True + - dump = boolean, dump profile data in csv file + - kwargs = keyword options associated with pandas.DataFrame.to_csv, such as: + sep, header, na_rep, index,...etc + Check doc. of "to_csv" for complete list of options + + *Notes* + - use time_ind or t_start and t_end, not both + """ + debug = debug or self._debug + if debug: + print 'Computing vertical shear at point...' + + # Find time interval to work in + argtime = [] + if not time_ind==[]: + argtime = time_ind + elif not t_start==[]: + if type(t_start)==str: + start = datetime.datetime.strptime(t_start, '%Y-%m-%d %H:%M:%S') + end = datetime.datetime.strptime(t_end, '%Y-%m-%d %H:%M:%S') + argtime = time_to_index(start, end, self._var.julianTime[:], debug=debug) + else: + argtime = np.arange(t_start, t_end) + + #Compute depth + depth = self._var.depth[:] + + #Extracting velocity at point + if not argtime==[]: + U = self._var.east_vel[argtime,:] + V = self._var.north_vel[argtime,:] + depths = depth[argtime,:] + else: + U = self._var.east_vel[:,:] + V = self._var.north_vel[:,:] + depths = depth[:,:] + + norm = ne.evaluate('sqrt(U**2 + V**2)').squeeze() + + # Compute shear + dz = depths[:,1:] - depths[:,:-1] + dvel = norm[:,1:] - norm[:,:-1] + dveldz = dvel / dz + + if debug: + print '...Passed' + + #Plot mean values + if graph: + mean_depth = np.mean((depths[:,1:] + depths[:,:-1]) / 2.0, axis=0) + mdat = np.ma.masked_array(dveldz,np.isnan(dveldz)) + mean_dveldz = np.mean(mdat,0) + error = np.std(mdat,axis=0) + self._plot.plot_xy(mean_dveldz, mean_depth, xerror=error[:], + title='Shear profile ', + xLabel='Shear (1/s) ', yLabel='Depth (m) ', + dump=dump, **kwargs) + + return dveldz + + def velo_norm(self, t_start=[], t_end=[], time_ind=[], + graph=True, dump=False, debug=False, **kwargs): + """ + Compute the velocity norm + + Outputs: + - velo_norm = velocity norm, 2D array (time, level) + + Options: + - t_start = start time, as a string ('yyyy-mm-ddThh:mm:ss'), + or time index as an integer + - t_end = end time, as a string ('yyyy-mm-ddThh:mm:ss'), + or time index as an integer + - time_ind = time indices to work in, list of integers + - graph = plots vertical profile averaged over time if True + - dump = boolean, dump profile data in csv file + - kwargs = keyword options associated with pandas.DataFrame.to_csv, such as: + sep, header, na_rep, index,...etc + Check doc. of "to_csv" for complete list of options + + *Notes* + - use time_ind or t_start and t_end, not both + """ + debug = debug or self._debug + if debug: + print 'Computing velocity norm at point...' + + # Find time interval to work in + argtime = [] + if not time_ind==[]: + argtime = time_ind + elif not t_start==[]: + if type(t_start)==str: + start = datetime.datetime.strptime(t_start, '%Y-%m-%d %H:%M:%S') + end = datetime.datetime.strptime(t_end, '%Y-%m-%d %H:%M:%S') + argtime = time_to_index(start, end, self._var.julianTime[:], debug=debug) + else: + argtime = np.arange(t_start, t_end) + + #Computing velocity norm + if not argtime==[]: + U = self._var.east_vel[argtime, :] + V = self._var.north_vel[argtime, :] + velo_norm = ne.evaluate('sqrt(U**2 + V**2)').squeeze() + else: + U = self._var.east_vel[:, :] + V = self._var.north_vel[:, :] + velo_norm = ne.evaluate('sqrt(U**2 + V**2)').squeeze() + + if debug: + print '...passed' + + #Plot mean values + if graph: + depth = self._var.depth + mdat = np.ma.masked_array(velo_norm,np.isnan(velo_norm)) + vel = np.mean(mdat,0) + error = np.std(mdat,axis=0) + self._plot.plot_xy(vel, depth.mean(axis=0), xerror=error[:], + title='Velocity norm profile ', + xLabel='Velocity (m/s) ', yLabel='Depth (m) ', + dump=dump, **kwargs) + + + return velo_norm + + def mattime2datetime(self, mattime, debug=False): + """ + Output the time (yyyy-mm-dd, hh:mm:ss) corresponding to + a given matlab time + + Inputs: + - mattime = matlab time (floats) + """ + time = mattime_to_datetime(mattime, debug=debug) + return time + +#TR_comments: templates +# def whatever(self, debug=False): +# if debug or self._debug: +# print 'Start whatever...' +# +# if debug or self._debug: +# print '...Passed' diff --git a/build/lib/pyseidon_dvt/adcpClass/plotsAdcp.py b/build/lib/pyseidon_dvt/adcpClass/plotsAdcp.py new file mode 100644 index 0000000..4d99900 --- /dev/null +++ b/build/lib/pyseidon_dvt/adcpClass/plotsAdcp.py @@ -0,0 +1,171 @@ +#!/usr/bin/python2.7 +# encoding: utf-8 + +from __future__ import division +import numpy as np +import matplotlib.pyplot as plt +import matplotlib.tri as Tri +import matplotlib.ticker as ticker +import matplotlib.patches as mpatches +import seaborn +import pandas as pd +from windrose import WindroseAxes +from interpolation_utils import * + +class PlotsAdcp: + """ **'Plots' subset of FVCOM class gathers plotting functions**""" + def __init__(self, variable, debug=False): + self._debug = debug + setattr(self, '_var', variable) + + return + + def _def_fig(self): + """Defines figure window""" + self._fig = plt.figure(figsize=(18,10)) + plt.rc('font',size='22') + + def plot_xy(self, x, y, xerror=[], yerror=[], + title=' ', xLabel=' ', yLabel=' ', dump=False, **kwargs): + """ + Simple X vs Y plot + + Inputs: + - x = 1D array + - y = 1D array + + Options: + - xerror = error on 'x', 1D array + - yerror = error on 'y', 1D array + - title = plot title, string + - xLabel = title of the x-axis, string + - yLabel = title of the y-axis, string + - dump = boolean, dump profile data in csv file + - kwargs = keyword options associated with pandas.DataFrame.to_csv, such as: + sep, header, na_rep, index,...etc + Check doc. of "to_csv" for complete list of options + """ + self._def_fig() + self._ax = self._fig.add_subplot(111) + self._ax.plot(x, y, label=title) + scale = 1 + self._ax.set_ylabel(yLabel) + self._ax.set_xlabel(xLabel) + self._ax.get_xaxis().set_minor_locator(ticker.AutoMinorLocator()) + self._ax.get_yaxis().set_minor_locator(ticker.AutoMinorLocator()) + self._ax.grid(b=True, which='major', color='w', linewidth=1.5) + self._ax.grid(b=True, which='minor', color='w', linewidth=0.5) + if not yerror==[]: + self._ax.fill_between(x, y-yerror, y+yerror, + alpha=0.2, edgecolor='#1B2ACC', facecolor='#089FFF', antialiased=True) + if not xerror==[]: + self._ax.fill_betweenx(y, x-xerror, x+xerror, + alpha=0.2, edgecolor='#1B2ACC', facecolor='#089FFF', antialiased=True) + if (not xerror==[]) or (not yerror==[]): + blue_patch = mpatches.Patch(color='#089FFF', + label='Standard deviation',alpha=0.2) + plt.legend(handles=[blue_patch],loc=1, fontsize=12) + #plt.legend([blue_patch],loc=1, fontsize=12) + + self._fig.show() + if dump: + self._dump_profile_data_as_csv(x, y,xerror=xerror, yerror=yerror, + title=title, xLabel=xLabel, + yLabel=yLabel, **kwargs) + + def Histogram(self, y, title=' ', xLabel=' ', yLabel=' ', dump=False, **kwargs): + """ + Histogram plot + + Inputs: + - bins = list of bin edges + - y = 1D array + + Options: + - title = plot title, string + - xLabel = title of the x-axis, string + - yLabel = title of the y-axis, string + - kwargs = keyword options associated with pandas.DataFrame.to_csv, such as: + sep, header, na_rep, index,...etc + Check doc. of "to_csv" for complete list of options + """ + self._def_fig() + self._ax = self._fig.add_subplot(111) + density, bins = np.histogram(y, bins=50, normed=True, density=True) + unity_density = density / density.sum() + widths = bins[:-1] - bins[1:] + # To plot correct percentages in the y axis + self._ax.bar(bins[1:], unity_density, width=widths) + formatter = ticker.FuncFormatter(lambda v, pos: str(v * 100)) + self._ax.yaxis.set_major_formatter(formatter) + self._ax.get_xaxis().set_minor_locator(ticker.AutoMinorLocator()) + self._ax.get_yaxis().set_minor_locator(ticker.AutoMinorLocator()) + self._ax.grid(b=True, which='major', color='w', linewidth=1.5) + self._ax.grid(b=True, which='minor', color='w', linewidth=0.5) + + plt.ylabel(yLabel) + plt.xlabel(xLabel) + + self._fig.show() + + if dump: self._dump_profile_data_as_csv(bins[1:], unity_density, + title=title, xLabel=xLabel, + yLabel=yLabel, **kwargs) + + def rose_diagram(self, direction, norm): + + """ + Plot rose diagram + + Inputs: + - direction = 1D array + - norm = 1D array + """ + #Convertion + #TR: not quite sure here, seems to change from location to location + # express principal axis in compass + direction = np.mod(90.0 - direction, 360.0) + + #Create new figure + self._def_fig() + rect = [0.1, 0.1, 0.8, 0.8] + ax = WindroseAxes(self._fig, rect)#, axisbg='w') + self._fig.add_axes(ax) + #Rose + ax.bar(direction, norm , normed=True, opening=0.8, edgecolor='white') + #adjust legend + l = ax.legend(shadow=True, bbox_to_anchor=[-0.1, 0], loc='lower left') + plt.setp(l.get_texts(), fontsize=10) + plt.xlabel('Rose diagram in % of occurrences - Colormap of norms') + self._fig.show() + + def _dump_profile_data_as_csv(self, x, y, xerror=[], yerror=[], + title=' ', xLabel=' ', yLabel=' ', **kwargs): + """ + Dumps profile data in csv file + + Inputs: + - x = 1D array + - y = 1D array + + Options: + - xerror = error on 'x', 1D array + - yerror = error on 'y', 1D array + - title = file name, string + - xLabel = name of the x-data, string + - yLabel = name of the y-data, string + - kwargs = keyword options associated with pandas.DataFrame.to_csv, such as: + sep, header, na_rep, index,...etc + Check doc. of "to_csv" for complete list of options + """ + if title == ' ': title = 'dump_profile_data' + filename=title + '.csv' + if xLabel == ' ': xLabel = 'X' + if yLabel == ' ': yLabel = 'Y' + if not xerror == []: + df = pd.DataFrame({xLabel:x[:], yLabel:y[:], 'error': xerror[:]}) + elif not yerror == []: + df = pd.DataFrame({xLabel:x[:], yLabel:y[:], 'error': yerror[:]}) + else: + df = pd.DataFrame({xLabel:x[:], yLabel:y[:]}) + df.to_csv(filename, encoding='utf-8', **kwargs) diff --git a/build/lib/pyseidon_dvt/adcpClass/rawADCPclass.py b/build/lib/pyseidon_dvt/adcpClass/rawADCPclass.py new file mode 100644 index 0000000..c638e43 --- /dev/null +++ b/build/lib/pyseidon_dvt/adcpClass/rawADCPclass.py @@ -0,0 +1,116 @@ +from __future__ import division +import scipy.io as sio +import h5py +from os import path + +class Struct: + def __init__(self, **entries): + self.__dict__.update(entries) + + +class rawADCP: + def __init__(self, filename): + self.QC = ['raw data'] + self.load(filename) + self.Params_Stn4_SWNSreport(filename) + self.load_rbrdata() + + ## set options + self.options = {} + self.options['showPA'] = 1 + self.options['showRBRavg'] = 1 + + ## save a flow file in BPformat + #save_FlowFile_BPFormat(fileinfo,adcp,rbr,saveparams,options) + + return + + def load(self, filename): + + try: + self.mat = sio.loadmat(filename, + struct_as_record=False, squeeze_me=True) + + self.adcp = self.mat['adcp'] + + except NotImplementedError: + self.mat = h5py.File(filename) + self.adcp = self.mat['adcp'] + #self.adcp = Struct(**self.mat['adcp']) + + def Params_Stn4_SWNSreport(self, filename): + fname = filename.split('/') + filebase = fname[-1].split('_')[0] + self.fileinfo = {} + self.fileinfo['datadir'] = path.join(*fname[:-1]) + '/' + self.fileinfo['ADCP'] = filebase + '_raw' + self.fileinfo['outdir'] = path.join(*fname[:-1]) + '/' + self.fileinfo['flowfile'] = filebase + '_Flow' + self.fileinfo['rbr']= 'station4_grandPassageII_RBRSN_011857.mat' + self.fileinfo['paramfile']= 'Params_Stn4_SWNSreport' + + #%% ADCP parameters + self.saveparams = {} + self.saveparams['tmin'] = 209 + self.saveparams['tmax'] = 240 + self.saveparams['zmin'] = 0 + self.saveparams['zmax'] = 20 + self.saveparams['approxdepth'] = 15.5 + self.saveparams['flooddir'] = 0 + self.saveparams['declination'] = -17.25 + self.saveparams['lat'] = 44.2605 + self.saveparams['lon'] = -66.3354 + self.saveparams['dabADCP'] = 0.5 + self.saveparams['dabPS'] = -0.6 + self.saveparams['rbr_hr_offset'] = 3 + + def load_rbrdata(self): + rbrFile = self.fileinfo['datadir'] + self.fileinfo['rbr'] + + try: + rbrMat = sio.loadmat(rbrFile, + struct_as_record=False, squeeze_me=True) + + except NotImplementedError: + rbrMat = h5py.File(rbrFile) + + rbr = rbrMat['rbr'] + rbrout = {} + rbrout['mtime'] = rbr.yd + + rbrout['temp'] = rbr.temperature + rbrout['pres'] = rbr.pressure + rbrout['depth'] = rbr.depth + rbrout['mtime'] = rbr.yd + self.rbr = rbrout + +if __name__ == '__main__': + #filename = 'GP-120726-BPd_raw.mat' + filename = '140703-EcoEII_database/data/GP-120726-BPd_raw.mat' + data = rawADCP(filename) + + + + +#stn = 'GP-120726-BPd'; +#%% File information +#fileinfo.datadir = '../data/'; %path to raw data files +#fileinfo.ADCP = [stn '_raw']; %name of ADCP file +#fileinfo.outdir = '../data/'; %path to output directory +#fileinfo.flowfile = [stn,'_Flow']; %name of output file with Flow data +#fileinfo.rbr = ['station4_grandPassageII_RBRSN_011857.mat']; +#fileinfo.paramfile = mfilename; +# +#%% ADCP parameters +#saveparams.tmin = 209; %tmin (year day) +#saveparams.tmax = 240; %tmax (year day) +#saveparams.zmin = 0; %minimum z to include in saves file +#saveparams.zmax = 20; +#saveparams.approxdepth = 15.5; %Approximate depth +#saveparams.flooddir= 0; %Flood direction (relative to true north, CW is positive) +#saveparams.declination = -17.25;%Declination angle +#saveparams.lat = 44.2605; %latitude +#saveparams.lon = -66.3354; %longitude +#saveparams.dabADCP = 0.5; %depth above bottom of ADCP +#saveparams.dabPS = -0.6; %depth above bottom of pressure sensor +#saveparams.rbr_hr_offset = 3; % hour offset to convert rbr time to UTC diff --git a/build/lib/pyseidon_dvt/adcpClass/variablesAdcp.py b/build/lib/pyseidon_dvt/adcpClass/variablesAdcp.py new file mode 100644 index 0000000..f72c0f7 --- /dev/null +++ b/build/lib/pyseidon_dvt/adcpClass/variablesAdcp.py @@ -0,0 +1,225 @@ +#!/usr/bin/python2.7 +# encoding: utf-8 + +from __future__ import division +import numpy as np +from numpy.ma import MaskError +import h5py +from pyseidon_dvt.utilities.miscellaneous import mattime_to_datetime + +# Custom error +from pyseidon_error import PyseidonError + +class _load_adcp: + """ + **'Variables' subset in ADCP class** + + It contains the following numpy arrays: :: + + _bins = depth of measurement bins, 1D array, shape=(bins) + |_depth = depth, negative from surface down, time serie, 2D array, shape=(time,bins) + |_dir_vel = velocity direction time serie, 2D array, shape=(time,bins) + |_east_vel = East velocity time serie, 2D array, shape=(time,bins) + |_lat = latitude, float, decimal degrees + |_lon = lontitude, float, decimal degrees + |_mag_signed_vel = signed velocity time serie, 2D array, shape=(time,bins) + |_matlabTime = matlab time, 1D array, shape=(time) + ADCP.Variables._|_north_vel = East velocity time serie, 2D array, shape=(time,bins) + |_percent_of_depth = percent of the water column measured by ADCP, float + |_surf = pressure at surface timeserie, 1D array, shape=(time) + |_el = elevation (m) at surface timeserie, 1D array, shape=(time) + |_ua = depth averaged velocity component timeserie, 1D array, shape=(time) + |_va = depth averaged velocity component timeserie, 1D array, shape=(time) + |_ucross = ???, 1D array, shape=(time) + |_ualong = ???, 1D array, shape=(time) + + """ + def __init__(self,cls, History, debug=False): + if debug: + print 'Loading variables...' + # Pointer to History + setattr(self, '_History', History) + + + # TR: fudge factor, squeeze out the 5 top % of the water column + self.percent_of_depth=0.95 + + # Convert some mat_struct objects into a dictionaries. + # This will enable compatible syntax to h5py files. + if not type(cls.Data) == h5py._hl.files.File: + try: + cls.Data['data'] = cls.Data['data'].__dict__ + except KeyError: + raise PyseidonError("Missing 'data' field from ADCP file.") + for key in cls.Data['data']: + if key is not '_fieldnames': + # in ther is a mat_struct in the mat_struct, like 'surf' + if hasattr(cls.Data['data'][key], '__module__') and \ + cls.Data['data'][key].__module__ == 'scipy.io.matlab.mio5_params': + cls.Data['data'][key] = cls.Data['data'][key].__dict__ + for kk in cls.Data['data'][key]: + if kk is not '_fieldnames': + cls.Data['data'][key][kk] = cls.Data['data'][key][kk].T + else: + cls.Data['data'][key] = cls.Data['data'][key].T + + # Convert other fields to dictionaries if they exist. + if 'pres' in cls.Data: + cls.Data['pres']=cls.Data['pres'].__dict__ + if 'time' in cls.Data: + cls.Data['time']=cls.Data['time'].__dict__ + + + try: + self.lat = np.ravel(cls.Data['lat'])[0] + self.lon = np.ravel(cls.Data['lon'])[0] + except KeyError: + if debug: + print 'Missing lon and/or lat data' + + try: + self.bins = cls.Data['data']['bins'][:].ravel() + except KeyError: + if debug: + print 'Missing bins' + + try: + self.north_vel = cls.Data['data']['north_vel'][:].T + self.east_vel = cls.Data['data']['east_vel'][:].T + self.v = self.north_vel + self.u = self.east_vel + try: + self.north_vel=np.ma.masked_array(self.north_vel,np.isnan(self.north_vel)) + self.east_vel=np.ma.masked_array(self.east_vel,np.isnan(self.east_vel)) + self.v=self.north_vel + self.u=self.east_vel + except MaskError: + print 'Failed to mask horizontal velocities (north_vel, east_vel)' + except KeyError: + if debug: + print 'Missing horizontal velocities (north_vel, east_vel)' + + try: + self.vert_vel = cls.Data['data']['vert_vel'][:].T + try: + self.vert_vel=np.ma.masked_array(self.vert_vel,np.isnan(self.vert_vel)) + except MaskError: + print 'Failed to mask vertical velocity (vert_vel)' + except KeyError: + if debug: + print 'Missing vertical velocity (vert_vel)' + + try: + self.dir_vel = cls.Data['data']['dir_vel'][:].T + try: + self.dir_vel=np.ma.masked_array(self.dir_vel,np.isnan(self.dir_vel)) + except MaskError: + print 'Failed to mask dir_vel' + except KeyError: + if debug: + print 'Missing dir_vel' + + try: + self.mag_signed_vel = cls.Data['data']['mag_signed_vel'][:].T + try: + self.mag_signed_vel=np.ma.masked_array(self.mag_signed_vel,np.isnan(self.mag_signed_vel)) + except MaskError: + print 'Failed to mask mag_signed_vel' + except KeyError: + if debug: + print 'Missing mag_signed_vel' + + try: + self.pressure = cls.Data['pres'] + self.surf = self.pressure['surf'][:].ravel() + self.el = self.surf + except KeyError: + if debug: + print 'Missing elevation data (pres.surf)' + + try: + self.time = cls.Data['time'] + self.matlabTime = self.time['mtime'][:].ravel() + except KeyError: + if debug: + print 'Missing time data (time.mtime)' + + try: + self.ucross = cls.Data['data']['Ucross'][:].T + self.ualong = cls.Data['data']['Ualong'][:].T + except KeyError: + if debug: + print 'Missing along/cross velocities (Ucross, Ualong)' + + try: + self.ua = cls.Data['data']['ua'][:].T + self.va = cls.Data['data']['va'][:].T + try: + self.ua=np.ma.masked_array(self.ua,np.isnan(self.ua)) + self.va=np.ma.masked_array(self.va,np.isnan(self.va)) + except MaskError: + print 'Failed to mask depth averaged velocities (ua, va)' + except KeyError: + if debug: + print 'Missing depth averaged velocities (ua, va)' + + + #-Append message to History field + try: + start = mattime_to_datetime(self.matlabTime[0]) + end = mattime_to_datetime(self.matlabTime[-1]) + text = 'Temporal domain from ' + str(start) +\ + ' to ' + str(end) + self._History.append(text) + except AttributeError: + if debug: + print 'Missing time variable failed to add history note' + + #Find the depth average of a variable based on percent_of_depth + #choosen by the user but only if not loaded from file directly. + # Currently only working for east_vel (u) and north_vel (v) + if (('ua' not in dir(self)) and ('va' not in dir(self))): + try: + #TR: alternative with percent of the depth + ind = np.argwhere(self.bins < self.percent_of_depth * self.surf[:,np.newaxis]) + #ind = np.argwhere(self.bins < self.surf[:,np.newaxis]) + index = ind[np.r_[ind[1:,0] != ind[:-1,0], True]] + try: + data_ma_u = np.ma.array(self.east_vel, + mask=np.arange(self.east_vel.shape[1]) > index[:, 1, np.newaxis]) + data_ma_u=np.ma.masked_array(data_ma_u,np.isnan(data_ma_u)) + except MaskError: + data_ma_u=np.ma.masked_array(self.east_vel,np.isnan(self.east_vel)) + + try: + data_ma_v = np.ma.array(self.north_vel, + mask=np.arange(self.north_vel.shape[1]) > index[:, 1, np.newaxis]) + data_ma_v=np.ma.masked_array(data_ma_v,np.isnan(data_ma_v)) + except MaskError: + data_ma_v=np.ma.masked_array(self.north_vel,np.isnan(self.north_vel)) + + self.ua = np.array(data_ma_u.mean(axis=1)) + self.va = np.array(data_ma_v.mean(axis=1)) + except AttributeError: + if debug: + print 'Missing atleast one variable required ' + \ + 'to compute depth averaged velocities' + + + # Compute depth with fvcom convention, negative from surface down + try: + self.depth = np.ones(self.north_vel.shape) * np.nan + for t in range(self.matlabTime.shape[0]): + #i = np.where(np.isnan(self.north_vel[t,:]))[0][0] + #z = self.bins[i] + self.depth[t, :] = self.bins[:] - self.surf[t] + self.depth[np.where(self.depth>0.0)] = np.nan + except AttributeError: + if debug: + print 'Missing atleast one variable required ' + \ + 'to compute depth with fvcom convention' + + if debug: + print '...Passed' + + return diff --git a/build/lib/pyseidon_dvt/drifterClass/__init__.py b/build/lib/pyseidon_dvt/drifterClass/__init__.py new file mode 100644 index 0000000..ff1dde5 --- /dev/null +++ b/build/lib/pyseidon_dvt/drifterClass/__init__.py @@ -0,0 +1,10 @@ +#!/usr/bin/python2.7 +# encoding: utf-8 + +#Local import +from drifterClass import Drifter + +__authors__ = ['Kody Crowell, Thomas Roc, Wesley Bowman, Jonathan Smith'] +__licence__ = 'GNU Affero GPL v3.0' +__copyright__ = 'Copyright (c) 2014 EcoEnergyII' + diff --git a/build/lib/pyseidon_dvt/drifterClass/drifterClass.py b/build/lib/pyseidon_dvt/drifterClass/drifterClass.py new file mode 100644 index 0000000..c6b21f2 --- /dev/null +++ b/build/lib/pyseidon_dvt/drifterClass/drifterClass.py @@ -0,0 +1,86 @@ +#!/usr/bin/python2.7 +# encoding: utf-8 + +from __future__ import division + +import scipy.io as sio +import h5py + +#Local import +from variablesDrifter import _load_drifter +# from functionsDrifter import FunctionsDrifter +from plotsDrifter import * + + +class Drifter: + """ + **A class/structure for Drifter data** + + Functionality structured as follows: :: + + _Data. = raw matlab file data + |_Variables. = useable drifter variables and quantities + |_History = Quality Control metadata + testAdcp._|_Utils. = set of useful functions + |_Plots. = plotting functions + |_method_1 + | ... = methods and analysis techniques intrinsic to drifters + |_method_n + + Inputs: + - Only takes a file name as input, ex: testDrifter=Drifter('./path_to_matlab_file/filename') + + Notes: + Only handle fully processed drifter matlab data previously quality-controlled at the mo. + + Throughout the package, the following conventions apply: + - Coordinates = decimal degrees East and North + - Directions = in degrees, between -180 and 180 deg., i.e. 0=East, 90=North, + +/-180=West, -90=South + - Depth = 0m is the free surface and depth is negative + """ + def __init__(self, filename, debug=False): + """ Initialize Drifter class. + Notes: only handle processed Drifter matlab data at the mo.""" + self._debug = debug + self._origin_file = filename + if debug: + print '-Debug mode on-' + #Load data from Matlab file + #TR_comments: find a way to dissociate raw and processed data + #TR_comments: *_Raw and *_10minavg open with h5py whereas *_davgBS + try: + self.Data = sio.loadmat(filename,struct_as_record=False, squeeze_me=True) + except NotImplementedError: + self.Data = h5py.File(filename) + + #Store info in "History" field + self.History = ['Created from ' + filename] + + # KC comment: for some reason, some drifter MATLAB files + # have 'Comments' as a key in the variables structure, + # while others have 'comments' as a key. + if debug: print '-adding comments to History-' + + if 'Comments' in self.Data: + for comment in self.Data['Comments'][:]: + self.History.append(str(comment)) + elif 'comments' in self.Data: + for comment in self.Data['comments'][:]: + self.History.append(str(comment)) + elif debug: + print '-no comments found-' + + + #Initialize class structure + self.Variables = _load_drifter(self, self.History, debug=self._debug) + self.Plots = PlotsDrifter(self.Variables, debug=self._debug) + #self.Utils = FunctionsAdcp(self.Variables, + # self.Plots, + # self.History, + # debug=self._debug) + + ##Re-assignement of utility functions as methods + self.dump_data_as_csv = self.Plots._dump_data_as_csv + + return \ No newline at end of file diff --git a/build/lib/pyseidon_dvt/drifterClass/functionsDrifter.py b/build/lib/pyseidon_dvt/drifterClass/functionsDrifter.py new file mode 100644 index 0000000..c8348c5 --- /dev/null +++ b/build/lib/pyseidon_dvt/drifterClass/functionsDrifter.py @@ -0,0 +1,20 @@ +#!/usr/bin/python2.7 +# encoding: utf-8 + +from __future__ import division + +class FunctionsDrifter: + """**'Utils' subset of Tidegauge class gathers useful functions**""" + def __init__(self,cls): + self._var = cls.Variables + self._debug = cls._debug + + return + +#TR_comments: templates +# def whatever(self, debug=False): +# if debug or self._debug: +# print 'Start whatever...' +# +# if debug or self._debug: +# print '...Passed' diff --git a/build/lib/pyseidon_dvt/drifterClass/plotsDrifter.py b/build/lib/pyseidon_dvt/drifterClass/plotsDrifter.py new file mode 100644 index 0000000..1e8991b --- /dev/null +++ b/build/lib/pyseidon_dvt/drifterClass/plotsDrifter.py @@ -0,0 +1,122 @@ +#!/usr/bin/python2.7 +# encoding: utf-8 + +from __future__ import division +import numpy as np +import matplotlib.pyplot as plt +import matplotlib.ticker as ticker +import seaborn +import pandas as pd + +class PlotsDrifter: + """ + **'Plots' subset of Drifter class gathers plotting functions** + """ + def __init__(self, variable, debug): + self._debug = debug + # Pointer + setattr(self, '_var', variable) + + return + + def _def_fig(self): + """Defines figure window""" + self._fig = plt.figure(figsize=(18,10)) + plt.rc('font',size='22') + + def trajectories(self, title='Drifter trajectories & speed (m/s)', + cmin=[], cmax=[], debug=False, dump=False, **kwargs): + """ + 2D xy colormap plot of all the trajectories. + Colors represent the drifter velocity + + Options: + - title = plot title, string + - cmin = minimum limit colorbar + - cmax = maximum limit colorbar + - dump = boolean, dump profile data in csv file + - kwargs = keyword options associated with pandas.DataFrame.to_csv, such as: + sep, header, na_rep, index,...etc + Check doc. of "to_csv" for complete list of options + """ + debug = debug or self._debug + if debug: print "Plotting drifter's trajectories..." + lon = self._var.lon + lat = self._var.lat + + #Compute drifter's speeds + u=self._var.u + v=self._var.v + norm=np.sqrt(u**2.0 + v**2.0) + + #setting limits and levels of colormap + if cmin==[]: + if debug: + print "Computing cmin..." + cmin=norm[:].min() + if cmax==[]: + if debug: + print "Computing cmax..." + cmax=norm[:].max() + + #Figure window params + self._def_fig() + self._ax = self._fig.add_subplot(111,aspect=(1.0/np.cos(np.mean(lat)*np.pi/180.0))) + + #Scatter plot + #sc = plt.scatter(lon, lat, c=norm, lw=0, cmap=plt.cm.gist_earth) + #sc.set_clim([cmin,cmax]) + + #Quiver plot + sc = plt.quiver(lon, lat, u, v, norm, lw=0.0, scale=100.0, + cmap=plt.cm.gist_earth) + sc.set_clim([cmin,cmax]) + + #Label and axis parameters + self._ax.set_ylabel('Latitude') + self._ax.set_xlabel('Longitude') + plt.gca().patch.set_facecolor('0.5') + cbar=self._fig.colorbar(sc,ax=self._ax) + cbar.set_label(title, rotation=-90,labelpad=30) + scale = 1 + ticks = ticker.FuncFormatter(lambda lon, pos: '{0:g}'.format(lon/scale)) + self._ax.xaxis.set_major_formatter(ticks) + self._ax.yaxis.set_major_formatter(ticks) + self._ax.grid() + self._fig.show() + if dump: + self._dump_data_as_csv(norm, u, v, lon, lat, title='drifter_velocity', **kwargs) + if debug or self._debug: + print '...Passed' + + def _dump_data_as_csv(self, var1, var2, var3, x, y, title=' ', **kwargs): + """ + Dumps map data in csv file + + Inputs: + - var1 = 1 D numpy array oh n elements + - var2 = 1 D numpy array oh n elements + - var3 = 1 D numpy array oh n elements + - x = coordinates, 1 D numpy array (nele or nnode) + - y = coordinates, 1 D numpy array (nele or nnode) + + Options: + - title = file name, string + - kwargs = keyword options associated with pandas.DataFrame.to_csv, such as: + sep, header, na_rep, index,...etc + Check doc. of "to_csv" for complete list of options + """ + if title == ' ': title = 'dump_map_data' + filename=title + '.csv' + df = pd.DataFrame({'norm': var1, 'u':var2, 'v':var3, + 'lon':x[:], 'lat':y[:]}) + + df.to_csv(filename, encoding='utf-8', **kwargs) + +#TR_comments: templates +# def whatever(self, debug=False): +# if debug or self._debug: +# print 'Start whatever...' +# +# if debug or self._debug: +# print '...Passed' diff --git a/build/lib/pyseidon_dvt/drifterClass/variablesDrifter.py b/build/lib/pyseidon_dvt/drifterClass/variablesDrifter.py new file mode 100644 index 0000000..1b2a43b --- /dev/null +++ b/build/lib/pyseidon_dvt/drifterClass/variablesDrifter.py @@ -0,0 +1,73 @@ +#!/usr/bin/python2.7 +# encoding: utf-8 + +from __future__ import division +import numpy as np +# from numpy.ma import MaskError +# import h5py +from pyseidon_dvt.utilities.miscellaneous import mattime_to_datetime +import sys + +class _load_drifter: + """ + **'Variables' subset in Tidegauge class** + It contains the following numpy arrays: :: + + _u = u velocity component (m/s), 1D array (ntime) + |_v = v velocity component (m/s), 1D array (ntime) + Drifter.Variables._|_matlabTime = matlab time, 1D array (ntime) + |_lon = longitudes (deg.), 1D array (ntime) + |_lat = latitudes (deg.), 1D array (ntime) + + """ + def __init__(self,cls, History, debug=False): + if debug: + print 'Loading variables...' + # Pointer to History + setattr(self, '_History', History) + try: + self.matlabTime = cls.Data['velocity'].vel_time[:] + #Sorting values with increasing time step + sortedInd = self.matlabTime.argsort() + self.matlabTime.sort() + self.lat = cls.Data['velocity'].vel_lat[sortedInd] + self.lon = cls.Data['velocity'].vel_lon[sortedInd] + self.u = cls.Data['velocity'].u[sortedInd] + self.v = cls.Data['velocity'].v[sortedInd] + # Luna Ocean Consulting Ltd. new drifter format + except KeyError: + self.matlabTime = cls.Data['time'] + self.original = {} + self.original['lat'] = cls.Data['lat'] + self.original['lon'] = cls.Data['lon'] + self.original['u'] = cls.Data['u'] + self.original['v'] = cls.Data['v'] + self.original['drift_start'] = cls.Data['drift_start'] + self.original['drift_stop'] = cls.Data['drift_stop'] + self.smooth = {} + self.smooth['lat'] = cls.Data['lat_smooth'] + self.smooth['lon'] = cls.Data['lon_smooth'] + self.smooth['u'] = cls.Data['u_smooth'] + self.smooth['v'] = cls.Data['v_smooth'] + self.smooth['drift_start'] = cls.Data['drift_start_smooth'] + self.smooth['drift_stop'] = cls.Data['drift_stop_smooth'] + except: + sys.exit('Drifter file format incompatible') + + #-Append message to History field + try: + start = mattime_to_datetime(self.matlabTime[0]) + end = mattime_to_datetime(self.matlabTime[-1]) + text = 'Temporal domain from ' + str(start) +\ + ' to ' + str(end) + self._History.append(text) + except ValueError: + start = mattime_to_datetime(np.nanmin(self.matlabTime)) + end = mattime_to_datetime(np.nanmax(self.matlabTime)) + text = 'Temporal domain from ' + str(start) +\ + ' to ' + str(end) + self._History.append(text) + + if debug: print '...Passed' + + return diff --git a/build/lib/pyseidon_dvt/fvcomClass/__init__.py b/build/lib/pyseidon_dvt/fvcomClass/__init__.py new file mode 100644 index 0000000..93446d3 --- /dev/null +++ b/build/lib/pyseidon_dvt/fvcomClass/__init__.py @@ -0,0 +1,10 @@ +#!/usr/bin/python2.7 +# encoding: utf-8 + +#Local import +from fvcomClass import FVCOM + +__authors__ = ['Thomas Roc, Wesley Bowman, Jonathan Smith'] +__licence__ = 'GNU Affero GPL v3.0' +__copyright__ = 'Copyright (c) 2014 EcoEnergyII' + diff --git a/build/lib/pyseidon_dvt/fvcomClass/functionsFvcom.py b/build/lib/pyseidon_dvt/fvcomClass/functionsFvcom.py new file mode 100644 index 0000000..6458786 --- /dev/null +++ b/build/lib/pyseidon_dvt/fvcomClass/functionsFvcom.py @@ -0,0 +1,1160 @@ +#!/usr/bin/python2.7 +# encoding: utf-8 + +from __future__ import division + +from scipy.interpolate import interp1d +import numexpr as ne +import datetime +from pyseidon_dvt.utilities.interpolation_utils import * +from pyseidon_dvt.utilities.miscellaneous import * +from pyseidon_dvt.utilities.BP_tools import * +from utide import solve, reconstruct +import time +import matplotlib.tri as Tri +from pydap.exceptions import ServerError +from pyseidon_dvt.utilities.pyseidon_error import PyseidonError + +class FunctionsFvcom: + """ + **'Util2D' subset of FVCOM class gathers useful functions and methods for 2D and 3D runs** + """ + def __init__(self, variable, grid, plot, History, debug): + self._debug = debug + self._plot = plot + #Create pointer to FVCOM class + setattr(self, '_var', variable) + setattr(self, '_grid', grid) + setattr(self, '_History', History) + + return + + #TR comment: I don't think I need this anymore + def _centers(self, debug=False): + """ + Create new variable 'bathy and elevation at center points' (m) + -> FVCOM.Grid.hc, elc + + *Notes* + - Can take time over the full domain + """ + debug = debug or self._debug + if debug: + print 'Computing central bathy...' + + #Interpolation at centers + size = self._grid.nele + size1 = self._grid.ntime + elc = np.zeros((size1, size)) + hc = np.zeros((size)) + #TR comment: I am dubeous about the interpolation method here + for ind, value in enumerate(self._grid.trinodes[:]): + value.sort()#due to new version of netCDF4 + elc[:, ind] = np.mean(self._var.el[:, value], axis=1) + hc[ind] = np.mean(self._grid.h[value]) + + #Custom return + setattr(self._grid, 'hc', hc) + setattr(self._var, 'elc', elc) + + # Add metadata entry + self._History.append('bathymetry at center points computed') + self._History.append('elevation at center points computed') + print '-Central bathy and elevation added to FVCOM.Grid.-' + + if debug: + print '...Passed' + + def slope(self, debug=False): + """ + This method computes a new variable: 'bathymetric slope' (degrees) + -> FVCOM.Grid.slope + """ + x = self._grid.x[:] + y = self._grid.y[:] + if not hasattr(self._grid, 'triangleXY'): + # Mesh triangle + if debug: + print "Computing triangulation..." + trinodes = self._grid.trinodes[:] + tri = Tri.Triangulation(x, y, triangles=trinodes) + self._grid.triangleXY = tri + else: + tri = self._grid.triangleXY + + if debug: print "Cubic interpolation..." + try: + tci = Tri.CubicTriInterpolator(tri, self._grid.h[:]) + except ValueError: # quick fix for library incompatibility on Acadia's server + tci = Tri.CubicTriInterpolator(tri, self._grid.h[:].copy()) + (Ex, Ey) = tci.gradient(tri.x, tri.y) + slope = np.sqrt(Ex**2 + Ey**2) + + if debug: print "Conversion to degrees..." + slope = np.rad2deg(np.arctan(slope)) + + # Custom return + setattr(self._grid, 'slope', slope) + + # Add metadata entry + self._History.append('bathymetric slope computed') + print '-Bathymetric slope added to FVCOM.Grid.-' + + def hori_velo_norm(self, debug=False): + """ + This method computes a new variable: 'horizontal velocity norm' (m/s) + -> FVCOM.Variables.hori_velo_norm + + Notes: + - Can take time over the full domain + """ + debug = debug or self._debug + if debug: + print 'Computing horizontal velocity norm...' + + try: + u = self._var.ua[:] + v = self._var.va[:] + vel = ne.evaluate('sqrt(u**2 + v**2)').squeeze() + + except (MemoryError, ServerError) as e: + if e == ServerError: + print '---Data too large for server---' + print 'Tip: Save data on your machine or use partial data' + elif e == MemoryError: + print '---Data too large for machine memory---' + print 'Tip: use ax or tx during class initialisation' + print '--- to use partial data' + raise + + #Custom return + setattr(self._var, 'hori_velo_norm', vel) + + # Add metadata entry + self._History.append('horizontal velocity norm computed') + print '-Horizontal velocity norm added to FVCOM.Variables.-' + + if debug: + print '...Passed' + + def flow_dir(self, debug=False): + """" + This method create new variable 'depth averaged flow directions' (deg.) + -> FVCOM.Variables.depth_av_flow_dir + + *Notes* + - directions between -180 and 180 deg., i.e. 0=East, 90=North, +/-180=West, -90=South + - Can take time over the full domain + """ + if debug or self._debug: + print 'Computing flow directions...' + + try: + u = self._var.ua[:] + v = self._var.va[:] + dirFlow = np.rad2deg(np.arctan2(v,u)) + + except (MemoryError, ServerError) as e: + if e == ServerError: + print '---Data too large for server---' + print 'Tip: save data on your machine or use partial data' + elif e == MemoryError: + print '---Data too large for machine memory---' + print 'Tip: use ax or tx during class initialisation' + print '--- to use partial data' + raise + + #Custom return + setattr(self._var, 'depth_av_flow_dir', dirFlow) + + # Add metadata entry + self._History.append('depth averaged flow directions computed') + print '-Depth averaged flow directions added to FVCOM.Variables.-' + + if debug or self._debug: + print '...Passed' + + def flow_dir_at_point(self, pt_lon, pt_lat, t_start=[], t_end=[], time_ind=[], + graph=True, exceedance=False, debug=False): + """ + This function computes flow directions and associated norm + at any give location. + + Inputs: + - pt_lon = longitude in decimal degrees East to find, float number + - pt_lat = latitude in decimal degrees North to find, float number + + Outputs: + - flowDir = flowDir at (pt_lon, pt_lat), 1D array + - norm = velocity norm at (pt_lon, pt_lat), 1D array + + Keywords: + - t_start = start time, as string ('yyyy-mm-ddThh:mm:ss'), or time index as an integer + - t_end = end time, as a string ('yyyy-mm-ddThh:mm:ss'), or time index as an integer + - time_ind = time indices to work in, list of integers + - excedance = True, compute associated exceedance curve + + *Notes* + - directions between -180 and 180 deg., i.e. 0=East, 90=North, +/-180=West, -90=South + """ + debug = debug or self._debug + if debug: + print 'Computing flow directions at point...' + + # Find time interval to work in + argtime = [] + if not time_ind==[]: + argtime = time_ind + elif not t_start==[]: + if type(t_start)==str: + start = datetime.datetime.strptime(t_start, '%Y-%m-%d %H:%M:%S') + end = datetime.datetime.strptime(t_end, '%Y-%m-%d %H:%M:%S') + argtime = time_to_index(start, end, self._var.julianTime[:], debug=debug) + else: + argtime = np.arange(t_start, t_end) + + #Choose the right pair of velocity components + if type(self._var.ua).__name__=='Variable': #Fix for netcdf4 lib + u = self._var.ua[:] + v = self._var.va[:] + else: + u = self._var.ua + v = self._var.va + + #Extraction at point + # Finding closest point + index = self.index_finder(pt_lon, pt_lat, debug=False) + if debug: + print 'Extraction of u and v at point...' + U = self.interpolation_at_point(u, pt_lon, pt_lat, index=index, + debug=debug) + V = self.interpolation_at_point(v, pt_lon, pt_lat, index=index, + debug=debug) + + #Compute directions + if debug: + print 'Computing arctan2 and norm...' + dirFlow = np.rad2deg(np.arctan2(V,U)) + + #Compute velocity norm + norm = ne.evaluate('sqrt(U**2 + V**2)').squeeze() + + #use only the time indices of interest + if not argtime==[]: + dirFlow = dirFlow[argtime[:]] + norm = norm[argtime[:]] + + if debug: + print '...Passed' + #Rose diagram + if graph: + self._plot.rose_diagram(dirFlow, norm) + if exceedance: + self.exceedance(norm, graph=True, debug=debug) + + return dirFlow, norm + + def bidirectionality_at_point(self, pt_lon, pt_lat, debug=False): + """" + This function computes the depth averaged bidirectionality (deg.) at any given point + + Inputs: + - pt_lon = longitude in decimal degrees East of the reference point, float number + - pt_lat = latitude in decimal degrees North of the reference point, float number + + Outputs: + - bidir = 1D array of depth averaged bidirectionality, (nele) + + *Notes* + - bidirectionality between 0 and 90 deg., i.e. 0=perfect alignment, 90 = perpendicular abb and flood + - bidirectionality is weighted by the flow speed to filter out slack water + """ + debug = debug or self._debug + if debug: + start = time.time() + print 'Computing bidirectonality...' + ##compute necessary fields + if not hasattr(self._var, 'hori_velo_norm'): + self.hori_velo_norm(debug=debug) + if not hasattr(self._var, 'depth_av_flow_dir'): + self.flow_dir(debug=debug) + ##Compute weights, function of flow velocity + #weights + fI,eI,pa,pav=self.ebb_flood_split_at_point(pt_lon,pt_lat, debug=debug) + weightF=self._var.hori_velo_norm[fI,:]/np.nansum(self._var.hori_velo_norm[fI,:],0) + weightE=self._var.hori_velo_norm[eI,:]/np.nansum(self._var.hori_velo_norm[eI,:],0) + #weighted directions + dirF=np.nansum(self._var.depth_av_flow_dir[fI,:]*weightF,0) + dirF[dirF < 0] += 180.0 + dirE=np.nansum(self._var.depth_av_flow_dir[eI,:]*weightE,0) + dirE[dirE < 0] += 180.0 + ##keep angle between 0-90 deg. + bidir = (dirF - dirE) + bidir[bidir < 0] += 180.0 + bidir[bidir > 90] = 180.0 - bidir[bidir > 90] + + if debug: + end = time.time() + print "...processing time: ", (end - start) + + return bidir + + def ebb_flood_split_at_point(self, pt_lon, pt_lat, + t_start=[], t_end=[], time_ind=[], debug=False): + """ + This functions computes time indices for ebb and flood but also the + principal flow directions and associated variances + at any given point. + + Inputs: + - pt_lon = longitude in decimal degrees East to find, float number + - pt_lat = latitude in decimal degrees North to find,float number + + Outputs: + - floodIndex = flood time index, 1D array of integers + - ebbIndex = ebb time index, 1D array of integers + - pr_axis = principal flow ax1s, float number in degrees + - pr_ax_var = associated variance, float number + + Options: + - t_start = start time, as a string ('yyyy-mm-ddThh:mm:ss'), or time index as an integer + - t_end = end time, as a string ('yyyy-mm-ddThh:mm:ss'), or time index as an integer + - time_ind = time indices to work in, 1D array of integers + + *Notes* + - may take time to compute if time period too long + - directions between -180 and 180 deg., i.e. 0=East, 90=North, +/-180=West, -90=South + - use time_ind or t_start and t_end, not both + - assume that flood is aligned with principal direction + """ + debug = debug or self._debug + if debug: + start = time.time() + print 'Computing principal flow directions...' + + # Find time interval to work in + argtime = [] + if not time_ind==[]: + argtime = time_ind + elif not t_start==[]: + if type(t_start)==str: + start = datetime.datetime.strptime(t_start, '%Y-%m-%d %H:%M:%S') + end = datetime.datetime.strptime(t_end, '%Y-%m-%d %H:%M:%S') + argtime = time_to_index(start, end, self._var.julianTime[:], debug=debug) + else: + argtime = np.arange(t_start, t_end) + + #Choose the right pair of velocity components + if type(self._var.ua).__name__=='Variable': #Fix for netcdf4 lib + u = self._var.ua[:] + v = self._var.va[:] + else: + u = self._var.ua + v = self._var.va + + #Extraction at point + # Finding closest point + index = self.index_finder(pt_lon, pt_lat, debug=False) + if debug: + print 'Extraction of u and v at point...' + U = self.interpolation_at_point(u, pt_lon, pt_lat, index=index, + debug=debug) + V = self.interpolation_at_point(v, pt_lon, pt_lat, index=index, + debug=debug) + + #use only the time indices of interest + if not argtime==[]: + U = U[argtime[:]] + V = V[argtime[:]] + + # WB version of BP's principal axis + # Assuming principal axis = flood heading + # determine principal axes - potentially a problem if axes are very kinked + # since this would misclassify part of ebb and flood + if debug: print 'Computing principal axis at point...' + pr_axis, pr_ax_var = principal_axis(U, V) + + if debug: print 'Computing ebb/flood intervals...' + #Defines interval + dir_all = np.rad2deg(np.arctan2(V,U)) + ind = np.where(dir_all < 0) + dir_all[ind] = dir_all[ind] + 360 + + # sign speed - eliminating wrap-around + dir_PA = dir_all - pr_axis + dir_PA[dir_PA < -90] += 360 + dir_PA[dir_PA > 270] -= 360 + + #general direction of flood passed as input argument + floodIndex = np.where((dir_PA >= -90) & (dir_PA < 90)) + ebbIndex = np.arange(dir_PA.shape[0]) + ebbIndex = np.delete(ebbIndex, floodIndex[:]) + + if debug: + end = time.time() + print "...processing time: ", (end - start) + + return floodIndex[0], ebbIndex, pr_axis, pr_ax_var + + def speed_histogram(self, pt_lon, pt_lat, t_start=[], t_end=[], time_ind=[], bins=50, + debug=False, dump=False, **kwargs): + """ + This function plots the histogram of occurrences for the signed + flow speed at any given point. + + Inputs: + - pt_lon = longitude in decimal degrees East to find, float number + - pt_lat = latitude in decimal degrees North to find,float number + + Options: + - t_start = start time, as a string ('yyyy-mm-ddThh:mm:ss'), or time index as an integer + - t_end = end time, as a string ('yyyy-mm-ddThh:mm:ss'), or time index as an integer + - time_ind = time indices to work in, 1D array of integers + - bins = number of bins, integer + - dump = boolean, dump profile data in csv file + - kwargs = keyword options associated with pandas.DataFrame.to_csv, such as: + sep, header, na_rep, index,...etc + Check doc. of "to_csv" for complete list of options + + *Notes* + - use time_ind or t_start and t_end, not both + """ + debug = debug or self._debug + if debug: + start = time.time() + print 'Computing speed histogram...' + + pI, nI, pa, pav = self.ebb_flood_split_at_point(pt_lon, pt_lat, + t_start=t_start, t_end=t_end, time_ind=time_ind, + debug=debug) + dirFlow, norm = self.flow_dir_at_point(pt_lon, pt_lat, + t_start=t_start, t_end=t_end, time_ind=time_ind, + exceedance=False, debug=debug) + norm[nI] = -1.0 * norm[nI] + + #compute bins + #minBound = norm.min() + #maxBound = norm.max() + #step = round((maxBound-minBound/51.0),1) + #bins = np.arange(minBound,maxBound,step) + + #plot histogram + self._plot.Histogram(norm, + title='Flow speed histogram', + xLabel='Signed flow speed (m/s)', + yLabel='Occurrences (%)', + bins=int(bins), + dump=dump, **kwargs) + + if debug: + end = time.time() + print "...processing time: ", (end - start) + + def index_finder(self, pt_lon, pt_lat, debug=False): + """ + Finds closest node index of any given point + + Inputs: + - pt_lon = longitude in decimal degrees East to find, float number + - pt_lat = latitude in decimal degrees North to find, float number + + Option: + - debug = debug flag, boolean + + Output: + - index = integer if within a triangle, -1 if outside of domain + """ + # Checking if point in domain + if not hasattr(self._grid, 'triangleLL'): + # Mesh triangle + if debug: print "Computing triangulation..." + tri = Tri.Triangulation(self._grid.lon[:], self._grid.lat[:], triangles=self._grid.trinodes[:]) + self._grid.triangleLL = tri + finder = self._grid.triangleLL.get_trifinder() + else: + finder = self._grid.triangleLL.get_trifinder() + index = int(finder(pt_lon,pt_lat)) + + return index + + def interpolation_at_point(self, var, pt_lon, pt_lat, index=[], nn=True, debug=False): + """ + This function interpolates any given variables at any give location. + + Inputs: + - var = any FVCOM grid data or variable, numpy array + - pt_lon = longitude in decimal degrees East to find, float number + - pt_lat = latitude in decimal degrees North to find, float number + + Outputs: + - varInterp = var interpolated at (pt_lon, pt_lat) + + Options: + - index = element index, integer. Use only if closest element index + is already known + - nn = if True then use the nearest location in the grid if the location is outside the grid. + + *Notes* + - use index if closest element already known + """ + debug = (debug or self._debug) + if debug: + print 'Interpolaling at point...' + if debug: start = time.time() + + if index == []: + index = self.index_finder(pt_lon, pt_lat, debug=False) + + if ((index == -1) and (nn==False)): + # nan array if outside of domain + varInterp = np.ones(var.shape[:-1]) * np.nan + elif ((index == -1) and (nn==True)): + #outside of domain with nn true use the closest node or element + if var.shape[-1] == self._grid.nnode: + idx=np.argmin((self._grid.lon[:]-pt_lon)**2+(self._grid.lat[:]-pt_lat)**2) + varInterp = var[:,idx] + obsloc=[pt_lon, pt_lat] + simloc=[self._grid.lon[idx], self._grid.lat[idx]] + print 'Using nearest location {} m from observation location'.format(distance(simloc, obsloc)) + else: + idx=np.argmin((self._grid.lonc[:]-pt_lon)**2+(self._grid.latc[:]-pt_lat)**2) + varInterp = var[:,idx] + obsloc=[pt_lon, pt_lat] + simloc=[self._grid.lonc[idx], self._grid.latc[idx]] + print 'Using nearest location {} m from observation location'.format(distance(simloc, obsloc)) + else: + lon = self._grid.lon + lat = self._grid.lat + trinodes = self._grid.trinodes + if type(index)==list: + index = index[0] + #Mitchell's method to convert deg. coordinates to relative coordinates in meters + lonweight = (lon[int(trinodes[index,0])]\ + + lon[int(trinodes[index,1])]\ + + lon[int(trinodes[index,2])]) / 3.0 + latweight = (lat[int(trinodes[index,0])]\ + + lat[int(trinodes[index,1])]\ + + lat[int(trinodes[index,2])]) / 3.0 + TPI=111194.92664455874 # earth radius * pi/180.0 + pt_y = TPI * (pt_lat - latweight) + dx_sph = pt_lon - lonweight + if (dx_sph > 180.0): + dx_sph=dx_sph-360.0 + elif (dx_sph < -180.0): + dx_sph =dx_sph+360.0 + pt_x = TPI * np.cos(np.deg2rad(pt_lat + latweight)*0.5) * dx_sph + + if debug: print "coordinates in meters: ", pt_x, pt_y + + if var.shape[-1] == self._grid.nnode: + varInterp = interpN_at_pt(var, pt_x, pt_y, index, trinodes, + self._grid.aw0, self._grid.awx, + self._grid.awy, debug=debug) + else: + triele = self._grid.triele[:] + varInterp = interpE_at_pt(var, pt_x, pt_y, index, triele, + self._grid.a1u, self._grid.a2u, + debug=debug) + + if debug: + end = time.time() + print "Processing time: ", (end - start) + + return varInterp + + def degree2metric_coordinates(self, pt_lon, pt_lat): + """ + Converts degree coordinates to relative coordinates in meters + + Inputs: + - pt_lon = longitude in deg., float + - pt_lat = latitude in deg., float + Outputs: + - pt_x = longitude in m, float + - pt_y = latitude in m, float + """ + index = self.index_finder(pt_lon, pt_lat, debug=False) + if type(index)==list: + index = index[0] + + lon = self._grid.lon + lat = self._grid.lat + trinodes = self._grid.trinodes + + lonweight = (lon[int(trinodes[index,0])]\ + + lon[int(trinodes[index,1])]\ + + lon[int(trinodes[index,2])]) / 3.0 + latweight = (lat[int(trinodes[index,0])]\ + + lat[int(trinodes[index,1])]\ + + lat[int(trinodes[index,2])]) / 3.0 + TPI=111194.92664455874 # earth radius * pi/180.0 + pt_y = TPI * (pt_lat - latweight) + dx_sph = pt_lon - lonweight + if (dx_sph > 180.0): + dx_sph=dx_sph-360.0 + elif (dx_sph < -180.0): + dx_sph =dx_sph+360.0 + pt_x = TPI * np.cos(np.deg2rad(pt_lat + latweight)*0.5) * dx_sph + + return pt_x + self._grid.xc[index], pt_y + self._grid.yc[index] + + def exceedance(self, var, pt_lon=[], pt_lat=[], + graph=True, dump=False, debug=False, **kwargs): + """ + This function calculates the excedence curve of a var(time) + at any given point. + + Inputs: + - var = given quantity, 1 or 2D array of n elements, i.e (time) or (time,ele) + Options: + - pt_lon, pt_lat = coordinates, float numbers. Necessary if var = 2D (i.e. [time, nnode or nele] + - graph: True->plots curve; False->does not + - dump = boolean, dump graph data in csv file + - kwargs = keyword options associated with pandas.DataFrame.to_csv, such as: + sep, header, na_rep, index,...etc + Check doc. of "to_csv" for complete list of options + Outputs: + - Exceedance = list of % of occurences, 1D array + - Ranges = list of signal amplitude bins, 1D array + *Notes* + - This method is not suitable for SSE + """ + debug = (debug or self._debug) + if debug: + print 'Computing exceedance...' + + #Distinguish between 1D and 2D var + if len(var.shape)>1: + if pt_lon==[] or pt_lat==[]: + print 'Lon, lat coordinates are needed' + sys.exit() + signal = self.interpolation_at_point(var, pt_lon, pt_lat, debug=debug) + else: + signal=var + + Max = max(signal) + dy = (Max/50.0) + Ranges = np.arange(0,(Max + dy), dy) + Exceedance = np.zeros(Ranges.shape[0]) + dt = self._var.julianTime[1] - self._var.julianTime[0] + Period = signal.shape[0] * dt + time = np.arange(0.0, Period, dt) + + N = len(signal) + M = len(Ranges) + + for i in range(M): + r = Ranges[i] + for j in range(N-1): + if signal[j] > r: + Exceedance[i] = Exceedance[i] + (time[j+1] - time[j]) + + Exceedance = (Exceedance * 100) / Period + + if debug: + print '...Passed' + + #Plot + if graph: + error=np.ones(Exceedance.shape) * np.std(var)/2.0 + #if debug: print "Error: ", str(np.std(Exceedance)) + self._plot.plot_xy(Exceedance, Ranges, yerror=error, + yLabel='Amplitudes', + xLabel='Exceedance probability in %', + dump=dump, **kwargs) + + return Exceedance, Ranges + + def vorticity(self, debug=False): + """ + This method creates a new variable: 'depth averaged vorticity (1/s)' + -> FVCOM.Variables.depth_av_vorticity + + *Notes* + - Can take time over the full domain + """ + debug = (debug or self._debug) + if debug: + print 'Computing vorticity...' + start = time.time() + + t = np.arange(self._grid.ntime) + + #Surrounding elements + n1 = self._grid.triele[:,0] + n2 = self._grid.triele[:,1] + n3 = self._grid.triele[:,2] + + ##change end bound indices + test = -1 + n1[np.where(n1==test)[0]] = 0 + n2[np.where(n2==test)[0]] = 0 + n3[np.where(n3==test)[0]] = 0 + # double check due to chunking and nans + test = self._grid.triele.shape[0] + n1[np.where(n1>=test)[0]] = 0 + n2[np.where(n2>=test)[0]] = 0 + n3[np.where(n3>=test)[0]] = 0 + #TR quick fix: due to error with pydap.proxy.ArrayProxy + # not able to cop with numpy.int + N1 = [] + N2 = [] + N3 = [] + + N1[:] = n1[:] + N2[:] = n2[:] + N3[:] = n3[:] + + if debug: + end = time.time() + print "Check element=0, computation time in (s): ", (end - start) + print "start np.multiply" + + dvdx = np.zeros((self._grid.ntime,self._grid.nele)) + dudy = np.zeros((self._grid.ntime,self._grid.nele)) + + try: + j=0 + for i in t: + dvdx[j,:] = np.multiply(self._grid.a1u[0,:], self._var.va[i,:]) \ + + np.multiply(self._grid.a1u[1,:], self._var.va[i,N1]) \ + + np.multiply(self._grid.a1u[2,:], self._var.va[i,N2]) \ + + np.multiply(self._grid.a1u[3,:], self._var.va[i,N3]) + dudy[j,:] = np.multiply(self._grid.a2u[0,:], self._var.ua[i,:]) \ + + np.multiply(self._grid.a2u[1,:], self._var.ua[i,N1]) \ + + np.multiply(self._grid.a2u[2,:], self._var.ua[i,N2]) \ + + np.multiply(self._grid.a2u[3,:], self._var.ua[i,N3]) + j+=1 + if debug: + print "loop number ", i + except IndexError: # Strange error due to netCDF4/utils.py + j=0 + N1 = np.asarray(N1).astype(int) + N2 = np.asarray(N2).astype(int) + N3 = np.asarray(N3).astype(int) + for i in t: + dvdx[j,:] = np.multiply(self._grid.a1u[0,:], self._var.va[i,:]) \ + + np.multiply(self._grid.a1u[1,:], self._var.va[i,N1]) \ + + np.multiply(self._grid.a1u[2,:], self._var.va[i,N2]) \ + + np.multiply(self._grid.a1u[3,:], self._var.va[i,N3]) + dudy[j,:] = np.multiply(self._grid.a2u[0,:], self._var.ua[i,:]) \ + + np.multiply(self._grid.a2u[1,:], self._var.ua[i,N1]) \ + + np.multiply(self._grid.a2u[2,:], self._var.ua[i,N2]) \ + + np.multiply(self._grid.a2u[3,:], self._var.ua[i,N3]) + j+=1 + if debug: + print "loop number ", i + + vort = dvdx - dudy + + # Add metadata entry + setattr(self._var, 'depth_av_vorticity', vort) + self._History.append('depth averaged vorticity computed') + print '-Depth averaged vorticity added to FVCOM.Variables.-' + + if debug: + end = time.time() + print "Computation time in (s): ", (end - start) + + def vorticity_over_period(self, time_ind=[], t_start=[], t_end=[], debug=False): + """ + This function computes the depth averaged vorticity for a time period. + + Outputs: + - vort = horizontal vorticity (1/s), 2D array (time, nele) + + Options: + - time_ind = time indices to work in, list of integers + - t_start = start time, as a string ('yyyy-mm-ddThh:mm:ss'), or time index as an integer + - t_end = end time, as a string ('yyyy-mm-ddThh:mm:ss'), or time index as an integer + *Notes* + - Can take time over the full domain + """ + debug = (debug or self._debug) + if debug: + print 'Computing vorticity...' + start = time.time() + + # Find time interval to work in + t = [] + if not time_ind==[]: + t = time_ind + elif not t_start==[]: + if type(t_start)==str: + start = datetime.datetime.strptime(t_start, '%Y-%m-%d %H:%M:%S') + end = datetime.datetime.strptime(t_end, '%Y-%m-%d %H:%M:%S') + t = time_to_index(start, end, self._var.julianTime[:], debug=debug) + else: + t = np.arange(t_start, t_end) + else: + t = np.arange(self._grid.ntime) + self.vorticity() + + #Checking if vorticity already computed + if not hasattr(self._var, 'depth_av_vorticity'): + #Surrounding elements + n1 = self._grid.triele[:,0] + n2 = self._grid.triele[:,1] + n3 = self._grid.triele[:,2] + + ##change end bound indices + #test = self._grid.triele.shape[0] + test = -1 + n1[np.where(n1==test)[0]] = 0 + n2[np.where(n2==test)[0]] = 0 + n3[np.where(n3==test)[0]] = 0 + #TR quick fix: due to error with pydap.proxy.ArrayProxy + # not able to cop with numpy.int + N1 = [] + N2 = [] + N3 = [] + + N1[:] = n1[:] + N2[:] = n2[:] + N3[:] = n3[:] + + + if debug: + end = time.time() + print "Check element=0, computation time in (s): ", (end - start) + print "start np.multiply" + + dvdx = np.zeros((t.shape[0],self._grid.nele)) + dudy = np.zeros((t.shape[0],self._grid.nele)) + + j=0 + for i in t: + dvdx[j,:] = np.multiply(self._grid.a1u[0,:], self._var.va[i,:]) \ + + np.multiply(self._grid.a1u[1,:], self._var.va[i,N1]) \ + + np.multiply(self._grid.a1u[2,:], self._var.va[i,N2]) \ + + np.multiply(self._grid.a1u[3,:], self._var.va[i,N3]) + dudy[j,:] = np.multiply(self._grid.a2u[0,:], self._var.ua[i,:]) \ + + np.multiply(self._grid.a2u[1,:], self._var.ua[i,N1]) \ + + np.multiply(self._grid.a2u[2,:], self._var.ua[i,N2]) \ + + np.multiply(self._grid.a2u[3,:], self._var.ua[i,N3]) + j+=1 + if debug: + print "loop number ", i + + vort = dvdx - dudy + else: + vort = self._var.depth_av_vorticity[t[:], :] + + if debug: + end = time.time() + print "Computation time in (s): ", (end - start) + return vort + + def depth(self, debug=False): + """ + This method creates a new grid variable: 'depth2D' (m) + -> FVCOM.Grid.depth2D + + *Notes* + - depth convention: 0 = free surface + - Can take time over the full domain + """ + debug = debug or self._debug + if debug: + start = time.time() + print "Computing depth..." + + #Compute depth + size = self._grid.nele + size1 = self._grid.ntime + size2 = self._grid.nlevel + elc = np.zeros((size1, size)) + hc = np.zeros((size)) + + try: + for ind, value in enumerate(self._grid.trinodes[:]): + value.sort()#due to new version of netCDF4 + elc[:, ind] = np.mean(self._var.el[:, value], axis=1) + hc[ind] = np.mean(self._grid.h[value]) + + dep = elc[:,:] + hc[None,:] + except MemoryError: + print '---Data too large for machine memory---' + print 'Tip: use ax or tx during class initialisation' + print '--- to use partial data' + raise + + if debug: + end = time.time() + print "Computation time in (s): ", (end - start) + + # Add metadata entry + setattr(self._grid, 'depth2D', dep) + self._History.append('depth 2D computed') + print '-Depth 2D added to FVCOM.Variables.-' + + def depth_at_point(self, pt_lon, pt_lat, index=[], debug=False): + """ + This function computes the depth at any given point. + + Inputs: + - pt_lon = longitude in decimal degrees East, float number + - pt_lat = latitude in decimal degrees North, float number + + Outputs: + - dep = depth, 2D array (ntime, nlevel) + + Options: + - index = element index, interger + + *Notes* + - depth convention: 0 = free surface + - index is used in case one knows already at which + element depth is requested + """ + debug = debug or self._debug + if debug: + print "Computing depth..." + start = time.time() + + #Finding index + if index==[]: + index = self.index_finder(pt_lon, pt_lat, debug=False) + + if not hasattr(self._grid, 'depth2D'): + #Compute depth + if type(self._grid.h).__name__=='Variable': #Fix for netcdf4 lib + H = self._grid.h[:] + EL = self._var.el[:] + else: + H = self._grid.h + EL = self._var.el + + h = self.interpolation_at_point(H, pt_lon, pt_lat, index=index, debug=debug) + el = self.interpolation_at_point(EL, pt_lon, pt_lat, index=index, debug=debug) + + dep = el + h + else: + if type(self._grid.depth2D).__name__=='Variable': #Fix for netcdf4 lib + d2D = self._grid.depth2D[:] + else: + d2D = self._grid.depth2D + dep = self.interpolation_at_point(d2D, pt_lon, pt_lat, index=index, debug=debug) + if debug: + end = time.time() + print "Computation time in (s): ", (end - start) + + return dep + + def depth_averaged_power_density(self, debug=False): + """ + This method creates a new variable: 'depth averaged power density' (W/m2) + -> FVCOM.Variables.depth_av_power_density + + *Notes* + - The power density (pd) is then calculated as follows: pd = 0.5*1025*(u**3) + - This may take some time to compute depending on the size of the data set + """ + debug = (debug or self._debug) + if debug: print "Computing depth averaged power density..." + + if not hasattr(self._var, 'hori_velo_norm'): + self.hori_velo_norm(debug=debug) + if debug: print "Computing powers of hori velo norm..." + u = self._var.hori_velo_norm[:] + pd = ne.evaluate('0.5*1025.0*(u**3)').squeeze() + #pd = 0.5*1025.0*np.power(self._var.hori_velo_norm[:],3.0) # TR: very slow + #pd = 0.5*1025.0*self._var.hori_velo_norm[:]*self._var.hori_velo_norm[:]*self._var.hori_velo_norm[:] + + # Add metadata entry + setattr(self._var, 'depth_av_power_density', pd) + self._History.append('depth averaged power density computed') + print '-Depth averaged power density to FVCOM.Variables.-' + + def depth_averaged_power_assessment(self, power_mat, rated_speed, + cut_in=1.0, cut_out=4.5, debug=False): + """ + This method creates a new variable: 'depth averaged power assessment' (W/m2) + -> FVCOM.Variables.depth_av_power_assessment + + Inputs: + - power_mat = power matrix (u,Ct(u)), 2D array (2,n), + u being power_mat[0,:] and Ct(u) being power_mat[1,:] + - rated_speed = rated speed speed in m/s, float number + + Options: + - cut_in = cut-in speed in m/s, float number + - cut_out = cut-out speed in m/s, float number + + *Notes* + - The power density (pd) is then calculated as follows: pd = Cp*(1/2)*1025*(u**3) + - This function performs tidal turbine power assessment by accounting for + cut-in and cut-out speed, power curve/function (pc): Cp = pc(u) (where u is the flow speed) + - This may take some time to compute depending on the size of the data set + """ + debug = (debug or self._debug) + if debug: print "Computing depth averaged power density..." + + if not hasattr(self._var, 'depth_av_power_density'): + if debug: print "Computing power density..." + self.depth_averaged_power_density(debug=debug) + + if debug: print "Initialising power curve..." + Cp = interp1d(power_mat[0,:],power_mat[1,:]) + + u = self._var.hori_velo_norm[:] + pd = self._var.depth_av_power_density[:] + + pa = Cp(u)*pd + + if debug: print "finding cut-in and out..." + #TR comment huge bottleneck here + #ind = np.where(pd cut_out): + # pa[i,j] = 0.0 + inM = np.ma.masked_where(ucut_out, u).mask + ioM = inM * outM * u.mask + pa=np.ma.masked_where(ioM, pa) + + if debug: print "finding rated speed..." + parated = Cp(rated_speed)*0.5*1025.0*(rated_speed**3.0) + #TR comment huge bottleneck here + #ind = np.where(pd>pdout)[0] + #if not ind.shape[0]==0: + # pd[ind] = pdout + #for i in range(pa.shape[0]): + # for j in range(pa.shape[1]): + # if u[i,j] > rated_speed: + # pa[i,j] = parated + pa[u>rated_speed] = parated + + # Add metadata entry + setattr(self._var, 'depth_av_power_assessment', pd) + self._History.append('depth averaged power assessment computed') + print '-Depth averaged power assessment to FVCOM.Variables.-' + + def Harmonic_analysis_at_point(self, pt_lon, pt_lat, + time_ind=[], t_start=[], t_end=[], + elevation=True, velocity=False, + debug=False, **kwarg): + """ + This function performs a harmonic analysis on the sea surface elevation + time series or the velocity components timeseries. + + Inputs: + - pt_lon = longitude in decimal degrees East, float number + - pt_lat = latitude in decimal degrees North, float number + + Outputs: + - harmo = harmonic coefficients, dictionary + + Options: + - time_ind = time indices to work in, list of integers + - t_start = start time, as a string ('yyyy-mm-ddThh:mm:ss'), or time index as an integer + - t_end = end time, as a string ('yyyy-mm-ddThh:mm:ss'), or time index as an integer + - elevation=True means that 'solve' will be done for elevation. + - velocity=True means that 'solve' will be done for velocity. + + Utide's options: + Options are the same as for 'solve', which are shown below with + their default values: + conf_int=True; cnstit='auto'; notrend=0; prefilt=[]; nodsatlint=0; + nodsatnone=0; gwchlint=0; gwchnone=0; infer=[]; inferaprx=0; + rmin=1; method='cauchy'; tunrdn=1; linci=0; white=0; nrlzn=200; + lsfrqosmp=1; nodiagn=0; diagnplots=0; diagnminsnr=2; + ordercnstit=[]; runtimedisp='yyy' + + *Notes* + For more detailed information about 'solve', please see + https://github.com/wesleybowman/UTide + + """ + debug = (debug or self._debug) + #TR_comments: Add debug flag in Utide: debug=self._debug + index = self.index_finder(pt_lon, pt_lat, debug=False) + argtime = [] + if not time_ind==[]: + argtime = time_ind + elif not t_start==[]: + if type(t_start)==str: + start = datetime.datetime.strptime(t_start, '%Y-%m-%d %H:%M:%S') + end = datetime.datetime.strptime(t_end, '%Y-%m-%d %H:%M:%S') + argtime = time_to_index(start, end, self._var.julianTime[:], debug=debug) + else: + argtime = np.arange(t_start, t_end) + + if velocity == elevation: + raise PyseidonError("---Can only process either velocities or elevation. Change options---") + + if velocity: + time = self._var.matlabTime[:] + if type(self._var.ua).__name__=='Variable': #Fix for netcdf4 lib + ua = self._var.ua[:] + va = self._var.va[:] + else: + ua = self._var.ua + va = self._var.va + + u = self.interpolation_at_point(ua, pt_lon, pt_lat, index=index, debug=debug) + v = self.interpolation_at_point(va, pt_lon, pt_lat, index=index, debug=debug) + if not argtime==[]: + time = time[argtime[:]] + u = u[argtime[:]] + v = v[argtime[:]] + + lat = self._grid.lat[index] + harmo = solve(time, u, v, lat, **kwarg) + + else: + time = self._var.matlabTime[:] + if type(self._var.el).__name__=='Variable': #Fix for netcdf4 lib + el = self._var.el[:] + else: + el = self._var.el + el = self.interpolation_at_point(el, pt_lon, pt_lat, + index=index, debug=debug) + + if not argtime==[]: + time = time[argtime[:]] + el = el[argtime[:]] + + lat = self._grid.lat[index] + harmo = solve(time, el, None, lat, **kwarg) + #Write meta-data only if computed over all the elements + + return harmo + + def Harmonic_reconstruction(self, harmo, recon_time=[], time_ind=slice(None), debug=False, **kwarg): + """ + This function reconstructs the velocity components or the surface elevation + from harmonic coefficients. + Harmonic_reconstruction calls 'reconstruct'. This function assumes harmonics + ('solve') has already been executed. + + Inputs: + - Harmo = harmonic coefficient from harmo_analysis + - time_ind = time indices to process, list of integers + - recon_time = time you want the harmonic coefficients to be reconstructed at + + Output: + - Reconstruct = reconstructed signal, dictionary + + Options: + Options are the same as for 'reconstruct', which are shown below with + their default values: + cnstit = [], minsnr = 2, minpe = 0 + + *Notes* + For more detailed information about 'reconstruct', please see + https://github.com/wesleybowman/UTide + + """ + debug = (debug or self._debug) + if recon_time == []: + time = self._var.matlabTime[time_ind] + else: + time = recon_time + #TR_comments: Add debug flag in Utide: debug=self._debug + Reconstruct = reconstruct(time,harmo) + + return Reconstruct diff --git a/build/lib/pyseidon_dvt/fvcomClass/functionsFvcomThreeD.py b/build/lib/pyseidon_dvt/fvcomClass/functionsFvcomThreeD.py new file mode 100644 index 0000000..11c1ca2 --- /dev/null +++ b/build/lib/pyseidon_dvt/fvcomClass/functionsFvcomThreeD.py @@ -0,0 +1,1262 @@ +#!/usr/bin/python2.7 +# encoding: utf-8 + +from __future__ import division +import numexpr as ne +import datetime +from scipy.interpolate import interp1d +from pyseidon_dvt.utilities.interpolation_utils import * +from pyseidon_dvt.utilities.miscellaneous import * +from pyseidon_dvt.utilities.BP_tools import * +from pyseidon_dvt.utilities.shortest_element_path import * +import time +import matplotlib.pyplot as plt +from pydap.exceptions import ServerError + +from utide import solve, reconstruct + +#TR comment: This all routine needs to be tested and debugged +class FunctionsFvcomThreeD: + """ + **'Utils3D' subset of FVCOM class gathers useful methods and functions for 3D runs** + """ + def __init__(self, variable, grid, plot, util, History, debug): + #Inheritance + self._debug = debug + self._plot = plot + self._util = util + self.interpolation_at_point = self._util.interpolation_at_point + self.index_finder = self._util.index_finder + self.hori_velo_norm = self._util.hori_velo_norm + + #Create pointer to FVCOM class + setattr(self, '_var', variable) + setattr(self, '_grid', grid) + setattr(self, '_History', History) + + return + + def depth(self, debug=False): + """ + This method computes new grid variable: 'depth' (m) + -> FVCOM.Grid.depth + + *Notes* + - depth convention: 0 = free surface + - Can take time over the full domain + """ + debug = debug or self._debug + if debug: + start = time.time() + print "Computing depth..." + + try: + elc = interpN(self._var.el[:], self._grid.trinodes[:], self._grid.aw0[:], debug=debug) + hc = interpN(self._grid.h[:], self._grid.trinodes[:], self._grid.aw0[:], debug=debug) + siglay = interpN(self._grid.siglay[:], self._grid.trinodes[:], self._grid.aw0[:], debug=debug) + zeta = elc[:,:] + hc[None,:] + dep = zeta[:,None,:]*siglay[None,:,:] + + except MemoryError: + print '---Data too large for machine memory---' + print 'Tip: use ax or tx during class initialisation' + print '--- to use partial data' + raise + + if debug: + end = time.time() + print "Computation time in (s): ", (end - start) + + # TR: need to find vectorized alternative + #Compute depth + # size = self._grid.nele + # size1 = self._grid.ntime + # size2 = self._grid.nlevel + # + # elc = np.zeros((size1, size)) + # hc = np.zeros((size) + # dep = np.zeros((size1, size2, size)) + # for ind in range(size): + # d = self.depth_at_point(self._grid.lonc[ind],self._grid.latc[ind],debug=debug) + # dep[:, :, ind] = d[:,:] + + # TR: does not work with netCDF4 lib + # elc = np.zeros((size1, size)) + # hc = np.zeros((size)) + # siglay = np.zeros((size2, size)) + # + # try: + # for ind, value in enumerate(self._grid.trinodes[:]): + # elc[:, ind] = np.mean(self._var.el[:, value], axis=1) + # hc[ind] = np.mean(self._grid.h[value]) + # siglay[:,ind] = np.mean(self._grid.siglay[:,value],1) + # + # #zeta = self._var.el[:,:] + self._grid.h[None,:] + # zeta = elc[:,:] + hc[None,:] + # dep = zeta[:,None,:]*siglay[None,:,:] + + # Add metadata entry + setattr(self._grid, 'depth', dep) + self._History.append('depth computed') + print '-Depth added to FVCOM.Grid.-' + + def depth_at_point(self, pt_lon, pt_lat, index=[], debug=False): + """ + This function computes depth at any given point. + + Inputs: + - pt_lon = longitude in decimal degrees East, float number + - pt_lat = latitude in decimal degrees North, float number + + Outputs: + - dep = depth, 2D array (ntime, nlevel) + + Options: + - index = element index, interger. Use only if closest element + index is already known + + *Notes* + - depth convention: 0 = free surface + - index is used in case one knows already at which + element depth is requested + """ + debug = debug or self._debug + if debug: + print "Computing depth..." + start = time.time() + + #Finding index + if index==[]: + index = self.index_finder(pt_lon, pt_lat, debug=False) + + if not hasattr(self._grid, 'depth'): + #Compute depth + h = self.interpolation_at_point(self._grid.h, pt_lon, pt_lat, + index=index, debug=debug) + el = self.interpolation_at_point(self._var.el, pt_lon, pt_lat, + index=index, debug=debug) + siglay = self.interpolation_at_point(self._grid.siglay, pt_lon, pt_lat, + index=index, debug=debug) + zeta = el + h + dep = zeta[:,None]*siglay[None,:] + else: + dep = self.interpolation_at_point(self._grid.depth[:], + pt_lon, pt_lat, index=index, + debug=debug) + if debug: + end = time.time() + print "Computation time in (s): ", (end - start) + + return dep + + def interp_at_depth(self, var, depth, ind=[], debug=False): + """ + This function interpolates any given FVCOM.Variables field + onto a specified depth plan + + Inputs: + - var = 3 dimensional (time, sigma level, element) variable, array + - depth = interpolation depth (float in meters), if negative = from + sea surface downwards, if positive = from sea bottom upwards + Options: + - ind = array of closest indexes to depth, 2D array (ntime, nele) + + Output: + - interpVar = 2 dimensional (time, element) variable, masked array + - ind = array of closest indexes to depth, 2D array (ntime, nele) + """ + debug = debug or self._debug + if debug: print 'Interpolating at '+str(depth)+' meter depth...' + + #checking if depth field already calculated + if not hasattr(self._grid, 'depth'): + self.depth() + Depth = self._grid.depth[:]#otherwise to slow with netcdf4 lib + if depth > 0.0: # Changing vertical axis convention + # for tt in range(Depth.shape[0]): + # for ii in range(Depth.shape[2]): + # mini = np.min(np.squeeze(Depth[tt, :, ii])) + # Depth[tt, :, ii] = Depth[tt, :, ii] - mini + # Alternative + mini = np.min(Depth, axis=1) + Depth = Depth - mini[:, None, :] + dep = Depth[:] - depth + #Finding closest values to specified depth + if ind==[]: + if debug: print 'Finding closest indexes to depth...' + #mask negative value + dep = np.ma.masked_where(dep<0.0, dep) + #find min argument in masked array + ind = dep.argmin(axis=1) + ind=ind.astype(float) + #set to nan to shallow elements + ind[ind==dep.shape[1]-1.0] = np.nan + + #ind = np.zeros((dep.shape[0],dep.shape[2])) + #for i in range(dep.shape[0]): + # for k in range(dep.shape[2]): + # test = dep[i,:,k] + # if not test[test>0.0].shape==test.shape: + # ind[i,k] = test[test>0.0].argmin() + # else: + # ind[i,k] = np.nan + + inddown = ind + 1 + + if debug: print 'Computing weights...' + ##weight matrix & interp + #interpVar = np.ones((var.shape[0], var.shape[2]))*np.nan + #for i in range(ind.shape[0]): + # for j in range(ind.shape[1]): + # iU = ind[i,j] + # iD = inddown[i,j] + # if not np.isnan(iU): + # iU = int(iU) + # iD = int(iD) + # length = np.abs(self._grid.depth[i,iU,j]\ + # - self._grid.depth[i,iD,j]) + # wU = np.abs(depth - self._grid.depth[i,iU,j])/length + # wD = np.abs(depth - self._grid.depth[i,iD,j])/length + # interpVar[i,j] = (wU * var[i,iU,j]) + (wD * var[i,iD,j]) + # else: + # interpVar[i,j] = np.nan + #if debug: print 'Computing nan mask...' + #interpVar = np.ma.masked_array(interpVar,np.isnan(interpVar)) + + ##Streamlining + I = []; J = []; U = []; D = [] + for i in range(ind.shape[0]): + for j in range(ind.shape[1]): + iU = ind[i,j] + iD = inddown[i,j] + I.append(i) + J.append(j) + U.append(iU) + D.append(iD) + if debug: print 'Convert lists to arrays...' + I=np.asarray(I); J=np.asarray(J); U=np.asarray(U); D=np.asarray(D) + if debug: print 'Find nan indices...' + nanI = np.ones(U.shape) + nanU = np.where(np.isnan(U)) + nanD = np.where(np.isnan(D)) + nanI[nanU] = np.nan + nanI[nanD] = np.nan + if debug: print 'convert to integer...' + U[nanU] = 0 + D[nanD] = 0 + I = I.astype(int) + J = J.astype(int) + U = U.astype(int) + D = D.astype(int) + + if type(var).__name__=='Variable': #Fix for netcdf4 lib + Var = var[:] + else: + Var = var + # dUp = Depth[I,U,J] + # varUp = Var[I,U,J] + # dDo = Depth[I,D,J] + # varDo = Var[I,D,J] + # TR: append to speed up caching + if debug: print 'Caching...' + for i in I: + dUp = Depth[i,U,J] + varUp = Var[i,U,J] + dDo = Depth[i,D,J] + varDo = Var[i,D,J] + if debug: print 'Compute weights...' + lengths = np.abs(dUp - dDo) + wU = np.abs(depth - dUp)/lengths + wD = np.abs(depth - dDo)/lengths + if debug: print 'interpolation...' + interpVar = nanI * ((wU * varUp) + (wD * varDo)) + if debug: print 'reshaping...' + interpVar = np.reshape(interpVar, (Var.shape[0], Var.shape[2])) + if debug: print 'masking...' + interpVar = np.ma.masked_array(interpVar,np.isnan(interpVar)) + if debug: print '...Passed' + + return interpVar, ind + + def verti_shear(self, debug=False): + """ + This method computes a new variable: 'vertical shear' (1/s) + -> FVCOM.Variables.verti_shear + + *Notes* + - Can take time over the full doma + """ + debug = debug or self._debug + if debug: + print 'Computing vertical shear...' + + #Compute depth if necessary + if not hasattr(self._grid, 'depth'): + depth = self.depth(debug=debug) + depth = self._grid.depth[:] + + # Checking if horizontal velocity norm already exists + if not hasattr(self._var, 'velo_norm'): + self.velo_norm() + vel = self._var.velo_norm[:] + + try: + #Sigma levels to consider + top_lvl = self._grid.nlevel - 1 + bot_lvl = 0 + sLvl = range(bot_lvl, top_lvl+1) + + # Compute shear + dz = depth[:,sLvl[1:],:] - depth[:,sLvl[:-1],:] + dvel = vel[:,sLvl[1:],:] - vel[:,sLvl[:-1],:] + dveldz = dvel / dz + except MemoryError: + print '---Data too large for machine memory---' + print 'Tip: use ax or tx during class initialisation' + print '--- to use partial data' + raise + + #Custom return + setattr(self._var, 'verti_shear', dveldz) + + # Add metadata entry + self._History.append('vertical shear computed') + print '-Vertical shear added to FVCOM.Variables.-' + + if debug: + print '...Passed' + + def verti_shear_at_point(self, pt_lon, pt_lat, t_start=[], t_end=[], time_ind=[], + bot_lvl=[], top_lvl=[], graph=True, dump=False, debug=False): + """ + This function computes vertical shear at any given point. + + Inputs: + - pt_lon = longitude in decimal degrees East, float number + - pt_lat = latitude in decimal degrees North, float number + + Outputs: + - dveldz = vertical shear (1/s), 2D array (time, nlevel - 1) + + Options: + - t_start = start time, as a string ('yyyy-mm-ddThh:mm:ss'), + or time index as an integer + - t_end = end time, as a string ('yyyy-mm-ddThh:mm:ss'), + or time index as an integer + - time_ind = time indices to work in, list of integers + - bot_lvl = index of the bottom level to consider, integer + - top_lvl = index of the top level to consider, integer + - graph = plots graph if True + - dump = boolean, dump profile data in csv file + + *Notes* + - use time_ind or t_start and t_end, not both + """ + debug = debug or self._debug + if debug: + print 'Computing vertical shear at point...' + + # Find time interval to work in + argtime = [] + if not time_ind==[]: + argtime = time_ind + elif not t_start==[]: + if type(t_start)==str: + start = datetime.datetime.strptime(t_start, '%Y-%m-%d %H:%M:%S') + end = datetime.datetime.strptime(t_end, '%Y-%m-%d %H:%M:%S') + argtime = time_to_index(start, end, self._var.julianTime[:], debug=debug) + else: + argtime = np.arange(t_start, t_end) + + # Finding closest point + index = self.index_finder(pt_lon, pt_lat, debug=False) + + #Compute depth + depth = self.depth_at_point(pt_lon, pt_lat, index=index, debug=debug) + + #Sigma levels to consider + if top_lvl==[]: + top_lvl = self._grid.nlevel - 1 + if bot_lvl==[]: + bot_lvl = 0 + sLvl = range(bot_lvl, top_lvl+1) + + + # Checking if vertical shear already exists + if not hasattr(self._var, 'verti_shear'): + if type(self._var.u).__name__=='Variable': + u = self._var.u[:] + v = self._var.v[:] + else: + u = self._var.u + v = self._var.v + + #Extraction at point + if debug: + print 'Extraction of u and v at point...' + U = self.interpolation_at_point(u, pt_lon, pt_lat, + index=index, debug=debug) + V = self.interpolation_at_point(v, pt_lon, pt_lat, + index=index, debug=debug) + norm = ne.evaluate('sqrt(U**2 + V**2)').squeeze() + + # Compute shear + dz = depth[:,sLvl[1:]] - depth[:,sLvl[:-1]] + dvel = norm[:,sLvl[1:]] - norm[:,sLvl[:-1]] + dveldz = dvel / dz + else: + if type(self._var.verti_shear).__name__=='Variable': + shear = self._var.verti_shear[:] + else: + shear = self._var.verti_shear + dveldz = self.interpolation_at_point(self._var.verti_shear, + pt_lon, pt_lat, + index=index, debug=debug) + + if debug: + print '...Passed' + #use time indices of interest + if not argtime==[]: + dveldz = dveldz[argtime,:] + depth = depth[argtime,:] + + #Plot mean values + if graph: + mean_depth = np.mean((depth[:,sLvl[1:]] + + depth[:,sLvl[:-1]]) / 2.0, 0) + mean_dveldz = np.mean(dveldz,0) + error = np.std(dveldz,axis=0)/2.0 + self._plot.plot_xy(mean_dveldz, mean_depth, xerror=error[:], + title='Shear profile ', + xLabel='Shear (1/s) ', yLabel='Depth (m) ', + dump=dump) + + return dveldz + + def velo_norm(self, debug=False): + """ + This method computes a new variable: 'velocity norm' (m/s) + -> FVCOM.Variables.velo_norm + + *Notes* + -Can take time over the full domain + """ + if debug or self._debug: + print 'Computing velocity norm...' + #Check if w if there + try: + try: + #Computing velocity norm + u = self._var.u[:] + v = self._var.v[:] + w = self._var.w[:] + vel = ne.evaluate('sqrt(u**2 + v**2 + w**2)').squeeze() + except (MemoryError, ServerError) as e: + print '---Data too large for machine memory or server---' + print 'Tip: Save data on your machine first' + print 'Tip: use ax or tx during class initialisation' + print '--- to use partial data' + raise + except AttributeError: + try: + #Computing velocity norm + u = self._var.u[:] + v = self._var.v[:] + vel = ne.evaluate('sqrt(u**2 + v**2)').squeeze() + except (MemoryError, ServerError) as e: + print '---Data too large for machine memory or server---' + print 'Tip: Save data on your machine first' + print 'Tip: use ax or tx during class initialisation' + print '--- to use partial data' + raise + + #Custom return + setattr(self._var, 'velo_norm', vel) + + # Add metadata entry + self._History.append('Velocity norm computed') + print '-Velocity norm added to FVCOM.Variables.-' + + if debug or self._debug: + print '...Passed' + + def velo_norm_at_point(self, pt_lon, pt_lat, t_start=[], t_end=[], time_ind=[], + graph=True, dump=False, debug=False): + """ + This function computes the velocity norm at any given point. + + Inputs: + - pt_lon = longitude in decimal degrees East, float number + - pt_lat = latitude in decimal degrees North, float number + + Outputs: + - velo_norm = velocity norm, 2D array (time, level) + + Options: + - t_start = start time, as a string ('yyyy-mm-ddThh:mm:ss'), + or time index as an integer + - t_end = end time, as a string ('yyyy-mm-ddThh:mm:ss'), + or time index as an integer + - time_ind = time indices to work in, list of integers + - graph = boolean, plots or not veritcal profile + - dump = boolean, dump profile data in csv file + + *Notes* + - use time_ind or t_start and t_end, not both + """ + debug = debug or self._debug + if debug: + print 'Computing velocity norm at point...' + + # Find time interval to work in + argtime = [] + if not time_ind==[]: + argtime = time_ind + elif not t_start==[]: + if type(t_start)==str: + start = datetime.datetime.strptime(t_start, '%Y-%m-%d %H:%M:%S') + end = datetime.datetime.strptime(t_end, '%Y-%m-%d %H:%M:%S') + argtime = time_to_index(start, end, self._var.julianTime[:], debug=debug) + else: + argtime = np.arange(t_start, t_end) + + try: + if not hasattr(self._var, 'velo_norm'): + if type(self._var.u).__name__=='Variable': #Fix for netcdf4 lib + u = self._var.u[:] + v = self._var.v[:] + w = self._var.w[:] + else: + u = self._var.u + v = self._var.v + w = self._var.w + else: + vel = self._var.velo_norm + except AttributeError: + if not hasattr(self._var, 'velo_norm'): + if type(self._var.u).__name__=='Variable': #Fix for netcdf4 lib + u = self._var.u[:] + v = self._var.v[:] + else: + u = self._var.u + v = self._var.v + else: + if type(self._var.velo_norm).__name__=='Variable': #Fix for netcdf4 lib: + vel = self._var.velo_norm[:] + else: + vel = self._var.velo_norm + + # Finding closest point + index = self.index_finder(pt_lon, pt_lat, debug=False) + + #Computing horizontal velocity norm + if debug: + print 'Extraction of u, v and w at point...' + if not hasattr(self._var, 'velo_norm'): + U = self.interpolation_at_point(u, pt_lon, pt_lat, + index=index, debug=debug) + V = self.interpolation_at_point(v, pt_lon, pt_lat, + index=index, debug=debug) + if 'w' in locals(): + W = self.interpolation_at_point(w, pt_lon, pt_lat, + index=index, debug=debug) + velo_norm = ne.evaluate('sqrt(U**2 + V**2 + W**2)').squeeze() + else: + velo_norm = ne.evaluate('sqrt(U**2 + V**2)').squeeze() + else: + velo_norm = self.interpolation_at_point(vel, pt_lon, pt_lat, + index=index, debug=debug) + if debug: + print '...passed' + + #use only the time indices of interest + if not argtime==[]: + velo_norm = velo_norm[argtime[:],:,:] + + #Plot mean values + if graph: + depth = self.depth_at_point(pt_lon, pt_lat, index=index) + mean_depth = np.mean(depth, 0) + mean_vel = np.mean(velo_norm,0) + error = np.std(velo_norm,axis=0)/2.0 + self._plot.plot_xy(mean_vel, mean_depth, xerror=error[:], + title='Flow speed vertical ', + xLabel='Flow speed (1/s) ', yLabel='Depth (m) ', + dump=dump) + + return velo_norm + + + def flow_dir_at_point(self, pt_lon, pt_lat, t_start=[], t_end=[], time_ind=[], + vertical=True, debug=False): + """ + This function computes flow directions and associated norm + at any given location. + + Inputs: + - pt_lon = longitude in decimal degrees East to find + - pt_lat = latitude in decimal degrees North to find + + Outputs: + - flowDir = flowDir at (pt_lon, pt_lat), 2D array (ntime, nlevel) + - norm = velocity norm at (pt_lon, pt_lat), 2D array (ntime, nlevel) + + Options: + - t_start = start time, as a string ('yyyy-mm-ddThh:mm:ss'), + or time index as an integer + - t_end = end time, as a string ('yyyy-mm-ddThh:mm:ss'), + or time index as an integer + - time_ind = time indices to work in, list of integers + - vertical = True, compute flowDir for each vertical level + + *Notes* + - directions between -180 and 180 deg., i.e. 0=East, 90=North, + +/-180=West, -90=South + - use time_ind or t_start and t_end, not both + """ + debug = debug or self._debug + if debug: + print 'Computing flow directions at point...' + + # Finding closest point + index = self.index_finder(pt_lon, pt_lat, debug=False) + + # Find time interval to work in + argtime = [] + if not time_ind==[]: + argtime = time_ind + elif not t_start==[]: + if type(t_start)==str: + start = datetime.datetime.strptime(t_start, '%Y-%m-%d %H:%M:%S') + end = datetime.datetime.strptime(t_end, '%Y-%m-%d %H:%M:%S') + argtime = time_to_index(start, end, self._var.julianTime[:], debug=debug) + else: + argtime = np.arange(t_start, t_end) + + #Choose the right pair of velocity components + if self._var._3D and vertical: + if type(self._var.u).__name__=='Variable': #Fix for netcdf4 lib + u = self._var.u[:] + v = self._var.v[:] + else: + u = self._var.u + v = self._var.v + else: + if type(self._var.ua).__name__=='Variable': #Fix for netcdf4 lib: + u = self._var.ua[:] + v = self._var.va[:] + else: + u = self._var.ua + v = self._var.va + + #Extraction at point + if debug: + print 'Extraction of u and v at point...' + U = self._util.interpolation_at_point(u, pt_lon, pt_lat, + index=index, debug=debug) + V = self._util.interpolation_at_point(v, pt_lon, pt_lat, + index=index, debug=debug) + + #Compute velocity norm + norm = ne.evaluate('sqrt(U**2 + V**2)').squeeze() + + #Compute directions + if debug: + print 'Computing arctan2...' + dirFlow = np.rad2deg(np.arctan2(V,U)) + + if debug: print '...Passed' + #use only the time indices of interest + if not argtime==[]: + dirFlow = dirFlow[argtime[:],:] + + return dirFlow, norm + + def flow_dir(self, debug=False): + """" + This method computes a new variable: 'flow directions' (deg.) + -> FVCOM.Variables.flow_dir + + *Notes* + - directions between -180 and 180 deg., i.e. 0=East, 90=North, + +/-180=West, -90=South + - Can take time over the full domain + """ + if debug or self._debug: + print 'Computing flow directions...' + + try: + u = self._var.u[:] + v = self._var.v[:] + dirFlow = np.rad2deg(np.arctan2(v,u)) + except (MemoryError, ServerError) as e: + print '---Data too large for machine memory or server---' + print 'Tip: Save data on your machine' + print 'Tip: use ax or tx during class initialisation' + print '--- to use partial data' + raise + + #Custom return + setattr(self._var, 'flow_dir', dirFlow) + + # Add metadata entry + self._History.append('flow directions computed') + print '-Flow directions added to FVCOM.Variables.-' + + if debug or self._debug: + print '...Passed' + + def vorticity(self, debug=False): + """ + This method creates a new variable: 'depth averaged vorticity' (1/s) + -> FVCOM.Variables.vorticity + + *Notes* + - Can take time over the full domain + """ + debug = (debug or self._debug) + if debug: + print 'Computing vorticity...' + start = time.time() + + t = np.arange(self._grid.ntime) + + #Surrounding elements + n1 = self._grid.triele[:,0] + n2 = self._grid.triele[:,1] + n3 = self._grid.triele[:,2] + #No need anymore + ##change end bound indices + #test = self._grid.triele.shape[0] + #n1[np.where(n1==test)[0]] = 0 + #n2[np.where(n2==test)[0]] = 0 + #n3[np.where(n3==test)[0]] = 0 + + #TR quick fix: due to error with pydap.proxy.ArrayProxy + # not able to cop with numpy.int + N1 = [] + N2 = [] + N3 = [] + + N1[:] = n1[:] + N2[:] = n2[:] + N3[:] = n3[:] + + if debug: + end = time.time() + print "Check element=0, computation time in (s): ", (end - start) + print "start np.multiply" + + x0 = self._grid.xc + y0 = self._grid.yc + + dvdx = np.zeros((self._grid.ntime,self._grid.nlevel,self._grid.nele)) + dudy = np.zeros((self._grid.ntime,self._grid.nlevel,self._grid.nele)) + nele = self._grid.nele + + j=0 + for i in t: + dvdx[j,:,:] = np.multiply(self._grid.a1u[0,:].reshape(1,nele), + self._var.v[i,:,:]) \ + + np.multiply(self._grid.a1u[1,:].reshape(nele,1), + self._var.v[i,:,N1]).T \ + + np.multiply(self._grid.a1u[2,:].reshape(nele,1), + self._var.v[i,:,N2]).T \ + + np.multiply(self._grid.a1u[3,:].reshape(nele,1), + self._var.v[i,:,N3]).T + dudy[j,:,:] = np.multiply(self._grid.a2u[0,:].reshape(1,nele), + self._var.u[i,:,:]) \ + + np.multiply(self._grid.a2u[1,:].reshape(nele,1), + self._var.u[i,:,N1]).T \ + + np.multiply(self._grid.a2u[2,:].reshape(nele,1), + self._var.u[i,:,N2]).T \ + + np.multiply(self._grid.a2u[3,:].reshape(nele,1), + self._var.u[i,:,N3]).T + j+=1 + if debug: + print "loop number ", i + + vort = dvdx - dudy + + # Add metadata entry + setattr(self._var, 'vorticity', vort) + self._History.append('vorticity computed') + print '-Vorticity added to FVCOM.Variables.-' + + if debug: + end = time.time() + print "Computation time in (s): ", (end - start) + + def vorticity_over_period(self, time_ind=[], t_start=[], t_end=[], debug=False): + """ + This function computes the vorticity for a time period. + + Outputs: + - vort = horizontal vorticity (1/s), 2D array (time, nele) + + Options: + - time_ind = time indices to work in, list of integers + - t_start = start time, as a string ('yyyy-mm-ddThh:mm:ss'), + or time index as an integer + - t_end = end time, as a string ('yyyy-mm-ddThh:mm:ss'), + or time index as an integer + *Notes* + - Can take time over the full domain + """ + debug = (debug or self._debug) + if debug: + print 'Computing vorticity...' + start = time.time() + + # Find time interval to work in + t = [] + if not time_ind==[]: + t = time_ind + elif not t_start==[]: + if type(t_start)==str: + start = datetime.datetime.strptime(t_start, '%Y-%m-%d %H:%M:%S') + end = datetime.datetime.strptime(t_end, '%Y-%m-%d %H:%M:%S') + t = time_to_index(start, end, self._var.julianTime[:], debug=debug) + else: + t = np.arange(t_start, t_end) + else: + t = np.arange(self._grid.ntime) + + #Checking if vorticity already computed + if not hasattr(self._var, 'vorticity'): + #Surrounding elements + n1 = self._grid.triele[:,0] + n2 = self._grid.triele[:,1] + n3 = self._grid.triele[:,2] + #No need anymore + ##change end bound indices + #test = self._grid.triele.shape[0] + #n1[np.where(n1==test)[0]] = 0 + #n2[np.where(n2==test)[0]] = 0 + #n3[np.where(n3==test)[0]] = 0 + #TR quick fix: due to error with pydap.proxy.ArrayProxy + # not able to cop with numpy.int + N1 = [] + N2 = [] + N3 = [] + + N1[:] = n1[:] + N2[:] = n2[:] + N3[:] = n3[:] + + if debug: + end = time.time() + print "Check element=0, computation time in (s): ", (end - start) + print "start np.multiply" + + x0 = self._grid.xc[:] + y0 = self._grid.yc[:] + + dvdx = np.zeros((t.shape[0],self._grid.nlevel,self._grid.nele)) + dudy = np.zeros((t.shape[0],self._grid.nlevel,self._grid.nele)) + + j=0 + for i in t: + dvdx[j,:,:] = np.multiply(self._grid.a1u[0,:], self._var.v[i,:,:]) \ + + np.multiply(self._grid.a1u[1,:], self._var.v[i,:,N1]) \ + + np.multiply(self._grid.a1u[2,:], self._var.v[i,:,N2]) \ + + np.multiply(self._grid.a1u[3,:], self._var.v[i,:,N3]) + dudy[j,:,:] = np.multiply(self._grid.a2u[0,:], self._var.u[i,:,:]) \ + + np.multiply(self._grid.a2u[1,:], self._var.u[i,:,N1]) \ + + np.multiply(self._grid.a2u[2,:], self._var.u[i,:,N2]) \ + + np.multiply(self._grid.a2u[3,:], self._var.u[i,:,N3]) + j+=1 + if debug: + print "loop number ", i + + vort = dvdx - dudy + else: + vort = self._var.vorticity[t[:],:,:] + + if debug: + end = time.time() + print "Computation time in (s): ", (end - start) + return vort + + def power_density(self, debug=False): + """ + This method creates a new variable: 'power density' (W/m2) + -> FVCOM.Variables.power_density + + The power density (pd) is then calculated as follows: + pd = 0.5*1025*(u**3) + + *Notes* + - This may take some time to compute depending on the size + of the data set + """ + debug = (debug or self._debug) + if debug: print "Computing power density..." + + if not hasattr(self._var, 'velo_norm'): + self.velo_norm(debug=debug) + if debug: print "Computing power density variable..." + u = self._var.velo_norm[:] + pd = ne.evaluate('0.5*1025.0*(u**3)').squeeze() + #pd = 0.5*1025.0*np.power(self._var.hori_velo_norm[:],3.0) # TR: very slow + #pd = 0.5*1025.0*self._var.hori_velo_norm[:]*self._var.hori_velo_norm[:]*self._var.hori_velo_norm[:] + + # Add metadata entry + setattr(self._var, 'power_density', pd) + self._History.append('power density computed') + print '-Power density to FVCOM.Variables.-' + + def power_assessment_at_depth(self, depth, power_mat, rated_speed, + cut_in=1.0, cut_out=4.5, debug=False): + """ + This function computes power assessment (W/m2) at given depth. + + Description: + This function performs tidal turbine power assessment by accounting for + cut-in and cut-out speed, power curve/function (pc): + Cp = pc(u) + (where u is the flow speed) + + The power density (pd) is then calculated as follows: + pd = Cp*(1/2)*1025*(u**3) + + Inputs: + - depth = given depth from the surface, float + - power_mat = power matrix (u,Cp(u)), 2D array (2,n), + u being power_mat[0,:] and Ct(u) being power_mat[1,:] + - rated_speed = rated speed speed in m/s, float number + + + Output: + - pa = power assessment in (W/m2), 2D masked array (ntime, nele) + + Options: + - cut_in = cut-in speed in m/s, float number + - cut_out = cut-out speed in m/s, float number + + *Notes* + - This may take some time to compute depending on the size + of the data set + """ + debug = (debug or self._debug) + if debug: print "Computing depth averaged power density..." + + if not hasattr(self._var, 'power_density'): + self.power_density(debug=debug) + + if debug: print "Initialising power curve..." + Cp = interp1d(power_mat[0,:],power_mat[1,:]) + + u, ind = self.interp_at_depth(self._var.velo_norm[:], depth, debug=debug) + pd, ind2 = self.interp_at_depth(self._var.power_density[:], depth, + ind=ind, debug=debug) + + pa = Cp(u)*pd + + if debug: print "finding cut-in..." + #TR comment huge bottleneck here + #ind = np.where(pd cut_out): + # pa[i,j] = 0.0 + + inM = np.ma.masked_where(ucut_out, u).mask + ioM = inM * outM * u.mask + pa=np.ma.masked_where(ioM, pa) + + if debug: print "finding rated speed..." + parated = Cp(rated_speed)*0.5*1025.0*(rated_speed**3.0) + #TR comment huge bottleneck here + #ind = np.where(pd>pdout)[0] + #if not ind.shape[0]==0: + # pd[ind] = pdout + #for i in range(pa.shape[0]): + # for j in range(pa.shape[1]): + # if u[i,j] > rated_speed: + # pa[i,j] = parated + pa[u>rated_speed] = parated + + return pa + + def _vertical_slice(self, var, start_pt, end_pt, + time_ind=[], t_start=[], t_end=[], + title='Title', cmax=[], cmin=[], debug=False): + """ + Draw vertical slice in var along the shortest path between + start_point, end_pt. + + Inputs: + - var = 2D dimensional (sigma level, element) variable, array + - start_pt = starting point, [longitude, latitude] + - end_pt = ending point, [longitude, latitude] + + Options: + - time_ind = reference time indices for surface elevation, list of integer + - t_start = start time, as a string ('yyyy-mm-ddThh:mm:ss'), + or time index as an integer + - t_end = end time, as a string ('yyyy-mm-ddThh:mm:ss'), + or time index as an integer + + Keywords for plot: + - title = plot title, string + - cmin = minimum limit colorbar + - cmax = maximum limit colorbar + """ + debug = debug or self._debug + if not self._var._3D: + print "Error: Only available for 3D runs." + raise + else: + lons = [start_pt[0], end_pt[0]] + lats = [start_pt[1], end_pt[1]] + #Finding the closest elements to start and end points + index = closest_points(lons, lats, + self._grid.lonc, + self._grid.latc, debug=debug) + + #Finding the shortest path between start and end points + if debug : print "Computing shortest path..." + short_path = shortest_element_path(self._grid.lonc[:], + self._grid.latc[:], + self._grid.lon[:], + self._grid.lat[:], + self._grid.trinodes[:], + self._grid.h[:], debug=debug) + el, _ = short_path.getTargets([index]) + # Plot shortest path + short_path.graphGrid(plot=True) + + # Find time interval to work in + argtime = [] + if not time_ind==[]: + argtime = time_ind + elif not t_start==[]: + if type(t_start)==str: + start = datetime.datetime.strptime(t_start, '%Y-%m-%d %H:%M:%S') + end = datetime.datetime.strptime(t_end, '%Y-%m-%d %H:%M:%S') + argtime = time_to_index(start, end, self._var.julianTime[:], debug=debug) + else: + argtime = np.arange(t_start, t_end) + + #Extract along line + ele=np.asarray(el[:])[0,:] + varP = var[:,ele] + # Depth along line + if debug : print "Computing depth..." + depth = np.zeros((self._grid.ntime, self._grid.nlevel, ele.shape[0])) + I=0 + for ind in ele: + value = self._grid.trinodes[ind] + h = np.mean(self._grid.h[value]) + zeta = np.mean(self._var.el[:,value],1) + h + siglay = np.mean(self._grid.siglay[:,value],1) + depth[:,:,I] = zeta[:,None]*siglay[None,:] + I+=1 + # Average depth over time + if not argtime==[]: + depth = np.mean(depth[argtime,:,:], 0) + else: + depth = np.mean(depth, 0) + + # Compute distance along line + x = self._grid.xc[ele] + y = self._grid.yc[ele] + # Pythagore + cumulative path + line = np.zeros(depth.shape) + dl = np.sqrt(np.square(x[1:]-x[:-1]) + np.square(y[1:]-y[:-1])) + for i in range(1,dl.shape[0]): + dl[i] = dl[i] + dl[i-1] + line[:,1:] = dl[:] + + #turn into gridded + #print 'Compute gridded data' + #nx, ny = 100, 100 + #xi = np.linspace(x.min(), x.max(), nx) + #yi = np.linspace(y.min(), y.max(), ny) + + #Plot features + #setting limits and levels of colormap + if cmax==[]: + cmax = varP[:].max() + if cmin==[]: + cmin = varP[:].min() + step = (cmax-cmin) / 20.0 + levels=np.arange(cmin, (cmax+step), step) + #plt.clf() + fig = plt.figure(figsize=(18,10)) + plt.rc('font',size='22') + ax = fig.add_subplot(111) #,aspect=(1.0/np.cos(np.mean(lat)*np.pi/180.0))) + #levels = np.linspace(0,3.3,34) + #cs = ax.contourf(line,depth,varP,levels=levels, cmap=plt.cm.jet) + cs = ax.contourf(line,depth,varP,levels=levels, vmax=cmax,vmin=cmin, + cmap=plt.get_cmap('jet')) + cbar = fig.colorbar(cs) + #cbar.set_label(title, rotation=-90,labelpad=30) + ax.contour(line,depth,varP,cs.levels) #, linewidths=0.5,colors='k') + #ax.set_title() + plt.title(title) + #scale = 1 + #ticks = ticker.FuncFormatter(lambda lon, pos: '{0:g}'.format(lon/scale)) + #ax.xaxis.set_major_formatter(ticks) + #ax.yaxis.set_major_formatter(ticks) + plt.xlabel('Distance along line (m)') + plt.ylabel('Depth (m)') + + def Harmonic_analysis_at_point(self, pt_lon, pt_lat, + time_ind=[], t_start=[], t_end=[], + elevation=True, velocity=False, + debug=False, **kwarg): + """ + This function performs a harmonic analysis on the sea surface elevation + time series or the velocity components timeseries. + + Inputs: + - pt_lon = longitude in decimal degrees East, float number + - pt_lat = latitude in decimal degrees North, float number + + Outputs: + - harmo = harmonic coefficients, dictionary + + Options: + - time_ind = time indices to work in, list of integers + - t_start = start time, as a string ('yyyy-mm-ddThh:mm:ss'), or time index as an integer + - t_end = end time, as a string ('yyyy-mm-ddThh:mm:ss'), or time index as an integer + - elevation=True means that 'solve' will be done for elevation. + - velocity=True means that 'solve' will be done for velocity. + + Utide's options: + Options are the same as for 'solve', which are shown below with + their default values: + conf_int=True; cnstit='auto'; notrend=0; prefilt=[]; nodsatlint=0; + nodsatnone=0; gwchlint=0; gwchnone=0; infer=[]; inferaprx=0; + rmin=1; method='cauchy'; tunrdn=1; linci=0; white=0; nrlzn=200; + lsfrqosmp=1; nodiagn=0; diagnplots=0; diagnminsnr=2; + ordercnstit=[]; runtimedisp='yyy' + + *Notes* + For more detailed information about 'solve', please see + https://github.com/wesleybowman/UTide + + """ + debug = (debug or self._debug) + #TR_comments: Add debug flag in Utide: debug=self._debug + index = self.index_finder(pt_lon, pt_lat, debug=False) + argtime = [] + if not time_ind==[]: + argtime = time_ind + elif not t_start==[]: + if type(t_start)==str: + start = datetime.datetime.strptime(t_start, '%Y-%m-%d %H:%M:%S') + end = datetime.datetime.strptime(t_end, '%Y-%m-%d %H:%M:%S') + argtime = time_to_index(start, end, self._var.julianTime[:], debug=debug) + else: + argtime = np.arange(t_start, t_end) + + if velocity == elevation: + raise PyseidonError("---Can only process either velocities or elevation. Change options---") + + if velocity: + harmo = {} + time = self._var.matlabTime[:] + for layerIndex in range(self._var.u.shape[1]): + #if type(self._var.ua).__name__=='Variable': #Fix for netcdf4 lib + # ua = self._var.ua[:] + # va = self._var.va[:] + #else: + U = self._var.u[:,layerIndex] + V = self._var.v[:,layerIndex] + + u = self.interpolation_at_point(U, pt_lon, pt_lat, index=index, debug=debug) + v = self.interpolation_at_point(V, pt_lon, pt_lat, index=index, debug=debug) + if not argtime==[]: + time = time[argtime[:]] + u = u[argtime[:]] + v = v[argtime[:]] + + lat = self._grid.lat[index] + harmo['Layer_'+str(layerIndex)] = solve(time, u, v, lat, **kwarg) + + else: + time = self._var.matlabTime[:] + if type(self._var.el).__name__=='Variable': #Fix for netcdf4 lib + el = self._var.el[:] + else: + el = self._var.el + el = self.interpolation_at_point(el, pt_lon, pt_lat, + index=index, debug=debug) + + if not argtime==[]: + time = time[argtime[:]] + el = el[argtime[:]] + + lat = self._grid.lat[index] + harmo = solve(time, el, None, lat, **kwarg) + #Write meta-data only if computed over all the elements + + return harmo + + def Harmonic_reconstruction(self, harmo, recon_time=[], time_ind=slice(None), debug=False, **kwarg): + """ + This function reconstructs the velocity components or the surface elevation + from harmonic coefficients. + Harmonic_reconstruction calls 'reconstruct'. This function assumes harmonics + ('solve') has already been executed. + + Inputs: + - Harmo = harmonic coefficients from harmo_analysis, dictionary if velocity + - time_ind = time indices to process, list of integers + - recon_time = time you want the harmonic coefficients to be reconstructed at + + Output: + - Reconstruct = reconstructed signal, dictionary + + Options: + Options are the same as for 'reconstruct', which are shown below with + their default values: + cnstit = [], minsnr = 2, minpe = 0 + + *Notes* + For more detailed information about 'reconstruct', please see + https://github.com/wesleybowman/UTide + + """ + debug = (debug or self._debug) + if recon_time == []: + time = self._var.matlabTime[time_ind] + else: + time = recon_time + #TR_comments: Add debug flag in Utide: debug=self._debug + if not 'Layer_0' in harmo: + Reconstruct = reconstruct(time, harmo) + else: + Reconstruct = {} + for layer in harmo: + Reconstruct[layer] = reconstruct(time, harmo[layer]) + + return Reconstruct + + def velo_stack(self, Reconstruct, debug=False): + """ + This function seperates and stacks u & v into respective matrices + from a 3D harmonically reconstructed dictionary + + Inputs: + - Reconstruct = reconstructed signal, dictionary from Harmonic_reconstruction() + + Outputs: + - u = stacked matrix of u velocity + - v = stacked matrix of v velocity + + """ + debug = (debug or self._debug) + u = Reconstruct['Layer_0']['u'] + v = Reconstruct['Layer_0']['v'] + for i in range(len(Reconstruct))[1:]: + u = np.vstack((u, Reconstruct['Layer_'+str(i)]['u'])) + v = np.vstack((v, Reconstruct['Layer_'+str(i)]['v'])) + + return u, v diff --git a/build/lib/pyseidon_dvt/fvcomClass/fvcomClass.py b/build/lib/pyseidon_dvt/fvcomClass/fvcomClass.py new file mode 100644 index 0000000..1ae6103 --- /dev/null +++ b/build/lib/pyseidon_dvt/fvcomClass/fvcomClass.py @@ -0,0 +1,322 @@ +#!/usr/bin/python2.7 +# encoding: utf-8 + +#Libs import +from __future__ import division +#TR comment: 2 alternatives +import netCDF4 as nc +from scipy.io import netcdf +from pydap.client import open_url +import cPickle as pkl +import pickle as Pkl +import copy +from os.path import isfile +import gc + +#Utility import +from pyseidon_dvt.utilities.object_from_dict import ObjectFromDict +from pyseidon_dvt.utilities.pyseidon2pickle import pyseidon_to_pickle +from pyseidon_dvt.utilities.pyseidon2matlab import pyseidon_to_matlab +from pyseidon_dvt.utilities.pyseidon2netcdf import pyseidon_to_netcdf + +# Custom error +from pyseidon_error import PyseidonError + +#Local import +from variablesFvcom import _load_var, _load_grid +from functionsFvcom import * +from functionsFvcomThreeD import * +from plotsFvcom import * + +class FVCOM: + """ + **A class/structure for FVCOM data** + Functionality structured as follows: :: + + _Data. = raw netcdf file data + |_Variables. = fvcom variables and quantities + |_Grid. = fvcom grid data + |_History = Quality Control metadata + FVCOM._|_Utils2D. = set of useful functions and methods for 2D and 3D runs + |_Utils3D. = set of useful functions and methods for 3D runs + |_Plots. = plotting functions + |_Save_as = "save as" methods + + Inputs: + - filename = path to file, string, + ex: testFvcom = FVCOM('./path_to_FVOM_output_file/filename') + Note that the file can be a pickle file (i.e. *.p) or a netcdf file (i.e. *.nc) + Additionally, either a file path or a OpenDap url could be used + + Options: + - ax = defines for a specific spatial region to work with, as such: + ax = [minimun longitude, maximun longitude, minimun latitude, maximum latitude] + or use one of the following pre-defined region: ax = 'GP', 'PP', 'DG' or 'MP' + Note that this option permits to extract partial data from the overall file + and therefore reduce memory and cpu use. + + - tx = defines for a specific temporal period to work with, as such: + tx = ['2012-11-07T12:00:00','2012.11.09 12:00:00'], string of 'yyyy-mm-dd hh:mm:ss' + Note that this option permits to extract partial data from the overall file + and therefore reduce memory and cpu use + + *Notes* + Throughout the package, the following conventions apply: + - Date = string of 'yyyy-mm-dd hh:mm:ss' + - Coordinates = decimal degrees East and North + - Directions = in degrees, between -180 and 180 deg., i.e. 0=East, 90=North, +/-180=West, -90=South + - Depth = 0m is the free surface and depth is negative + """ + + def __init__(self, filename, ax=[], tx=[], debug=False): + """ Initialize FVCOM class.""" + self._debug = debug + if debug: print '-Debug mode on-' + #Force garbage collector when fvcom object created + gc.collect() + + #Loading pickle file + if filename.endswith('.p'): + f = open(filename, "rb") + try: + data = pkl.load(f) + except MemoryError: + try: + data = Pkl.load(f) + except KeyError: + data = pkl.load(f,2) + self._origin_file = data['Origin'] + self.History = data['History'] + if debug: print "Turn keys into attributs" + self.Grid = ObjectFromDict(data['Grid']) + self.Variables = ObjectFromDict(data['Variables']) + try: + if self._origin_file.startswith('http'): + #Look for file through OpenDAP server + print "Retrieving data through OpenDap server..." + self.Data = open_url(data['Origin']) + #Create fake attribut to be consistent with the rest of the code + self.Data.variables = self.Data + else: + #WB_Alternative: self.Data = sio.netcdf.netcdf_file(filename, 'r') + #WB_comments: scipy has causes some errors, and even though can be + # faster, can be unreliable + if isfile(data['Origin']): + try: + self.Data = netcdf.netcdf_file(data['Origin'], 'r',mmap=True) + #due to mmap not coping with big array > 4Gib + except (OverflowError, TypeError, ValueError) as e: + self.Data = nc.Dataset(data['Origin'], 'r', + format='NETCDF4_CLASSIC') + else: + print "the original *.nc file has not been found" + except: #TR: need to precise the type of error here + print "the original *.nc file has not been found" + pass + #Loading netcdf file + elif filename.endswith('.nc'): + if filename.startswith('http'): + #Look for file through OpenDAP server + print "Retrieving data through OpenDap server..." + self.Data = open_url(filename) + #Create fake attribut to be consistent with the rest of the code + self.Data.variables = self.Data + else: + #Look for file locally + print "Retrieving data from " + filename + " ..." + #WB_Alternative: self.Data = sio.netcdf.netcdf_file(filename, 'r') + #WB_comments: scipy has causes some errors, and even though can be + # faster, can be unreliable + try: + self.Data = netcdf.netcdf_file(filename, 'r',mmap=True) + #due to mmap not coping with big array > 4Gib + except (OverflowError, TypeError, ValueError) as e: + self.Data = nc.Dataset(filename, 'r', format='NETCDF4_CLASSIC') + text = 'Created from ' + filename + self._origin_file = filename + #Metadata + self.History = [text] + # Calling sub-class + print "Initialisation..." + #print "This might take some time..." + try: + self.Grid = _load_grid(self.Data, + ax, + self.History, + debug=self._debug) + self.Variables = _load_var(self.Data, + self.Grid, + tx, + self.History, + debug=self._debug) + except MemoryError: + print '---Data too large for machine memory---' + print 'Tip: use ax or tx during class initialisation' + print '--- to use partial data' + raise + + elif filename.endswith('.mat'): + raise PyseidonError("---Functionality not yet implemented---") + else: + raise PyseidonError("---Wrong file format---") + + self.Plots = PlotsFvcom(self.Variables, + self.Grid, + self._debug) + self.Util2D = FunctionsFvcom(self.Variables, + self.Grid, + self.Plots, + self.History, + self._debug) + + if self.Variables._3D: + self.Util3D = FunctionsFvcomThreeD(self.Variables, + self.Grid, + self.Plots, + self.Util2D, + self.History, + self._debug) + self.Plots.vertical_slice = self.Util3D._vertical_slice + + ##Re-assignement of utility functions as methods + #self.dump_profile_data = self.Plots._dump_profile_data_as_csv + #self.dump_map_data = self.Plots._dump_map_data_as_csv + + return + + #Special methods + def __del__(self): + """making sure that all opened files are closed when deleted or overwritten""" + #TR: not sure __del__ is the best approach for that + try: + if type(self.Data).__name__ == "netcdf_file": + try: + self.Data.close() + except AttributeError: + pass + elif type(self.Data).__name__ == "Dataset": + self.Data.close() + else: + try: + f.close() + except (NameError,AttributeError) as e: + pass + except AttributeError: + try: + f.close() + except (NameError,AttributeError) as e: + pass + + def __new__(self): + """Force garbage collector when new fvcom object created""" + gc.collect() + + def __add__(self, FvcomClass, debug=False): + """ + This special method permits to stack variables + of 2 FVCOM objects through a simple addition: :: + fvcom1 += fvcom2 + + *Notes* + - fvcom1 and fvcom2 have to cover the exact + same spatial domain + - last time step of fvcom1 must be <= to the + first time step of fvcom2 + """ + debug = debug or self._debug + #Define bounding box + if debug: + print "Computing bounding box..." + if self.Grid._ax == []: + lon = self.Grid.lon[:] + lat = self.Grid.lat[:] + self.Grid._ax = [lon.min(), lon.max(), + lat.min(), lat.max()] + if FvcomClass.Grid._ax == []: + lon = FvcomClass.Grid.lon[:] + lat = FvcomClass.Grid.lat[:] + FvcomClass.Grid._ax = [lon.min(), lon.max(), + lat.min(), lat.max()] + #series of test before stacking + if not (self.Grid._ax == FvcomClass.Grid._ax): + raise PyseidonError("---Spatial regions do not match---") + elif not ((self.Grid.nele == FvcomClass.Grid.nele) and + (self.Grid.nnode == FvcomClass.Grid.nnode) and + (self.Variables._3D == FvcomClass.Variables._3D)): + raise PyseidonError("---Data dimensions do not match---") + else: + if not (self.Variables.julianTime[-1]<= + FvcomClass.Variables.julianTime[0]): + raise PyseidonError("---Data not consecutive in time---") + #Copy self to newself + newself = copy.copy(self) + #TR comment: it still points toward self and modifies it + # so cannot do fvcom3 = fvcom1 + fvcom2 + if debug: + print 'Stacking variables...' + #keyword list for hstack + kwl=['matlabTime', 'julianTime'] + for key in kwl: + tmpN = getattr(newself.Variables, key) + tmpO = getattr(FvcomClass.Variables, key) + setattr(newself.Variables, key, + np.hstack((tmpN[:], tmpO[:]))) + + #keyword list for vstack + kwl=['u', 'v', 'w', 'ua', 'va', 'el', 'tke', 'gls'] + for key in kwl: + try: + tmpN = getattr(newself.Variables, key) + tmpO = getattr(FvcomClass.Variables, key) + setattr(newself.Variables, key, + np.vstack((tmpN[:], tmpO[:]))) + except AttributeError: + continue + #New time dimension + newself.Grid.ntime = newself.Grid.ntime + FvcomClass.Grid.ntime + #Append to new object history + text = 'Data from ' + FvcomClass.History[0].split('/')[-1] \ + + ' has been stacked' + newself.History.append(text) + + return newself + + #Methods + def save_as(self, filename, fileformat='netcdf', exceptions=[], compression=False, debug=False): + """ + This method saves the current FVCOM structure as: + - *.nc, i.e. netcdf file + - *.p, i.e. python file + - *.mat, i.e. Matlab file + + Inputs: + - filename = path + name of the file to be saved, string + + Options: + - fileformat = format of the file to be saved, i.e. 'pickle', .netcdf. or 'matlab' + - exceptions = list of variables to exclude from output file + , list of strings + - compresion = compresses data with zlib and uses at least 3 significant digits, boolean + Note: Works only with netcdf format + """ + debug = debug or self._debug + if debug: + print 'Saving file...' + #Save as different formats + if fileformat=='pickle': + pyseidon_to_pickle(self, filename, exceptions=exceptions, debug=debug) + elif fileformat=='matlab': + pyseidon_to_matlab(self, filename, exceptions=exceptions, debug=debug) + elif fileformat=='netcdf': + pyseidon_to_netcdf(self, filename, exceptions=exceptions, compression=compression, debug=debug) + else: + print "---Wrong file format---" + +#Test section when running in shell >> python fvcomClass.py +#if __name__ == '__main__': + + #filename = './test_file/dn_coarse_0001.nc' + #test = FVCOM(filename) + #test.harmonics(0, cnstit='auto', notrend=True, nodiagn=True) + #WB_COMMENTS: fixed matlabttime to matlabtime + #test.reconstr(test.Variables.matlabTime) diff --git a/build/lib/pyseidon_dvt/fvcomClass/plotsFvcom.py b/build/lib/pyseidon_dvt/fvcomClass/plotsFvcom.py new file mode 100644 index 0000000..f397105 --- /dev/null +++ b/build/lib/pyseidon_dvt/fvcomClass/plotsFvcom.py @@ -0,0 +1,682 @@ +#!/usr/bin/python2.7 +# encoding: utf-8 + +from __future__ import division +import numpy as np +import os +import matplotlib.pyplot as plt +import matplotlib.tri as Tri +import matplotlib.ticker as ticker +import matplotlib.patches as mpatches +import seaborn +import pandas as pd +try: + from osgeo import ogr + from osgeo import osr + havegdal=True +except ImportError: + print 'Gdal is not installed, osgeo cannot be imported. Saving to shape files will not be possible.' + havegdal=False +import zipfile +# Local import +from pyseidon_dvt.utilities.windrose import WindroseAxes +from pyseidon_dvt.utilities.interpolation_utils import * +#from miscellaneous import depth_at_FVCOM_element as depth_at_ind + +# Kml header +### +kml_groundoverlay = ''' + + + + __NAME__ + __COLOR__ + __VISIBILITY__ + + overlay.png + + + __SOUTH__ + __NORTH__ + __WEST__ + __EAST__ + + + + Legend + + legend.png + + + + + + + + +''' +### + +class PlotsFvcom: + """ + **'Plots' subset of FVCOM class gathers plotting functions** + """ + def __init__(self, variable, grid, debug): + self._debug = debug + #Back pointer + setattr(self, '_var', variable) + setattr(self, '_grid', grid) + + return + + def _def_fig(self): + """Defines figure window""" + self._fig = plt.figure(figsize=(18,10)) + plt.rc('font',size='22') + + + def colormap_var(self, var, title=' ', cmin=[], cmax=[], cmap=[], bb = None, + degree=True, mesh=False, isoline = 'bathy', isostep = None, + dump=False, png=False, shapefile=False, kmz=False, debug=False, **kwargs): + """ + 2D xy colormap plot of any given variable and mesh. + + Input: + - var = gridded variable, 1 D numpy array (nele or nnode) + + Options: + - title = plot title, string + - cmin = minimum limit colorbar + - cmax = maximum limit colorbar + - cmap = matplolib colormap + - units = string, var's units + - bb = bounding box, ex.: + bb = [minimun longitude, maximun longitude, minimun latitude, maximum latitude] + bb = [minimun x, maximun x, minimun y, maximum y] + - mesh = True, with mesh; False, without mesh + - degree = boolean, coordinates in degrees (True) or meters (False) + - isoline = 'bathy': bathymetric isolines, 'var': variable isolines, 'none': no isolines + - isostep = increment between isolines, float + - dump = boolean, dump profile data in csv file + - png = boolean, save map as png + - shapefile = boolean, save map as shapefile + - kmz = boolean, save map as kmz + - kwargs = keyword options associated with pandas.DataFrame.to_csv, such as: + sep, header, na_rep, index,...etc + Check doc. of "to_csv" for complete list of options + """ + debug = debug or self._debug + if debug: + print 'Plotting grid...' + # Figure if var had nele or nnode dimensions + if var.shape[0] == self._grid.nele: + dim = self._grid.nele + elif var.shape[0] == self._grid.nnode: + dim = self._grid.nnode + else: + print "Var has the wrong dimension, var.shape[0]= Grid.nele or nnode" + return + + # Bounding box nodes, elements and variable + if degree: + lon = self._grid.lon[:] + lat = self._grid.lat[:] + if debug: + print "Computing bounding box..." + if bb is None: + if self._grid._ax == []: + self._grid._ax = [lon.min(), lon.max(), + lat.min(), lat.max()] + bb = self._grid._ax + + if not hasattr(self._grid, 'triangleLL'): + # Mesh triangle + if debug: + print "Computing triangulation..." + trinodes = self._grid.trinodes[:] + tri = Tri.Triangulation(lon, lat, triangles=trinodes) + self._grid.triangleLL = tri + else: + tri = self._grid.triangleLL + + else: + x = self._grid.x[:] + y = self._grid.y[:] + if debug: + print "Computing bounding box..." + if bb is None: + bb = [x.min(), x.max(), y.min(), y.max()] + + if not hasattr(self._grid, 'triangleXY'): + # Mesh triangle + if debug: + print "Computing triangulation..." + trinodes = self._grid.trinodes[:] + tri = Tri.Triangulation(x, y, triangles=trinodes) + self._grid.triangleXY = tri + else: + tri = self._grid.triangleXY + + #setting limits and levels of colormap + if cmin==[]: + if debug: + print "Computing cmin..." + cmin=var[:].min() + if cmax==[]: + if debug: + print "Computing cmax..." + cmax=var[:].max() + step = (cmax-cmin) / 50.0 + + #Figure window params + self._def_fig() + if degree: + self._ax = self._fig.add_subplot(111, + aspect=(1.0/np.cos(np.mean(lat)*np.pi/180.0))) + else: + self._ax = self._fig.add_subplot(111, aspect=1.0) + + #Plotting functions + if debug: + print "Computing colormap..." + if cmap==[]: + cmap = plt.cm.gist_earth + f = self._ax.tripcolor(tri, var[:],vmax=cmax,vmin=cmin,cmap=cmap) + if mesh: + plt.triplot(tri, color='white', linewidth=0.5) + + #Label and axis parameters + if degree: + self._ax.set_ylabel('Latitude') + self._ax.set_xlabel('Longitude') + else: + self._ax.set_ylabel('Distance (m)') + self._ax.set_xlabel('Distance (m)') + self._ax.patch.set_facecolor('0.5') + self._ax.set_title(title) + units = kwargs.pop('units', '-') + cbar=self._fig.colorbar(f, ax=self._ax) + cbar.set_label(units, rotation=-90,labelpad=30) + scale = 1 + + if degree: + ticks = ticker.FuncFormatter(lambda lon, pos: '{0:g}'.format(lon/scale)) + self._ax.xaxis.set_major_formatter(ticks) + self._ax.yaxis.set_major_formatter(ticks) + + self._ax.set_xlim([bb[0],bb[1]]) + self._ax.set_ylim([bb[2],bb[3]]) + + # Isolines + if not isoline == 'none': + if isoline == 'bathy': + if not isostep == None: + levels = list(np.arange(round(self._grid.h.min()), round(self._grid.h.max()), isostep)) + cs = self._ax.tricontour(tri, self._grid.h, colors='w', linewidths=1.0, levels=levels) + else: + cs = self._ax.tricontour(tri, self._grid.h, colors='w', linewidths=1.0) + plt.clabel(cs, fontsize=11, inline=1) + plt.figtext(.12, .95, "Notes: white lines = bathymetric isolines", size='x-small') + elif isoline == 'var': + if var.shape[0] == self._grid.nele: + vari = interp_linear_to_nodes(var, self._grid.xc, self._grid.yc, self._grid.x, self._grid.y) + else: + vari = var + bounds=np.linspace(cmin,cmax,11) + if not isostep == None: + levels = list(np.arange(cmin, cmax, isostep)) + cs = self._ax.tricontour(tri, vari[:], vmin=cmin, vmax=cmax, colors='w', linewidths=1.0, levels=levels) + else: + cs = self._ax.tricontour(tri, vari[:], vmin=cmin, vmax=cmax, colors='w', linewidths=1.0, levels=bounds) + plt.clabel(cs, fontsize=11, inline=1) + plt.figtext(.12, .95, "Notes: white lines = isolines", size='x-small') + + # Show plot + self._ax.grid() + self._fig.show() + + # Saving + title = title.replace(" ", "_") + title = title.replace("(", "_") + title = title.replace(")", "_") + title = title.replace("-", "_") + title = title.replace("/", "_") + title = title.replace(".", "_") + savename=title.lower() + if png: + #if kmz: + # self._fig.savefig("overlay.png", bbox_inches='tight', transparent=True) + self._fig.savefig(savename+".png", bbox_inches='tight') + + if dump: + if degree: + self._dump_map_data_as_csv(var, self._grid.lonc, self._grid.latc, + title=savename, varLabel='map', + xLabel=' ', yLabel=' ', **kwargs) + else: + self._dump_map_data_as_csv(var, self._grid.xc, self._grid.yc, title=savename, + varLabel='map', xLabel=' ', yLabel=' ', **kwargs) + if debug or self._debug: + print '...Passed' + + if shapefile and havegdal: + if var.shape[0] == self._grid.nnode: + if debug: print "Interpolating var..." + var = interpN(var, self._grid.trinodes, self._grid.aw0, debug=debug) + if degree: + self._save_map_as_shapefile(var, self._grid.lon, self._grid.lat, + title=savename, varLabel='map', debug=debug) + else: + self._save_map_as_shapefile(var, self._grid.x, self._grid.y, + title=savename, varLabel='map', debug=debug) + elif shapefile and not havegdal: + print 'Shape file cannot be saved. Missing gdal.' + + if kmz: + name = kwargs.pop('name', title) + color = kwargs.pop('color', '9effffff') + visibility = str( kwargs.pop('visibility', 1) ) + kmzfile = kwargs.pop('kmzfile', savename+'.kmz') + pixels = kwargs.pop('pixels', 2048) # pixels of the max. dimension + units = kwargs.pop('units', '-') + + # Deformation dur to projection + # if degree: + # geo_aspect = np.cos(lat.mean()*np.pi/180.0) + # xsize = lon.ptp()*geo_aspect + # ysize = lat.ptp() + # xmax = lon.max() + # ymax = lat.max() + # xmin = lon.min() + # ymin = lat.min() + # else: + # geo_aspect = np.cos(self._grid.lat[:].mean()*np.pi/180.0) + # xsize = x.ptp()*geo_aspect + # ysize = y.ptp() + # xmax = x.max() + # ymax = y.max() + # xmin = x.min() + # ymin = y.min() + geo_aspect = np.cos(self._grid.lat[:].mean()*np.pi/180.0) + xsize = np.abs(bb[1] - bb[0]) * geo_aspect + ysize = np.abs(bb[3] - bb[2]) + aspect = ysize/xsize + if aspect > 1.0: + figsize = (30.0/aspect, 30.0) + else: + figsize = (30.0, 30.0*aspect) + + plt.ioff() + fig = plt.figure(figsize=figsize, facecolor=None, frameon=False, dpi=pixels // 5) + # fig = figure(facecolor=None, frameon=False, dpi=pixels//10) + ax = fig.add_axes([0, 0, 1, 1]) + pc = ax.tripcolor(tri, var[:], vmax=cmax, vmin=cmin, cmap=cmap) + + # Isolines + if not isoline == 'none': + if isoline == 'bathy': + if not isostep == None: + levels = list(np.arange(round(self._grid.h.min()), round(self._grid.h.max()), isostep)) + cs = ax.tricontour(tri, self._grid.h, colors='w', linewidths=1.0, levels=levels) + else: + cs = ax.tricontour(tri, self._grid.h, colors='w', linewidths=1.0) + units += " & bathymetric isolines" + elif isoline == 'var': + if not isostep == None: + levels = list(np.arange(cmin, cmax, isostep)) + cs = ax.tricontour(tri, vari[:], vmin=cmin, vmax=cmax, colors='w', linewidths=1.0, levels=levels) + else: + cs = ax.tricontour(tri, vari[:], vmin=cmin, vmax=cmax, colors='w', linewidths=1.0, levels=bounds) + plt.clabel(cs, fontsize=11, inline=1) + + ax.set_xlim([bb[0], bb[1]]) + ax.set_ylim([bb[2], bb[3]]) + ax.set_axis_off() + fig.savefig('overlay.png', dpi=200, transparent=True) + + # Write kmz + fz = zipfile.ZipFile(kmzfile, 'w') + fz.writestr(savename+'.kml', kml_groundoverlay.replace('__NAME__', name)\ + .replace('__COLOR__', color)\ + .replace('__VISIBILITY__', visibility)\ + .replace('__SOUTH__', str(bb[2]))\ + .replace('__NORTH__', str(bb[3]))\ + .replace('__EAST__', str(bb[1]))\ + .replace('__WEST__', str(bb[0]))) + fz.write('overlay.png') + os.remove('overlay.png') + + # colorbar png + fig = plt.figure(figsize=(1.0, 4.0), facecolor=None, frameon=False) + ax = fig.add_axes([0.0, 0.05, 0.2, 0.9]) + cb = fig.colorbar(pc, cax=ax) + cb.set_label(units, color='0.0') + for lab in cb.ax.get_yticklabels(): + plt.setp(lab, 'color', '0.0') + + fig.savefig('legend.png', transparent=True) + fz.write('legend.png') + os.remove('legend.png') + fz.close() + + def rose_diagram(self, direction, norm, png=False, title="rose_diagram"): + + """ + Plots rose diagram + + Inputs: + - direction = 1D array + - norm = 1D array + + Options: + - title = plot title, string + - png = boolean, saves rose diagram as png + """ + #Convertion + #TR: not quite sure here, seems to change from location to location + # express principal axis in compass + direction = np.mod(90.0 - direction, 360.0) + + #Create new figure + #fig = plt.figure(figsize=(18,10)) + #plt.rc('font',size='22') + self._def_fig() + rect = [0.1, 0.1, 0.8, 0.8] + ax = WindroseAxes(self._fig, rect)#, axisbg='w') + self._fig.add_axes(ax) + #Rose + ax.bar(direction, norm , normed=True, opening=0.8, edgecolor='white') + #adjust legend + l = ax.legend(shadow=True, bbox_to_anchor=[-0.1, 0], loc='lower left') + plt.setp(l.get_texts(), fontsize=10) + plt.xlabel('Rose diagram in % of occurrences - Colormap of norms') + self._fig.show() + + # Saving + title = title.replace(" ", "_") + title = title.replace("(", "_") + title = title.replace(")", "_") + title = title.replace("-", "_") + title = title.replace("/", "_") + title = title.replace(".", "_") + savename=title.lower() + if png: + self._fig.savefig(savename+".png", bbox_inches='tight') + + def plot_xy(self, x, y, xerror=[], yerror=[], + title='xy_plot', xLabel=' ', yLabel=' ', + png=False, dump=False, **kwargs): + """ + Simple X vs Y plot + + Inputs: + - x = 1D array + - y = 1D array + + Options: + - xerror = error on 'x', 1D array + - yerror = error on 'y', 1D array + - title = plot title, string + - xLabel = title of the x-axis, string + - yLabel = title of the y-axis, string + - png = boolean, saves map as png + - dump = boolean, dump profile data in csv file + - kwargs = keyword options associated with pandas.DataFrame.to_csv, such as: + sep, header, na_rep, index,...etc + Check doc. of "to_csv" for complete list of options + """ + #fig = plt.figure(figsize=(18,10)) + #plt.rc('font',size='22') + self._def_fig() + self._ax = self._fig.add_subplot(111) + self._ax.plot(x, y, label=title) + scale = 1 + self._ax.set_ylabel(yLabel) + self._ax.set_xlabel(xLabel) + self._ax.get_xaxis().set_minor_locator(ticker.AutoMinorLocator()) + self._ax.get_yaxis().set_minor_locator(ticker.AutoMinorLocator()) + self._ax.grid(b=True, which='major', color='w', linewidth=1.5) + self._ax.grid(b=True, which='minor', color='w', linewidth=0.5) + if not yerror==[]: + self._ax.fill_between(x, y-yerror, y+yerror, + alpha=0.2, edgecolor='#1B2ACC', facecolor='#089FFF', antialiased=True) + if not xerror==[]: + self._ax.fill_betweenx(y, x-xerror, x+xerror, + alpha=0.2, edgecolor='#1B2ACC', facecolor='#089FFF', antialiased=True) + if (not xerror==[]) or (not yerror==[]): + blue_patch = mpatches.Patch(color='#089FFF', + label='Standard deviation',alpha=0.2) + plt.legend(handles=[blue_patch],loc=1, fontsize=12) + #plt.legend([blue_patch],loc=1, fontsize=12) + + self._fig.show() + # Saving + title = title.replace(" ", "_") + title = title.replace("(", "_") + title = title.replace(")", "_") + title = title.replace("-", "_") + title = title.replace("/", "_") + title = title.replace(".", "_") + savename=title.lower().replace(" ","_") + if png: + self._fig.savefig(savename+".png", bbox_inches='tight') + + if dump: self._dump_profile_data_as_csv(x, y,xerror=xerror, yerror=yerror, + title=savename, xLabel=xLabel, + yLabel=yLabel, **kwargs) + + def Histogram(self, y, title='Histogram', xLabel=' ', yLabel=' ', bins=50, + png=False, dump=False, **kwargs): + """ + Histogram plot + + Inputs: + - bins = list of bin edges + - y = 1D array + + Options: + - title = plot title, string + - xLabel = title of the x-axis, string + - yLabel = title of the y-axis, string + - bins = number of bins, integer + - png = boolean, saves histogram as png + - dump = boolean, dump profile data in csv file + - kwargs = keyword options associated with pandas.DataFrame.to_csv, such as: + sep, header, na_rep, index,...etc + Check doc. of "to_csv" for complete list of options + """ + ## the histogram of the data + #fig = plt.figure(figsize=(18,10)) + self._def_fig() + self._ax = self._fig.add_subplot(111) + density, bins = np.histogram(y, bins=bins, normed=True, density=True) + unity_density = density / density.sum() + widths = bins[:-1] - bins[1:] + # To plot correct percentages in the y axis + self._ax.bar(bins[1:], unity_density, width=widths) + formatter = ticker.FuncFormatter(lambda v, pos: str(v * 100)) + self._ax.yaxis.set_major_formatter(formatter) + self._ax.get_xaxis().set_minor_locator(ticker.AutoMinorLocator()) + self._ax.get_yaxis().set_minor_locator(ticker.AutoMinorLocator()) + self._ax.grid(b=True, which='major', color='w', linewidth=1.5) + self._ax.grid(b=True, which='minor', color='w', linewidth=0.5) + + plt.ylabel(yLabel) + plt.xlabel(xLabel) + + self._fig.show() + + # Saving + title = title.replace(" ", "_") + title = title.replace("(", "_") + title = title.replace(")", "_") + title = title.replace("-", "_") + title = title.replace("/", "_") + title = title.replace(".", "_") + savename=title.lower() + if png: + self._fig.savefig(savename+".png", bbox_inches='tight') + if dump: self._dump_profile_data_as_csv(bins[1:], unity_density, + title=savename, xLabel=xLabel, + yLabel=yLabel, **kwargs) + + def add_points(self, x, y, label=' ', color='black'): + """ + Adds scattered points (x,y) on current figure, + where x and y are 1D arrays of the same lengths. + + Inputs: + - x = float number or list of float numbers + - y = float number or list of float numbers + + Options: + - Label = a string + - Color = a string, 'red', 'green', etc. or gray shades like '0.5' + """ + plt.scatter(x, y, s=50, color=color) + #TR : annotate does not work on my machine !? + plt.annotate(label, xy=(x, y), xycoords='data', xytext=(-20, 20), + textcoords='offset points', ha='right', + arrowprops=dict(arrowstyle="->", shrinkA=0), + fontsize=12) + + def _dump_profile_data_as_csv(self, x, y, xerror=[], yerror=[], + title=' ', xLabel=' ', yLabel=' ', **kwargs): + """ + Dumps profile data in csv file + + Inputs: + - x = 1D array + - y = 1D array + + Options: + - xerror = error on 'x', 1D array + - yerror = error on 'y', 1D array + - title = file name, string + - xLabel = name of the x-data, string + - yLabel = name of the y-data, string + - kwargs = keyword options associated with pandas.DataFrame.to_csv, such as: + sep, header, na_rep, index,...etc + Check doc. of "to_csv" for complete list of options + """ + if title == ' ': title = 'dump_profile_data' + filename=title + '.csv' + if xLabel == ' ': xLabel = 'X' + if yLabel == ' ': yLabel = 'Y' + if not xerror == []: + df = pd.DataFrame({xLabel:x[:], yLabel:y[:], 'error': xerror[:]}) + elif not yerror == []: + df = pd.DataFrame({xLabel:x[:], yLabel:y[:], 'error': yerror[:]}) + else: + df = pd.DataFrame({xLabel:x[:], yLabel:y[:]}) + df.to_csv(filename, encoding='utf-8', **kwargs) + + def _dump_map_data_as_csv(self, var, x, y, title=' ', + varLabel=' ', xLabel=' ', yLabel=' ', **kwargs): + """ + Dumps map data in csv file + + Inputs: + - var = gridded variable, 1 D numpy array (nele or nnode) + - x = coordinates, 1 D numpy array (nele or nnode) + - y = coordinates, 1 D numpy array (nele or nnode) + + Options: + - title = file name, string + - xLabel = name of the x-data, string + - yLabel = name of the y-data, string + - kwargs = keyword options associated with pandas.DataFrame.to_csv, such as: + sep, header, na_rep, index,...etc + Check doc. of "to_csv" for complete list of options + """ + if title == ' ': title = 'dump_map_data' + filename=title + '.csv' + if varLabel == ' ': varLabel = 'Z' + if xLabel == ' ': xLabel = 'X' + if yLabel == ' ': yLabel = 'Y' + df = pd.DataFrame({xLabel:x[:], yLabel:y[:], varLabel: var[:]}) + df.to_csv(filename, encoding='utf-8', **kwargs) + + def _save_map_as_shapefile(self, var, x, y, title=' ', varLabel=' ', debug=False): + """ + Saves map as shapefile + + Inputs: + - var = gridded variable, 1 D numpy array (nele or nnode) + - x = coordinates, 1 D numpy array (nele or nnode) + - y = coordinates, 1 D numpy array (nele or nnode) + + Options: + - title = file name, string + - kwargs = keyword options associated with ??? + """ + + debug = debug or self._debug + if debug: + print 'Converting map to shapefile...' + if title == ' ': + title = 'save_map_data' + else: # reformat file name + title = title.replace(" ", "_") + title = title.replace("(", "_") + title = title.replace(")", "_") + title = title.replace("-", "_") + title = title.replace("/", "_") + title = title.replace(".", "_") + + filename=title + '.shp' + + # Projection + #epsg_in=4326 + epsg_in = 3857 # Google Projection + + # give alternative file name is already exists + driver = ogr.GetDriverByName('ESRI Shapefile') + if os.path.exists(filename): + filename = filename[:-4] + "_bis.shp" + + shapeData = driver.CreateDataSource(filename) + + spatialRefi = osr.SpatialReference() + spatialRefi.ImportFromEPSG(epsg_in) + + lyr = shapeData.CreateLayer("poly_layer", spatialRefi, ogr.wkbPolygon) + + #Features + if varLabel==' ': varLabel = 'var' + lyr.CreateField(ogr.FieldDefn(varLabel, ogr.OFTReal)) + + if debug: print "Writing ESRI Shapefile %s..." % filename + lon = x[:] + lat = y[:] + trinodes = self._grid.trinodes[:] + + if debug: print "Writing Node Array" + cnt = 0 + for row in trinodes: + val1 = -999 + ring = ogr.Geometry(ogr.wkbLinearRing) + for val in row: + if val1 == -999: + val1 = val + ring.AddPoint(lon[val], lat[val]) + #Add 1st point to close ring + ring.AddPoint(lon[val1], lat[val1]) + + poly = ogr.Geometry(ogr.wkbPolygon) + poly.AddGeometry(ring) + + #Now add field values from array + feat = ogr.Feature(lyr.GetLayerDefn()) + feat.SetGeometry(poly) + feat.SetField(varLabel, float(var[cnt])) + + lyr.CreateFeature(feat) + feat.Destroy() + poly.Destroy() + + val1 = -999 + cnt += 1 + + shapeData.Destroy() + if debug: print "Finished writing Shapefile Mesh. [Total Nodes: %d]" % cnt diff --git a/build/lib/pyseidon_dvt/fvcomClass/variablesFvcom.py b/build/lib/pyseidon_dvt/fvcomClass/variablesFvcom.py new file mode 100644 index 0000000..eff00ea --- /dev/null +++ b/build/lib/pyseidon_dvt/fvcomClass/variablesFvcom.py @@ -0,0 +1,713 @@ +#!/usr/bin/python2.7 +# encoding: utf-8 + +from __future__ import division +from itertools import groupby +from operator import itemgetter +import datetime +import gc +# Parallel computing +#import multiprocessing as mp +#Local import +from pyseidon_dvt.utilities.regioner import * +from pyseidon_dvt.utilities.miscellaneous import time_to_index +from pyseidon_dvt.utilities.miscellaneous import mattime_to_datetime + +class _load_grid: + """ + **'Grid' subset in FVCOM class contains grid related quantities** + + Some grid data are directly passed on from FVCOM output: :: + _lon = longitudes at nodes (deg.), 2D array (ntime, nnode) + |_lonc = longitudes at elements (deg.), 2D array (ntime, nele) + |_lat = latitudes at nodes (deg.), 2D array (ntime, nnode) + |_latc = latitudes at elements (deg.), 2D array (ntime, nele) + |_x = x coordinates at nodes (m), 2D array (ntime, nnode) + |_xc = x coordinates at elements (m), 2D array (ntime, nele) + |_y = y coordinates at nodes (m), 2D array (ntime, nnode) + |_yc = y coordinates at nodes (m), 2D array (ntime, nele) + FVCOM.Grid._|_h = bathymetry (m), 2D array (ntime, nnode) + |_nele = element dimension, integer + |_nnode = node dimension, integer + |_nlevel = vertical level dimension, integer + |_ntime = time dimension, integer + |_trinodes = surrounding node indices, 2D array (3, nele) + |_triele = surrounding element indices, 2D array (3, nele) + |_siglay = sigma layers, 2D array (nlevel, nnode) + |_siglay = sigma levels, 2D array (nlevel+1, nnode) + |_and a all bunch of grid parameters... + | i.e. a1u, a2u, aw0, awx, awy + + Some others shall be generated as methods are being called, ex: :: + ... + |_triangle = triangulation object for plotting purposes + + """ + def __init__(self, data, ax, History, debug=False): + self._debug = debug + if debug: + print 'Loading grid...' + #Pointer to History + setattr(self, '_History', History) + + #list of required grid variable + gridvar = ['lon','lat','lonc','latc','x','y','xc','yc', + 'a1u','a2u','aw0','awx','awy'] + + # Figure out which quantity to treat + self._gridvar = [] + for key in gridvar: + if key in data.variables.keys(): + self._gridvar.append(key) + else: + if key in ["a1u", "a2u", "aw0", "awx", "awy"]: + print "--- "+key+" is missing. Some interpolation functions will not work ---" + if key in ["lonc", "latc", "xc", "yc"]: + print "--- "+key+" is missing. Some element based functions will not work ---" + + for key in self._gridvar: + try: + setattr(self, key, data.variables[key].data) + except AttributeError: #exception for nc.dataset type data + setattr(self, key, data.variables[key])#[:]) + + #special treatment for triele & trinodes due to Save_as(netcdf) + datavar = data.variables.keys() + if "trinodes" in datavar: + try: + setattr(self, 'trinodes', data.variables['trinodes'].data) + except AttributeError: #exception for nc.dataset type data + setattr(self, 'trinodes', data.variables['trinodes'])#[:]) + elif "nv" in datavar: + try: + self.trinodes = np.transpose(data.variables['nv'].data) - 1 + except AttributeError: #exception for nc.dataset type data + self.trinodes = np.transpose(data.variables['nv'][:]) - 1 + else: + print "--- surrounding node indices (nv) missing. Some functions will not work ---" + if "triele" in datavar: + try: + setattr(self, 'triele', data.variables['triele'].data) + except AttributeError: #exception for nc.dataset type data + setattr(self, 'triele', data.variables['triele'])#[:]) + elif "nbe" in datavar: + try: + self.triele = np.transpose(data.variables['nbe'].data) - 1 + except AttributeError: #exception for nc.dataset type data + self.triele = np.transpose(data.variables['nbe'][:]) - 1 + else: + print "--- surrounding element indices (nbe) missing. Some functions will not work ---" + + #special treatment for depth2D & depth due to Save_as(netcdf) + if "depth2D" in datavar: + setattr(self, "depth2D", data.variables["depth2D"])#[:]) + if "depth" in datavar: + setattr(self, "depth", data.variables["depth"])#[:]) + + if ax==[]: + #Define bounding box + self._ax = [] + #Append message to History field + text = 'Full spatial domain' + self._History.append(text) + #Define the rest of the grid variables + try: self.h = data.variables['h'][:] + except KeyError: pass + try: self.siglay = data.variables['siglay'][:] + except KeyError: pass + try: self.siglev = data.variables['siglev'][:] + except KeyError: pass + try: self.nlevel = self.siglay.shape[0] + except AttributeError: pass + try: self.nele = self.lonc.shape[0] + except AttributeError: pass + try: self.nnode = self.lon.shape[0] + except: pass + else: + #Checking for pre-defined regions + if ax=='GP': ax=[-66.36, -66.31, 44.24, 44.3] + elif ax=='PP': ax=[-66.23, -66.19, 44.37, 44.41] + elif ax=='DG': ax=[-65.84, -65.73, 44.64, 44.72] + elif ax=='MP': ax=[-65.5, -63.3, 45.0, 46.0] + + print 'Re-indexing may take some time...' + Data = regioner(self, ax, debug=debug) + #list of grid variable + gridvar = ['lon','lat','lonc','latc','x','y','xc','yc', + 'a1u','a2u','aw0','awx','awy','nv','nbe'] + + # Figure out which quantity to treat + self._gridvar = [] + for key in gridvar: + if key in data.variables.keys(): + self._gridvar.append(key) + else: + if debug: print "Grid related field "+key+" is missing !" + + for key in self._gridvar: + setattr(self, key, Data[key][:]) + # Special treatment here + self.trinodes = Data['nv'][:] + self.triele = Data['nbe'][:] + self.triangle = Data['triangle'] + # Only load the element within the box + self._node_index = Data['node_index'] + self._element_index = Data['element_index'] + + # different loading technique if using OpenDap server + if type(data.variables).__name__ == 'DatasetType': + # Split into consecutive integers to optimise loading + # TR comment: data.variables['ww'].data[:,:,region_n] doesn't + # work with non consecutive indices + H=0 + for k, g in groupby(enumerate(self._node_index), lambda (i,x):i-x): + ID = map(itemgetter(1), g) + # if debug: print 'Index bound: ' + str(ID[0]) + '-' + str(ID[-1]+1) + if H==0: + try: + self.h = data.variables['h'].data[ID[0]:(ID[-1]+1)] + self.siglay = data.variables['siglay'].data[:,ID[0]:(ID[-1]+1)] + self.siglev = data.variables['siglev'].data[:,ID[0]:(ID[-1]+1)] + except AttributeError: #exception for nc.dataset type data + self.h = data.variables['h'][ID[0]:(ID[-1]+1)] + self.siglay = data.variables['siglay'][:,ID[0]:(ID[-1]+1)] + self.siglev = data.variables['siglev'][:,ID[0]:(ID[-1]+1)] + else: + try: + self.h = np.hstack((self.h, + data.variables['h'].data[ID[0]:(ID[-1]+1)])) + self.siglay = np.hstack((self.siglay, + data.variables['siglay'].data[:,ID[0]:(ID[-1]+1)])) + self.siglev = np.hstack((self.siglev, + data.variables['siglev'].data[:,ID[0]:(ID[-1]+1)])) + except AttributeError: #exception for nc.dataset type data + self.h = np.hstack((self.h, + data.variables['h'][ID[0]:(ID[-1]+1)])) + self.siglay = np.hstack((self.siglay, + data.variables['siglay'][:,ID[0]:(ID[-1]+1)])) + self.siglev = np.hstack((self.siglev, + data.variables['siglev'][:,ID[0]:(ID[-1]+1)])) + H=1 + else: + try: + self.h = data.variables['h'].data[self._node_index] + self.siglay = data.variables['siglay'].data[:,self._node_index] + self.siglev = data.variables['siglev'].data[:,self._node_index] + except AttributeError: #exception for nc.dataset type data + self.h = data.variables['h'][self._node_index] + self.siglay = data.variables['siglay'][:,self._node_index] + self.siglev = data.variables['siglev'][:,self._node_index] + #Dimensions + self.nlevel = self.siglay.shape[0] + self.nele = Data['element_index'].shape[0] + self.nnode = Data['node_index'].shape[0] + + del Data + #Define bounding box + self._ax = ax + # Add metadata entry + text = 'Bounding box =' + str(ax) + self._History.append(text) + print '-Now working in bounding box-' + + if debug: + print '...Passed' + + return + +class _load_var: + """ + **'Variables' subset in FVCOM class contains the numpy arrays** + + Some variables are directly passed on from FVCOM output: :: + + _el = elevation (m), 2D array (ntime, nnode) + |_julianTime = julian date, 1D array (ntime) + |_matlabTime = matlab time, 1D array (ntime) + |_tauc = bottom shear stress (m2/s2), + | 2D array (ntime, nele) + |_ua = depth averaged u velocity component (m/s), + | 2D array (ntime, nele) + |_va = depth averaged v velocity component (m/s), + FVCOM.Variables._| 2D array (ntime, nele) + |_u = u velocity component (m/s), + | 3D array (ntime, nlevel, nele) + |_v = v velocity component (m/s), + | 3D array (ntime, nlevel, nele) + |_w = w velocity component (m/s), + | 3D array (ntime, nlevel, nele) + + Some others shall be generated as methods are being called, ex: :: + ... + |_hori_velo_norm = horizontal velocity norm (m/s), + | 2D array (ntime, nele) + |_velo_norm = velocity norm (m/s), + | 3D array (ntime, nlevel, nele) + |_verti_shear = vertical shear (1/s), + | 3D array (ntime, nlevel, nele) + |_vorticity... + + """ + def __init__(self, data, grid, tx, History, debug=False): + self._debug = debug + self._3D = False + self._opendap = type(data.variables).__name__=='DatasetType' + + # Pointer to History + setattr(self, '_History', History) + + # Parallel computing attributs + #self._cpus = mp.cpu_count() + + #List of keywords + kwl2D = ['ua', 'va', 'zeta','depth_av_flow_dir', 'hori_velo_norm', + 'depth_av_vorticity', 'depth_av_power_density', + 'depth_av_power_assessment', 'tauc'] + kwl3D = ['ww', 'u', 'v', 'gls', 'tke', 'flow_dir', 'velo_norm', + 'verti_shear', 'vorticity', 'power_density'] + #List of aliaSes + al2D = ['ua', 'va', 'el','depth_av_flow_dir', 'hori_velo_norm', + 'depth_av_vorticity', 'depth_av_power_density', + 'depth_av_power_assessment', 'tauc'] + al3D = ['w', 'u', 'v', 'gls', 'tke', 'flow_dir', 'velo_norm', + 'verti_shear', 'vorticity', 'power_density'] + + # Figure out which quantity to treat + self._kwl2D = [] + self._al2D = [] + for key, aliaS in zip(kwl2D, al2D): + if key in data.variables.keys(): + self._kwl2D.append(key) + self._al2D.append(aliaS) + else: + if debug: print key, " is missing !" + + + self._kwl3D = [] + self._al3D = [] + for key, aliaS in zip(kwl3D, al3D): + if key in data.variables.keys(): + self._kwl3D.append(key) + self._al3D.append(aliaS) + else: + if debug: print key, " is missing !" + + if not len(self._kwl3D)==0: self._3D = True + + #Loading time stamps + try: + julianTime = data.variables['julianTime'] + except KeyError: + # exception due to Save_as(netcdf) + julianTime=data.variables['time'] + # Work out if time is in julian time + timeFlag = 0.0 + try: + for key in julianTime.attributes: + if "julian" in julianTime.attributes[key].lower(): + timeFlag += 1.0 + except AttributeError: # pydap lib error + if "julian" in julianTime.format.lower(): + timeFlag += 1.0 + # if not julian time, convert in days in needed + if timeFlag == 0.0: + try: + for key in julianTime.attributes: + if "second" in julianTime.attributes[key].lower(): + timeFlag += 1.0 + except AttributeError: # pydap lib error + if "second" in julianTime.units.lower(): + timeFlag += 1.0 + if not timeFlag == 0.0: # TR: this conversion needs to be improved by introducing the right epoch + dayTime = julianTime[:] / (24*60*60) # convert in days + julianTime = dayTime + + if tx==[]: + # get time and adjust it to matlab datenum + self.julianTime = julianTime[:] + self.matlabTime = self.julianTime[:] + 678942.0 + #-Append message to History field + start = mattime_to_datetime(self.matlabTime[0]) + end = mattime_to_datetime(self.matlabTime[-1]) + text = 'Full temporal domain from ' + str(start) +\ + ' to ' + str(end) + self._History.append(text) + #Add time dimension to grid variables + grid.ntime = self.julianTime.shape[0] + if debug: print 'Full temporal domain' + else: + #Time period + region_t = self._t_region(tx, julianTime, debug=debug) + self._region_time = region_t + ts = self._region_time[0] + te = self._region_time[-1] + 1 + # get time and adjust it to matlab datenum + self.julianTime = julianTime[ts:te] + self.matlabTime = self.julianTime + 678942.0 + #-Append message to History field + start = mattime_to_datetime(self.matlabTime[0]) + end = mattime_to_datetime(self.matlabTime[-1]) + text = 'Temporal domain from ' + str(start) +\ + ' to ' + str(end) + self._History.append(text) + #Add time dimension to grid variables + grid.ntime = self.julianTime.shape[0] + if debug: print "ntime: ", grid.ntime + if debug: print "region_t shape: ", region_t.shape + + # Define which loading function to use + if grid._ax==[] and tx==[]: + loadVar = self._load_full_time_full_region + for key, aliaS in zip(self._kwl2D, self._al2D): + loadVar(data, key, aliaS, debug=debug) + for key, aliaS in zip(self._kwl3D, self._al3D): + loadVar(data, key, aliaS, debug=debug) + else: + + if grid._ax!=[] and tx!=[]: + loadVar = self._load_partial_time_partial_region + elif grid._ax==[] and tx!=[]: + loadVar = self._load_partial_time_full_region + else: + loadVar = self._load_full_time_partial_region + + # Loading 2D variables + for key, aliaS in zip(self._kwl2D, self._al2D): + gc.collect() # force garbage collector in an attempt to free up some RAM + loadVar(data, grid, key, aliaS, debug=debug) + + # Loading 3D variables + for key, aliaS in zip(self._kwl3D, self._al3D): + gc.collect() # force garbage collector in an attempt to free up some RAM + loadVar(data, grid, key, aliaS, debug=debug) + + ##-------Parallelized loading block------- + # if debug: startT = time.time() + # + # divisor = len(self._kwl2D)//self._cpus + # remainder = len(self._kwl2D)%self._cpus + # + # if debug: print "Parallel loading 2D vars..." + # if debug: start2D = time.time() + # + # for i in range(divisor): + # start = self._cpus * i + # end = start + (self._cpus-1) + # processes = [mp.Process(target=loadVar, args=(data, grid, key, aliaS, debug))\ + # for key, aliaS in zip(self._kwl2D[start:end], self._al2D[start:end])] + # # Run processes + # for p in processes: + # p.start() + # # Exit the completed processes + # for p in processes: + # p.join() + # + # # Remaining vars + # if remainder != 0: + # start = int(-1 * remainder) + # processes = [mp.Process(target=loadVar, args=(data, grid, key, aliaS, debug))\ + # for key, aliaS in zip(self._kwl2D[start:], self._al2D[start:])] + # # Run processes + # for p in processes: + # p.start() + # # Exit the completed processes + # for p in processes: + # p.join() + # + # if debug: end2D = time.time() + # if debug: print "...processing time: ", (end2D - start2D) + # + # if debug: print "Parallel loading 3D vars..." + # if debug: start3D = time.time() + # + # for i in range(divisor): + # start = self._cpus * i + # end = start + (self._cpus-1) + # processes = [mp.Process(target=loadVar, args=(data, grid,key, aliaS, debug))\ + # for key, aliaS in zip(self._kwl3D[start:end], self._al3D[start:end])] + # # Run processes + # for p in processes: + # p.start() + # # Exit the completed processes + # for p in processes: + # p.join() + # + # # Remaining vars + # if remainder != 0: + # start = int(-1 * remainder) + # processes = [mp.Process(target=loadVar, args=(data, grid, key, aliaS, debug))\ + # for key, aliaS in zip(self._kwl3D[start:], self._al3D[start:])] + # # Run processes + # for p in processes: + # p.start() + # # Exit the completed processes + # for p in processes: + # p.join() + # + # if debug: end3D = time.time() + # if debug: endT = time.time() + # if debug: print "...-loading 3D- processing time: ", (end3D - start3D) + # if debug: print "...-loading 2D & 3D- processing time: ", (endT - startT) + # #-------end------- + + if debug: print '...Passed' + + # Define method loadvar for use in import + self._loadVar = loadVar + + return + + def _load_full_time_full_region(self, data, key, aliaS, debug=False): + """ + loading variables for full time and space domains + + Inputs: + - key = FVCOM variable name, str + - aliaS = PySeidon variable alias, str + + Options: + - debug = debug flag, boolean + """ + if debug: print "loading " + str(aliaS) +"..." + try: + setattr(self, aliaS, data.variables[key].data) + except AttributeError: #exeception due nc.Dataset + setattr(self, aliaS, data.variables[key]) + + def _load_partial_time_partial_region(self, data, grid, key, aliaS, debug=False): + """ + loading variables for partial time and space domains + + Inputs: + - key = FVCOM variable name, str + - aliaS = PySeidon variable alias, str + + Options: + - debug = debug flag, boolean + """ + if debug: print "loading " + str(aliaS) +"..." + + # define time bounds + ts = self._region_time[0] + te = self._region_time[-1] + 1 + + if key == 'zeta': + region = grid._node_index + horiDim = grid.nnode + else: + region = grid._element_index + horiDim = grid.nele + + if key == 'verti_shear': + vertiDim = grid.nlevel-1 + else: + vertiDim = grid.nlevel + + # Find out if using netCDF4 or scipy + try: + Test = data.variables[key].data + self._scipynetcdf = True + except AttributeError: # exeception due nc.Dataset + self._scipynetcdf = False + + if self._opendap: + # loop over contiguous indexes for opendap + H = 0 #local counter + for k, g in groupby(enumerate(region), lambda (i,x):i-x): + ID = map(itemgetter(1), g) + #if debug: print 'Index bound: ' + str(ID[0]) + '-' + str(ID[-1]+1) + if key in self._kwl2D: + if self._scipynetcdf: + #TR : Don't I need to transpose here? + var = data.variables[key].data[ts:te,ID[0]:(ID[-1]+1)] + else: + var = data.variables[key][ts:te,ID[0]:(ID[-1]+1)] + if H == 0: + setattr(self, aliaS,var) + H = 1 + else: + setattr(self, aliaS, np.hstack((getattr(self, aliaS), var))) + else: + if self._scipynetcdf: + #TR : Don't I need to transpose here? + var = data.variables[key].data[ts:te,:,ID[0]:(ID[-1]+1)] + else: + var = data.variables[key][ts:te,:,ID[0]:(ID[-1]+1)] + if H == 0: + setattr(self, aliaS,var) + H = 1 + else: + setattr(self, aliaS, np.dstack((getattr(self, aliaS), var))) + # TR comment: looping on time indices is a trick from Mitchell O'Flaherty-Sproul to improve loading time + else: + I = 0 + if key in self._kwl2D: + setattr(self, aliaS, np.zeros((grid.ntime, horiDim))) + for i in self._region_time: + if self._scipynetcdf: + getattr(self, aliaS)[I,:] = np.transpose(data.variables[key].data[i, region]) + else: + getattr(self, aliaS)[I,:] = (data.variables[key][i, region]) + I += 1 + else: + setattr(self, aliaS, np.zeros((grid.ntime, vertiDim, horiDim))) + for i in self._region_time: + if self._scipynetcdf: + getattr(self, aliaS)[I,:,:] = np.transpose(data.variables[key].data[i, :, region]) + else: + getattr(self, aliaS)[I,:,:] = (data.variables[key][i, :, region]) + I += 1 + + + def _load_full_time_partial_region(self, data, grid, key, aliaS, debug=False): + """ + loading variables for full time domain and partial space domain + + Inputs: + - key = FVCOM variable name, str + - aliaS = PySeidon variable alias, str + + Options: + - debug = debug flag, boolean + """ + if debug: print "loading " + str(aliaS) +"..." + if key == 'zeta': + region = grid._node_index + horiDim = grid.nnode + else: + region = grid._element_index + horiDim = grid.nele + + if key == 'verti_shear': + vertiDim = grid.nlevel-1 + else: + vertiDim = grid.nlevel + + # Find out if using netCDF4 or scipy + try: + Test = data.variables[key].data + self._scipynetcdf = True + except AttributeError: # exeception due nc.Dataset + self._scipynetcdf = False + + if self._opendap: + # loop over contiguous indexes for opendap + H = 0 #local counter + for k, g in groupby(enumerate(region), lambda (i,x):i-x): + ID = map(itemgetter(1), g) + #if debug: print 'Index bound: ' + str(ID[0]) + '-' + str(ID[-1]+1) + if key in self._kwl2D: + if self._scipynetcdf: + #TR : Don't I need to transpose here? + var = data.variables[key].data[:,ID[0]:(ID[-1]+1)] + else: + var = data.variables[key][:,ID[0]:(ID[-1]+1)] + if H == 0: + setattr(self, aliaS,var) + H = 1 + else: + setattr(self, aliaS, np.hstack((getattr(self, aliaS), var))) + else: + if self._scipynetcdf: + #TR : Don't I need to transpose here? + var = data.variables[key].data[:,:,ID[0]:(ID[-1]+1)] + else: + var = data.variables[key][:,:,ID[0]:(ID[-1]+1)] + if H == 0: + setattr(self, aliaS,var) + H = 1 + else: + setattr(self, aliaS, np.dstack((getattr(self, aliaS), var))) + else: + # TR comment: looping on time indices is a trick from Mitchell O'Flaherty-Sproul to improve loading time + if key in self._kwl2D: + setattr(self, aliaS, np.zeros((grid.ntime, horiDim))) + for i in range(grid.ntime): + if self._scipynetcdf: + getattr(self, aliaS)[i,:] = np.transpose(data.variables[key].data[i, region]) + else: + getattr(self, aliaS)[i,:] = (data.variables[key][i, region]) + else: + setattr(self, aliaS, np.zeros((grid.ntime, vertiDim, horiDim))) + for i in range(grid.ntime): + if self._scipynetcdf: + getattr(self, aliaS)[i,:,:] = np.transpose(data.variables[key].data[i, :, region]) + else: + getattr(self, aliaS)[i,:,:] = (data.variables[key][i, :, region]) + + def _load_partial_time_full_region(self, data, grid, key, aliaS, debug=False): + """ + loading variables for partial time domain and full space domain + + Inputs: + - key = FVCOM variable name, str + - aliaS = PySeidon variable alias, str + + Options: + - debug = debug flag, boolean + """ + if debug: print "loading " + str(aliaS) +"..." + + # define time bounds + ts = self._region_time[0] + te = self._region_time[-1] + 1 + + if key == 'zeta': + horiDim = grid.nnode + else: + horiDim = grid.nele + + if key == 'verti_shear': + vertiDim = grid.nlevel-1 + else: + vertiDim = grid.nlevel + + # Find out if using netCDF4 or scipy + try: + Test = data.variables[key].data + self._scipynetcdf = True + except AttributeError: # exeception due nc.Dataset + self._scipynetcdf = False + + if self._opendap: + if key in self._kwl2D: + if self._scipynetcdf: + var = data.variables[key].data[ts:te,:] + else: + var = data.variables[key][ts:te,:] + else: + if self._scipynetcdf: + var = data.variables[key].data[ts:te,:,:] + else: + var = data.variables[key][ts:te,:,:] + setattr(self, aliaS,var) + else: + I = 0 + # TR comment: looping on time indices is a trick from Mitchell O'Flaherty-Sproul to improve loading time + if key in self._kwl2D: + setattr(self, aliaS, np.zeros((grid.ntime, horiDim))) + for i in self._region_time: + if self._scipynetcdf: + getattr(self, aliaS)[I,:] = np.transpose(data.variables[key].data[i, :]) + else: + getattr(self, aliaS)[I,:] = (data.variables[key][i, :]) + I += 1 + else: + setattr(self, aliaS, np.zeros((grid.ntime, vertiDim, horiDim))) + for i in self._region_time: + if self._scipynetcdf: + getattr(self, aliaS)[I,:,:] = np.transpose(data.variables[key].data[i, :, :]) + else: + getattr(self, aliaS)[I,:,:] = (data.variables[key][i, :, :]) + I += 1 + + def _t_region(self, tx, julianTime, debug=False): + """Return time indices included in time period, aka tx""" + debug = debug or self._debug + if debug: print 'Computing region_t...' + start = datetime.datetime.strptime(tx[0], '%Y-%m-%d %H:%M:%S') + end = datetime.datetime.strptime(tx[1], '%Y-%m-%d %H:%M:%S') + region_t = time_to_index(start, end, julianTime[:], debug=debug) + if debug: print '...Passed' + print '-Now working in time box-' + return region_t diff --git a/build/lib/pyseidon_dvt/stationClass/__init__.py b/build/lib/pyseidon_dvt/stationClass/__init__.py new file mode 100644 index 0000000..5afd3fc --- /dev/null +++ b/build/lib/pyseidon_dvt/stationClass/__init__.py @@ -0,0 +1,10 @@ +#!/usr/bin/python2.7 +# encoding: utf-8 + +#Local import +from stationClass import Station + +__authors__ = ['Wesley Bowman, Thomas Roc, Jonathan Smith'] +__licence__ = 'GNU Affero GPL v3.0' +__copyright__ = 'Copyright (c) 2014 EcoEnergyII' + diff --git a/build/lib/pyseidon_dvt/stationClass/functionsStation.py b/build/lib/pyseidon_dvt/stationClass/functionsStation.py new file mode 100644 index 0000000..764824f --- /dev/null +++ b/build/lib/pyseidon_dvt/stationClass/functionsStation.py @@ -0,0 +1,483 @@ +#!/usr/bin/python2.7 +# encoding: utf-8 + +from __future__ import division +import numpy as np +import sys +import numexpr as ne +from pyseidon_dvt.utilities.miscellaneous import * +from pyseidon_dvt.utilities.BP_tools import * +from utide import solve, reconstruct +import time + +# Custom error +from pyseidon_error import PyseidonError + +class FunctionsStation: + """ + **'Util2D' subset of Station class gathers useful functions for 2D and 3D runs** + """ + def __init__(self, variable, grid, plot, History, debug): + self._debug = debug + self._plot = plot + #Create pointer to Station class + setattr(self, '_var', variable) + setattr(self, '_grid', grid) + setattr(self, '_History', History) + + def search_index(self, station): + """Search for the station index""" + if type(station)==int: + index = station + elif type(station).__name__ in ['str', 'ndarray']: + station = "".join(station).strip().upper() + for i in range(self._grid.nele): + if station=="".join(self._grid.name[i]).strip().upper(): + index=i + else: + raise PyseidonError("---Wrong station input---") + if not 'index' in locals(): + raise PyseidonError("---Wrong station input---") + + return index + + def flow_dir(self, station, t_start=[], t_end=[], time_ind=[], + exceedance=False, debug=False): + """ + Flow directions and associated norm at any give location. + + Inputs: + - station = either station index (interger) or name (string) + + Outputs: + - flowDir = flowDir at station, 1D array + - norm = velocity norm at station, 1D array + + Options: + - t_start = start time, as string ('yyyy-mm-ddThh:mm:ss'), or time index as an integer + - t_end = end time, as a string ('yyyy-mm-ddThh:mm:ss'), or time index as an integer + - time_ind = time indices to work in, list of integers + - excedance = True, compute associated exceedance curve + + *Notes* + - directions between -180 and 180 deg., i.e. 0=East, 90=North, + +/-180=West, -90=South + """ + debug = debug or self._debug + if debug: + print 'Computing flow directions at point...' + + # Find time interval to work in + argtime = [] + if not time_ind==[]: + argtime = time_ind + elif not t_start==[]: + if type(t_start)==str: + argtime = time_to_index(t_start, t_end, self._var.matlabTime, debug=debug) + else: + argtime = np.arange(t_start, t_end) + + #Search for the station + index = self.search_index(station) + + #Choose the right pair of velocity components + if not argtime==[]: + U = self._var.ua[argtime,index] + V = self._var.va[argtime,index] + else: + U = self._var.ua[:,index] + V = self._var.va[:,index] + + #Compute directions + if debug: + print 'Computing arctan2 and norm...' + dirFlow = np.rad2deg(np.arctan2(V,U)) + + #Compute velocity norm + norm = ne.evaluate('sqrt(U**2 + V**2)').squeeze() + if debug: + print '...Passed' + #Rose diagram + self._plot.rose_diagram(dirFlow, norm) + if exceedance: + self.exceedance(norm) + + return dirFlow, norm + + def ebb_flood_split(self, station, + t_start=[], t_end=[], time_ind=[], debug=False): + """ + Compute time indices for ebb and flood but also the + principal flow directions and associated variances for (lon, lat) point + + Inputs: + - station = either station index (interger) or name (string) + + Outputs: + - floodIndex = flood time index, 1D array of integers + - ebbIndex = ebb time index, 1D array of integers + - pr_axis = principal flow ax1s, float number in degrees + - pr_ax_var = associated variance, float number + + Options: + - t_start = start time, as string ('yyyy-mm-ddThh:mm:ss'), or time index as an integer + - t_end = end time, as a string ('yyyy-mm-ddThh:mm:ss'), or time index as an integer + - time_ind = time indices to work in, list of integers + + *Notes* + - may take time to compute if time period too long + - directions between -180 and 180 deg., i.e. 0=East, 90=North, + +/-180=West, -90=South + - use time_ind or t_start and t_end, not both + - assume that flood is aligned with principal direction + """ + debug = debug or self._debug + if debug: + start = time.time() + print 'Computing principal flow directions...' + + # Find time interval to work in + argtime = [] + if not time_ind==[]: + argtime = time_ind + elif not t_start==[]: + if type(t_start)==str: + argtime = time_to_index(t_start, t_end, + self._var.matlabTime, + debug=debug) + else: + argtime = np.arange(t_start, t_end) + + #Search for the station + index = self.search_index(station) + + #Choose the right pair of velocity components + if not argtime==[]: + U = self._var.ua[argtime,index] + V = self._var.va[argtime,index] + else: + U = self._var.ua[:,index] + V = self._var.va[:,index] + + #WB version of BP's principal axis + if debug: print 'Computin principal axis at point...' + pr_axis, pr_ax_var = principal_axis(U, V) + + #ebb/flood split + if debug: print 'Splitting ebb and flood at point...' + ###TR: version 1 + ## reverse 0-360 deg convention + #ra = (-pr_axis - 90.0) * np.pi /180.0 + #if ra>np.pi: + # ra = ra - (2.0*np.pi) + #elif ra<-np.pi: + # ra = ra + (2.0*np.pi) + #dirFlow = np.arctan2(V,U) + ##Define bins of angles + #if ra == 0.0: + # binP = [0.0, np.pi/2.0] + # binP = [0.0, -np.pi/2.0] + #elif ra > 0.0: + # if ra == np.pi: + # binP = [np.pi/2.0 , np.pi] + # binM = [-np.pi, -np.pi/2.0 ] + # elif ra < (np.pi/2.0): + # binP = [0.0, ra + (np.pi/2.0)] + # binM = [-((np.pi/2.0)-ra), 0.0] + # else: + # binP = [ra - (np.pi/2.0), np.pi] + # binM = [-np.pi, -np.pi + (ra-(np.pi/2.0))] + #else: + # if ra == -np.pi: + # binP = [np.pi/2.0 , np.pi] + # binM = [-np.pi, -np.pi/2.0] + # elif ra > -(np.pi/2.0): + # binP = [0.0, ra + (np.pi/2.0)] + # binM = [ ((-np.pi/2.0)+ra), 0.0] + # else: + # binP = [np.pi - (ra+(np.pi/2.0)) , np.pi] + # binM = [-np.pi, ra + (np.pi/2.0)] + # + #test = (((dirFlow > binP[0]) * (dirFlow < binP[1])) + + # ((dirFlow > binM[0]) * (dirFlow < binM[1]))) + #floodIndex = np.where(test == True)[0] + #ebbIndex = np.where(test == False)[0] + + #Defines interval + flood_heading = np.array([-90, 90]) + pr_axis + dir_all = np.rad2deg(np.arctan2(V,U)) + ind = np.where(dir_all<0) + dir_all[ind] = dir_all[ind] + 360 + + # sign speed - eliminating wrap-around + dir_PA = dir_all - pr_axis + dir_PA[dir_PA < -90] += 360 + dir_PA[dir_PA > 270] -= 360 + + #general direction of flood passed as input argument + floodIndex = np.where((dir_PA >= -90) & (dir_PA<90)) + ebbIndex = np.arange(dir_PA.shape[0]) + ebbIndex = np.delete(ebbIndex,floodIndex[:]) + + if debug: + end = time.time() + print "...processing time: ", (end - start) + + return floodIndex, ebbIndex, pr_axis, pr_ax_var + + def exceedance(self, var, station=[], debug=False): + """ + This function calculate the excedence curve of a var(time). + + Inputs: + - var = given quantity, 1 or 2D array of n elements, i.e (time) or (time,ele) + + Options: + - station = either station index (interger) or name (string) + Necessary if var = 2D (i.e. [time, nnode or nele] + - graph: True->plots curve; False->does not + + Outputs: + - Exceedance = list of % of occurences, 1D array + - Ranges = list of signal amplitude bins, 1D array + + *Notes* + - This method is not suitable for SSE + """ + debug = (debug or self._debug) + if debug: + print 'Computing exceedance...' + + #Distinguish between 1D and 2D var + if len(var.shape)>1: + if station==[]: + print 'Lon, lat coordinates are needed' + sys.exit() + #Search for the station + index = self.search_index(station) + signal = var[:,index] + else: + signal=var + + Max = max(signal) + dy = (Max/30.0) + Ranges = np.arange(0,(Max + dy), dy) + Exceedance = np.zeros(Ranges.shape[0]) + dt = self._var.julianTime[1] - self._var.julianTime[0] + if dt==0: + dt = self._var.secondTime[1] - self._var.secondTime[0] + Period = var.shape[0] * dt + time = np.arange(0.0, Period, dt) + + N = len(signal) + M = len(Ranges) + + for i in range(M): + r = Ranges[i] + for j in range(N-1): + if signal[j] > r: + Exceedance[i] = Exceedance[i] + (time[j+1] - time[j]) + + Exceedance = (Exceedance * 100) / Period + + if debug: + print '...Passed' + + #Plot + self._plot.plot_xy(Exceedance, Ranges, yLabel='Amplitudes', + xLabel='Exceedance probability in %') + + return Exceedance, Ranges + + def depth(self, station, debug=False): + """ + Compute depth at given point + + Inputs: + - station = either station index (interger) or name (string) + + Outputs: + - dep = depth, 2D array (ntime, nlevel) + + *Notes* + - depth convention: 0 = free surface + """ + debug = debug or self._debug + if debug: + print "Computing depth..." + start = time.time() + + #Search for the station + index = self.search_index(station) + + #Compute depth + h = self._grid.h[index] + el = self._var.el[:,index] + dep = el + h + + if debug: + end = time.time() + print "Computation time in (s): ", (end - start) + + return dep + + def speed_histogram(self, station, t_start=[], t_end=[], time_ind=[], debug=False): + """ + This function plots the histogram of occurrences for the signed + flow speed at any given point. + + Inputs: + - station = either station index (interger) or name (string) + + Options: + - t_start = start time, as string ('yyyy-mm-ddThh:mm:ss'), or time index as an integer + - t_end = end time, as a string ('yyyy-mm-ddThh:mm:ss'), or time index as an integer + - time_ind = time indices to work in, list of integers + + *Notes* + - use time_ind or t_start and t_end, not both + """ + debug = debug or self._debug + if debug: + start = time.time() + print 'Computing speed histogram...' + + pI, nI, pa, pav = self.ebb_flood_split(station, + t_start=t_start, t_end=t_end, time_ind=time_ind, + debug=debug) + dirFlow, norm = self.flow_dir(station, + t_start=t_start, t_end=t_end, time_ind=time_ind, + exceedance=False, debug=debug) + norm[nI] = -1.0 * norm[nI] + + #compute bins + #minBound = norm.min() + #maxBound = norm.max() + #step = round((maxBound-minBound/51.0),1) + #bins = np.arange(minBound,maxBound,step) + + #plot histogram + self._plot.Histogram(norm, + xLabel='Signed flow speed (m/s)', + yLabel='Occurrences (%)') + + if debug: + end = time.time() + print "...processing time: ", (end - start) + + + def Harmonic_analysis_at_point(self, station, + time_ind=[], t_start=[], t_end=[], + elevation=True, velocity=False, + debug=False, **kwarg): + """ + This function performs a harmonic analysis on the sea surface elevation + time series or the velocity components timeseries. + + Inputs: + - station = either station index (interger) or name (string) + + Outputs: + - harmo = harmonic coefficients, dictionary + + Options: + - t_start = start time, as string ('yyyy-mm-ddThh:mm:ss'), or time index as an integer + - t_end = end time, as a string ('yyyy-mm-ddThh:mm:ss'), or time index as an integer + - time_ind = time indices to work in, list of integers + - elevation=True means that 'solve' will be done for elevation. + - velocity=True means that 'solve' will be done for velocity. + + Utide's options: + Options are the same as for 'solve', which are shown below with + their default values: + conf_int=True; cnstit='auto'; notrend=0; prefilt=[]; nodsatlint=0; + nodsatnone=0; gwchlint=0; gwchnone=0; infer=[]; inferaprx=0; + rmin=1; method='cauchy'; tunrdn=1; linci=0; white=0; nrlzn=200; + lsfrqosmp=1; nodiagn=0; diagnplots=0; diagnminsnr=2; + ordercnstit=[]; runtimedisp='yyy' + + *Notes* + For more detailed information about 'solve', please see + https://github.com/wesleybowman/UTide + + """ + debug = (debug or self._debug) + + #Search for the station + index = self.search_index(station) + + argtime = [] + if not time_ind==[]: + argtime = time_ind + elif not t_start==[]: + if type(t_start)==str: + argtime = time_to_index(t_start, t_end, + self._var.matlabTime, + debug=debug) + else: + argtime = np.arange(t_start, t_end) + + if velocity == elevation: + raise PyseidonError("---Can only process either velocities or elevation. Change options---") + + if velocity: + time = self._var.matlabTime[:] + u = self._var.ua[:,index] + v = self._var.va[:,index] + + if not argtime==[]: + time = time[argtime[:]] + u = u[argtime[:]] + v = v[argtime[:]] + + lat = self._grid.lat[index] + harmo = solve(time, u, v, lat, **kwarg) + + if elevation: + time = self._var.matlabTime[:] + el = self._var.el[:,index] + + if not argtime==[]: + time = time[argtime[:]] + el = el[argtime[:]] + + lat = self._grid.lat[index] + harmo = solve(time, el, None, lat, **kwarg) + #Write meta-data only if computed over all the elements + + return harmo + + def Harmonic_reconstruction(self, harmo, recon_time=[], time_ind=slice(None), debug=False, **kwarg): + """ + This function reconstructs the velocity components or the surface elevation + from harmonic coefficients. + Harmonic_reconstruction calls 'reconstruct'. This function assumes harmonics + ('solve') has already been executed. + + Inputs: + - Harmo = harmonic coefficient from harmo_analysis + - time_ind = time indices to process, list of integers + - recon_time = time you want the harmonic coefficients to be reconstructed at + + Output: + - Reconstruct = reconstructed signal, dictionary + + Options: + Options are the same as for 'reconstruct', which are shown below with + their default values: + cnstit = [], minsnr = 2, minpe = 0 + + *Notes* + For more detailed information about 'reconstruct', please see + https://github.com/wesleybowman/UTide + + """ + debug = (debug or self._debug) + if recon_time == []: + time = self._var.matlabTime[time_ind] + else: + time = recon_time + Reconstruct = reconstruct(time, harmo) + + + return Reconstruct diff --git a/build/lib/pyseidon_dvt/stationClass/functionsStationThreeD.py b/build/lib/pyseidon_dvt/stationClass/functionsStationThreeD.py new file mode 100644 index 0000000..177d1ca --- /dev/null +++ b/build/lib/pyseidon_dvt/stationClass/functionsStationThreeD.py @@ -0,0 +1,456 @@ +#!/usr/bin/python2.7 +# encoding: utf-8 + +from __future__ import division + +import numpy as np +import numexpr as ne +from pyseidon_dvt.utilities.miscellaneous import * +from pyseidon_dvt.utilities.BP_tools import * +import time + +from utide import solve, reconstruct + +# Custom error +from pyseidon_error import PyseidonError + +class FunctionsStationThreeD: + """ + **'Utils3D' subset of Station class gathers useful functions for 3D runs** + """ + def __init__(self, variable, grid, plot, History, debug): + #Inheritance + self._debug = debug + self._plot = plot + + #Create pointer to FVCOM class + setattr(self, '_var', variable) + setattr(self, '_grid', grid) + setattr(self, '_History', History) + + def search_index(self, station): + """Search for the station index""" + if type(station)==int: + index = station + elif type(station).__name__ in ['str', 'ndarray']: + station = "".join(station).strip().upper() + for i in range(self._grid.nele): + if station=="".join(self._grid.name[i]).strip().upper(): + index=i + else: + raise PyseidonError("---Wrong station input---") + if not 'index' in locals(): + raise PyseidonError("---Wrong station input---") + + return index + + def depth(self, station, debug=False): + """ + Compute depth at given point + + Inputs: + - station = either station index (interger) or name (string) + + Outputs: + - dep = depth, 2D array (ntime, nlevel) + + Notes: + - depth convention: 0 = free surface + - index is used in case one knows already at which + element depth is requested + """ + debug = debug or self._debug + if debug: + print "Computing depth..." + start = time.time() + + #Search for the station + index = self.search_index(station) + + #Compute depth + h = self._grid.h[index] + el = self._var.el[:,index] + zeta = el + h + siglay = self._grid.siglay[:,index] + dep = zeta[:, np.newaxis]*siglay[np.newaxis, :] + + if debug: + end = time.time() + print "Computation time in (s): ", (end - start) + + return np.squeeze(dep) + + def verti_shear(self, station, t_start=[], t_end=[], time_ind=[], + bot_lvl=[], top_lvl=[], graph=True, debug=False): + """ + Compute vertical shear at any given location + + Inputs: + - station = either station index (interger) or name (string) + + Outputs: + - dveldz = vertical shear (1/s), 2D array (time, nlevel - 1) + + Options: + - t_start = start time, as string ('yyyy-mm-ddThh:mm:ss'), or time index as an integer + - t_end = end time, as a string ('yyyy-mm-ddThh:mm:ss'), or time index as an integer + - time_ind = time indices to work in, list of integers + - bot_lvl = index of the bottom level to consider, integer + - top_lvl = index of the top level to consider, integer + - graph = plots graph if True + + *Notes* + - use time_ind or t_start and t_end, not both + """ + debug = debug or self._debug + if debug: + print 'Computing vertical shear at point...' + + # Find time interval to work in + argtime = [] + if not time_ind==[]: + argtime = time_ind + elif not t_start==[]: + if type(t_start)==str: + start = datetime.datetime.strptime(t_start, '%Y-%m-%d %H:%M:%S') + end = datetime.datetime.strptime(t_end, '%Y-%m-%d %H:%M:%S') + argtime = time_to_index(start, end, self._var.julianTime[:], debug=debug) + else: + argtime = np.arange(t_start, t_end) + + #Search for the station + index = self.search_index(station) + + #Compute depth + dep = self.depth(station, debug=debug) + + if not argtime==[]: + depth = dep[argtime,:] + else: + depth = dep + + #Sigma levels to consider + if top_lvl==[]: + top_lvl = self._grid.nlevel - 1 + if bot_lvl==[]: + bot_lvl = 0 + sLvl = range(bot_lvl, top_lvl+1) + + #Extracting velocity at point + if not argtime==[]: + U = self._var.u[argtime,:,index] + V = self._var.v[argtime,:,index] + else: + U = self._var.u[:,:,index] + V = self._var.v[:,:,index] + + norm = ne.evaluate('sqrt(U**2 + V**2)').squeeze() + + # Compute shear + dz = depth[:,sLvl[1:]] - depth[:,sLvl[:-1]] + dvel = norm[:,sLvl[1:]] - norm[:,sLvl[:-1]] + dveldz = dvel / dz + + if debug: + print '...Passed' + + #Plot mean values + if graph: + mean_depth = np.mean((depth[:,sLvl[1:]] + + depth[:,sLvl[:-1]]) / 2.0, 0) + mean_dveldz = np.mean(dveldz,0) + error = np.std(dveldz,axis=0) + self._plot.plot_xy(mean_dveldz, mean_depth, xerror=error[:], + title='Shear profile ', + xLabel='Shear (1/s) ', yLabel='Depth (m) ') + + return np.squeeze(dveldz) + + def velo_norm(self, station, t_start=[], t_end=[], time_ind=[], + graph=True, debug=False): + """ + Compute the velocity norm at any given location + + Inputs: + - station = either station index (interger) or name (string) + + Outputs: + - velo_norm = velocity norm, 2D array (time, level) + + Options: + - t_start = start time, as string ('yyyy-mm-ddThh:mm:ss'), or time index as an integer + - t_end = end time, as a string ('yyyy-mm-ddThh:mm:ss'), or time index as an integer + - time_ind = time indices to work in, list of integers + - graph = plots vertical profile averaged over time if True + + *Notes* + - use time_ind or t_start and t_end, not both + """ + debug = debug or self._debug + if debug: + print 'Computing velocity norm at point...' + + # Find time interval to work in + argtime = [] + if not time_ind==[]: + argtime = time_ind + elif not t_start==[]: + if type(t_start)==str: + start = datetime.datetime.strptime(t_start, '%Y-%m-%d %H:%M:%S') + end = datetime.datetime.strptime(t_end, '%Y-%m-%d %H:%M:%S') + argtime = time_to_index(start, end, self._var.julianTime[:], debug=debug) + else: + argtime = np.arange(t_start, t_end) + + #Search for the station + index = self.search_index(station) + + #Computing velocity norm + try: + if not argtime==[]: + U = self._var.u[argtime, :, index] + V = self._var.v[argtime, :, index] + W = self._var.w[argtime, :, index] + velo_norm = ne.evaluate('sqrt(U**2 + V**2 + W**2)').squeeze() + else: + U = self._var.u[:, :, index] + V = self._var.v[:, :, index] + W = self._var.w[:, :, index] + velo_norm = ne.evaluate('sqrt(U**2 + V**2 + W**2)').squeeze() + except AttributeError: + if not argtime==[]: + U = self._var.u[argtime, :, index] + V = self._var.v[argtime, :, index] + velo_norm = ne.evaluate('sqrt(U**2 + V**2)').squeeze() + else: + U = self._var.u[:, :, index] + V = self._var.v[:, :, index] + velo_norm = ne.evaluate('sqrt(U**2 + V**2)').squeeze() + + if debug: + print '...passed' + + #Plot mean values + if graph: + depth = np.mean(self.depth(station),axis=0) + vel = np.mean(velo_norm,axis=0) + error = np.std(velo_norm,axis=0) + self._plot.plot_xy(vel, depth, xerror=error[:], + title='Velocity norm profile ', + xLabel='Velocity (m/s) ', yLabel='Depth (m) ') + + + return velo_norm + + def flow_dir(self, station, t_start=[], t_end=[], time_ind=[], + vertical=True, debug=False): + """ + Compute flow directions and associated norm at any given location. + + Inputs: + - station = either station index (interger) or name (string) + + Outputs: + - flowDir = flowDir at (pt_lon, pt_lat), 2D array (ntime, nlevel) + + Options: + - t_start = start time, as string ('yyyy-mm-ddThh:mm:ss'), or time index as an integer + - t_end = end time, as a string ('yyyy-mm-ddThh:mm:ss'), or time index as an integer + - time_ind = time indices to work in, list of integers + - vertical = True, compute flowDir for each vertical level + + *Notes* + - directions between -180 and 180 deg., i.e. 0=East, 90=North, + +/-180=West, -90=South + - use time_ind or t_start and t_end, not both + """ + debug = debug or self._debug + if debug: + print 'Computing flow directions at point...' + + #Search for the station + index = self.search_index(station) + + # Find time interval to work in + argtime = [] + if not time_ind==[]: + argtime = time_ind + elif not t_start==[]: + if type(t_start)==str: + start = datetime.datetime.strptime(t_start, '%Y-%m-%d %H:%M:%S') + end = datetime.datetime.strptime(t_end, '%Y-%m-%d %H:%M:%S') + argtime = time_to_index(start, end, self._var.julianTime[:], debug=debug) + else: + argtime = np.arange(t_start, t_end) + + #Choose the right pair of velocity components + if not argtime==[]: + if self._var._3D and vertical: + u = self._var.u[argtime,:,index] + v = self._var.v[argtime,:,index] + else: + u = self._var.ua[argtime,index] + v = self._var.va[argtime,index] + + #Compute directions + if debug: print 'Computing arctan2 and norm...' + dirFlow = np.rad2deg(np.arctan2(v,u)) + + if debug: + print '...Passed' + + return np.squeeze(dirFlow) + + def Harmonic_analysis_at_point(self, station, + time_ind=[], t_start=[], t_end=[], + elevation=True, velocity=False, + debug=False, **kwarg): + """ + This function performs a harmonic analysis on the sea surface elevation + time series or each layer of the velocity components timeseries. + + Inputs: + - station = either station index (interger) or name (string) + + Outputs: + - harmo = harmonic coefficients, dictionary + + Options: + - t_start = start time, as string ('yyyy-mm-ddThh:mm:ss'), or time index as an integer + - t_end = end time, as a string ('yyyy-mm-ddThh:mm:ss'), or time index as an integer + - time_ind = time indices to work in, list of integers + - elevation=True means that 'solve' will be done for elevation. + - velocity=True means that 'solve' will be done for velocity. + + Utide's options: + Options are the same as for 'solve', which are shown below with + their default values: + conf_int=True; cnstit='auto'; notrend=0; prefilt=[]; nodsatlint=0; + nodsatnone=0; gwchlint=0; gwchnone=0; infer=[]; inferaprx=0; + rmin=1; method='cauchy'; tunrdn=1; linci=0; white=0; nrlzn=200; + lsfrqosmp=1; nodiagn=0; diagnplots=0; diagnminsnr=2; + ordercnstit=[]; runtimedisp='yyy' + + *Notes* + For more detailed information about 'solve', please see + https://github.com/wesleybowman/UTide + + """ + debug = (debug or self._debug) + + #Search for the station + index = self.search_index(station) + + argtime = [] + if not time_ind==[]: + argtime = time_ind + elif not t_start==[]: + if type(t_start)==str: + argtime = time_to_index(t_start, t_end, + self._var.matlabTime, + debug=debug) + else: + argtime = np.arange(t_start, t_end) + + if velocity == elevation: + raise PyseidonError("---Can only process either velocities or elevation. Change options---") + + if velocity: + harmo = {} + for layerIndex in range(self._var.u.shape[1]): + time = self._var.matlabTime[:] + u = self._var.u[:,layerIndex,index] + v = self._var.v[:,layerIndex,index] + + if not argtime==[]: + time = time[argtime[:]] + u = u[argtime[:]] + v = v[argtime[:]] + + lat = self._grid.lat[index] + harmo['Layer_'+str(layerIndex)] = solve(time, u, v, lat, **kwarg) + + if elevation: + time = self._var.matlabTime[:] + el = self._var.el[:,index] + + if not argtime==[]: + time = time[argtime[:]] + el = el[argtime[:]] + + lat = self._grid.lat[index] + harmo = solve(time, el, None, lat, **kwarg) + #Write meta-data only if computed over all the elements + + return harmo + + def Harmonic_reconstruction(self, harmo, recon_time=[], time_ind=slice(None), debug=False, **kwarg): + """ + This function reconstructs the velocity components or the surface elevation + from harmonic coefficients. + Harmonic_reconstruction calls 'reconstruct'. This function assumes harmonics + ('solve') has already been executed. + + Inputs: + - Harmo = harmonic coefficient from harmo_analysis + - time_ind = time indices to process, list of integers + - recon_time = time you want the harmonic coefficients to be reconstructed at + + Output: + - Reconstruct = reconstructed signal, dictionary + + Options: + Options are the same as for 'reconstruct', which are shown below with + their default values: + cnstit = [], minsnr = 2, minpe = 0 + + *Notes* + For more detailed information about 'reconstruct', please see + https://github.com/wesleybowman/UTide + + """ + debug = (debug or self._debug) + if recon_time == []: + time = self._var.matlabTime[time_ind] + else: + time = recon_time + + if not 'Layer_0' in harmo: + Reconstruct = reconstruct(time, harmo) + else: + Reconstruct = {} + for layer in harmo: + Reconstruct[layer] = reconstruct(time, harmo[layer]) + + return Reconstruct + + def velo_stack(self, Reconstruct, debug=False): + """ + This function seperates and stacks u & v into respective matrices + from a 3D harmonically reconstructed dictionary + + Inputs: + - Reconstruct = reconstructed signal, dictionary from Harmonic_reconstruction() + + Outputs: + - u = stacked matrix of u velocity + - v = stacked matrix of v velocity + + """ + debug = (debug or self._debug) + u = Reconstruct['Layer_0']['u'] + v = Reconstruct['Layer_0']['v'] + for i in range(len(Reconstruct))[1:]: + u = np.vstack((u, Reconstruct['Layer_'+str(i)]['u'])) + v = np.vstack((v, Reconstruct['Layer_'+str(i)]['v'])) + + return u, v + + +#TR_comments: templates +# def whatever(self, debug=False): +# if debug or self._debug: +# print 'Start whatever...' +# +# if debug or self._debug: +# print '...Passed' diff --git a/build/lib/pyseidon_dvt/stationClass/plotsStation.py b/build/lib/pyseidon_dvt/stationClass/plotsStation.py new file mode 100644 index 0000000..b76b861 --- /dev/null +++ b/build/lib/pyseidon_dvt/stationClass/plotsStation.py @@ -0,0 +1,218 @@ +#!/usr/bin/python2.7 +# encoding: utf-8 + +from __future__ import division +import numpy as np +import matplotlib.pyplot as plt +import matplotlib.ticker as ticker +import matplotlib.patches as mpatches +import seaborn +import pandas as pd +from windrose import WindroseAxes +from interpolation_utils import * +#from miscellaneous import depth_at_FVCOM_element as depth_at_ind + +class PlotsStation: + """ + Description: + 'Plots' subset of Station class gathers plotting functions + """ + def __init__(self, variable, grid, debug): + self._debug = debug + #Back pointer + setattr(self, '_var', variable) + setattr(self, '_grid', grid) + + def _def_fig(self): + """Defines figure window""" + self._fig = plt.figure(figsize=(18,10)) + plt.rc('font',size='22') + + def rose_diagram(self, direction, norm): + + """ + Plot rose diagram + + Inputs: + - direction = 1D array + - norm = 1D array + """ + #Convertion + #TR: not quite sure here, seems to change from location to location + # express principal axis in compass + direction = np.mod(90.0 - direction, 360.0) + + #Create new figure + self._def_fig() + rect = [0.1, 0.1, 0.8, 0.8] + ax = WindroseAxes(self._fig, rect)#, axisbg='w') + self._fig.add_axes(ax) + #Rose + ax.bar(direction, norm , normed=True, opening=0.8, edgecolor='white') + #adjust legend + l = ax.legend(shadow=True, bbox_to_anchor=[-0.1, 0], loc='lower left') + plt.setp(l.get_texts(), fontsize=10) + plt.xlabel('Rose diagram in % of occurrences - Colormap of norms') + self._plt.show() + + def plot_xy(self, x, y, xerror=[], yerror=[], + title=' ', xLabel=' ', yLabel=' ', dump=False, **kwargs): + """ + Simple X vs Y plot + + Inputs: + - x = 1D array + - y = 1D array + + Options: + - xerror = error on 'x', 1D array + - yerror = error on 'y', 1D array + - title = plot title, string + - xLabel = title of the x-axis, string + - yLabel = title of the y-axis, string + - dump = boolean, dump profile data in csv file + - kwargs = keyword options associated with pandas.DataFrame.to_csv, such as: + sep, header, na_rep, index,...etc + Check doc. of "to_csv" for complete list of options + """ + self._def_fig() + self._ax = self._fig.add_subplot(111) + self._ax.plot(x, y, label=title) + scale = 1 + self._ax.set_ylabel(yLabel) + self._ax.set_xlabel(xLabel) + self._ax.get_xaxis().set_minor_locator(ticker.AutoMinorLocator()) + self._ax.get_yaxis().set_minor_locator(ticker.AutoMinorLocator()) + self._ax.grid(b=True, which='major', color='w', linewidth=1.5) + self._ax.grid(b=True, which='minor', color='w', linewidth=0.5) + if not yerror==[]: + self._ax.fill_between(x, y-yerror, y+yerror, + alpha=0.2, edgecolor='#1B2ACC', facecolor='#089FFF', antialiased=True) + if not xerror==[]: + self._ax.fill_betweenx(y, x-xerror, x+xerror, + alpha=0.2, edgecolor='#1B2ACC', facecolor='#089FFF', antialiased=True) + if (not xerror==[]) or (not yerror==[]): + blue_patch = mpatches.Patch(color='#089FFF', + label='Standard deviation',alpha=0.2) + plt.legend(handles=[blue_patch],loc=1, fontsize=12) + #plt.legend([blue_patch],loc=1, fontsize=12) + + self._fig.show() + if dump: self._dump_profile_data_as_csv(x, y,xerror=xerror, yerror=yerror, + title=title, xLabel=xLabel, + yLabel=yLabel, **kwargs) + + def Histogram(self, y, title=' ', xLabel=' ', yLabel=' '): + """ + Histogram plot + + Inputs: + - bins = list of bin edges + - y = 1D array + + Options: + - title = plot title, string + - xLabel = title of the x-axis, string + - yLabel = title of the y-axis, string + """ + self._def_fig() + self._ax = self._fig.add_subplot(111) + density, bins = np.histogram(y, bins=50, normed=True, density=True) + unity_density = density / density.sum() + widths = bins[:-1] - bins[1:] + # To plot correct percentages in the y axis + self._ax.bar(bins[1:], unity_density, width=widths) + formatter = ticker.FuncFormatter(lambda v, pos: str(v * 100)) + self._ax.yaxis.set_major_formatter(formatter) + + plt.ylabel(yLabel) + plt.xlabel(xLabel) + + self._fig.show() + + def add_points(self, x, y, label=' ', color='black'): + """ + Add scattered points (x,y) on current figure, + where x and y are 1D arrays of the same lengths. + + Inputs: + - x = float number + - y = float numbe + + Options: + - Label = a string + - Color = a string, 'red', 'green', etc. or gray shades like '0.5' + """ + plt.scatter(x, y, s=100, color=color) + #TR : annotate does not work on my machine !? + plt.annotate(label, xy=(x, y), xycoords='data', xytext=(-20, 20), + textcoords='offset points', ha='right', + arrowprops=dict(arrowstyle="->", shrinkA=0)) + + def rose_diagram(self, direction, norm): + + """ + Plot rose diagram + + Inputs: + - direction = 1D array + - norm = 1D array + """ + #Convertion + #TR: not quite sure here, seems to change from location to location + # express principal axis in compass + direction = np.mod(90.0 - direction, 360.0) + + #Create new figure + #fig = plt.figure(figsize=(18,10)) + #plt.rc('font',size='22') + self._def_fig() + rect = [0.1, 0.1, 0.8, 0.8] + ax = WindroseAxes(self._fig, rect)#, axisbg='w') + self._fig.add_axes(ax) + #Rose + ax.bar(direction, norm , normed=True, opening=0.8, edgecolor='white') + #adjust legend + l = ax.legend(shadow=True, bbox_to_anchor=[-0.1, 0], loc='lower left') + plt.setp(l.get_texts(), fontsize=10) + plt.xlabel('Rose diagram in % of occurrences - Colormap of norms') + self._fig.show() + + def _dump_profile_data_as_csv(self, x, y, xerror=[], yerror=[], + title=' ', xLabel=' ', yLabel=' ', **kwargs): + """ + Dumps profile data in csv file + + Inputs: + - x = 1D array + - y = 1D array + + Options: + - xerror = error on 'x', 1D array + - yerror = error on 'y', 1D array + - title = file name, string + - xLabel = name of the x-data, string + - yLabel = name of the y-data, string + - kwargs = keyword options associated with pandas.DataFrame.to_csv, such as: + sep, header, na_rep, index,...etc + Check doc. of "to_csv" for complete list of options + """ + if title == ' ': title = 'dump_profile_data' + filename=title + '.csv' + if xLabel == ' ': xLabel = 'X' + if yLabel == ' ': yLabel = 'Y' + if not xerror == []: + df = pd.DataFrame({xLabel:x[:], yLabel:y[:], 'error': xerror[:]}) + elif not yerror == []: + df = pd.DataFrame({xLabel:x[:], yLabel:y[:], 'error': yerror[:]}) + else: + df = pd.DataFrame({xLabel:x[:], yLabel:y[:]}) + df.to_csv(filename, encoding='utf-8', **kwargs) + +#TR_comments: templates +# def whatever(self, debug=False): +# if debug or self._debug: +# print 'Start whatever...' +# +# if debug or self._debug: +# print '...Passed' diff --git a/build/lib/pyseidon_dvt/stationClass/stationClass.py b/build/lib/pyseidon_dvt/stationClass/stationClass.py new file mode 100644 index 0000000..007e6c8 --- /dev/null +++ b/build/lib/pyseidon_dvt/stationClass/stationClass.py @@ -0,0 +1,446 @@ +#!/usr/bin/python2.7 +# encoding: utf-8 + +from __future__ import division + +import numpy as np +import netCDF4 as nc +from scipy.io import netcdf +from scipy.io import savemat +from pydap.client import open_url +import cPickle as pkl +import copy + +#Utility import +from pyseidon_dvt.utilities.object_from_dict import ObjectFromDict +from pyseidon_dvt.utilities.miscellaneous import findFiles + +# Custom error +from pyseidon_dvt.utilities.pyseidon_error import PyseidonError + +#Local import +from variablesStation import _load_var, _load_grid +from functionsStation import * +from functionsStationThreeD import * +from plotsStation import * + +class Station: + """ + **A class/structure for Station data** + + Functionality structured as follows: :: + + _Data. = raw netcdf file data + |_Variables. = fvcom station variables and quantities + |_Grid. = fvcom station grid data + |_History = Quality Control metadata + Station._|_Utils2D. = set of useful functions for 2D and 3D station + |_Utils3D. = set of useful functions for 3D station + |_Plots. = plotting functions + |_Harmonic_analysis = harmonic analysis based UTide package + |_Harmonic_reconstruction = harmonic reconstruction based UTide package + + Inputs: + - filename = path to netcdf file or folder, string, + ex: testFvcom=Station('./path_to_FVOM_output_file/filename') + testFvcom=Station('./path_to_FVOM_output_file/folder/') + + Note that if the path point to a folder all the similar netCDF station files + will be stack together. + Note that the file can be a pickle file (i.e. *.p) or a netcdf file (i.e. *.nc). + + Options: + - elements = indices to extract, list of integers + + *Notes* + Throughout the package, the following conventions apply: + - Date = string of 'yyyy-mm-dd hh:mm:ss' + - Coordinates = decimal degrees East and North + - Directions = in degrees, between -180 and 180 deg., i.e. 0=East, 90=North, + +/-180=West, -90=South + - Depth = 0m is the free surface and depth is negative + + """ + def __init__(self, filename, elements=slice(None), debug=False): + #Class attributs + self._debug = debug + self._isMulti(filename) + if not self._multi: + self._load(filename, elements, debug=debug ) + self.Plots = PlotsStation(self.Variables, + self.Grid, + self._debug) + self.Util2D = FunctionsStation(self.Variables, + self.Grid, + self.Plots, + self.History, + self._debug) + if self.Variables._3D: + self.Util3D = FunctionsStationThreeD( + self.Variables, + self.Grid, + self.Plots, + self.History, + self._debug) + else: + print "---Finding matching files---" + self._matches = findFiles(filename, 'STATION') + filename = self._matches.pop(0) + self._load(filename, elements, debug=debug ) + self.Plots = PlotsStation(self.Variables, + self.Grid, + self._debug) + self.Util2D = FunctionsStation(self.Variables, + self.Grid, + self.Plots, + self.History, + self._debug) + if self.Variables._3D: + self.Util3D = FunctionsStationThreeD( + self.Variables, + self.Grid, + self.Plots, + self.History, + self._debug) + for entry in self._matches: + #Define new + text = 'Created from ' + entry + tmp = {} + tmp['Data'] = self._load_nc(entry) + tmp['History'] = [text] + tmp['Grid'] = _load_grid(tmp['Data'], elements, [], debug=self._debug) + tmp['Variables'] = _load_var(tmp['Data'], elements, tmp['Grid'], [], + debug=self._debug) + tmp = ObjectFromDict(tmp) + self = self.__add__(tmp) + + ##Re-assignement of utility functions as methods + self.dump_profile_data = self.Plots._dump_profile_data_as_csv + + return + + def _isMulti(self, filename): + """Tells if filename point to a file or a folder""" + split = filename.split('/') + if split[-1]: + self._multi = False + else: + self._multi = True + + def _load(self, filename, elements, debug=False): + """Loads data from *.nc, *.p and OpenDap url""" + #Loading pickle file + if filename.endswith('.p'): + f = open(filename, "rb") + data = pkl.load(f) + self._origin_file = data['Origin'] + self.History = data['History'] + if debug: print "Turn keys into attributs" + self.Grid = ObjectFromDict(data['Grid']) + self.Variables = ObjectFromDict(data['Variables']) + try: + if self._origin_file.startswith('http'): + #Look for file through OpenDAP server + print "Retrieving data through OpenDap server..." + self.Data = open_url(data['Origin']) + #Create fake attribut to be consistent with the rest of the code + self.Data.variables = self.Data + else: + self.Data = self._load_nc(data['Origin']) + except: #TR: need to precise the type of error here + print "the original *.nc file has not been found" + pass + #Loading netcdf file + elif filename.endswith('.nc'): + if filename.startswith('http'): + #Look for file through OpenDAP server + print "Retrieving data through OpenDap server..." + self.Data = open_url(filename) + #Create fake attribut to be consistent with the rest of the code + self.Data.variables = self.Data + else: + #Look for file locally + print "Retrieving data from " + filename + " ..." + self.Data = self._load_nc(filename) + #Metadata + text = 'Created from ' + filename + self._origin_file = filename + self.History = [text] + # Calling sub-class + print "Initialisation..." + try: + self.Grid = _load_grid(self.Data, + elements, + self.History, + debug=self._debug) + self.Variables = _load_var(self.Data, + elements, + self.Grid, + self.History, + debug=self._debug) + + except MemoryError: + print '---Data too large for machine memory---' + print 'Tip: use ax or tx during class initialisation' + print '--- to use partial data' + raise + + elif filename.endswith('.mat'): + raise PyseidonError("---Functionality not yet implemented---") + else: + raise PyseidonError("---Wrong file format---") + + def _load_nc(self, filename): + """loads netcdf file""" + #Look for file locally + #print "Retrieving data from " + filename + " ..." + # WB_Alternative: self.Data = sio.netcdf.netcdf_file(filename, 'r') + # WB_comments: scipy has causes some errors, and even though can be + # faster, can be unreliable + try: + Data = netcdf.netcdf_file(filename, 'r', mmap=True) + except ValueError: #TR: quick fix due to mmap + Data = nc.Dataset(filename, 'r') + return Data + + #Special methods + def __add__(self, StationClass, debug=False): + """ + This special method permit to stack variables + of 2 Station objects through a simple addition: :: + + station1 += station2 + + *Notes* + - station1 and station2 have to cover the exact + same spatial domain + - last time step of station1 must be <= to the + first time step of station2 + """ + debug = debug or self._debug + if debug: print "Find matching elements..." + #Find matching elements + origNele = self.Grid.nele + origEle = [] + origName = self.Grid.name + #origX = self.Grid.x[:] + #origY = self.Grid.y[:] + newNele = StationClass.Grid.nele + newEle = [] + newName = StationClass.Grid.name + #newX = StationClass.Grid.x[:] + #newY = StationClass.Grid.y[:] + for i in range(origNele): + for j in range(newNele): + #Match based on names + #if (all(origName[i,:]==newName[j,:])): + if origName[i]==newName[j]: + origEle.append(i) + newEle.append(j) + #Match based on coordinates + #if ((origX[i]==newX[j]) and (origY[i]==newY[j])): + # origEle.append(i) + # newEle.append(j) + + print len(origEle), " points will be stacked..." + + if len(origEle)==0: + raise PyseidonError("---No matching element found---") + elif not (self.Variables._3D == StationClass.Variables._3D): + raise PyseidonError("---Data dimensions do not match---") + else: + if not (self.Variables.julianTime[-1]<= + StationClass.Variables.julianTime[0]): + raise PyseidonError("---Data not consecutive in time---") + #Copy self to newself + newself = copy.copy(self) + #TR comment: it still points toward self and modifies it + # so cannot do Station3 = Station1 + Station2 + if debug: + print 'Stacking variables...' + #keyword list for hstack + kwl=['matlabTime', 'julianTime', 'secondTime'] + for key in kwl: + tmpN = getattr(newself.Variables, key) + tmpO = getattr(StationClass.Variables, key) + setattr(newself.Variables, key, + np.hstack((tmpN[:], tmpO[:]))) + + #keyword list for vstack + kwl=['u', 'v', 'w', 'tke', 'gls', 'ua', 'va','el'] + kwl2D=['ua', 'va','el'] + for key in kwl: + try: + if key in kwl2D: + tmpN = getattr(newself.Variables, key)\ + [:,newEle[:]] + tmpO = getattr(StationClass.Variables, key)\ + [:,origEle[:]] + setattr(newself.Variables, key, + np.vstack((tmpN[:], tmpO[:]))) + if debug: print "Stacking " + key + "..." + else: + tmpN = getattr(newself.Variables, key)\ + [:,:,newEle[:]] + tmpO = getattr(StationClass.Variables, key)\ + [:,:,origEle[:]] + setattr(newself.Variables, key, + np.vstack((tmpN[:], tmpO[:]))) + if debug: print "Stacking " + key + "..." + except AttributeError: + continue + + #New code added by RK Aug 20 + #get unique, sorted time + if debug: + print 'Getting unique, sorted time...' + + #keyword list for unique + + time = getattr(newself.Variables, 'matlabTime') + uniquetime, unique_index=np.unique(time,return_index=True) + sort_index=np.argsort(uniquetime) + index=unique_index[sort_index] + time=time[index] + #New time dimension + newself.Grid.ntime = time.shape + if debug: print "New time length: " +str(newself.Grid.ntime[0]) + + #keyword list for vstack + kwl=['matlabTime', 'julianTime', 'secondTime'] + for key in kwl: + tmpN = getattr(newself.Variables, key) + setattr(newself.Variables, key, tmpN[index]) + + kwl=['u', 'v', 'w', 'tke', 'gls', 'ua', 'va','el'] + kwl2D=['ua', 'va','el'] + for key in kwl: + try: + if key in kwl2D: + tmpN = getattr(newself.Variables, key)\ + [:,newEle[:]] + setattr(newself.Variables, key,tmpN[index,:]) + if debug: print "Sorting " + key + "..." + else: + tmpN = getattr(newself.Variables, key)\ + [:,:,newEle[:]] + setattr(newself.Variables, key,tmpN[index,:]) + if debug: print "Sorting " + key + "..." + except AttributeError: + continue + #End new code added by RK Aug 20 + + #Keep only matching names + newself.Grid.name = self.Grid.name[origEle[:]] + #Append to new object history + text = 'Data from ' + StationClass.History[0].split('/')[-1] \ + + ' has been stacked' + newself.History.append(text) + + return newself + + #Methods + def Save_as(self, filename, fileformat='pickle', debug=False): + """ + Save the current Station structure as: + - *.p, i.e. python file + - *.mat, i.e. Matlab file + + Inputs: + - filename = path + name of the file to be saved, string + + Options: + - fileformat = format of the file to be saved, i.e. 'pickle' or 'matlab' + """ + debug = debug or self._debug + if debug: + print 'Saving file...' + #Define bounding box + if debug: + print "Computing bounding box..." + if self.Grid._ax == []: + lon = self.Grid.lon[:] + lat = self.Grid.lat[:] + self.Grid._ax = [lon.min(), lon.max(), + lat.min(), lat.max()] + #Save as different formats + if fileformat=='pickle': + filename = filename + ".p" + f = open(filename, "wb") + data = {} + data['Origin'] = self._origin_file + data['History'] = self.History + data['Grid'] = self.Grid.__dict__ + data['Variables'] = self.Variables.__dict__ + #TR: Force caching Variables otherwise error during loading + # with 'netcdf4.Variable' type (see above) + for key in data['Variables']: + listkeys=['Variable', 'ArrayProxy', 'BaseType'] + if any([type(data['Variables'][key]).__name__==x for x in listkeys]): + if debug: + print "Force caching for " + key + data['Variables'][key] = data['Variables'][key][:] + #Unpickleable objects + data['Grid'].pop("triangle", None) + #TR: Force caching Variables otherwise error during loading + # with 'netcdf4.Variable' type (see above) + for key in data['Grid']: + listkeys=['Variable', 'ArrayProxy', 'BaseType'] + if any([type(data['Grid'][key]).__name__==x for x in listkeys]): + if debug: + print "Force caching for " + key + data['Grid'][key] = data['Grid'][key][:] + #Save in pickle file + if debug: + print 'Dumping in pickle file...' + try: + pkl.dump(data, f, protocol=pkl.HIGHEST_PROTOCOL) + except SystemError: + print '---Data too large for machine memory---' + print 'Tip: use ax or tx during class initialisation' + print '--- to use partial data' + raise + + f.close() + elif fileformat=='matlab': + filename = filename + ".mat" + #TR comment: based on MitchellO'Flaherty-Sproul's code + dtype = float + data = {} + Grd = {} + Var = {} + data['Origin'] = self._origin_file + data['History'] = self.History + Grd = self.Grid.__dict__ + Var = self.Variables.__dict__ + #TR: Force caching Variables otherwise error during loading + # with 'netcdf4.Variable' type (see above) + for key in Var: + listkeys=['Variable', 'ArrayProxy', 'BaseType'] + if any([type(Var[key]).__name__==x for x in listkeys]): + if debug: + print "Force caching for " + key + Var[key] = Var[key][:] + #keyV = key + '-var' + #data[keyV] = Var[key] + data[key] = Var[key] + #Unpickleable objects + Grd.pop("triangle", None) + for key in Grd: + listkeys=['Variable', 'ArrayProxy', 'BaseType'] + if any([type(Grd[key]).__name__==x for x in listkeys]): + if debug: + print "Force caching for " + key + Grd[key] = Grd[key][:] + #keyG = key + '-grd' + #data[keyG] = Grd[key] + data[key] = Grd[key] + + #Save in mat file file + if debug: + print 'Dumping in matlab file...' + savemat(filename, data, oned_as='column') + else: + print "---Wrong file format---" + +#if __name__ == '__main__': diff --git a/build/lib/pyseidon_dvt/stationClass/variablesStation.py b/build/lib/pyseidon_dvt/stationClass/variablesStation.py new file mode 100644 index 0000000..797cd2d --- /dev/null +++ b/build/lib/pyseidon_dvt/stationClass/variablesStation.py @@ -0,0 +1,293 @@ +#!/usr/bin/python2.7 +# encoding: utf-8 + +from __future__ import division + +import numpy as np +from itertools import groupby +from operator import itemgetter +#Local import +from pyseidon_dvt.utilities.miscellaneous import mattime_to_datetime + +class _load_grid: + """ + **'Grid' subset in Station class contains grid related quantities** + + Some grid data are directly passed on from Station output: :: + + _lon = longitudes at nodes (deg.), 2D array (ntime, nnode) + |_lat = latitudes at nodes (deg.), 2D array (ntime, nnode) + |_x = x coordinates at nodes (m), 2D array (ntime, nnode) + |_y = y coordinates at nodes (m), 2D array (ntime, nnode) + |_h = bathymetry (m), 2D array (ntime, nnode) + |_nele = element dimension, integer + Station.Grid._|_nnode = node dimension, integer + |_nlevel = vertical level dimension, integer + |_ntime = time dimension, integer + |_siglay = sigma layers, 2D array (nlevel, nnode) + |_siglay = sigma levels, 2D array (nlevel+1, nnode) + |_name = name of the stations, (nnode, 20) + + Some others shall be generated as methods are being called, ex: :: + + ... + |_triangle = triangulation object for plotting purposes + + """ + def __init__(self, data, elements, History, debug=False): + if debug: + print 'Loading grid...' + #Pointer to History + setattr(self, '_History', History) + + self.x = data.variables['x'][elements] + self.y = data.variables['y'][elements] + self.lon = data.variables['lon'][elements] + self.lat = data.variables['lat'][elements] + self.siglay = data.variables['siglay'][:,elements] + self.siglev = data.variables['siglev'][:,elements] + self.h = data.variables['h'][elements] + self.name = data.variables['name_station'][elements,:] + self.nlevel = self.siglay.shape[0] + self.nele = self.x.shape[0] + self.nnode = self.x.shape[0] + + # formatting names + if len(self.name.shape) > 1: + newNames = np.arange(self.nele).astype(str) + for i in range(self.nele): + newNames[i]="".join(self.name[i,:]).strip() + self.name = newNames + + #Computing bounding box + lon = self.lon[:] + lat = self.lat[:] + if debug: print "Computing bounding box..." + ax = [lon.min(), lon.max(), lat.min(), lat.max()] + self._ax = ax + # Add metadata entry + text = 'Bounding box =' + str(ax) + self._History.append(text) + + if debug: + print '...Passed' + +class _load_var: + """ + **'Variables' subset in Station class contains the hydrodynamic related quantities** + + Some variables are directly passed on from Station output: :: + + _el = elevation (m), 2D array (ntime, nnode) + |_julianTime = julian date, 1D array (ntime) + |_matlabTime = matlab time, 1D array (ntime) + |_ua = depth averaged u velocity component (m/s), + | 2D array (ntime, nele) + |_va = depth averaged v velocity component (m/s), + Station.Variables._| 2D array (ntime, nele) + |_u = u velocity component (m/s), + | 3D array (ntime, nlevel, nele) + |_v = v velocity component (m/s), + | 3D array (ntime, nlevel, nele) + |_w = w velocity component (m/s), + 3D array (ntime, nlevel, nele) + + Some others shall be generated as methods are being called, ex: :: + + ... + |_hori_velo_norm = horizontal velocity norm (m/s), + | 2D array (ntime, nele) + |_velo_norm = velocity norm (m/s), + | 3D array (ntime, nlevel, nele) + |_verti_shear = vertical shear (1/s), + 3D array (ntime, nlevel, nele) + + """ + def __init__(self, data, elements, grid, History, debug=False): + if debug: print 'Loading variables...' + + #Pointer to History + setattr(self, '_grid', grid) + setattr(self, '_History', History) + + #List of keywords + kwl2D = ['ua', 'va', 'zeta'] + kwl3D = ['ww', 'u', 'v', 'gls', 'tke'] + #List of aliaSes + al2D = ['ua', 'va', 'el'] + al3D = ['w', 'u', 'v', 'gls', 'tke'] + + if debug: print '...time variables...' + self.julianTime = data.variables['time_JD'][:] + self.secondTime = data.variables['time_second'][:] + self.matlabTime = self.julianTime[:] + 678942.0 + self.secondTime[:] / (24*3600) + # Append message to History field + start = mattime_to_datetime(self.matlabTime[0]) + end = mattime_to_datetime(self.matlabTime[-1]) + text = 'Full temporal domain from ' + str(start) +\ + ' to ' + str(end) + self._History.append(text) + # Add time dimension to grid variables + self._grid.ntime = self.matlabTime.shape[0] + + if debug: print '...hydro variables...' + + region_e = elements + region_n = elements + #Redefine variables in bounding box + #Check if OpenDap variables or not + if type(data.variables).__name__=='DatasetType': + # TR: fix for issue wih elements = slice(none) + if elements == slice(None): + region_e = np.arange(self._grid.nele) + region_n = np.arange(self._grid.nnode) + #loading hori data + keyCount = 0 + for key, aliaS in zip(kwl2D, al2D): + #Special loading for zeta + H = 0 #local counter + if key == 'zeta': + for k, g in groupby(enumerate(region_n), lambda (i,x):i-x): + ID = map(itemgetter(1), g) + if debug: print 'Index bound: ' +\ + str(ID[0]) + '-' + str(ID[-1]+1) + if H==0: + try: + setattr(self, aliaS, data.variables[key].data[:,ID[0]:(ID[-1]+1)]) + except AttributeError: #exeception due nc.Dataset + setattr(self, aliaS, data.variables[key][:,ID[0]:(ID[-1]+1)]) + H=1 + else: + try: + setattr(self, aliaS, + np.hstack((getattr(self, aliaS), + data.variables[key].data[:,ID[0]:(ID[-1]+1)]))) + except AttributeError: #exeception due nc.Dataset + setattr(self, aliaS, + np.hstack((getattr(self, aliaS), + data.variables[key][:,ID[0]:(ID[-1]+1)]))) + else: + try: + for k, g in groupby(enumerate(region_e), lambda (i,x):i-x): + ID = map(itemgetter(1), g) + if debug: print 'Index bound: ' + str(ID[0]) + '-' + str(ID[-1]+1) + if H==0: + setattr(self, aliaS, + data.variables[key].\ + data[:,ID[0]:(ID[-1]+1)]) + H=1 + else: + try: + setattr(self, aliaS, + np.hstack((getattr(self, aliaS), + data.variables[key].data[:,ID[0]:(ID[-1]+1)]))) + except AttributeError: #exeception due nc.Dataset + setattr(self, aliaS, + np.hstack((getattr(self, aliaS), + data.variables[key][:,ID[0]:(ID[-1]+1)]))) + keyCount +=1 + except KeyError: + if debug: print key, " is missing !" + continue + if keyCount==0: + print "---Horizontal variables are missing---" + self._3D = False + + #loading verti data + keyCount = 0 + for key, aliaS in zip(kwl3D, al3D): + try: + H = 0 #local counter + for k, g in groupby(enumerate(region_e), lambda (i,x):i-x): + ID = map(itemgetter(1), g) + if debug: print 'Index bound: ' + str(ID[0]) + '-' + str(ID[-1]+1) + if H==0: + try: + setattr(self, aliaS, + data.variables[key].data[:,:,ID[0]:(ID[-1]+1)]) + except AttributeError: #exeception due nc.Dataset + setattr(self, aliaS, + data.variables[key][:,:,ID[0]:(ID[-1]+1)]) + H=1 + else: + try: + setattr(self, aliaS, + np.dstack((getattr(self, aliaS), + data.variables[key].data[:,:,ID[0]:(ID[-1]+1)]))) + except AttributeError: #exeception due nc.Dataset + setattr(self, aliaS, + np.dstack((getattr(self, aliaS), + data.variables[key][:,:,ID[0]:(ID[-1]+1)]))) + keyCount +=1 + except KeyError: + if debug: print key, " is missing !" + continue + if keyCount==0: + print "---Vertical variables are missing---" + else: + self._3D = True + #Not OpenDap + else: + #loading hori data + keyCount = 0 + for key, aliaS in zip(kwl2D, al2D): + try: + if key=='zeta': + setattr(self, aliaS, np.zeros((self._grid.ntime, self._grid.nnode))) + for i in range(self._grid.ntime): + try: + #TR comment: looping on time indices is a trick from + # Mitchell to improve loading time + getattr(self, aliaS)[i,:] =\ + np.transpose(data.variables[key].data[i,region_n]) + except AttributeError: #exeception due nc.Dataset + getattr(self, aliaS)[i,:] = data.variables[key][i,region_n] + else: + setattr(self, aliaS, np.zeros((self._grid.ntime, self._grid.nele))) + for i in range(self._grid.ntime): + try: + #TR comment: looping on time indices is a trick from + # Mitchell to improve loading time + getattr(self, aliaS)[i,:] =\ + np.transpose(data.variables[key].data[i,region_e]) + except AttributeError: #exeception due nc.Dataset + getattr(self, aliaS)[i,:] = data.variables[key][i,region_e] + keyCount +=1 + except KeyError: + if debug: print key, " is missing !" + continue + if keyCount==0: + print "---Horizontal variables are missing---" + self._3D = False + + #loading verti data + keyCount = 0 + for key, aliaS in zip(kwl3D, al3D): + try: + testKey = data.variables[key] + del testKey + setattr(self, aliaS, + np.zeros((self._grid.ntime,self._grid.nlevel, self._grid.nele))) + for i in range(self._grid.ntime): + #TR comment: looping on time indices is a trick from + # Mitchell to improve loading time + try: + try: + getattr(self, aliaS)[i,:,:] =\ + np.transpose(data.variables[key].data[i,:,region_e]) + except ValueError: # TR: some issue with transpose...quite puzzling + getattr(self, aliaS)[i,:,:] = data.variables[key].data[i,:,region_e] + except AttributeError: #exeception due nc.Dataset + getattr(self, aliaS)[i,:,:] =\ + data.variables[key][i,:,region_e] + keyCount +=1 + except KeyError: + if debug: print key, " is missing !" + continue + if keyCount==0: + print "---Vertical variables are missing---" + else: + self._3D = True + if debug: + print '...Passed' + diff --git a/build/lib/pyseidon_dvt/tidegaugeClass/__init__.py b/build/lib/pyseidon_dvt/tidegaugeClass/__init__.py new file mode 100644 index 0000000..e7ebb2a --- /dev/null +++ b/build/lib/pyseidon_dvt/tidegaugeClass/__init__.py @@ -0,0 +1,10 @@ +#!/usr/bin/python2.7 +# encoding: utf-8 + +#Local import +from tidegaugeClass import TideGauge + +__authors__ = ['Thomas Roc, Wesley Bowman, Jonathan Smith'] +__licence__ = 'GNU Affero GPL v3.0' +__copyright__ = 'Copyright (c) 2014 EcoEnergyII' + diff --git a/build/lib/pyseidon_dvt/tidegaugeClass/functionsTidegauge.py b/build/lib/pyseidon_dvt/tidegaugeClass/functionsTidegauge.py new file mode 100644 index 0000000..dae60f8 --- /dev/null +++ b/build/lib/pyseidon_dvt/tidegaugeClass/functionsTidegauge.py @@ -0,0 +1,95 @@ +#!/usr/bin/python2.7 +# encoding: utf-8 + +from __future__ import division + +from utide import solve, reconstruct +from pyseidon_dvt.utilities.miscellaneous import mattime_to_datetime + +class FunctionsTidegauge: + """ + **'Utils' subset of TideGauge class gathers useful functions** + """ + def __init__(self, variable, plot, History, debug=False): + self._plot = plot + #Create pointer to FVCOM class + setattr(self, '_var', variable) + setattr(self, '_History', History) + + def harmonics(self, time_ind=slice(None), **kwarg): + """ + This function performs a harmonic analysis on the sea surface elevation + time series or the velocity components timeseries. + + Outputs: + - harmo = harmonic coefficients, dictionary + + Options: + - time_ind = time indices to work in, list of integers + + Utide's options: + Options are the same as for 'solve', which are shown below with + their default values: + conf_int=True; cnstit='auto'; notrend=0; prefilt=[]; nodsatlint=0; + nodsatnone=0; gwchlint=0; gwchnone=0; infer=[]; inferaprx=0; + rmin=1; method='cauchy'; tunrdn=1; linci=0; white=0; nrlzn=200; + lsfrqosmp=1; nodiagn=0; diagnplots=0; diagnminsnr=2; + ordercnstit=[]; runtimedisp='yyy' + + *Notes* + For more detailed information about 'solve', please see + https://github.com/wesleybowman/UTide + """ + harmo = solve(self._var.matlabTime[time_ind], + self._var.el, None, + self._var.lat, **kwarg) + return harmo + + def reconstr(self, harmo, time_ind=slice(None), **kwarg): + """ + This function reconstructs the velocity components or the surface elevation + from harmonic coefficients. + Harmonic_reconstruction calls 'reconstruct'. This function assumes harmonics + ('solve') has already been executed. + + Inputs: + - Harmo = harmonic coefficient from harmo_analysis + + Output: + - Reconstruct = reconstructed signal, dictionary + + Keywords: + - time_ind = time indices to process, list of integers + + Utide's options: + Options are the same as for 'reconstruct', which are shown below with + their default values: + cnstit = [], minsnr = 2, minpe = 0 + + *Notes* + For more detailed information about 'reconstruct', please see + https://github.com/wesleybowman/UTide + """ + time = self._var.matlabTime[time_ind] + ts_recon = reconstruct(time, harmo, **kwarg) + return ts_recon + + def mattime2datetime(self, mattime, debug=False): + """ + Description: + Output the time (yyyy-mm-dd, hh:mm:ss) corresponding to + a given matlab time + + Inputs: + - mattime = matlab time (floats) + """ + time = mattime_to_datetime(mattime, debug=debug) + return time + +#TR_comments: templates +# def whatever(self, debug=False): +# if debug or self._debug: +# print 'Start whatever...' +# +# if debug or self._debug: +# print '...Passed' diff --git a/build/lib/pyseidon_dvt/tidegaugeClass/plotsTidegauge.py b/build/lib/pyseidon_dvt/tidegaugeClass/plotsTidegauge.py new file mode 100644 index 0000000..941ab8f --- /dev/null +++ b/build/lib/pyseidon_dvt/tidegaugeClass/plotsTidegauge.py @@ -0,0 +1,112 @@ +#!/usr/bin/python2.7 +# encoding: utf-8 + +from __future__ import division +import numpy as np +import matplotlib.pyplot as plt +import matplotlib.tri as Tri +import matplotlib.ticker as ticker +import matplotlib.patches as mpatches +import seaborn +import pandas as pd + +class PlotsTidegauge: + """ + **'Plots' subset of Tidegauge class gathers plotting functions** + """ + def __init__(self, variable, debug=False): + self._debug = debug + setattr(self, '_var', variable) + + def _def_fig(self): + """Defines figure window""" + self._fig = plt.figure(figsize=(18,10)) + plt.rc('font',size='22') + + def plot_xy(self, x, y, xerror=[], yerror=[], + title=' ', xLabel=' ', yLabel=' ', dump=False, **kwargs): + """ + Simple X vs Y plot + + Inputs: + - x = 1D array + - y = 1D array + + Options: + - xerror = error on 'x', 1D array + - yerror = error on 'y', 1D array + - title = plot title, string + - xLabel = title of the x-axis, string + - yLabel = title of the y-axis, string + - dump = boolean, dump profile data in csv file + - kwargs = keyword options associated with pandas.DataFrame.to_csv, such as: + sep, header, na_rep, index,...etc + Check doc. of "to_csv" for complete list of options + """ + #fig = plt.figure(figsize=(18,10)) + #plt.rc('font',size='22') + self._def_fig() + self._ax = self._fig.add_subplot(111) + self._ax.plot(x, y, label=title) + scale = 1 + self._ax.set_ylabel(yLabel) + self._ax.set_xlabel(xLabel) + self._ax.get_xaxis().set_minor_locator(ticker.AutoMinorLocator()) + self._ax.get_yaxis().set_minor_locator(ticker.AutoMinorLocator()) + self._ax.grid(b=True, which='major', color='w', linewidth=1.5) + self._ax.grid(b=True, which='minor', color='w', linewidth=0.5) + if not yerror==[]: + self._ax.fill_between(x, y-yerror, y+yerror, + alpha=0.2, edgecolor='#1B2ACC', facecolor='#089FFF', antialiased=True) + if not xerror==[]: + self._ax.fill_betweenx(y, x-xerror, x+xerror, + alpha=0.2, edgecolor='#1B2ACC', facecolor='#089FFF', antialiased=True) + if (not xerror==[]) or (not yerror==[]): + blue_patch = mpatches.Patch(color='#089FFF', + label='Standard deviation',alpha=0.2) + plt.legend(handles=[blue_patch],loc=1, fontsize=12) + #plt.legend([blue_patch],loc=1, fontsize=12) + + self._fig.show() + if dump: self._dump_profile_data_as_csv(x, y,xerror=xerror, yerror=yerror, + title=title, xLabel=xLabel, + yLabel=yLabel, **kwargs) + + def _dump_profile_data_as_csv(self, x, y, xerror=[], yerror=[], + title=' ', xLabel=' ', yLabel=' ', **kwargs): + """ + Dumps profile data in csv file + + Inputs: + - x = 1D array + - y = 1D array + + Options: + - xerror = error on 'x', 1D array + - yerror = error on 'y', 1D array + - title = file name, string + - xLabel = name of the x-data, string + - yLabel = name of the y-data, string + - kwargs = keyword options associated with pandas.DataFrame.to_csv, such as: + sep, header, na_rep, index,...etc + Check doc. of "to_csv" for complete list of options + """ + if title == ' ': title = 'dump_profile_data' + filename=title + '.csv' + if xLabel == ' ': xLabel = 'X' + if yLabel == ' ': yLabel = 'Y' + if not xerror == []: + df = pd.DataFrame({xLabel:x[:], yLabel:y[:], 'error': xerror[:]}) + elif not yerror == []: + df = pd.DataFrame({xLabel:x[:], yLabel:y[:], 'error': yerror[:]}) + else: + df = pd.DataFrame({xLabel:x[:], yLabel:y[:]}) + df.to_csv(filename, encoding='utf-8', **kwargs) + +#TR_comments: templates +# def whatever(self, debug=False): +# if debug or self._debug: +# print 'Start whatever...' +# +# if debug or self._debug: +# print '...Passed' diff --git a/build/lib/pyseidon_dvt/tidegaugeClass/tidegaugeClass.py b/build/lib/pyseidon_dvt/tidegaugeClass/tidegaugeClass.py new file mode 100644 index 0000000..9743ae4 --- /dev/null +++ b/build/lib/pyseidon_dvt/tidegaugeClass/tidegaugeClass.py @@ -0,0 +1,57 @@ +#!/usr/bin/python2.7 +# encoding: utf-8 + +from __future__ import division + +import scipy.io as sio + +#Local import +from variablesTidegauge import _load_tidegauge +from functionsTidegauge import * +from plotsTidegauge import * + +class TideGauge: + """ + **A class/structure for tide gauge data** + + Functionality structured as follows: :: + + _Data. = raw matlab file data + |_Variables. = useable tide gauge variables and quantities + TideGauge._|_History = Quality Control metadata + |_Utils. = set of useful functions + |_Plots. = plotting functions + + Inputs: + - Only takes a file name as input, ex: testTG=TideGauge('./path_to_matlab_file/filename') + + *Notes* + - Only handle fully processed tide gauge matlab data at the mo. + Throughout the package, the following conventions apply: + - Coordinates = decimal degrees East and North + - Directions = in degrees, between -180 and 180 deg., i.e. 0=East, 90=North, + +/-180=West, -90=South + - Depth = 0m is the free surface and depth is negative + """ + def __init__(self, filename, debug=False): + self._debug = debug + if debug: print '-Debug mode on-' + if debug: print 'Loading...' + #Metadata + self._origin_file = filename + self.History = ['Created from ' + filename] + + self.Data = sio.loadmat(filename, + struct_as_record=False, squeeze_me=True) + self.Variables = _load_tidegauge(self.Data, self.History, debug=self._debug) + + self.Plots = PlotsTidegauge(self.Variables, debug=self._debug) + + self.Utils = FunctionsTidegauge(self.Variables, + self.Plots, + self.History, + debug=self._debug) + + ##Re-assignement of utility functions as methods + self.dump_profile_data = self.Plots._dump_profile_data_as_csv + diff --git a/build/lib/pyseidon_dvt/tidegaugeClass/variablesTidegauge.py b/build/lib/pyseidon_dvt/tidegaugeClass/variablesTidegauge.py new file mode 100644 index 0000000..2d70dec --- /dev/null +++ b/build/lib/pyseidon_dvt/tidegaugeClass/variablesTidegauge.py @@ -0,0 +1,41 @@ +#!/usr/bin/python2.7 +# encoding: utf-8 + +from __future__ import division +import numpy as np +from pyseidon_dvt.utilities.miscellaneous import mattime_to_datetime + +class _load_tidegauge: + """ + **'Variables' subset in TideGauge class** + + It contains the following numpy arrays: :: + + _lat = latitude, float, decimal degrees + |_lon = lontitude, float, decimal degrees + TideGauge.Variables._|_RBR = Raw recording and sampling features + |_matlabTime = matlab time, 1D array + |_el = sea surface elevation (m) timeserie, 1D array + """ + def __init__(self, cls, History, debug=False): + if debug: print 'Loading variables...' + + # Pointer to History + setattr(self, '_History', History) + + self.RBR = cls['RBR'] + self.data = self.RBR.data + self.matlabTime = self.RBR.date_num_Z + self.lat = self.RBR.lat + self.lon = self.RBR.lon + self.el = self.data - np.mean(self.data) + + #-Append message to History field + start = mattime_to_datetime(self.matlabTime[0]) + end = mattime_to_datetime(self.matlabTime[-1]) + text = 'Temporal domain from ' + str(start) +\ + ' to ' + str(end) + self._History.append(text) + + if debug: print '...Done' + diff --git a/build/lib/pyseidon_dvt/utilities/BP_tools.py b/build/lib/pyseidon_dvt/utilities/BP_tools.py new file mode 100644 index 0000000..3992544 --- /dev/null +++ b/build/lib/pyseidon_dvt/utilities/BP_tools.py @@ -0,0 +1,360 @@ +from __future__ import division +import numpy as np +from math import atan2 +#from rawADCPclass import rawADCP +from datetime import datetime +from datetime import timedelta +import scipy.io as sio +import scipy.interpolate as sip +import matplotlib.pyplot as plt + +def date2py(matlab_datenum): + python_datetime = datetime.fromordinal(int(matlab_datenum)) + \ + timedelta(days=matlab_datenum%1) - timedelta(days = 366) + + return python_datetime + + +def py2date(dt): + mdn = dt + timedelta(days = 366) + frac_seconds = (dt-datetime(dt.year,dt.month,dt.day,0,0,0)).seconds / (24.0 * 60.0 * 60.0) + frac_microseconds = dt.microsecond / (24.0 * 60.0 * 60.0 * 1000000.0) + return mdn.toordinal() + frac_seconds + frac_microseconds + +def calc_ensemble(x, ens, ens_dim): + + #initialize input + ens = int(ens) + #x = x[:, None] + + if ens_dim == 1: + ens_size = np.floor(x.shape[0]/60) + else: + pass + + #x_ens = np.empty((ens_size, 1, ens)) + x_ens = np.empty((ens_size, ens)) + x_ens[:] = np.nan + + for j in xrange(ens): + if ens_dim == 1: + ind_ens = np.arange(j, x.shape[0] - (ens - j), ens) + #x_ens[..., j] = x[ind_ens] + x_ens[..., j] = x[ind_ens] + + else: + pass + + #x_ens = np.nanmean(x_ens, axis=2) + x_ens = np.nanmean(x_ens, axis=1) + return x_ens + + +def rotate_coords(x, y, theta): + ''' + Similar to "rotate_to_channelcoords.m" code, + theta is now the angle + between the old axis and the new x-axis (CCw is positive) + ''' + + xnew = x * np.cos(theta) + y * np.sin(theta) + ynew = -x * np.sin(theta) + y * np.cos(theta) + + return xnew, ynew + +def rotate_to_true(X, Y, theta=-19): + ''' + % X,Y are the X and Y coordinates (could be speeds) relative to magnetic + % north -- inputs can be vectors + % x,y are the coordinates relative to true north + % This function assumes the measured location is Nova Scotia where the + % declination angle is -19 degrees. + % + % Sept 29, 2012: Changed print statement + % + % Sept 20, 2012: Modified the function to allow for theta to be input. + % Default will remain at -19 degrees, but this may not be accurate for all + % places in Nova Scotia. + ''' + + print 'Rotating velocities to be relative to true north (declination = {0})'.format(theta) + + Theta = theta * np.pi / 180 + + x = X * np.cos(Theta) + Y * np.sin(Theta) + y = -X * np.sin(Theta) + Y * np.cos(Theta) + + return x, y + + +def get_DirFromN(u,v): + ''' + #This function computes the direction from North with the output in degrees + #and measured clockwise from north. + # + # Inputs: + # u: eastward component + # v: northward component + ''' + + #theta = np.arctan2(u,v) * 180 / np.pi + theta = atan2(u,v) * 180 / np.pi + + ind = np.where(theta<0) + theta[ind] = theta[ind] + 360 + return theta + +def sign_speed(u_all, v_all, s_all, dir_all, flood_heading): + + if type(flood_heading)==int: + flood_heading += np.array([-90, 90]) + + s_signed_all = np.empty(spd_all.shape) + s_signed_all.fill(np.nan) + + PA_all = np.zeros(s_all.shape[-1]) + for i in xrange(s_all.shape[-1]): + u = u_all[:, i] + v = v_all[:, i] + dir = dir_all[:, i] + s = s_all[:, i] + + #determine principal axes - potentially a problem if axes are very kinked + # since this would misclassify part of ebb and flood + PA, _ = principal_axis(u, v) + PA_all[i] = PA + + # sign speed - eliminating wrap-around + dir_PA = dir - PA + + dir_PA[dir_PA < -90] += 360 + dir_PA[dir_PA > 270] -= 360 + + #dir_PA[dir_PA<-90] = dir_PA(dir_PA<-90) + 360; + #dir_PA(dir_PA>270) = dir_PA(dir_PA>270) - 360; + + #general direction of flood passed as input argument + #if PA >= flood_heading[0] and PA <= flood_heading[1]: + if flood_heading[0] <= PA <= flood_heading[1]: + #ind_fld = find(dir_PA >= -90 & dir_PA<90) + ind_fld = np.where((dir_PA >= -90) & (dir_PA<90)) + s_signed = -s + s_signed[ind_fld] = s[ind_fld] + else: + ind_ebb = np.where((dir_PA >= -90) & (dir_PA<90)) + s_signed = s + s_signed[ind_ebb] = -s[ind_ebb] + + s_signed_all[:, i] = s_signed + + return s_signed_all, PA_all + +def principal_axis(u, v): + + #create velocity matrix + U = np.vstack((u,v)).T + #eliminate NaN values + U = U[~np.isnan(U[:, 0]), :] + #convert matrix to deviate form + rep = np.tile(np.mean(U, axis=0), [len(U), 1]) + U -= rep + #compute covariance matrix + R = np.dot(U.T, U) / (len(U) - 1) + + #calculate eigenvalues and eigenvectors for covariance matrix + R[np.where(np.isnan(R))] = 0.0 + R[np.where(np.isinf(R))] = 0.0 + R[np.where(np.isneginf(R))] = 0.0 + lamb, V = np.linalg.eig(R) + #sort eignvalues in descending order so that major axis is given by first eigenvector + # sort in descending order with indices + ilamb = sorted(range(len(lamb)), key=lambda k: lamb[k], reverse=True) + lamb = sorted(lamb, reverse=True) + # reconstruct the eigenvalue matrix + lamb = np.diag(lamb) + #reorder the eigenvectors + V = V[:, ilamb] + + #rotation angle of major axis in radians relative to cartesian coordiantes + #ra = np.arctan2(V[0,1], V[1,1]) + ra = atan2(V[0,1], V[1,1]) + #express principal axis in compass coordinates + #PA = -ra * 180 / np.pi + 90 + #TR: still not sure here + PA = -ra * 180 / np.pi + #variance captured by principal + varxp_PA = np.diag(lamb[0]) / np.trace(lamb) + + return PA, varxp_PA + + +class Struct: + def __init__(self, **entries): + self.__dict__.update(entries) + +def save_FlowFile_BPFormat(fileinfo, adcp, rbr, params, options): + print adcp.mtime[0] + day1 = date2py(adcp.mtime[0][0]) + print day1 + #date_time = [date2py(tval[0]) for tval in adcp.mtime[:]] + datenum = datetime(day1.year,1,1) + timedelta(365) + datenum = datenum.toordinal() + + yd = adcp.mtime[:].ravel() - datenum + tind = np.where((yd > params.tmin) & (yd < params.tmax))[0] + + time = {} + time['mtime'] = adcp.mtime[:].ravel()[tind] + dt = np.nanmean(np.diff(time['mtime'])) + + if not rbr: + print 'Depths measured by ADCP not yet coded.' + else: + print 'Ensemble averaging rbr data' + + nens = round(dt/(rbr.mtime[1] - rbr.mtime[0])) + + + +# mini = timedelta(days=params.tmin) +# maxi = timedelta(days=params.tmax) +# +# nmin = datetime(day1.year,1,1) + mini +# nmax = datetime(day1.year,1,1) + maxi +# print yd + + + +if __name__ == '__main__': + filename = '140703-EcoEII_database/data/GP-120726-BPd_raw.mat' + data = rawADCP(filename) + rawdata = rawADCP(filename) + #adcp = Struct(**data.adcp) + rawADCP = data.adcp + adcp = data.adcp + params = Struct(**data.saveparams) + params = data.saveparams + rbr = Struct(**data.rbr) + +# save_FlowFile_BPFormat(data.fileinfo, data.adcp, data.rbr, +# data.saveparams, data.options) + + debug = False + #day1 = date2py(adcp.mtime[0][0]) + day1 = date2py(adcp['mtime'][0][0]) + #date_time = [date2py(tval[0]) for tval in adcp.mtime[:]] + datenum = datetime(day1.year,1,1) + timedelta(365) + datenum = datenum.toordinal() + #yd = adcp.mtime[:].ravel() - datenum + yd = adcp['mtime'][:].ravel() - datenum + #tind = np.where((yd > params.tmin) & (yd < params.tmax))[0] + tind = np.where((yd > params['tmin']) & (yd < params['tmax']))[0] + + time = {} + time['mtime'] = adcp['mtime'][:].ravel()[tind] + dt = np.nanmean(np.diff(time['mtime'])) + pres = {} + + if not rbr: + print 'Depths measured by ADCP not yet coded.' + else: + print 'Ensemble averaging rbr data' + + nens = round(dt/(rbr.mtime[1] - rbr.mtime[0])) + temp = np.arange(rbr.mtime[nens/2-1], rbr.mtime[-1-nens/2], dt) + temp2 = np.r_[rbr.mtime[nens/2-1]: rbr.mtime[-1-nens/2]: dt] + + # Load in matlab values + filename = './140703-EcoEII_database/scripts_examples/mtime.mat' + mat = sio.loadmat(filename, struct_as_record=False, squeeze_me=True) + matTimes = mat['mtimeens'] + filename = './140703-EcoEII_database/scripts_examples/dt.mat' + mat = sio.loadmat(filename, struct_as_record=False, squeeze_me=True) + matdt = mat['dt'] + + + mtimeens = np.arange(rbr.mtime[nens/2-1], rbr.mtime[-1-nens/2], dt) + #mtimeens = mtimeens + params.rbr_hr_offset / 24 + mtimeens = mtimeens + params['rbr_hr_offset'] / 24 + depthens = calc_ensemble(rbr.depth, nens, 1) + + filename = './140703-EcoEII_database/scripts_examples/depthens.mat' + mat = sio.loadmat(filename, struct_as_record=False, squeeze_me=True) + matdepthens = mat['depthens'] + + filename = './140703-EcoEII_database/scripts_examples/time.mat' + mat = sio.loadmat(filename, struct_as_record=False, squeeze_me=True) + matmtime = mat['mtime'] + + + debug = True + if debug: + print matTimes.shape + print temp - matTimes + print temp2 - matTimes + print dt - matdt + print depthens - matdepthens + print 'time' + print time['mtime'] - matmtime + + temp = sip.interp1d(mtimeens, depthens, kind='linear') + + #pres['surf']= temp(time['mtime']) + params.dabPS + pres['surf']= temp(time['mtime']) + params['dabPS'] + + #if data.options['showRBRavg']: + if debug: + #plt.plot(rbr.mtime+params.rbr_hr_offset/24, rbr.depth+params.dabPS, + # label='RBR') + #plt.plot(time['mtime'], pres['surf'], 'r', label='AVG') + plt.plot(rbr.mtime+params['rbr_hr_offset']/24, rbr.depth+params['dabPS'], + label='RBR') + plt.plot(time['mtime'], pres['surf'], 'r', label='AVG') + plt.xlabel('Time') + plt.ylabel('Elevation') + plt.legend(bbox_to_anchor=(0, 0, 1, 1), bbox_transform=plt.gcf().transFigure) + + plt.show() + + ## zlevels + data = {} + z = adcp['config']['ranges'][:] + params['dabADCP'] + z = z.ravel() + zind = np.where((z > params['zmin']) & (z < params['zmax']))[0] + data['bins'] = z[zind] + + ## Currents + #data['vert_vel'] = adcp['vert_vel'][:][zind, tind].T + #data['error_vel'] = adcp['error_vel'][:][zind, tind].T + data['vert_vel'] = adcp['vert_vel'][:][tind][:, zind] + data['error_vel'] = adcp['error_vel'][:][tind][:, zind] + + # If compass wasn't calibrated + #if isfield(params,'hdgmod'): + if 'hdgmod' in params: + adcp['east_vel'][:], adcp['north_vel'][:] = rotate_coords(adcp['east_vel'][:], + adcp['north_vel'][:], + params['hdgmod']) + #Comments{end+1} = 'East and north velocity rotated by params.hdgmod'; + + # Rotate east_vel and north_vel to be relative to true north + data['east_vel'], data['north_vel'] = \ + rotate_to_true(adcp['east_vel'][:][tind][:, zind], + adcp['north_vel'][:][tind][:, zind], + params['declination']) + + # Direction + data['dir_vel'] = get_DirFromN(data['east_vel'],data['north_vel']) + + # Signed Speed + spd_all = np.sqrt(data['east_vel']**2+data['north_vel']**2) + + # Determine flood and ebb based on principal direction (Polagye Routine) + print 'Getting signed speed (Principal Direction Method) -- used all speeds' + s_signed_all, PA_all = sign_speed(data['east_vel'], data['north_vel'], + spd_all, data['dir_vel'], params['flooddir']) + + data['mag_signed_vel'] = s_signed_all + +## save_FlowFile_BPFormat(data.fileinfo, adcp, data.rbr, +## params, data.options) diff --git a/build/lib/pyseidon_dvt/utilities/__init__.py b/build/lib/pyseidon_dvt/utilities/__init__.py new file mode 100644 index 0000000..2223df3 --- /dev/null +++ b/build/lib/pyseidon_dvt/utilities/__init__.py @@ -0,0 +1,10 @@ +#!/usr/bin/python2.7 +# encoding: utf-8 + +from __future__ import division + + +__authors__ = ['Wesley Bowman, Thomas Roc, Jonathan Smith'] +__licence__ = 'GNU Affero GPL v3.0' +__copyright__ = 'Copyright (c) 2014 EcoEnergyII' + diff --git a/build/lib/pyseidon_dvt/utilities/createNC.py b/build/lib/pyseidon_dvt/utilities/createNC.py new file mode 100644 index 0000000..34e6f65 --- /dev/null +++ b/build/lib/pyseidon_dvt/utilities/createNC.py @@ -0,0 +1,87 @@ +import netCDF4 as nc + + +def createNC(data): + + ncFile = nc.Dataset('test.nc', 'w', format='NETCDF4') + + #ncgrp = ncFile.createGroup('regioned') + #ncgrp.createDimension('dim', None) + #ncgrp = ncFile.createGroup('regioned') + #ncFile.createDimension('dimTest', None) + + ncFile.createDimension('dim', None) + + time = ncFile.createVariable('time', 'f8', ('dim',)) + time[:] = data['time'] + + x = ncFile.createVariable('x', 'f8', ('dim',)) + x[:] = data['x'] + + y = ncFile.createVariable('y', 'f8', ('dim',)) + y[:] = data['y'] + + xc = ncFile.createVariable('xc', 'f8', ('dim',)) + xc[:] = data['xc'] + + yc = ncFile.createVariable('yc', 'f8', ('dim',)) + yc[:] = data['yc'] + + h = ncFile.createVariable('h', 'f8', ('dim',)) + h[:] = data['h'] + + lon = ncFile.createVariable('lon', 'f8', ('dim',)) + lon[:] = data['lon'] + + lat = ncFile.createVariable('lat', 'f8', ('dim',)) + lat[:] = data['lat'] + + lonc = ncFile.createVariable('lonc', 'f8', ('dim',)) + lonc[:] = data['lonc'] + + latc = ncFile.createVariable('latc', 'f8', ('dim',)) + latc[:] = data['latc'] + + elev = ncFile.createVariable('elev', 'f8', ('dim', 'dim')) + elev[:] = data['elev'] + + ua = ncFile.createVariable('ua', 'f8', ('dim', 'dim')) + ua[:] = data['ua'] + + va = ncFile.createVariable('va', 'f8', ('dim', 'dim')) + va[:] = data['va'] + + node_index = ncFile.createVariable('node_index', 'f8', ('dim',)) + node_index[:] = data['node_index'] + + element_index = ncFile.createVariable('element_index', 'f8', ('dim',)) + element_index[:] = data['element_index'] + + nbe = ncFile.createVariable('nbe', 'f8', ('dim', 'dim')) + nbe[:] = data['nbe'] + + nv = ncFile.createVariable('nv', 'f8', ('dim', 'dim')) + nv[:] = data['nv'] + + a1u = ncFile.createVariable('a1u', 'f8', ('dim', 'dim')) + a1u[:] = data['a1u'] + + a2u = ncFile.createVariable('a2u', 'f8', ('dim', 'dim')) + a2u[:] = data['a2u'] + + aw0 = ncFile.createVariable('aw0', 'f8', ('dim', 'dim')) + aw0[:] = data['aw0'] + + awx = ncFile.createVariable('awx', 'f8', ('dim', 'dim')) + awx[:] = data['awx'] + + awy = ncFile.createVariable('awy', 'f8', ('dim', 'dim')) + awy[:] = data['awy'] + + siglay = ncFile.createVariable('siglay', 'f8', ('dim', 'dim')) + siglay[:] = data['siglay'] + + siglev = ncFile.createVariable('siglev', 'f8', ('dim', 'dim')) + siglev[:] = data['siglev'] + + ncFile.close() diff --git a/build/lib/pyseidon_dvt/utilities/interpolation_utils.py b/build/lib/pyseidon_dvt/utilities/interpolation_utils.py new file mode 100644 index 0000000..f2918d0 --- /dev/null +++ b/build/lib/pyseidon_dvt/utilities/interpolation_utils.py @@ -0,0 +1,597 @@ +#!/usr/bin/python2.7 +# encoding: utf-8 + +import sys +import numpy as np +import matplotlib.pyplot as plt +import matplotlib.tri as Tri +import matplotlib.ticker as ticker +from matplotlib.path import Path +from scipy.spatial import KDTree +import scipy.interpolate as interpolate + +def closest_point(pt_lon, pt_lat, lon, lat, lonc, latc, tri, + debug=False): + ''' + Finds the closest exact lon, lat centre indexes of an FVCOM class + to given lon, lat coordinates. + + Inputs: + - pt_lon = list of longitudes in degrees to find + - pt_lat = list of latitudes in degrees to find + - lon = list of longitudes in degrees to search in + - lat = list of latitudes in degrees to search in + Outputs: + - closest_point_indexes = numpy array of grid indexes + ''' + if debug: + print 'Computing closest_point_indexes...' + points = np.array([[pt_lon], [pt_lat]]).T + point_list = np.array([lonc[:], latc[:]]).T + + #closest_dist = (np.square((point_list[:, 0] - points[:, 0, np.newaxis])) + + # np.square((point_list[:, 1] - points[:, 1, np.newaxis]))) + + #closest_point_indexes = np.argmin(closest_dist, axis=1) + + #Wesley's optimized version of this bottleneck + point_list0 = point_list[:, 0] + points0 = points[:, 0, np.newaxis] + point_list1 = point_list[:, 1] + points1 = points[:, 1, np.newaxis] + + closest_dist = ((point_list0 - points0) * + (point_list0 - points0) + + (point_list1 - points1) * + (point_list1 - points1) + ) + + #Check if it is the right index + if tri[:].max() == (lonc[:].shape[0]-1): + lo = lonc[:] + la = latc[:] + else: + lo = lon[:] + la = lat[:] + + index = np.argmin(closest_dist, axis=1)[0] + triIndex = tri[index] + triIndex.sort()#due to new version of netCDF4 + trig = Tri.Triangulation(lo[triIndex], la[triIndex], np.array([[0,1,2]])) + trif = trig.get_trifinder() + test = -1 * trif.__call__(pt_lon, pt_lat) + if test: index = np.nan #freak point + + #Thomas' optimized version of this bottleneck + #closest_point_indexes = np.zeros(points.shape[0]) + #for i in range(points.shape[0]): + # dist=((point_list-points[i,:])**2).sum(axis=1) + # ndx = d.argsort() + # closest_point_indexes[i] = ndx[0] + #if debug: print 'Closest dist: ', closest_dist + if debug: + print 'closest_point_indexes', index + print '...Passed' + + return index + +def closest_points( pt_lon, pt_lat, lon, lat, debug=False): + ''' + Finds the closest exact lon, lat centre indexes of an FVCOM class + to given lon, lat coordinates. + + Inputs: + - pt_lon = list of longitudes in degrees to find + - pt_lat = list of latitudes in degrees to find + - lon = list of longitudes in degrees to search in + - lat = list of latitudes in degrees to search in + Outputs: + - closest_point_indexes = numpy array of grid indexes + ''' + if debug: + print 'Computing closest_point_indexes...' + + lonc=lon[:] + latc=lat[:] + if not type(pt_lon)==list: + points = np.array([[pt_lon], [pt_lat]]).T + else: + points = np.array([pt_lon, pt_lat]).T + point_list = np.array([lonc, latc]).T + + #closest_dist = (np.square((point_list[:, 0] - points[:, 0, np.newaxis])) + + # np.square((point_list[:, 1] - points[:, 1, np.newaxis]))) + + #closest_point_indexes = np.argmin(closest_dist, axis=1) + + #Wesley's optimized version of this bottleneck + point_list0 = point_list[:, 0] + points0 = points[:, 0, np.newaxis] + point_list1 = point_list[:, 1] + points1 = points[:, 1, np.newaxis] + + closest_dist = ((point_list0 - points0) * + (point_list0 - points0) + + (point_list1 - points1) * + (point_list1 - points1) + ) + closest_point_indexes = np.argmin(closest_dist, axis=1) + + #Thomas' optimized version of this bottleneck + #closest_point_indexes = np.zeros(points.shape[0]) + #for i in range(points.shape[0]): + # dist=((point_list-points[i,:])**2).sum(axis=1) + # ndx = d.argsort() + # closest_point_indexes[i] = ndx[0] + if debug: + print 'Closest dist: ', closest_dist + + + if debug: + print 'closest_point_indexes', closest_point_indexes + print '...Passed' + + return closest_point_indexes + +def interpN_at_pt(var, pt_x, pt_y, index, trinodes, + aw0, awx, awy, debug=False): + """ + Interpol node variable any given variables at any give location. + Inputs: + - var = variable, numpy array, dim=(node) or (time, node) or (time, level, node) + - pt_x = x coordinate in m to find + - pt_y = y coordinate in m to find + - xc = list of x coordinates of var, numpy array, dim= ele + - yc = list of y coordinates of var, numpy array, dim= ele + - trinodes = FVCOM trinodes, numpy array, dim=(3,nele) + - index = index of the nearest element + - aw0, awx, awy = grid parameters + Outputs: + - varInterp = var interpolate at (pt_lon, pt_lat) + """ + if debug: + print 'Interpolating at node...' + + n1 = int(trinodes[index,0]) + n2 = int(trinodes[index,1]) + n3 = int(trinodes[index,2]) + # x0 = pt_x - xc[index] + # y0 = pt_y - yc[index] + #due to Mitchell's alternative, conversion made in functionsFvcom.py + x0 = pt_x + y0 = pt_y + + if len(var.shape)==1: + var0 = (aw0[0,index] * var[n1]) \ + + (aw0[1,index] * var[n2]) \ + + (aw0[2,index] * var[n3]) + varX = (awx[0,index] * var[n1]) \ + + (awx[1,index] * var[n2]) \ + + (awx[2,index] * var[n3]) + varY = (awy[0,index] * var[n1]) \ + + (awy[1,index] * var[n2]) \ + + (awy[2,index] * var[n3]) + elif len(var.shape)==2: + var0 = (aw0[0,index] * var[:,n1]) \ + + (aw0[1,index] * var[:,n2]) \ + + (aw0[2,index] * var[:,n3]) + varX = (awx[0,index] * var[:,n1]) \ + + (awx[1,index] * var[:,n2]) \ + + (awx[2,index] * var[:,n3]) + varY = (awy[0,index] * var[:,n1]) \ + + (awy[1,index] * var[:,n2]) \ + + (awy[2,index] * var[:,n3]) + else: + var0 = (aw0[0,index] * var[:,:,n1]) \ + + (aw0[1,index] * var[:,:,n2]) \ + + (aw0[2,index] * var[:,:,n3]) + varX = (awx[0,index] * var[:,:,n1]) \ + + (awx[1,index] * var[:,:,n2]) \ + + (awx[2,index] * var[:,:,n3]) + varY = (awy[0,index] * var[:,:,n1]) \ + + (awy[1,index] * var[:,:,n2]) \ + + (awy[2,index] * var[:,:,n3]) + varPt = var0 + (varX * x0) + (varY * y0) + + if debug: + if len(var.shape)==1: + zi = varPt + print 'Varpt: ', zi + print '...Passed' + + #TR comment: squeeze seems to resolve my problem with pydap + return varPt.squeeze() + +def interpN(var,trinodes,aw0,debug=False): + """ + Interpol node variable at elements. + Inputs: + - var = variable, numpy array, dim=(node) or (time, node) or (time, level, node) + - trinodes = FVCOM trinodes, numpy array, dim=(3,nele) + - aw0, awx, awy = grid parameters + Outputs: + - varInterp = var interpolated + """ + if debug: + print 'Interpolating at nodes...' + + n1 = [int(number) for number in trinodes[:,0]] + n2 = [int(number) for number in trinodes[:,1]] + n3 = [int(number) for number in trinodes[:,2]] + + if len(var.shape)==1: + var0 = (aw0[0,:] * var[n1]) \ + + (aw0[1,:] * var[n2]) \ + + (aw0[2,:] * var[n3]) + elif len(var.shape)==2: + var0 = (aw0[0,:] * var[:,n1]) \ + + (aw0[1,:] * var[:,n2]) \ + + (aw0[2,:] * var[:,n3]) + else: + var0 = (aw0[0,:] * var[:,:,n1]) \ + + (aw0[1,:] * var[:,:,n2]) \ + + (aw0[2,:] * var[:,:,n3]) + varPt = var0 + + if debug: print '...Passed' + + #TR comment: squeeze seems to resolve my problem with pydap + return varPt.squeeze() + +def interpE_at_pt(var, pt_x, pt_y, index, triele, + a1u, a2u, debug=False): + """ + Interpol element variable any given variables at any give location. + Inputs: + - var = variable, numpy array, dim=(nele) or (time, nele) or (time, level, nele) + - pt_x = x coordinate in m to find + - pt_y = y coordinate in m to find + - xc = list of x coordinates of var, numpy array, dim= nele + - yc = list of y coordinates of var, numpy array, dim= nele + - triele = FVCOM triele, numpy array, dim=(3,nele) + - index = index of the nearest element + - a1u, a2u = grid parameters + Outputs: + - varInterp = var interpolate at (pt_lon, pt_lat) + """ + if debug: + print 'Interpolating at element...' + + n1 = int(triele[index,0]) + n2 = int(triele[index,1]) + n3 = int(triele[index,2]) + + #Test for ghost points + test = [-1, var.shape[-1]] + + #TR quick fix: due to error with pydap.proxy.ArrayProxy + # not able to cop with numpy.int + n1 = int(n1) + n2 = int(n2) + n3 = int(n3) + + # x0 = pt_x - xc[index] + # y0 = pt_y - yc[index] + #due to Mitchell's alternative, conversion made in functionsFvcom.py + x0 = pt_x + y0 = pt_y + + if len(var.shape)==1: + # Treatment of ghost points + if n1 in test: + V1 = 0.0 + else: + V1 = var[n1] + if n2 in test: + V2 = 0.0 + else: + V2 = var[n2] + if n3 in test: + V3 = 0.0 + else: + V3 = var[n3] + + dvardx = (a1u[0,index] * var[index]) \ + + (a1u[1,index] * V1) \ + + (a1u[2,index] * V2) \ + + (a1u[3,index] * V3) + dvardy = (a2u[0,index] * var[index]) \ + + (a2u[1,index] * V1) \ + + (a2u[2,index] * V2) \ + + (a2u[3,index] * V3) + varPt = var[index] + (dvardx * x0) + (dvardy * y0) + elif len(var.shape)==2: + # Treatment of ghost points + if n1 in test: + V1 = np.zeros(var.shape[0]) + else: + V1 = var[:,n1] + if n2 in test: + V2 = np.zeros(var.shape[0]) + else: + V2 = var[:,n2] + if n3 in test: + V3 = np.zeros(var.shape[0]) + else: + V3 = var[:,n3] + + dvardx = (a1u[0,index] * var[:,index]) \ + + (a1u[1,index] * V1) \ + + (a1u[2,index] * V2) \ + + (a1u[3,index] * V3) + dvardy = (a2u[0,index] * var[:,index]) \ + + (a2u[1,index] * V1) \ + + (a2u[2,index] * V2) \ + + (a2u[3,index] * V3) + varPt = var[:,index] + (dvardx * x0) + (dvardy * y0) + else: + # Treatment of ghost points + if n1 in test: + V1 = np.zeros((var.shape[0], var.shape[1])) + else: + V1 = var[:,:,n1] + if n2 in test: + V2 = np.zeros((var.shape[0], var.shape[1])) + else: + V2 = var[:,:,n2] + if n3 in test: + V3 = np.zeros((var.shape[0], var.shape[1])) + else: + V3 = var[:,:,n3] + + dvardx = (a1u[0,index] * var[:,:,index]) \ + + (a1u[1,index] * V1) \ + + (a1u[2,index] * V2) \ + + (a1u[3,index] * V3) + dvardy = (a2u[0,index] * var[:,:,index]) \ + + (a2u[1,index] * V1) \ + + (a2u[2,index] * V2) \ + + (a2u[3,index] * V3) + varPt = var[:,:,index] + (dvardx * x0) + (dvardy * y0) + + if debug: + if len(var.shape)==1: + zi = varPt + print 'Varpt: ', zi + print '...Passed' + #TR comment: squeeze seems to resolve my problem with pydap + return varPt.squeeze() + +def interpE(var, xc, yc, triele, + a1u, a2u, debug=False): + """ + Interpol element variable at node locations. + Inputs: + - var = variable, numpy array, dim=(nele) or (time, nele) or (time, level, nele) + - xc = list of x coordinates of var, numpy array, dim= nele + - yc = list of y coordinates of var, numpy array, dim= nele + - triele = FVCOM triele, numpy array, dim=(3,nele) + - a1u, a2u = grid parameters + Outputs: + - varInterp = var interpolate at (pt_lon, pt_lat) + """ + if debug: + print 'Interpolating at element...' + + n1 = [int(number) for number in triele[:,0]] + n2 = [int(number) for number in triele[:,1]] + n3 = [int(number) for number in triele[:,2]] + + #TR quick fix: due to error with pydap.proxy.ArrayProxy + # not able to cop with numpy.int + + x0 = xc[:] + y0 = yc[:] + + if len(var.shape)==1: + # Treatment of ghost points + Var = np.hstack((var,0)) + V1 = Var[n1] + V2 = Var[n2] + V3 = Var[n3] + + dvardx = (a1u[0,:] * var[:]) \ + + (a1u[1,:] * V1) \ + + (a1u[2,:] * V2) \ + + (a1u[3,:] * V3) + dvardy = (a2u[0,:] * var[:]) \ + + (a2u[1,:] * V1) \ + + (a2u[2,:] * V2) \ + + (a2u[3,:] * V3) + varPt = var[:] + (dvardx * x0) + (dvardy * y0) + elif len(var.shape)==2: + # Treatment of ghost points + Var = np.zeros((var.shape[0], var.shape[1]+1)) + Var[:,:-1] = var[:] + V1 = Var[:,n1] + V2 = Var[:,n2] + V3 = Var[:,n3] + + dvardx = (a1u[0,:] * var[:,:]) \ + + (a1u[1,:] * V1) \ + + (a1u[2,:] * V2) \ + + (a1u[3,:] * V3) + dvardy = (a2u[0,:] * var[:,:]) \ + + (a2u[1,:] * V1) \ + + (a2u[2,:] * V2) \ + + (a2u[3,:] * V3) + varPt = var[:,:] + (dvardx * x0) + (dvardy * y0) + else: + # Treatment of ghost points + Var = np.zeros((var.shape[0], var.shape[1], var.shape[2]+1)) + Var[:,:,:-1] = var[:] + V1 = Var[:,:,n1] + V2 = Var[:,:,n2] + V3 = Var[:,:,n3] + + dvardx = (a1u[0,:] * var[:,:,:]) \ + + (a1u[1,:] * V1) \ + + (a1u[2,:] * V2) \ + + (a1u[3,:] * V3) + dvardy = (a2u[0,:] * var[:,:,:]) \ + + (a2u[1,:] * V1) \ + + (a2u[2,:] * V2) \ + + (a2u[3,:] * V3) + varPt = var[:,:,:] + (dvardx * x0) + (dvardy * y0) + + if debug: + if len(var.shape)==1: + zi = varPt + print 'Varpt: ', zi + print '...Passed' + #TR comment: squeeze seems to resolve my problem with pydap + return varPt.squeeze() + +def interp_at_point(var, pt_lon, pt_lat, lon, lat, + index, trinodes, tri=[], debug=False): + """ + Interpol any given variables at any give location. + + Inputs: + - var = variable, numpy array, dim=(time, nele or node) + - pt_lon = longitude in degrees to find + - pt_lat = latitude in degrees to find + - lon = list of longitudes of var, numpy array, dim=(nele or node) + - lat = list of latitudes of var, numpy array, dim=(nele or node) + - trinodes = FVCOM trinodes, numpy array, dim=(3,nele) + Keywords: + - tri = triangulation object + Outputs: + - varInterp = var interpolate at (pt_lon, pt_lat) + """ + if debug: + print 'Interpolating at point...' + #Finding the right indexes + + #Triangulation + #if debug: + # print triIndex, lon[triIndex], lat[triIndex] + triIndex = trinodes[index] + triIndex.sort()#due to new version of netCDF4 + if tri==[]: + tri = Tri.Triangulation(lon[triIndex], lat[triIndex], np.array([[0,1,2]])) + + trif = tri.get_trifinder() + trif.__call__(pt_lon, pt_lat) + if debug: + if len(var.shape)==1: + averEl = var[triIndex] + print 'Var', averEl + inter = Tri.LinearTriInterpolator(tri, averEl) + zi = inter(pt_lon, pt_lat) + print 'zi', zi + + #Choose the right interpolation depending on the variable + if len(var.shape)==1: + triVar = np.zeros(triIndex.shape) + triVar[:] = var[triIndex] + inter = Tri.LinearTriInterpolator(tri, triVar[:]) + varInterp = inter(pt_lon, pt_lat) + elif len(var.shape)==2: + triVar = np.zeros((var.shape[0], triIndex.shape[0])) + triVar[:] = var[:, triIndex] + varInterp = np.ones(triVar.shape[0]) + for i in range(triVar.shape[0]): + inter = Tri.LinearTriInterpolator(tri, triVar[i,:]) + varInterp[i] = inter(pt_lon, pt_lat) + else: + triVar = np.zeros((var.shape[0], var.shape[1], triIndex.shape[0])) + triVar[:] = var[:, :, triIndex] + varInterp = np.ones(triVar.shape[:-1]) + for i in range(triVar.shape[0]): + for j in range(triVar.shape[1]): + inter = Tri.LinearTriInterpolator(tri, triVar[i,j,:]) + varInterp[i,j] = inter(pt_lon, pt_lat) + + if debug: + print '...Passed' + + #TR comment: squeeze seems to resolve my problem with pydap + return varInterp.squeeze() + +def interpE_at_point_bis(var, pt_lon, pt_lat, lonc, latc, debug=False): + """ + Interpol at awkward locations. + + Inputs: + - var = variable, numpy array, dim=(time, nele) + - pt_lon = longitude to find + - pt_lat = latitude to find + - lonc = list of longitudes of var, numpy array, dim=(nele) + - lonc = list of latitudes of var, numpy array, dim=(nele) + + Outputs: + - varInterp = var interpolate at (pt_lon, pt_lat) + """ + if debug: + print 'Interpolating at awkward point...' + #Finding the right indexes + points = np.array([[pt_lon], [pt_lat]]).T + point_list = np.array([lonc[:], latc[:]]).T + point_list0 = point_list[:, 0] + points0 = points[:, 0, np.newaxis] + point_list1 = point_list[:, 1] + points1 = points[:, 1, np.newaxis] + + closest_dist = ((point_list0 - points0) * + (point_list0 - points0) + + (point_list1 - points1) * + (point_list1 - points1) + ) + #Finding closest elements + triIndex = [0,0,0] + triIndex[0] = np.argmin(closest_dist, axis=1)[0] + closest_dist[:,triIndex[0]]=np.inf + triIndex[1] = np.argmin(closest_dist, axis=1)[0] + closest_dist[:,triIndex[1]]=np.inf + #test if inside triangle + test=1 + while test: + trig = Tri.Triangulation(lonc[triIndex], latc[triIndex], np.array([[0,1,2]])) + triIndex[2] = np.argmin(closest_dist, axis=1)[0] + trif = trig.get_trifinder() + test = -1 * trif.__call__(pt_lon, pt_lat) + if test: closest_dist[:,triIndex[2]]=np.inf + #new scheme + #linear equation based on plane equation + x1 = lonc[triIndex[0]]; x2 = lonc[triIndex[1]]; x3 = lonc[triIndex[2]] + y1 = latc[triIndex[0]]; y2 = latc[triIndex[1]]; y3 = latc[triIndex[2]] + a = np.array([[x2-x1,x3-x1],[y2-y1,y3-y1]]) + b = np.array([pt_lon-x1,pt_lat-y1]) + A, B = np.linalg.solve(a, b) + #applying weights + if len(var.shape)==1: + z1 = var[triIndex[0]] + z2 = var[triIndex[1]] + z3 = var[triIndex[2]] + elif len(var.shape)==2: + z1 = var[:,triIndex[0]] + z2 = var[:,triIndex[1]] + z3 = var[:,triIndex[2]] + else: + z1 = var[:,:,triIndex[0]] + z2 = var[:,:,triIndex[1]] + z3 = var[:,:,triIndex[2]] + varInterp = z1 + A * (z2 - z1) + B * (z3 - z1) + #end new scheme + if debug: + print '...Passed' + + #TR comment: squeeze seems to resolve my problem with pydap + return varInterp.squeeze() + +def interp_linear_to_nodes(var, xc, yc, x, y): + """Linear interpolation from elements to nodes""" + L = xc.shape[0] + M = x.shape[0] + orig = np.zeros((L, 2)) + ask = np.zeros((M, 2)) + orig[:, 0] = xc + orig[:, 1] = yc + ask[:, 0] = x + ask[:, 1] = y + interpol = interpolate.LinearNDInterpolator(orig, var) + varinterp = interpol(ask) + varinterp[np.where(varinterp==np.nan)]=0.0 + + return varinterp \ No newline at end of file diff --git a/build/lib/pyseidon_dvt/utilities/miscellaneous.py b/build/lib/pyseidon_dvt/utilities/miscellaneous.py new file mode 100644 index 0000000..f54b30b --- /dev/null +++ b/build/lib/pyseidon_dvt/utilities/miscellaneous.py @@ -0,0 +1,174 @@ +#!/usr/bin/python2.7 +# encoding: utf-8 + +from __future__ import division +import numpy as np +from datetime import datetime +from datetime import timedelta +import fnmatch +import os +import time +from scipy.io import netcdf +from pydap.client import open_url + +# Custom error +from pyseidon_error import PyseidonError + +def date2py(matlab_datenum): + python_datetime = datetime.fromordinal(int(matlab_datenum)) + \ + timedelta(days=matlab_datenum%1) - timedelta(days = 366) + + return python_datetime + +def op_angles_from_vectors(u, v, debug=False): + """ + This function takes in vectors in the form (u,v) and compares them in + order to find the angles of the vectors without any wrap-around issues. + This is accomplished by finding the smallest difference between angles + compared at different wrap-around values. + This appears to work correctly. + + Inputs: + -u = velocity component along x (West-East) direction, 1D array + -v = velocity component along y (South-North) direction, 1D array + + Outputs: + -angle = corresponidng angle in degrees, 1D array + + Notes: + -Angles are reported in compass coordinates, i.e. 0 and 360 deg., + 0/360=East, 90=North, 180=West, 270=South + """ + if debug: + print 'Computing angles from velocity component...' + start = time.time() + + phi = np.mod((-1.0*np.arctan2(v,u)) * (180.0/np.pi) + 90.0, 360.0) + if len(phi.shape)==1:#Assuming the only dimension is time + #Compute difference between angles + diff1 = np.abs(phi[:-1]-phi[1:]) #initial difference between angles + diff2 = np.abs(phi[:-1]-phi[1:]-360.0) #diff when moved down a ring + diff3 = np.abs(phi[:-1]-phi[1:]+360.0) #diff when moved up a ring + + index1 = np.where((diff2 < diff1) & (diff2 < diff3))[0] + index2 = np.where((diff3 < diff1) & (diff3 < diff2))[0] + + phi[index1] = np.mod(phi[index1] - 360.0, 360.0) + phi[index2] = np.mod(phi[index2] + 360.0, 360.0) + elif len(phi.shape)==2:#Assuming the only dimension is time and sigma level + #Compute difference between angles + diff1 = np.abs(phi[:-1,:]-phi[1:,:]) #initial difference between angles + diff2 = np.abs(phi[:-1,:]-phi[1:,:]-360.0) #diff when moved down a ring + diff3 = np.abs(phi[:-1,:]-phi[1:,:]+360.0) #diff when moved up a ring + + index1 = np.where((diff2 < diff1) & (diff2 < diff3))[0] + index2 = np.where((diff3 < diff1) & (diff3 < diff2))[0] + + phi[index1] = phi[index1] - 360.0 + phi[index2] = phi[index2] + 360.0 + else: #Assuming the only dimension is time ,sigma level and element + #Compute difference between angles + diff1 = np.abs(phi[:-1,:,:]-phi[1:,:,:]) #initial difference between angles + diff2 = np.abs(phi[:-1,:,:]-phi[1:,:,:]-360.0) #diff when moved down a ring + diff3 = np.abs(phi[:-1,:,:]-phi[1:,:,:]+360.0) #diff when moved up a ring + + index1 = np.where((diff2 < diff1) & (diff2 < diff3))[0] + index2 = np.where((diff3 < diff1) & (diff3 < diff2))[0] + + phi[index1] = phi[index1] - 360.0 + phi[index2] = phi[index2] + 360.0 + + if debug: + end = time.time() + print "...processing time: ", (end - start) + + return phi + +def time_to_index(t_start, t_end, time, debug=False): + """ + Convert datetime64[us] string in FVCOM index + + Inputs: + - t_start = start time in datetime + - t_end = end time in datetime + - time = array of julian days + + Outputs: + - argtime = arry of indices + """ + start = date_to_julian_day(t_start) + end = date_to_julian_day(t_end) + + t_slice = [start, end] + + argtime = np.argwhere((time>=t_slice[0])&(time<=t_slice[-1])).ravel() + if debug: + print 'Argtime: ', argtime + if argtime == []: + raise PyseidonError("Wrong time input") + return argtime + +def mattime_to_datetime(mattime, debug=False): + """Convert matlab time to datetime64[us] """ + date = datetime.fromordinal(int(mattime)) + \ + timedelta(days=mattime%1)-timedelta(days=366) + time = np.array(date,dtype='datetime64[us]') + + return time + +def datetime_to_mattime(dt, debug=False): + """Convert datetime64[us] to matlab time""" + mdn = dt + timedelta(days = 366) + s = (dt.hour * (60.0*60.0)) + (dt.minute * 60.0) + dt.second + day = 24.0*60.0*60.0 + frac = s/day + + return mdn.toordinal() + frac + +def findFiles(filename, name): + """ + Wesley comment[elements] the name needs to be a linux expression to find files + you want. For multiple station files, this would work + name = '*station*.nc' + + For just dngrid_0001 and no restart files: + name = 'dngrid_0*.nc' + will work + """ + + name = '*' + name + '*.nc' + matches = [] + for root, dirnames, filenames in os.walk(filename): + for filename in fnmatch.filter(filenames, name): + matches.append(os.path.join(root, filename)) + filenames.remove(filename) + for filename in fnmatch.filter(filenames, name.lower()): + matches.append(os.path.join(root, filename)) + + return sorted(matches) + +def date_to_julian_day(my_date): + """Returns the Julian day number of a date.""" + # a = (14 - my_date.month)//12 + # y = my_date.year + 4800 - a + # m = my_date.month + 12*a - 3 + # s = (my_date.hour * (60.0*60.0)) + (my_date.minute * 60.0) + my_date.second + # day = 24.0*60.0*60.0 + # jtime = my_date.day + ((153*m + 2)//5) + 365*y + y//4 - y//100 + y//400 - 32045 + s/day + jtime = datetime_to_mattime(my_date) - 678942.0 + return jtime + +def distance(locs,loce): + """Returns the distance in meters between two locations in long/lat.""" + TPI=111194.92664455874 + y0c = TPI * (loce[1] - locs[1]) + dx_sph = loce[0] - locs[0] + if (dx_sph > 180.0): + dx_sph=dx_sph-360.0 + elif (dx_sph < -180.0): + dx_sph =dx_sph+360.0 + x0c = TPI * np.cos(np.deg2rad(loce[1] + locs[1])*0.5) * dx_sph + + dist=np.linalg.norm([x0c, y0c]) + + return dist diff --git a/build/lib/pyseidon_dvt/utilities/object_from_dict.py b/build/lib/pyseidon_dvt/utilities/object_from_dict.py new file mode 100644 index 0000000..333c06b --- /dev/null +++ b/build/lib/pyseidon_dvt/utilities/object_from_dict.py @@ -0,0 +1,6 @@ +class ObjectFromDict(object): + """ + Turns any given class object into a dictionnary + """ + def __init__(self, d): + self.__dict__ = d diff --git a/build/lib/pyseidon_dvt/utilities/pyseidon2matlab.py b/build/lib/pyseidon_dvt/utilities/pyseidon2matlab.py new file mode 100644 index 0000000..7275894 --- /dev/null +++ b/build/lib/pyseidon_dvt/utilities/pyseidon2matlab.py @@ -0,0 +1,74 @@ +#!/usr/bin/python2.7 +# encoding: utf-8 + +#Libs import +from __future__ import division +import numpy as np +import sys +from scipy.io import savemat + +def pyseidon_to_matlab(fvcom, filename, exceptions=[], debug=False): + """ + Saves fvcom object in a pickle file + + inputs: + - fvcom = fvcom pyseidon object + - filename = file name, string + options: + - exceptions = list of variables to exclude from output file + , list of strings + """ + #Define bounding box + if debug: + print "Computing bounding box..." + if fvcom.Grid._ax == []: + lon = fvcom.Grid.lon[:] + lat = fvcom.Grid.lat[:] + fvcom.Grid._ax = [lon.min(), lon.max(), + lat.min(), lat.max()] + + filename = filename + ".mat" + #TR comment: based on MitchellO'Flaherty-Sproul's code + dtype = float + data = {} + Grd = {} + Var = {} + data['Origin'] = fvcom._origin_file + data['History'] = fvcom.History + Grd = fvcom.Grid.__dict__ + Var = fvcom.Variables.__dict__ + # delete exceptions + for key in exceptions: Var.pop(key, None) + for key in exceptions: Grd.pop(key, None) + #TR: Force caching Variables otherwise error during loading + # with 'netcdf4.Variable' type (see above) + for key in Var: + listkeys=['Variable', 'ArrayProxy', 'BaseType'] + if any([type(Var[key]).__name__==x for x in listkeys]): + if debug: + print "Force caching for " + key + Var[key] = Var[key][:] + #keyV = key + '-var' + try: + data[key] = np.float64(Var[key].copy()) + except AttributeError: + data[key] = Var[key] + #Unpickleable objects + Grd.pop("triangle", None) + for key in Grd: + listkeys=['Variable', 'ArrayProxy', 'BaseType'] + if any([type(Grd[key]).__name__==x for x in listkeys]): + if debug: + print "Force caching for " + key + Grd[key] = Grd[key][:] + #keyG = key + '-grd' + + try: + data[key] = np.float64(Grd[key].copy()) + except AttributeError: + data[key] = Grd[key] + + #Save in mat file file + if debug: + print 'Dumping in matlab file...' + savemat(filename, data, oned_as='column') \ No newline at end of file diff --git a/build/lib/pyseidon_dvt/utilities/pyseidon2netcdf.py b/build/lib/pyseidon_dvt/utilities/pyseidon2netcdf.py new file mode 100644 index 0000000..b3c90a6 --- /dev/null +++ b/build/lib/pyseidon_dvt/utilities/pyseidon2netcdf.py @@ -0,0 +1,125 @@ +#!/usr/bin/python2.7 +# encoding: utf-8 + +#Libs import +from __future__ import division +import numpy as np +import sys +#TR comment: 2 alternatives +import netCDF4 as nc +#from scipy.io import netcdf + +def pyseidon_to_netcdf(fvcom, filename, exceptions=[], compression=False, debug=False): + """ + Saves fvcom object in a pickle file + + inputs: + - fvcom = fvcom pyseidon object + - filename = file name, string + options: + - exceptions = list of variables to exclude from output file + , list of strings + - compresion = compresses data with zlib and uses at least 3 significant digits, boolean + Note: Works only with netcdf format + """ + #Define bounding box + if debug: print "Computing bounding box..." + if compression: + zlib = True + least_significant_digit = 3 + else: + zlib = False + least_significant_digit = None + + # Check if netcdffilename has an extension + if not filename[-3:] == '.nc': + filename = filename + '.nc' + f = nc.Dataset(filename, 'w', format='NETCDF4_CLASSIC') + #history attribut + f.history = fvcom.History[:] + + #create dimensions + if not fvcom.Variables._3D: + ##2D dimensions + dims = {'three':3, 'four':4, + 'nele': fvcom.Grid.nele, 'node': fvcom.Grid.nnode, + 'siglay': 2, 'siglev': 3, + 'time': fvcom.Variables.julianTime.shape[0]} + else: + ##3D dimensions + dims = {'three':3, 'four':4, + 'nele': fvcom.Grid.nele, 'node': fvcom.Grid.nnode, + 'siglay': fvcom.Grid.nlevel, 'siglev': fvcom.Grid.nlevel+1, + 'vertshear':fvcom.Grid.nlevel-1, + 'time': fvcom.Variables.julianTime.shape[0]} + for key in dims.keys(): + f.createDimension(key, dims[key]) + + # list of var + varname = fvcom.Variables.__dict__.keys() + gridname = fvcom.Grid.__dict__.keys() + # getting rid of the "_var" kind + iterlist = varname[:] + for key in iterlist: + if key[0] == "_": varname.remove(key) + iterlist = gridname[:] # getting rid of the "_var" kind + for key in iterlist: + if key[0] == "_": gridname.remove(key) + mstrList = varname + gridname + + #load in netcdf file + if debug: print "Loading in nc file..." + for var in mstrList: + if not var in exceptions: # check if in list of exceptions + if var in varname: + data = fvcom.Variables + else: + data = fvcom.Grid + zlib = False + least_significant_digit = None + try: + if hasattr(data, var): + dim = [] + s = getattr(data, var).shape + for d in s: + flag = 1 + count = 0 + while flag: + if count == len(dims.keys()): # when two dimensions are the same by coincidence + #dim.append(dim[-1]) # TODO search in the entire list == d rather than last index + for k in dim: + if dims[k] == d: + newkey = k + dim.append(newkey) + flag = 0 + else: + key = dims.keys()[count] + if dims[key] == d: + if len(dim) == len(s): # in case of similar dims + count += 1 + pass + else: + if key not in dim: # make sure dimension doesn't get recorded twice + dim.append(key) + flag = 0 + count += 1 + else: + count += 1 + else: + count += 1 + dim = tuple(dim) + # exceptions which need name replacement + if var == 'w': + keyAlias = 'ww' + elif var == 'el': + keyAlias = 'zeta' + else: + keyAlias = var + tmp_var = f.createVariable(keyAlias, 'float', dim, + zlib=zlib, least_significant_digit=least_significant_digit) + tmp_var[:] = getattr(data, var)[:] + except (AttributeError, IndexError) as e: + pass + if debug: print "..."+var+" loaded..." + # clean exit + f.close() diff --git a/build/lib/pyseidon_dvt/utilities/pyseidon2pickle.py b/build/lib/pyseidon_dvt/utilities/pyseidon2pickle.py new file mode 100644 index 0000000..cdc865d --- /dev/null +++ b/build/lib/pyseidon_dvt/utilities/pyseidon2pickle.py @@ -0,0 +1,75 @@ +#!/usr/bin/python2.7 +# encoding: utf-8 + +#Libs import +from __future__ import division +import cPickle as pkl + +#Local import +from functionsFvcomThreeD import * + +# Custom error +from pyseidon_error import PyseidonError + +def pyseidon_to_pickle(fvcom, filename, exceptions=[], debug=False): + """ + Saves fvcom object in a pickle file + + inputs: + - fvcom = fvcom pyseidon object + - filename = file name, string + options: + - exceptions = list of variables to exclude from output file + , list of strings + """ + #Define bounding box + if debug: + print "Computing bounding box..." + if fvcom.Grid._ax == []: + lon = fvcom.Grid.lon[:] + lat = fvcom.Grid.lat[:] + fvcom.Grid._ax = [lon.min(), lon.max(), + lat.min(), lat.max()] + filename = filename + ".p" + f = open(filename, "wb") + data = {} + data['Origin'] = fvcom._origin_file + data['History'] = fvcom.History + data['Grid'] = fvcom.Grid.__dict__ + data['Variables'] = fvcom.Variables.__dict__ + # delete exceptions + for key in exceptions: data['Variables'].pop(key, None) + for key in exceptions: data['Grid'].pop(key, None) + #TR: Force caching Variables otherwise error during loading + # with 'netcdf4.Variable' type (see above) + for key in data['Variables']: + listkeys=['Variable', 'ArrayProxy', 'BaseType'] + if any([type(data['Variables'][key]).__name__==x for x in listkeys]): + if debug: + print "Force caching for " + key + data['Variables'][key] = data['Variables'][key][:] + #Unpickleable objects + data['Grid'].pop("triangle", None) + #TR: Force caching Variables otherwise error during loading + # with 'netcdf4.Variable' type (see above) + for key in data['Grid']: + listkeys=['Variable', 'ArrayProxy', 'BaseType'] + if any([type(data['Grid'][key]).__name__==x for x in listkeys]): + if debug: + print "Force caching for " + key + data['Grid'][key] = data['Grid'][key][:] + #Save in pickle file + if debug: + print 'Dumping in pickle file...' + try: + pkl.dump(data, f, protocol=pkl.HIGHEST_PROTOCOL) + except SystemError: + try: + print "---Very large data, this may take a while---" + pkl.dump(data, f) + except SystemError: + raise PyseidonError("---Data too large for machine memory---\n"\ + "Tip: use ax or tx during class initialisation\n"\ + " to use partial data") + + f.close() diff --git a/build/lib/pyseidon_dvt/utilities/pyseidon_error.py b/build/lib/pyseidon_dvt/utilities/pyseidon_error.py new file mode 100644 index 0000000..b5d299c --- /dev/null +++ b/build/lib/pyseidon_dvt/utilities/pyseidon_error.py @@ -0,0 +1,9 @@ +# encoding: utf-8 + +class PyseidonError(Exception): + """ + Custom error for PySeidon library + """ + def __init__(self, arg): + # Call the base class constructor with the parameters it needs + super(PyseidonError, self).__init__(arg) diff --git a/build/lib/pyseidon_dvt/utilities/regioner.py b/build/lib/pyseidon_dvt/utilities/regioner.py new file mode 100644 index 0000000..f3b5d2f --- /dev/null +++ b/build/lib/pyseidon_dvt/utilities/regioner.py @@ -0,0 +1,178 @@ +from __future__ import division +import numpy as np +from bisect import bisect_left, bisect_right +import matplotlib.pyplot as plt +import matplotlib.tri as Tri +#quick fix +#import netCDF4 as nc +import scipy.io.netcdf as nc + +def node_region(ax, lon, lat): + + region_n = np.argwhere((lon >= ax[0]) & + (lon <= ax[1]) & + (lat >= ax[2]) & + (lat <= ax[3])) + + region_n = region_n.ravel() + + return region_n + +def element_region(ax, lonc, latc): + + region_e = np.argwhere((lonc >= ax[0]) & + (lonc <= ax[1]) & + (latc >= ax[2]) & + (latc <= ax[3])) + + region_e = region_e.ravel() + + return region_e + + +def regioner(gridVar, ax, debug=False): + """ + Takes as input a region (given by a four elemenTakes as input a region + (given by a four element NumPy array), + and some standard data output by ncdatasort and loadnc2d_python + and returns only the data that lies within the region specified + in the region arrayt NumPy array), + and some standard data output by ncdatasort and loadnc2d_python + and returns only the data that lies within the region specified + in the region array + + Inputs: + - region = four element array containing the four corners of the + region box. Entires should be in the following form: + [long1, long2, lat1, lat2] with the following property: + abs(long1) < abs(long2), etc. + - data = standard python data dictionary for these files + - name = what should the region be called in the output file + - savedir = where should the resultant data be saved? Default is + none, i.e. the data will not be saved, only returned. + + **dim = {'2D', '3D'}** the dimension of the data to use regioner + on. Default is 2D. + """ + if debug: + print 'Reindexing...' + lon = gridVar.lon[:] + lat = gridVar.lat[:] + nbe = gridVar.triele[:] + nv = gridVar.trinodes[:] + a1u = gridVar.a1u[:] + a2u = gridVar.a2u[:] + aw0 = gridVar.aw0[:] + awx = gridVar.awx[:] + awy = gridVar.awy[:] + x = gridVar.x[:] + xc = gridVar.xc[:] + y = gridVar.y[:] + yc = gridVar.yc[:] + lonc = gridVar.lonc[:] + latc = gridVar.latc[:] + + l = nv.shape[0] + + idx = node_region(ax, lon, lat) + + #first, reindex elements in the region + element_index_tmp = np.zeros(l, int) + nv_rs = nv.reshape(l*3, order='F') + #find indices that sort nv_rs + nv_sortedind = nv_rs.argsort() + #sort the array + nv_sortd = nv_rs[nv_sortedind] + #pick out the values in the region + if debug: + print 'Extracting values from box...' + #TR comment: very slow...gonna need optimisation down the line + for i in xrange(len(idx)): + i1 = bisect_left(nv_sortd, idx[i]) + i2 = bisect_right(nv_sortd, idx[i]) + inds = nv_sortedind[i1:i2] + element_index_tmp[inds % l] = 1 + element_index = np.where(element_index_tmp == 1)[0] + element_index = element_index.astype(int) + + #TR needs to be inside the loop? + node_index = np.unique(nv[element_index,:]).astype(int) + #create new linkage arrays + nv_tmp = nv[element_index,:] + L = len(nv_tmp[:,0]) + nv_tmp2 = np.empty((1, L*3)) + + #make a new array of the node labellings for the tri's in the region + if debug: + print 'Re-labelling nodes...' + nv2 = nv_tmp.reshape(L * 3, order='F') + nv2_sortedind = nv2.argsort() + nv2_sortd = nv2[nv2_sortedind] + + for i in xrange(len(node_index)): + i1 = bisect_left(nv2_sortd, node_index[i]) + i2 = bisect_right(nv2_sortd, node_index[i]) + inds = nv2_sortedind[i1:i2] + nv_tmp2[0, inds] = i + + nv_new = np.reshape(nv_tmp2, (L, 3), 'F') + + #now do the same for nbe...sort of + nbe_index = np.unique(nbe[element_index, :]) + #ghost points + ghost=np.asarray(list(set(nbe_index) - set(element_index))) + + nbe_tmp = nbe[element_index,:] + lnbe = len(nbe_tmp[:,0]) + #nbe_tmp2 = np.empty((1, lnbe*3)) + #TR: np.empty sometimes generates freak values + nbe_tmp2 = np.ones((1, lnbe*3)) * l # ghost point default value + + if debug: + print 'Re-labelling elements...' + nbe2 = nbe_tmp.reshape(lnbe*3, order='F') + nbe_sortedind = nbe2.argsort() + nbe_sortd = nbe2[nbe_sortedind] + + #TR: iterator + I = 0 + for i in xrange(len(nbe_index)): + #TR: check if ghost point + if not nbe_index[i] in ghost: + i1 = bisect_left(nbe_sortd, nbe_index[i]) + i2 = bisect_right(nbe_sortd, nbe_index[i]) + inds = nbe_sortedind[i1:i2] + nbe_tmp2[0, inds] = I + I += 1 + + nbe_new = np.reshape(nbe_tmp2, (lnbe,3), 'F') + + #create new variables for the region + + data = {} + data['node_index'] = node_index + data['element_index'] = element_index + data['nbe'] = nbe_new.astype(int) + data['nv'] = nv_new.astype(int) + + data['a1u'] = a1u[:, element_index] + data['a2u'] = a2u[:, element_index] + data['aw0'] = aw0[:, element_index] + data['awx'] = awx[:, element_index] + data['awy'] = awy[:, element_index] + + data['x'] = x[node_index] + data['y'] = y[node_index] + data['xc'] = xc[element_index] + data['yc'] = yc[element_index] + + data['lon'] = lon[node_index] + data['lat'] = lat[node_index] + data['lonc'] = lonc[element_index] + data['latc'] = latc[element_index] + + data['triangle'] = Tri.Triangulation(data['lon'], data['lat'], \ + data['nv']) + + return data + diff --git a/build/lib/pyseidon_dvt/utilities/save_FlowFile_BPFormat.py b/build/lib/pyseidon_dvt/utilities/save_FlowFile_BPFormat.py new file mode 100644 index 0000000..8ccd3c0 --- /dev/null +++ b/build/lib/pyseidon_dvt/utilities/save_FlowFile_BPFormat.py @@ -0,0 +1,388 @@ +from __future__ import division +import numpy as np +#from rawADCPclass import rawADCP +from datetime import datetime +from datetime import timedelta +import scipy.io as sio +import scipy.interpolate as sip +import matplotlib.pyplot as plt +import seaborn + +def date2py(matlab_datenum): + """ + Converts matlab's datenum time to datetime time + """ + python_datetime = datetime.fromordinal(int(matlab_datenum)) + \ + timedelta(days=matlab_datenum%1) - timedelta(days = 366) + + return python_datetime + + +def py2date(dt): + """ + Converts datetime time to matlab's datenum time + """ + mdn = dt + timedelta(days = 366) + frac_seconds = (dt-datetime(dt.year,dt.month,dt.day,0,0,0)).seconds / (24.0 * 60.0 * 60.0) + frac_microseconds = dt.microsecond / (24.0 * 60.0 * 60.0 * 1000000.0) + return mdn.toordinal() + frac_seconds + frac_microseconds + +def calc_ensemble(x, ens, ens_dim, debug=False, debug_plot=False): + if debug: "calc_ensemble..." + #initialize input + ens = int(ens) + #x = x[:, None] + + if ens_dim == 1: + ens_size = np.floor(x.shape[0]/60) + else: + pass + + #x_ens = np.empty((ens_size, 1, ens)) + x_ens = np.empty((ens_size, ens)) + x_ens[:] = np.nan + + for j in xrange(ens): + if ens_dim == 1: + ind_ens = np.arange(j, x.shape[0] - (ens - j), ens) + #x_ens[..., j] = x[ind_ens] + x_ens[..., j] = x[ind_ens] + + else: + pass + + #x_ens = np.nanmean(x_ens, axis=2) + x_ens = np.nanmean(x_ens, axis=1) + + if debug: "...calc_ensemble done." + + return x_ens + + +def rotate_coords(x, y, theta, debug=False, debug_plot=False ): + """ + Similar to "rotate_to_channelcoords.m" code, + theta is now the angle + between the old axis and the new x-axis (CCw is positive) + """ + if debug: "rotate_coords..." + xnew = x * np.cos(theta) + y * np.sin(theta) + ynew = -x * np.sin(theta) + y * np.cos(theta) + + if debug: "...rotate_coords done." + + return xnew, ynew + +def rotate_to_true(X, Y, theta=-19.0): + """ + X,Y are the X and Y coordinates (could be speeds) relative to magnetic + north -- inputs can be vectors + x,y are the coordinates relative to true north + This function assumes the measured location is Nova Scotia where the + declination angle is -19 degrees. + + + Inputs: + - X = longitudes in deg., array or list + - Y = latitudes in deg., array or list + + Outputs: + - X = true-north longitudes in deg., array or list + - Y = true-north latitudes in deg., array or list + + Options: + - theta = declination angle in deg., float + """ + + print 'Rotating velocities to be relative to true north (declination = {0})'.format(theta) + + Theta = theta * np.pi / 180 + + x = X * np.cos(Theta) + Y * np.sin(Theta) + y = -X * np.sin(Theta) + Y * np.cos(Theta) + + return x, y + + +def get_DirFromN(u,v): + """ + This function computes the direction from North with the output in degrees + and measured clockwise from north. + + Inputs: + - u = eastward component + - v = northward component + """ + + theta = np.arctan2(u,v) * 180 / np.pi + + ind = np.where(theta<0) + theta[ind] = theta[ind] + 360 + return theta + +def sign_speed(u_all, v_all, s_all, dir_all, flood_heading): + """ + Computes the signed speed + Inputs: + - u_all = u velocity component time series + - v_all = v velocity component time series + - s_all = ??? + - dir_all = direction time series + - flood_heading = direction of flood + Outputs: + - s_signed_all = signed speed + - PA_all = principal axis + """ + + if type(flood_heading)==int: + flood_heading += np.array([-90, 90]) + + s_signed_all = np.empty(s_all.shape) + s_signed_all.fill(np.nan) + + PA_all = np.zeros(s_all.shape[-1]) + for i in xrange(s_all.shape[-1]): + u = u_all[:, i] + v = v_all[:, i] + dir = dir_all[:, i] + s = s_all[:, i] + + #determine principal axes - potentially a problem if axes are very kinked + # since this would misclassify part of ebb and flood + PA, _ = principal_axis(u, v) + PA_all[i] = PA + + # sign speed - eliminating wrap-around + dir_PA = dir - PA + + dir_PA[dir_PA < -90] += 360 + dir_PA[dir_PA > 270] -= 360 + + #general direction of flood passed as input argument + if flood_heading[0] <= PA <= flood_heading[1]: + ind_fld = np.where((dir_PA >= -90) & (dir_PA<90)) + s_signed = -s + s_signed[ind_fld] = s[ind_fld] + else: + ind_ebb = np.where((dir_PA >= -90) & (dir_PA<90)) + s_signed = s + s_signed[ind_ebb] = -s[ind_ebb] + + s_signed_all[:, i] = s_signed + + return s_signed_all, PA_all + +def principal_axis(u, v): + """ + Computes the principal axis angle and its variance. + Inputs: + - u = u velocity component time series + - v = v velocity component time series + Outputs: + - PA = principal axis angle in deg., float + - varxp_PA = principal axis variance + """ + + #create velocity matrix + U = np.vstack((u,v)).T + #eliminate NaN values + U = U[~np.isnan(U[:, 0]), :] + #convert matrix to deviate form + rep = np.tile(np.mean(U, axis=0), [len(U), 1]) + U -= rep + #compute covariance matrix + R = np.dot(U.T, U) / (len(U) - 1) + + #calculate eigenvalues and eigenvectors for covariance matrix + lamb, V = np.linalg.eig(R) + #sort eignvalues in descending order so that major axis is given by first eigenvector + # sort in descending order with indices + ilamb = sorted(range(len(lamb)), key=lambda k: lamb[k], reverse=True) + lamb = sorted(lamb, reverse=True) + # reconstruct the eigenvalue matrix + lamb = np.diag(lamb) + #reorder the eigenvectors + V = V[:, ilamb] + + #rotation angle of major axis in radians relative to cartesian coordiantes + ra = np.arctan2(V[0,1], V[1,1]) + #express principal axis in compass coordinates + # WES_COMMENT: may need to change this, cause in original is -ra + PA = ra * 180 / np.pi + 90 + #variance captured by principal + varxp_PA = np.diag(lamb[0]) / np.trace(lamb) + + return PA, varxp_PA + + +class Struct: + def __init__(self, **entries): + self.__dict__.update(entries) + + +def save_FlowFile_BPFormat(fileinfo, adcp, rbr, params, options, debug=False): + """ + Processes and formats raw ADCP data into the BP's format defined by Dalhousie university + """ + + comments = ['data is in Polagye Tools format', + 'data.east_vel and data.north_vel are relative to true north', + 'The parameters were set by ' + fileinfo['paramfile']] + + day1 = date2py(adcp['mtime'][0][0]) + print day1 + #date_time = [date2py(tval[0]) for tval in adcp.mtime[:]] + datenum = datetime(day1.year,1,1) + timedelta(365) + datenum = datenum.toordinal() + + yd = adcp['mtime'][:].ravel() - datenum + tind = np.where((yd > params['tmin']) & (yd < params['tmax']))[0] + + pres = {} + time = {} + time['mtime'] = adcp['mtime'][:].ravel()[tind] + dt = np.nanmean(np.diff(time['mtime'])) + + if not rbr: + print 'Depths measured by ADCP not yet coded.' + comments.append('Depths as measured by ADCP') + else: + print 'Ensemble averaging rbr data' + comments.append('Depths as measured by RBR sensor') + + nens = round(dt/(rbr.mtime[1] - rbr.mtime[0])) + temp = np.arange(rbr.mtime[nens/2-1], rbr.mtime[-1-nens/2], dt) + #temp2 = np.r_[rbr.mtime[nens/2-1]: rbr.mtime[-1-nens/2]: dt] + + mtimeens = np.arange(rbr.mtime[nens/2-1], rbr.mtime[-1-nens/2], dt) + mtimeens = mtimeens + params['rbr_hr_offset'] / 24 + depthens = calc_ensemble(rbr.depth, nens, 1) + + temp = sip.interp1d(mtimeens, depthens, kind='linear') + + pres['surf']= temp(time['mtime']) + params['dabPS'] + + if debug: + # Load in matlab values for testing + filename = './140703-EcoEII_database/scripts_examples/mtime.mat' + mat = sio.loadmat(filename, struct_as_record=False, squeeze_me=True) + matTimes = mat['mtimeens'] + filename = './140703-EcoEII_database/scripts_examples/dt.mat' + mat = sio.loadmat(filename, struct_as_record=False, squeeze_me=True) + matdt = mat['dt'] + + + filename = './140703-EcoEII_database/scripts_examples/depthens.mat' + mat = sio.loadmat(filename, struct_as_record=False, squeeze_me=True) + matdepthens = mat['depthens'] + + filename = './140703-EcoEII_database/scripts_examples/time.mat' + mat = sio.loadmat(filename, struct_as_record=False, squeeze_me=True) + matmtime = mat['mtime'] + + print matTimes.shape + print temp - matTimes + print temp2 - matTimes + print dt - matdt + print depthens - matdepthens + print 'time' + print time['mtime'] - matmtime + + ## zlevels + data = {} + z = adcp['config']['ranges'][:] + params['dabADCP'] + z = z.ravel() + zind = np.where((z > params['zmin']) & (z < params['zmax']))[0] + data['bins'] = z[zind] + + ## Currents + data['vert_vel'] = adcp['vert_vel'][:][tind][:, zind] + data['error_vel'] = adcp['error_vel'][:][tind][:, zind] + + # If compass wasn't calibrated + if 'hdgmod' in params: + adcp['east_vel'][:], adcp['north_vel'][:] = rotate_coords(adcp['east_vel'][:], + adcp['north_vel'][:], + params['hdgmod']) + + comments.append('East and north velocity rotated by params.hdgmod') + + # Rotate east_vel and north_vel to be relative to true north + data['east_vel'], data['north_vel'] = \ + rotate_to_true(adcp['east_vel'][:][tind][:, zind], + adcp['north_vel'][:][tind][:, zind], + params['declination']) + + # Direction + data['dir_vel'] = get_DirFromN(data['east_vel'],data['north_vel']) + + # Signed Speed + spd_all = np.sqrt(data['east_vel']**2+data['north_vel']**2) + + # Determine flood and ebb based on principal direction (Polagye Routine) + print 'Getting signed speed (Principal Direction Method) -- used all speeds' + s_signed_all, PA_all = sign_speed(data['east_vel'], data['north_vel'], + spd_all, data['dir_vel'], params['flooddir']) + + data['mag_signed_vel'] = s_signed_all + + if options['showRBRavg'] or debug: + print 'Plotting RBR vs average' + plt.plot(rbr.mtime + params['rbr_hr_offset'] / 24, rbr.depth+params['dabPS'], + label='RBR') + plt.plot(time['mtime'], pres['surf'], 'r', label='AVG') + plt.xlabel('Time') + plt.ylabel('Elevation') + plt.legend(bbox_to_anchor=(0, 0, 1, 1), bbox_transform=plt.gcf().transFigure) + + plt.show() + + if options['showPA'] or debug: + print 'Plotting PA vs mean' + plt.plot(PA_all, data['bins'], label='PA') + plt.plot(np.array([PA_all[0], PA_all[-1]]), + np.array([np.mean(pres['surf']), np.mean(pres['surf'])]), + label='mean') + + plt.xlabel('Principal Axis Direction\n(clockwise from north)') + plt.ylabel('z (m)') + plt.legend(bbox_to_anchor=(0, 0, 1, 1), bbox_transform=plt.gcf().transFigure) + plt.show() + + ## save + lon = params['lon'] + lat = params['lat'] + + outfile = fileinfo['outdir'] + fileinfo['flowfile'] + print 'Saving data to {0}'.format(outfile) + + saveDict = {'data':data, 'pres':pres, 'time':time, 'lon':lon, 'lat':lat, + 'params':params, 'comments':comments} + #save(outfile,'data','pres','time','lon','lat','params','Comments') + + ## Save metadata + #metadata.progname=[mfilename('fullpath')]; + #metadata.date = datestr(now); + #metadata.paramfile = fileinfo.paramfile; + #save(outfile,'metadata','-append') + return saveDict + + + +if __name__ == '__main__': + filename = '140703-EcoEII_database/data/GP-120726-BPd_raw.mat' + data = rawADCP(filename) + rawdata = rawADCP(filename) + #adcp = Struct(**data.adcp) + #rawADCP = data.adcp + adcp = data.adcp + #params = Struct(**data.saveparams) + params = data.saveparams + rbr = Struct(**data.rbr) + +# save_FlowFile_BPFormat(data.fileinfo, data.adcp, data.rbr, +# data.saveparams, data.options) + + saveDict = \ + save_FlowFile_BPFormat(data.fileinfo, adcp, rbr, + params, data.options) diff --git a/build/lib/pyseidon_dvt/utilities/shortest_element_path.py b/build/lib/pyseidon_dvt/utilities/shortest_element_path.py new file mode 100644 index 0000000..098cb30 --- /dev/null +++ b/build/lib/pyseidon_dvt/utilities/shortest_element_path.py @@ -0,0 +1,200 @@ +#from __future__ import division +#import netCDF4 as nc +import numpy as np +import scipy.spatial +import networkx as nx +import matplotlib.pyplot as plt +import matplotlib.tri as Tri +import matplotlib.ticker as ticker +import seaborn + +class shortest_element_path: + """ + Class that mostly computes the shortest path from A to B + by hopping from an element to the next + """ + def __init__(self, lonc, latc, lon, lat, trinodes, h, debug=False): + + #self.data = nc.Dataset(filename,'r') + + #latc = self.data.variables['latc'][:] + #lonc = self.data.variables['lonc'][:] + self.lonc = lonc[:] + self.latc = latc[:] + self.lat = lat[:] + self.lon = lon[:] + self.trinodes = trinodes[:] + self.h = h[:] + + #z = np.vstack((latc,lonc)).T + z = np.vstack((lonc,latc)).T + #z = np.vstack((xc, yc)).T + + self.points = map(tuple,z) + + if debug : print 'File Loaded' + + # make a Delaunay triangulation of the point data + self.delTri = scipy.spatial.Delaunay(self.points) + if debug : print 'Delaunay Triangulation Done' + + # create a set for edges that are indexes of the points + self.edges = [] + self.weight = [] + # for each Delaunay triangle + for n in xrange(self.delTri.nsimplex): + # for each edge of the triangle + # sort the vertices + # (sorting avoids duplicated edges being added to the set) + # and add to the edges set + + self.edge = sorted([self.delTri.vertices[n,0], self.delTri.vertices[n,1]]) + a = self.points[self.edge[0]] + b = self.points[self.edge[1]] + self.weight = (np.sqrt((a[0]-b[0])**2+(a[1]-b[1])**2)) + self.edges.append((self.edge[0], self.edge[1],{'weight':self.weight})) + + self.edge = sorted([self.delTri.vertices[n,0], self.delTri.vertices[n,2]]) + a = self.points[self.edge[0]] + b = self.points[self.edge[1]] + self.weight = (np.sqrt((a[0]-b[0])**2+(a[1]-b[1])**2)) + self.edges.append((self.edge[0], self.edge[1],{'weight':self.weight})) + + + self.edge = sorted([self.delTri.vertices[n,1], self.delTri.vertices[n,2]]) + a = self.points[self.edge[0]] + b = self.points[self.edge[1]] + self.weight = (np.sqrt((a[0]-b[0])**2+(a[1]-b[1])**2)) + self.edges.append((self.edge[0], self.edge[1],{'weight':self.weight})) + + if debug : print 'Edges and Weighting Done' + + # make a graph based on the Delaunay triangulation edges + self.graph = nx.Graph(self.edges) + #print(graph.edges()) + + if debug : print 'Graph Constructed' + + self.pointIDXY = dict(zip(range(len(self.points)), self.points)) + + def getTargets(self, source_target, coords=False): + + self.elements = [] + self.coordinates = [] + self.maxcoordinates = [] + self.mincoordinates = [] + for i in source_target: + source = i[0] + target = i[1] + +# print '\n' +# print 'Source' +# print source + s = source +# +# print 'Target' +# print target + t = target + + if coords: + for key, value in self.pointIDXY.items(): + if value==source: + print 'Source' + print key + s = key + + if value==target: + print 'Target' + print key + t = key + + #print s,t + shortest = nx.shortest_path(self.graph,source=s,target=t,weight='weight') +# dist = nx.shortest_path_length(self.graph,source=s,target=t,weight='weight') + +# print 'Shortest Path (by elements)' +# print shortest + + self.elements.append(shortest) + + coords = [self.pointIDXY[i] for i in shortest] + self.coordinates.append(coords) + self.maxcoordinates.append(np.max(np.array(coords),axis=0)) + self.mincoordinates.append(np.min(np.array(coords),axis=0)) + +# print 'Shortest Distance (by coordinates)' +# print dist + + return self.elements, self.coordinates + + def graphGrid(self,narrowGrid=False, plot=False): + #nx.draw(self.graph, self.pointIDXY) + #plt.show() + + #lat = self.data.variables['lat'][:] + #lon = self.data.variables['lon'][:] + #nv = self.data.variables['nv'][:].T -1 + #h = self.data.variables['h'][:] + #lat = self.self.lat + #lon = self.lon + #trinodes = self.trinodes[:] + #h = self.h + + tri = Tri.Triangulation(self.lon, self.lat, triangles=self.trinodes) + # xy or latlon based on how you are #Grand Passage + + #levels=np.arange(-38,6,1) # depth contours to plot + + fig = plt.figure(figsize=(18,10)) + plt.rc('font',size='22') + ax = fig.add_subplot(111,aspect=(1.0/np.cos(np.mean(self.lat)*np.pi/180.0))) + #plt.tricontourf(tri,-self.h,shading='faceted',cmap=plt.cm.gist_earth) + plt.triplot(tri, color='white', linewidth=0.5) + plt.ylabel('Latitude') + plt.xlabel('Longitude') + plt.gca().patch.set_facecolor('0.5') + #cbar=plt.colorbar() + #cbar.set_label('Water Depth (m)', rotation=-90,labelpad=30) + + scale = 1 + ticks = ticker.FuncFormatter(lambda lon, pos: '{0:g}'.format(lon/scale)) + ax.xaxis.set_major_formatter(ticks) + ax.yaxis.set_major_formatter(ticks) + plt.grid() + + maxlat, maxlon = np.max(self.maxcoordinates,axis=0) + minlat, minlon = np.min(self.mincoordinates,axis=0) + if narrowGrid: + ax.set_xlim(minlon,maxlon) + ax.set_ylim(minlat,maxlat) + + + zz = len(self.elements) + for i,v in enumerate(self.elements): + source = self.pointIDXY[v[0]] + target = self.pointIDXY[v[-1]] + lab = '({:.6},{:.6})-({:.6},{:.6})'.format(source[0], source[1], + target[0], target[1]) + + plt.scatter(self.lonc[v], self.latc[v], + s=80, label=lab, c=plt.cm.Set1(i/zz)) + + #plt.legend(bbox_to_anchor=(0., 1.02, 1., .102), loc=2, ncol=3,fontsize='14', borderaxespad=0.) + plt.legend(bbox_to_anchor=(0., 1.02, 1., .102), loc=2, ncol=3) + #plt.legend() + if plot: + plt.ylabel('Latitude') + plt.xlabel('Longitude') + plt.show() + + +if __name__ == '__main__': + + filename = '/home/wesley/ncfiles/smallcape_force_0001.nc' + + test = shortest_element_path(filename) + + test.getTargets([[41420,39763],[48484,53441],[27241,24226],[21706,17458],[14587,5416]]) + test.graphGrid(narrowGrid=True) + + element_path, coordinates_path = test.getTargets([[41420,39763]]) diff --git a/build/lib/pyseidon_dvt/utilities/windrose.py b/build/lib/pyseidon_dvt/utilities/windrose.py new file mode 100644 index 0000000..13af860 --- /dev/null +++ b/build/lib/pyseidon_dvt/utilities/windrose.py @@ -0,0 +1,548 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +__version__ = '1.4' +__author__ = 'Lionel Roubeyrie' +__mail__ = 'lionel.roubeyrie@gmail.com' +__license__ = 'CeCILL-B' + +import matplotlib +import matplotlib.cm as cm +import numpy as np +from matplotlib.patches import Rectangle, Polygon +from matplotlib.ticker import ScalarFormatter, AutoLocator +from matplotlib.text import Text, FontProperties +from matplotlib.projections.polar import PolarAxes +from numpy.lib.twodim_base import histogram2d +import matplotlib.pyplot as plt +from pylab import poly_between + +RESOLUTION = 100 +ZBASE = -1000 #The starting zorder for all drawing, negative to have the grid on + +class WindroseAxes(PolarAxes): + """ + + Create a windrose axes + + """ + + def __init__(self, *args, **kwargs): + """ + See Axes base class for args and kwargs documentation + """ + + #Uncomment to have the possibility to change the resolution directly + #when the instance is created + #self.RESOLUTION = kwargs.pop('resolution', 100) + PolarAxes.__init__(self, *args, **kwargs) + self.set_aspect('equal', adjustable='box', anchor='C') + self.radii_angle = 67.5 + self.cla() + + + def cla(self): + """ + Clear the current axes + """ + PolarAxes.cla(self) + + self.theta_angles = np.arange(0, 360, 45) + self.theta_labels = ['E', 'N-E', 'N', 'N-W', 'W', 'S-W', 'S', 'S-E'] + self.set_thetagrids(angles=self.theta_angles, labels=self.theta_labels) + + self._info = {'dir' : list(), + 'bins' : list(), + 'table' : list()} + + self.patches_list = list() + + + def _colors(self, cmap, n): + ''' + Returns a list of n colors based on the colormap cmap + + ''' + return [cmap(i) for i in np.linspace(0.0, 1.0, n)] + + + def set_radii_angle(self, **kwargs): + """ + Set the radii labels angle + """ + + null = kwargs.pop('labels', None) + angle = kwargs.pop('angle', None) + if angle is None: + angle = self.radii_angle + self.radii_angle = angle + radii = np.linspace(0.1, self.get_rmax(), 6) + radii_labels = [ "%.1f" %r for r in radii ] + radii_labels[0] = "" #Removing label 0 + null = self.set_rgrids(radii=radii, labels=radii_labels, + angle=self.radii_angle, **kwargs) + + + def _update(self): + self.set_rmax(rmax=np.max(np.sum(self._info['table'], axis=0))) + self.set_radii_angle(angle=self.radii_angle) + + + def legend(self, loc='lower left', **kwargs): + """ + Sets the legend location and her properties. + The location codes are + + 'best' : 0, + 'upper right' : 1, + 'upper left' : 2, + 'lower left' : 3, + 'lower right' : 4, + 'right' : 5, + 'center left' : 6, + 'center right' : 7, + 'lower center' : 8, + 'upper center' : 9, + 'center' : 10, + + If none of these are suitable, loc can be a 2-tuple giving x,y + in axes coords, ie, + + loc = (0, 1) is left top + loc = (0.5, 0.5) is center, center + + and so on. The following kwargs are supported: + + isaxes=True # whether this is an axes legend + prop = FontProperties(size='smaller') # the font property + pad = 0.2 # the fractional whitespace inside the legend border + shadow # if True, draw a shadow behind legend + labelsep = 0.005 # the vertical space between the legend entries + handlelen = 0.05 # the length of the legend lines + handletextsep = 0.02 # the space between the legend line and legend text + axespad = 0.02 # the border between the axes and legend edge + """ + + def get_handles(): + handles = list() + for p in self.patches_list: + if isinstance(p, matplotlib.patches.Polygon) or \ + isinstance(p, matplotlib.patches.Rectangle): + color = p.get_facecolor() + elif isinstance(p, matplotlib.lines.Line2D): + color = p.get_color() + else: + raise AttributeError("Can't handle patches") + handles.append(Rectangle((0, 0), 0.2, 0.2, + facecolor=color, edgecolor='black')) + return handles + + def get_labels(): + labels = np.copy(self._info['bins']) + labels = ["[%.1f : %0.1f[" %(labels[i], labels[i+1]) \ + for i in range(len(labels)-1)] + return labels + + null = kwargs.pop('labels', None) + null = kwargs.pop('handles', None) + handles = get_handles() + labels = get_labels() + self.legend_ = matplotlib.legend.Legend(self, handles, labels, + loc, **kwargs) + return self.legend_ + + + def _init_plot(self, dir, var, **kwargs): + """ + Internal method used by all plotting commands + """ + #self.cla() + null = kwargs.pop('zorder', None) + + #Init of the bins array if not set + bins = kwargs.pop('bins', None) + if bins is None: + bins = np.linspace(np.min(var), np.max(var), 6) + if isinstance(bins, int): + bins = np.linspace(np.min(var), np.max(var), bins) + bins = np.asarray(bins) + nbins = len(bins) + + #Number of sectors + nsector = kwargs.pop('nsector', None) + if nsector is None: + nsector = 16 + + #Sets the colors table based on the colormap or the "colors" argument + colors = kwargs.pop('colors', None) + cmap = kwargs.pop('cmap', None) + if colors is not None: + if isinstance(colors, str): + colors = [colors]*nbins + if isinstance(colors, (tuple, list)): + if len(colors) != nbins: + raise ValueError("colors and bins must have same length") + else: + if cmap is None: + cmap = cm.jet + colors = self._colors(cmap, nbins) + + #Building the angles list + angles = np.arange(0, -2*np.pi, -2*np.pi/nsector) + np.pi/2 + + normed = kwargs.pop('normed', False) + blowto = kwargs.pop('blowto', False) + + #Set the global information dictionnary + self._info['dir'], self._info['bins'], self._info['table'] = histogram(dir, var, bins, nsector, normed, blowto) + + return bins, nbins, nsector, colors, angles, kwargs + + + def contour(self, dir, var, **kwargs): + """ + Plot a windrose in linear mode. For each var bins, a line will be + draw on the axes, a segment between each sector (center to center). + Each line can be formated (color, width, ...) like with standard plot + pylab command. + + Mandatory: + * dir : 1D array - directions the wind blows from, North centred + * var : 1D array - values of the variable to compute. Typically the wind + speeds + Optional: + * nsector: integer - number of sectors used to compute the windrose + table. If not set, nsectors=16, then each sector will be 360/16=22.5°, + and the resulting computed table will be aligned with the cardinals + points. + * bins : 1D array or integer- number of bins, or a sequence of + bins variable. If not set, bins=6, then + bins=linspace(min(var), max(var), 6) + * blowto : bool. If True, the windrose will be pi rotated, + to show where the wind blow to (usefull for pollutant rose). + * colors : string or tuple - one string color ('k' or 'black'), in this + case all bins will be plotted in this color; a tuple of matplotlib + color args (string, float, rgb, etc), different levels will be plotted + in different colors in the order specified. + * cmap : a cm Colormap instance from matplotlib.cm. + - if cmap == None and colors == None, a default Colormap is used. + + others kwargs : see help(pylab.plot) + + """ + + bins, nbins, nsector, colors, angles, kwargs = self._init_plot(dir, var, + **kwargs) + + #closing lines + angles = np.hstack((angles, angles[-1]-2*np.pi/nsector)) + vals = np.hstack((self._info['table'], + np.reshape(self._info['table'][:,0], + (self._info['table'].shape[0], 1)))) + + offset = 0 + for i in range(nbins): + val = vals[i,:] + offset + offset += vals[i, :] + zorder = ZBASE + nbins - i + patch = self.plot(angles, val, color=colors[i], zorder=zorder, + **kwargs) + self.patches_list.extend(patch) + self._update() + + + def contourf(self, dir, var, **kwargs): + """ + Plot a windrose in filled mode. For each var bins, a line will be + draw on the axes, a segment between each sector (center to center). + Each line can be formated (color, width, ...) like with standard plot + pylab command. + + Mandatory: + * dir : 1D array - directions the wind blows from, North centred + * var : 1D array - values of the variable to compute. Typically the wind + speeds + Optional: + * nsector: integer - number of sectors used to compute the windrose + table. If not set, nsectors=16, then each sector will be 360/16=22.5°, + and the resulting computed table will be aligned with the cardinals + points. + * bins : 1D array or integer- number of bins, or a sequence of + bins variable. If not set, bins=6, then + bins=linspace(min(var), max(var), 6) + * blowto : bool. If True, the windrose will be pi rotated, + to show where the wind blow to (usefull for pollutant rose). + * colors : string or tuple - one string color ('k' or 'black'), in this + case all bins will be plotted in this color; a tuple of matplotlib + color args (string, float, rgb, etc), different levels will be plotted + in different colors in the order specified. + * cmap : a cm Colormap instance from matplotlib.cm. + - if cmap == None and colors == None, a default Colormap is used. + + others kwargs : see help(pylab.plot) + + """ + + bins, nbins, nsector, colors, angles, kwargs = self._init_plot(dir, var, + **kwargs) + null = kwargs.pop('facecolor', None) + null = kwargs.pop('edgecolor', None) + + #closing lines + angles = np.hstack((angles, angles[-1]-2*np.pi/nsector)) + vals = np.hstack((self._info['table'], + np.reshape(self._info['table'][:,0], + (self._info['table'].shape[0], 1)))) + offset = 0 + for i in range(nbins): + val = vals[i,:] + offset + offset += vals[i, :] + zorder = ZBASE + nbins - i + xs, ys = poly_between(angles, 0, val) + patch = self.fill(xs, ys, facecolor=colors[i], + edgecolor=colors[i], zorder=zorder, **kwargs) + self.patches_list.extend(patch) + + + def bar(self, dir, var, **kwargs): + """ + Plot a windrose in bar mode. For each var bins and for each sector, + a colored bar will be draw on the axes. + + Mandatory: + * dir : 1D array - directions the wind blows from, North centred + * var : 1D array - values of the variable to compute. Typically the wind + speeds + Optional: + * nsector: integer - number of sectors used to compute the windrose + table. If not set, nsectors=16, then each sector will be 360/16=22.5°, + and the resulting computed table will be aligned with the cardinals + points. + * bins : 1D array or integer- number of bins, or a sequence of + bins variable. If not set, bins=6 between min(var) and max(var). + * blowto : bool. If True, the windrose will be pi rotated, + to show where the wind blow to (usefull for pollutant rose). + * colors : string or tuple - one string color ('k' or 'black'), in this + case all bins will be plotted in this color; a tuple of matplotlib + color args (string, float, rgb, etc), different levels will be plotted + in different colors in the order specified. + * cmap : a cm Colormap instance from matplotlib.cm. + - if cmap == None and colors == None, a default Colormap is used. + edgecolor : string - The string color each edge bar will be plotted. + Default : no edgecolor + * opening : float - between 0.0 and 1.0, to control the space between + each sector (1.0 for no space) + + """ + + bins, nbins, nsector, colors, angles, kwargs = self._init_plot(dir, var, + **kwargs) + null = kwargs.pop('facecolor', None) + edgecolor = kwargs.pop('edgecolor', None) + if edgecolor is not None: + if not isinstance(edgecolor, str): + raise ValueError('edgecolor must be a string color') + opening = kwargs.pop('opening', None) + if opening is None: + opening = 0.8 + dtheta = 2*np.pi/nsector + opening = dtheta*opening + + for j in range(nsector): + offset = 0 + for i in range(nbins): + if i > 0: + offset += self._info['table'][i-1, j] + val = self._info['table'][i, j] + zorder = ZBASE + nbins - i + patch = Rectangle((angles[j]-opening/2, offset), opening, val, + facecolor=colors[i], edgecolor=edgecolor, zorder=zorder, + **kwargs) + self.add_patch(patch) + if j == 0: + self.patches_list.append(patch) + self._update() + + + def box(self, dir, var, **kwargs): + """ + Plot a windrose in proportional bar mode. For each var bins and for each + sector, a colored bar will be draw on the axes. + + Mandatory: + * dir : 1D array - directions the wind blows from, North centred + * var : 1D array - values of the variable to compute. Typically the wind + speeds + Optional: + * nsector: integer - number of sectors used to compute the windrose + table. If not set, nsectors=16, then each sector will be 360/16=22.5°, + and the resulting computed table will be aligned with the cardinals + points. + * bins : 1D array or integer- number of bins, or a sequence of + bins variable. If not set, bins=6 between min(var) and max(var). + * blowto : bool. If True, the windrose will be pi rotated, + to show where the wind blow to (usefull for pollutant rose). + * colors : string or tuple - one string color ('k' or 'black'), in this + case all bins will be plotted in this color; a tuple of matplotlib + color args (string, float, rgb, etc), different levels will be plotted + in different colors in the order specified. + * cmap : a cm Colormap instance from matplotlib.cm. + - if cmap == None and colors == None, a default Colormap is used. + edgecolor : string - The string color each edge bar will be plotted. + Default : no edgecolor + + """ + + bins, nbins, nsector, colors, angles, kwargs = self._init_plot(dir, var, + **kwargs) + null = kwargs.pop('facecolor', None) + edgecolor = kwargs.pop('edgecolor', None) + if edgecolor is not None: + if not isinstance(edgecolor, str): + raise ValueError('edgecolor must be a string color') + opening = np.linspace(0.0, np.pi/16, nbins) + + for j in range(nsector): + offset = 0 + for i in range(nbins): + if i > 0: + offset += self._info['table'][i-1, j] + val = self._info['table'][i, j] + zorder = ZBASE + nbins - i + patch = Rectangle((angles[j]-opening[i]/2, offset), opening[i], + val, facecolor=colors[i], edgecolor=edgecolor, + zorder=zorder, **kwargs) + self.add_patch(patch) + if j == 0: + self.patches_list.append(patch) + self._update() + +def histogram(dir, var, bins, nsector, normed=False, blowto=False): + """ + Returns an array where, for each sector of wind + (centred on the north), we have the number of time the wind comes with a + particular var (speed, polluant concentration, ...). + * dir : 1D array - directions the wind blows from, North centred + * var : 1D array - values of the variable to compute. Typically the wind + speeds + * bins : list - list of var category against we're going to compute the table + * nsector : integer - number of sectors + * normed : boolean - The resulting table is normed in percent or not. + * blowto : boolean - Normaly a windrose is computed with directions + as wind blows from. If true, the table will be reversed (usefull for + pollutantrose) + + """ + + if len(var) != len(dir): + raise ValueError, "var and dir must have same length" + + angle = 360./nsector + + dir_bins = np.arange(-angle/2 ,360.+angle, angle, dtype=np.float) + dir_edges = dir_bins.tolist() + dir_edges.pop(-1) + dir_edges[0] = dir_edges.pop(-1) + dir_bins[0] = 0. + + var_bins = bins.tolist() + var_bins.append(np.inf) + + if blowto: + dir = dir + 180. + dir[dir>=360.] = dir[dir>=360.] - 360 + + table = histogram2d(x=var, y=dir, bins=[var_bins, dir_bins], + normed=False)[0] + # add the last value to the first to have the table of North winds + table[:,0] = table[:,0] + table[:,-1] + # and remove the last col + table = table[:, :-1] + if normed: + table = table*100/table.sum() + + return dir_edges, var_bins, table + + +def wrcontour(dir, var, **kwargs): + fig = plt.figure() + rect = [0.1, 0.1, 0.8, 0.8] + ax = WindroseAxes(fig, rect) + fig.add_axes(ax) + ax.contour(dir, var, **kwargs) + l = ax.legend(axespad=-0.10) + plt.setp(l.get_texts(), fontsize=8) + plt.draw() + plt.show() + return ax + +def wrcontourf(dir, var, **kwargs): + fig = plt.figure() + rect = [0.1, 0.1, 0.8, 0.8] + ax = WindroseAxes(fig, rect) + fig.add_axes(ax) + ax.contourf(dir, var, **kwargs) + l = ax.legend(axespad=-0.10) + plt.setp(l.get_texts(), fontsize=8) + plt.draw() + plt.show() + return ax + +def wrbox(dir, var, **kwargs): + fig = plt.figure() + rect = [0.1, 0.1, 0.8, 0.8] + ax = WindroseAxes(fig, rect) + fig.add_axes(ax) + ax.box(dir, var, **kwargs) + l = ax.legend(axespad=-0.10) + plt.setp(l.get_texts(), fontsize=8) + plt.draw() + plt.show() + return ax + +def wrbar(dir, var, **kwargs): + fig = plt.figure() + rect = [0.1, 0.1, 0.8, 0.8] + ax = WindroseAxes(fig, rect) + fig.add_axes(ax) + ax.bar(dir, var, **kwargs) + l = ax.legend(axespad=-0.10) + plt.setp(l.get_texts(), fontsize=8) + plt.draw() + plt.show() + return ax + +def clean(dir, var): + """ + Remove masked values in the two arrays, where if a direction data is masked, + the var data will also be removed in the cleaning process (and vice-versa) + """ + dirmask = dir.mask==False + varmask = var.mask==False + ind = dirmask*varmask + return dir[ind], var[ind] + +if __name__=='__main__': + from pylab import figure, show, setp, random, grid, draw + vv=random(500)*6 + dv=random(500)*360 + fig = figure(figsize=(8, 8), dpi=80, facecolor='w', edgecolor='w') + rect = [0.1, 0.1, 0.8, 0.8] + ax = WindroseAxes(fig, rect, axisbg='w') + fig.add_axes(ax) + +# ax.contourf(dv, vv, bins=np.arange(0,8,1), cmap=cm.hot) +# ax.contour(dv, vv, bins=np.arange(0,8,1), colors='k') +# ax.bar(dv, vv, normed=True, opening=0.8, edgecolor='white') + ax.box(dv, vv, normed=True) + l = ax.legend(axespad=-0.10) + setp(l.get_texts(), fontsize=8) + draw() + #print ax._info + show() + + + + + + diff --git a/build/lib/pyseidon_dvt/validationClass/__init__.py b/build/lib/pyseidon_dvt/validationClass/__init__.py new file mode 100644 index 0000000..7f20917 --- /dev/null +++ b/build/lib/pyseidon_dvt/validationClass/__init__.py @@ -0,0 +1,12 @@ +#!/usr/bin/python2.7 +# encoding: utf-8 + +from __future__ import division + +#Local import +from validationClass import Validation + +__authors__ = ['Jonathan Smith, Thomas Roc, Wesley Bowman'] +__licence__ = 'GNU Affero GPL v3.0' +__copyright__ = 'Copyright (c) 2014 EcoEnergyII' + diff --git a/build/lib/pyseidon_dvt/validationClass/compareData.py b/build/lib/pyseidon_dvt/validationClass/compareData.py new file mode 100644 index 0000000..61a817d --- /dev/null +++ b/build/lib/pyseidon_dvt/validationClass/compareData.py @@ -0,0 +1,305 @@ +#!/usr/bin/python2.7 +# encoding: utf-8 +from __future__ import division + +import numpy as np +from os import mkdir +from os.path import exists +from tidalStats import TidalStats +from smooth import smooth +from depthInterp import depthFromSurf, depthFromBott +from datetime import datetime, timedelta + +# Custom error +from pyseidon_dvt.utilities.pyseidon_error import PyseidonError + +# Local import +from plotsValidation import * + +def dn2dt(datenum): + ''' + Convert matlab datenum to python datetime. + ''' + return datetime.fromordinal(int(datenum)) + timedelta(days=datenum%1) - \ + timedelta(days=366) + +def compareOBS(data, save_path, threeDim=False, depth=5, slack_velo=0.8, plot=False, save_csv=False, + phase_shift=False, debug=False, debug_plot=False): + """ + Does a comprehensive validation process between modeled and observed. + Outputs a list of important statistics for each variable, calculated + using the TidalStats class + + Inputs: + - data = dictionary containing all necessary observed and model data + - threeDim = boolean flag, 3D or not + Outputs: + - elev_suite = dictionary of useful statistics for sea elevation + - speed_suite = dictionary of useful statistics for flow speed + - dir_suite = dictionary of useful statistics for flow direction + - u_suite = dictionary of useful statistics for u velocity component + - v_suite = dictionary of useful statistics for v velocity component + - vel_suite = dictionary of useful statistics for signed flow velocity + - csp_suite = dictionary of useful statistics for cubic flow speed + Options: + - depth = interpolation depth (float in meters), if negative = from + water column top downwards, if positive = from sea bottom upwards + - slack_velo = slack water's velocity (m/s), float, everything below will be dumped out + - plot = boolean flag for plotting results + - save_csv = boolean flag for saving statistical benchmarks in csv file + """ + if debug: print "CompareOBS..." + + hasEL=False + hasUV=False + if 'el' in data['_commonlist_data']: + hasEL=True + + ulist=[var for var in ['ua', 'u'] if var in data['_commonlist_data']] + vlist=[var for var in ['va', 'v'] if var in data['_commonlist_data']] + if len(ulist)>0 and len(vlist)>0: + hasUV=True + + # take data from input dictionary + mod_time = data['mod_time'] + if not data['type'] == 'Drifter': + obs_time = data['obs_time'] + else: + obs_time = data['mod_time'] + + if hasEL: + mod_el = data['mod_timeseries']['el'] + obs_el = data['obs_timeseries']['el'] + + # Check if 3D simulation and for velocity data + if hasUV: + if threeDim: + obs_u_all = data['obs_timeseries']['u'] + obs_v_all = data['obs_timeseries']['v'] + mod_u_all = data['mod_timeseries']['u'] + mod_v_all = data['mod_timeseries']['v'] + bins = data['obs_timeseries']['bins'] + siglay = data['mod_timeseries']['siglay'] + # use depth interpolation to get a single timeseries + #mod_depth = mod_el + np.mean(obs_el[~np.isnan(obs_el)]) + mod_depth = mod_el + data['mod_timeseries']['h'] + if depth < 0.0: + (mod_u, obs_u) = depthFromSurf(mod_u_all, mod_depth, siglay, + obs_u_all, obs_el, bins, depth=depth, + debug=debug, debug_plot=debug_plot) + (mod_v, obs_v) = depthFromSurf(mod_v_all, mod_depth, siglay, + obs_v_all, obs_el, bins, depth=depth, + debug=debug, debug_plot=debug_plot) + else: + (mod_u, obs_u) = depthFromBott(mod_u_all, mod_depth, siglay, + obs_u_all, obs_el, bins, depth=depth, + debug=debug, debug_plot=debug_plot) + (mod_v, obs_v) = depthFromBott(mod_v_all, mod_depth, siglay, + obs_v_all, obs_el, bins, depth=depth, + debug=debug, debug_plot=debug_plot) + else: + if not data['type'] == 'Drifter': + obs_u = data['obs_timeseries']['ua'] + obs_v = data['obs_timeseries']['va'] + mod_u = data['mod_timeseries']['ua'] + mod_v = data['mod_timeseries']['va'] + else: + obs_u = data['obs_timeseries']['u'] + obs_v = data['obs_timeseries']['v'] + mod_u = data['mod_timeseries']['u'] + mod_v = data['mod_timeseries']['v'] + + + if debug: print "...convert times to datetime..." + mod_dt, obs_dt = [], [] + for i in mod_time: + mod_dt.append(dn2dt(i)) + for j in obs_time: + obs_dt.append(dn2dt(j)) + + if debug: print "...put data into a useful format..." + if hasUV: + mod_spd = np.sqrt(mod_u**2.0 + mod_v**2.0) + obs_spd = np.sqrt(obs_u**2.0 + obs_v**2.0) + mod_dir = np.arctan2(mod_v, mod_u) * 180.0 / np.pi + obs_dir = np.arctan2(obs_v, obs_u) * 180.0 / np.pi + if 'el' in data['_commonlist_data']: + obs_el = obs_el - np.mean(obs_el[~np.isnan(obs_el)]) + # Chose the component with the biggest variance as sign reference + if np.var(mod_v) > np.var(mod_u): + mod_signed = np.sign(mod_v) + obs_signed = np.sign(obs_v) + else: + mod_signed = np.sign(mod_u) + obs_signed = np.sign(obs_u) + + if debug: print "...check if the modeled data lines up with the observed data..." + if (mod_time[-1] < obs_time[0] or obs_time[-1] < mod_time[0]): + raise PyseidonError("---time periods do not match up---") + + else: + if debug: print "...interpolate the data onto a common time step for each data type..." + if not data['type'] == 'Drifter': + # elevation + if hasEL: + (mod_el_int, obs_el_int, step_el_int, start_el_int) = smooth(mod_el, mod_dt, obs_el, obs_dt, + debug=debug, debug_plot=debug_plot) + if hasUV: + # speed + (mod_sp_int, obs_sp_int, step_sp_int, start_sp_int) = smooth(mod_spd, mod_dt, obs_spd, obs_dt, + debug=debug, debug_plot=debug_plot) + # direction + (mod_dr_int, obs_dr_int, step_dr_int, start_dr_int) = smooth(mod_dir, mod_dt, obs_dir, obs_dt, + debug=debug, debug_plot=debug_plot) + # u velocity + (mod_u_int, obs_u_int, step_u_int, start_u_int) = smooth(mod_u, mod_dt, obs_u, obs_dt, + debug=debug, debug_plot=debug_plot) + # v velocity + (mod_v_int, obs_v_int, step_v_int, start_v_int) = smooth(mod_v, mod_dt, obs_v, obs_dt, + debug=debug, debug_plot=debug_plot) + # velocity i.e. signed speed + (mod_ve_int, obs_ve_int, step_ve_int, start_ve_int) = smooth(mod_spd * mod_signed, mod_dt, + obs_spd * obs_signed, obs_dt, + debug=debug, debug_plot=debug_plot) + # cubic signed speed + #mod_cspd = mod_spd**3.0 + #obs_cspd = obs_spd**3.0 + mod_cspd = mod_signed * mod_spd**3.0 + obs_cspd = obs_signed * obs_spd**3.0 + (mod_cspd_int, obs_cspd_int, step_cspd_int, start_cspd_int) = smooth(mod_cspd, mod_dt, obs_cspd, obs_dt, + debug=debug, debug_plot=debug_plot) + else: + # Time steps + step = mod_time[1] - mod_time[0] + start = mod_time[0] + + # Already interpolated, so no need to use smooth... + # speed + (mod_sp_int, obs_sp_int, step_sp_int, start_sp_int) = (mod_spd, obs_spd, step, start) + # direction + (mod_dr_int, obs_dr_int, step_dr_int, start_dr_int) = (mod_dir, obs_dir, step, start) + # u velocity + (mod_u_int, obs_u_int, step_u_int, start_u_int) = (mod_u, obs_u, step, start) + # v velocity + (mod_v_int, obs_v_int, step_v_int, start_v_int) = (mod_v, obs_v, step, start) + # velocity i.e. signed speed + (mod_ve_int, obs_ve_int, step_ve_int, start_ve_int) = (mod_spd, obs_spd, step, start) + # cubic signed speed + #mod_cspd = mod_spd**3.0 + #obs_cspd = obs_spd**3.0 + mod_cspd = mod_signed * mod_spd**3.0 + obs_cspd = obs_signed * obs_spd**3.0 + (mod_cspd_int, obs_cspd_int, step_cspd_int, start_cspd_int) = (mod_cspd, obs_cspd, step, start) + + if debug: print "...remove directions where velocities are small..." + if hasUV: + MIN_VEL = slack_velo + indexMin = np.where(obs_sp_int < MIN_VEL) + obs_dr_int[indexMin] = np.nan + obs_u_int[indexMin] = np.nan + obs_v_int[indexMin] = np.nan + obs_ve_int[indexMin] = np.nan + obs_cspd_int[indexMin] = np.nan + + indexMin = np.where(mod_sp_int < MIN_VEL) + mod_dr_int[indexMin] = np.nan + mod_u_int[indexMin] = np.nan + mod_v_int[indexMin] = np.nan + mod_ve_int[indexMin] = np.nan + mod_cspd_int[indexMin] = np.nan + + if debug: print "...get stats for each tidal variable..." + gear = data['type'] # Type of measurement gear (drifter, adcp,...) + + suites={} + + if hasEL: + suites['el'] = tidalSuite(gear, mod_el_int, obs_el_int, step_el_int, start_el_int, + [], [], [], [], [], [], + kind='elevation', plot=plot, + save_csv=save_csv, save_path=save_path, phase_shift=phase_shift, + debug=debug, debug_plot=debug_plot) + if hasUV: + suites['speed'] = tidalSuite(gear, mod_sp_int, obs_sp_int, step_sp_int, start_sp_int, + [], [], [], [], [], [], + kind='speed', plot=plot, + save_csv=save_csv, save_path=save_path, phase_shift=phase_shift, + debug=debug, debug_plot=debug_plot) + suites['dir'] = tidalSuite(gear, mod_dr_int, obs_dr_int, step_dr_int, start_dr_int, + mod_u, obs_u, mod_v, obs_v, + mod_dt, obs_dt, + kind='direction', plot=plot, + save_csv=save_csv, save_path=save_path, phase_shift=phase_shift, + debug=debug, debug_plot=debug_plot) + suites['u'] = tidalSuite(gear, mod_u_int, obs_u_int, step_u_int, start_u_int, + [], [], [], [], [], [], + kind='u velocity', plot=plot, save_csv=save_csv, save_path=save_path, phase_shift=phase_shift, + debug=debug, debug_plot=debug_plot) + suites['v'] = tidalSuite(gear, mod_v_int, obs_v_int, step_v_int, start_v_int, + [], [], [], [], [], [], + kind='v velocity', plot=plot, save_csv=save_csv, save_path=save_path, phase_shift=phase_shift, + debug=debug, debug_plot=debug_plot) + + # TR: requires special treatments from here on + suites['vel'] = tidalSuite(gear, mod_ve_int, obs_ve_int, step_ve_int, start_ve_int, + mod_u, obs_u, mod_v, obs_v, + mod_dt, obs_dt, + kind='velocity', plot=plot, save_csv=save_csv, save_path=save_path, phase_shift=phase_shift, + debug=debug, debug_plot=debug_plot) + suites['cubic_speed'] = tidalSuite(gear, mod_cspd_int, obs_cspd_int, step_cspd_int, start_cspd_int, + mod_u, obs_u, mod_v, obs_v, + mod_dt, obs_dt, + kind='cubic speed', plot=plot, save_csv=save_csv, save_path=save_path, phase_shift=phase_shift, + debug=debug, debug_plot=debug_plot) + + # output statistics in useful format + + if debug: print "...CompareOBS done." + + + return suites + + +def tidalSuite(gear, model, observed, step, start, + model_u, observed_u, model_v, observed_v, + model_time, observed_time, + kind='', plot=False, save_csv=False, save_path='./', phase_shift=False, + debug=False, debug_plot=False): + """ + Create stats classes for a given tidal variable. + + Accepts interpolated model and observed data, the timestep, and start + time. kind is a string representing the kind of data. If plot is set + to true, a time plot and regression plot will be produced. + + Returns a dictionary containing all the stats. + """ + if debug: print "tidalSuite..." + stats = TidalStats(gear, model, observed, step, start, + model_u = model_u, observed_u = observed_u, model_v = model_v, observed_v = observed_v, + model_time = model_time, observed_time = observed_time, phase_shift=phase_shift, + kind=kind, debug=debug, debug_plot=debug_plot) + stats_suite = stats.getStats(phase_shift=phase_shift) + stats_suite['r_squared'] = stats.linReg()['r_2'] + # calling special methods + if kind == 'direction': + rmse, nrmse = stats.statsForDirection(debug=debug) + stats_suite['RMSE'] = rmse + stats_suite['NRMSE'] = nrmse + try: #Fix for Drifter's data + stats_suite['phase'] = stats.getPhase(phase_shift=phase_shift, debug=debug) + except: + stats_suite['phase'] = 0.0 + + if plot or debug_plot: + plotData(stats) + plotRegression(stats, stats.linReg()) + + if save_csv: + stats.save_data(path=save_path) + plotData(stats, savepath=save_path, fname=kind+"_"+gear+"_time_series.png") + plotRegression(stats, stats.linReg(), savepath=save_path, fname=kind+"_"+gear+"_linear_regression.png") + + if debug: print "...tidalSuite done." + + return stats_suite diff --git a/build/lib/pyseidon_dvt/validationClass/depthInterp.py b/build/lib/pyseidon_dvt/validationClass/depthInterp.py new file mode 100644 index 0000000..ad30fe0 --- /dev/null +++ b/build/lib/pyseidon_dvt/validationClass/depthInterp.py @@ -0,0 +1,203 @@ +#!/usr/bin/python2.7 +# encoding: utf-8 +import numpy as np +from scipy.interpolate import interp1d + +''' +ASSUMPTIONS: +first dimension of matrices identify the timestep, second the depth +column vectors are organized from bottom to top +i.e. data[3] is the column at the third timestep + data[3][10] is the tenth layer from the bottom at the third timestep +ADCP bins are the depths of each ADCP layer +ADCP bin width is constant +The top ADCP value of any column is no greater than 95% of the total depth +''' + +ADCP_TOP_SURF = 0.95 + +def depthToSigma(obs_data, obs_depth, siglay, bins, debug=False, debug_plot=False): + ''' + Performs linear interpolation on 3D ADCP data to change it into a sigma + layer format, similar to an FVCOM run. + + Outputs a 2D numpy array representing the ADCP data in sigma layer + format. + ''' + if debug: print "depthToSigma..." + sig_obs = np.zeros(obs_data.shape[0], siglay.size) + + if debug: print "...map old depths to between 0 and 1, then interpol..." + # loop through columns/steps + for i, column in enumerate(obs_data): + # map old depths to between 0 and 1, make interpolation function + col_nonan = column[np.where(~np.isnan(column))[0]] + old_depths = bins[np.where(~np.isnan(column))[0]] + mapped_depths = old_depths / obs_depth[i] + f_obs = interp1d(col_nonan, mapped_depths) + # perform interpolation + sig_obs[i] = f_obs(siglay) + + if debug: print "...depthToSigma done." + + return sig_obs + +def sigmaToDepth(mod_data, mod_depth, siglay, bins, debug=False, debug_plot=False): + ''' + Performs linear interpolation on 3D FVCOM output to change it into the + same format as ADCP output (i.e. constant depths, NaNs above surface) + + Outputs a 2D numpy array representing the FVCOM matrix in ADCP format. + ''' + if debug: print "sigmaToDepth..." + bin_mod = np.zeros(mod_data.shape[0], bins.size) + bin_width = bins[1] - bins[0] + + if debug: print "...loop through columns/steps and interpol..." + # loop through columns/steps + for i, column in enumerate(mod_data): + # create interpolation function + f_mod = interp1d(column, siglay) + depth = mod_depth[i] + # loop through bins + for j in np.arange(bins.size): + # check if location is above ADCP_TOP_SURF + loc = float(bins_width * j + bins[0]) / float(depth) + if (loc <= ADCP_TOP_SURF): + bin_mod[i][j] = f_mod(loc) + else: + bin_mod[i][j] = np.nan + + if debug: print "...sigmaToDepth done." + + return bin_mod + +def depthFromSurf(mod_data, mod_depth, siglay, obs_data, obs_depth, bins, depth=5, + debug=False, debug_plot=False): + ''' + Performs linear interpolation on 3D ocean data to obtain data at a + specific distance from the surface. + + Inputs: + - mod_data = 2D numpy array of FVCOM model data + - mod_depth = 1D numpy array of model depths at each timestep + - siglay = array containing values between 0 and 1 representing the + respective percentage of depths for each sigma layer + - obs_data = 2D numpy array of observed ADCP data + - obs_depth = 1D numpy array of observed depths at each timestep + - depth = number of metres from surface of output timeseries. + Defaults to 5m + + Outputs: + - (new_mod, new_obs) = timeseries representing model and observed data + at 'depth' metres from the surface. + ''' + if debug: print "depthFromSurf..." + new_mod = np.zeros(mod_data.shape[0]) + new_obs = np.zeros(obs_data.shape[0]) + depth = np.abs(depth) + + if debug: print "...loop through simulation columns and interpol at specified depth..." + # loop over mod_data columns + for i, step in enumerate(mod_data): + # create interpolation function + #TR: quick fix + try: + f_mod = interp1d(np.abs(siglay), step) #, bounds_error=False) + # find location of specified depth and perform interpolation + location = mod_depth[i] - depth + sig_loc = float(location) / float(mod_depth[i]) + new_mod[i] = f_mod(sig_loc) + except ValueError: + f_mod = interp1d(np.abs(siglay)[::-1], step[::-1], bounds_error=False) + # find location of specified depth and perform interpolation + location = mod_depth[i] - depth + sig_loc = float(location) / float(mod_depth[i]) + new_mod[i] = f_mod(sig_loc) + + if debug: print "...loop through measurement columns and interpol at specified depth..." + # loop over obs_data columns + for ii, column in enumerate(obs_data): + # create interpolation function + col_nonan = column[np.where(~np.isnan(column))[0]] + bin_nonan = bins[np.where(~np.isnan(column))[0]] + + if not col_nonan.shape[0]==0: + # find location of specified depth and perform interpolation + try: + f_obs = interp1d(bin_nonan, col_nonan) + location = obs_depth[ii] - depth + new_obs[ii] = f_obs(location) + except ValueError: + f_obs = interp1d(bin_nonan[::-1], col_nonan[::-1], bounds_error=False) + location = obs_depth[ii] - depth + new_obs[ii] = f_obs(location) + else: + new_obs[ii] = np.nan + + if debug: print "...depthFromSurf done." + + return (new_mod, new_obs) + +def depthFromBott(mod_data, mod_depth, siglay, obs_data, obs_depth, bins, depth=5, + debug=False, debug_plot=False): + ''' + Performs linear interpolation on 3D ocean data to obtain data at a + specific distance from the surface. + + Inputs: + - mod_data = 2D numpy array of FVCOM model data + - mod_depth = 1D numpy array of model depths at each timestep + - siglay = array containing values between 0 and 1 representing the + respective percentage of depths for each sigma layer + - obs_data = 2D numpy array of observed ADCP data + - obs_depth = 1D numpy array of observed depths at each timestep + - depth = number of metres from surface of output timeseries. + Defaults to 5m + + Outputs: + - (new_mod, new_obs) = timeseries representing model and observed data + at 'depth' metres from the surface. + ''' + if debug: print "depthFromBott..." + new_mod = np.zeros(mod_data.shape[0]) + new_obs = np.zeros(obs_data.shape[0]) + depth = np.abs(depth) + + if debug: print "...loop through simulation columns and interpol at specified depth..." + # loop over mod_data columns + for i, step in enumerate(mod_data): + # create interpolation function + #TR: quick fix + try: + f_mod = interp1d(np.abs(siglay), step) #, bounds_error=False) + # find location of specified depth and perform interpolation + sig_loc = float(depth) / float(mod_depth[i]) + new_mod[i] = f_mod(sig_loc) + except ValueError: + f_mod = interp1d(np.abs(siglay)[::-1], step[::-1], bounds_error=False) + # find location of specified depth and perform interpolation + sig_loc = float(depth) / float(mod_depth[i]) + new_mod[i] = f_mod(sig_loc) + + if debug: print "...loop through measurement columns and interpol at specified depth..." + # loop over obs_data columns + for ii, column in enumerate(obs_data): + # create interpolation function + col_nonan = column[np.where(~np.isnan(column))[0]] + bin_nonan = bins[np.where(~np.isnan(column))[0]] + + if not col_nonan.shape[0]==0: + # find location of specified depth and perform interpolation + try: + f_obs = interp1d(bin_nonan, col_nonan) #, bounds_error=False) + new_obs[ii] = f_obs(depth) + except ValueError: + f_obs = interp1d(bin_nonan[::-1], col_nonan[::-1], bounds_error=False) + new_obs[ii] = f_obs(depth) + else: + new_obs[ii] = np.nan + + if debug: print "...depthFromBott done." + + return (new_mod, new_obs) \ No newline at end of file diff --git a/build/lib/pyseidon_dvt/validationClass/interpolate.py b/build/lib/pyseidon_dvt/validationClass/interpolate.py new file mode 100644 index 0000000..4802b19 --- /dev/null +++ b/build/lib/pyseidon_dvt/validationClass/interpolate.py @@ -0,0 +1,59 @@ +#!/usr/bin/python2.7 +# encoding: utf-8 +from scipy.interpolate import interp1d +import numpy as np +from datetime import timedelta +import time + + +def interpol(data_1, data_2, time_step=timedelta(minutes=5), + debug=False, debug_plot=False): + ''' + Interpolates between two datasets so their points line up in the time + domain. + + Accepts two sets of data, each of which are dictionaries containing two + values: + time- array containing the datetimes corresponding to the points + pts- 1D numpy array containing the data + + Third optional argument sets the time between data points in the output + data. Is a timedelta object, defaults to 10 minutes. + ''' + if debug: print "interpol..." + if debug: print "...line up points in the time, step=5min..." + + dt_1 = data_1['time'] + dt_2 = data_2['time'] + + # create POSIX timestamp array corresponding to each dataset + times_1, times_2 = np.zeros(len(dt_1)), np.zeros(len(dt_2)) + for i in np.arange(times_1.size): + times_1[i] = time.mktime(dt_1[i].timetuple()) + for v in np.arange(times_2.size): + times_2[v] = time.mktime(dt_2[v].timetuple()) + + # generate interpolation functions using linear interpolation + f1 = interp1d(times_1, data_1['pts']) + f2 = interp1d(times_2, data_2['pts']) + + # choose interval on which to interpolate + start = max(times_1[0], times_2[0]) + end = min(times_1[-1], times_2[-1]) + length = end - start + + # determine number of steps in the interpolation interval + step_sec = time_step.total_seconds() + steps = int(length / step_sec) + + # create POSIX timestamp array for new data and perform interpolation + output_times = start + np.arange(steps) * step_sec + + series_1 = f1(output_times) + series_2 = f2(output_times) + + dt_start = max(dt_1[0], dt_2[0]) + + if debug: print "...interpol done." + + return (series_1, series_2, time_step, dt_start) diff --git a/build/lib/pyseidon_dvt/validationClass/plotsValidation.py b/build/lib/pyseidon_dvt/validationClass/plotsValidation.py new file mode 100644 index 0000000..ffa0bed --- /dev/null +++ b/build/lib/pyseidon_dvt/validationClass/plotsValidation.py @@ -0,0 +1,329 @@ +#!/usr/bin/python2.7 +# encoding: utf-8 + +from __future__ import division + +import os.path as os +import matplotlib.pyplot as plt +import numpy as np +from matplotlib.projections import PolarAxes +import mpl_toolkits.axisartist.floating_axes as fa +import mpl_toolkits.axisartist.grid_finder as gf + +def plotRegression(tidalStatClass, lr, savepath='', fname='', debug=False): + """ + Plots a visualization of the output from linear regression, + including confidence intervals for predictands and slope. + + If a savepath and filename is defined, exports the plot as an image file + to that location. Filenames should include the image file name and extension. + + Returns: + fig, ax: maplotlib objects + """ + if debug : print "Plotting linear regression" + + #define figure frame + fig = plt.figure(figsize=(18,10)) + plt.rc('font',size='22') + ax = fig.add_subplot(111) + + ax.scatter(tidalStatClass.model, tidalStatClass.observed, c='b', marker='+', alpha=0.5) + + ## plot regression line + mod_max = np.amax(tidalStatClass.model) + mod_min = np.amin(tidalStatClass.model) + upper_intercept = lr['intercept'] + lr['pred_CI_width'] + lower_intercept = lr['intercept'] - lr['pred_CI_width'] + ax.plot([mod_min, mod_max], [mod_min * lr['slope'] + lr['intercept'], + mod_max * lr['slope'] + lr['intercept']], + color='k', linestyle='-', linewidth=2, label='Linear fit') + + ## plot CI's for slope + ax.plot([mod_min, mod_max], [mod_min * lr['slope_CI'][0] + lr['intercept_CI'][0], + mod_max * lr['slope_CI'][0] + lr['intercept_CI'][0]], + color='r', linestyle='--', linewidth=2) + ax.plot([mod_min, mod_max], [mod_min * lr['slope_CI'][1] + lr['intercept_CI'][1], + mod_max * lr['slope_CI'][1] + lr['intercept_CI'][1]], + color='r', linestyle='--', linewidth=2, label='Slope CI') + + ## plot CI's for predictands + ax.plot([mod_min, mod_max], [mod_min * lr['slope'] + upper_intercept, + mod_max * lr['slope'] + upper_intercept], + color='g', linestyle='--', linewidth=2) + ax.plot([mod_min, mod_max], [mod_min * lr['slope'] + lower_intercept, + mod_max * lr['slope'] + lower_intercept], + color='g', linestyle='--', linewidth=2, label='Predictand CI') + + ax.set_xlabel('Modeled Data') + ax.set_ylabel('Observed Data') + fig.suptitle('Modeled vs. Observed {}: Linear Fit'.format(tidalStatClass.kind)) + plt.legend(loc='lower right', shadow=True) + + r_string = 'R Squared: {}'.format(np.around(lr['r_2'], decimals=3)) + plt.title(r_string) + + # Pretty plot + # df = DataFrame(data={'model': tidalStatClass.model.ravel(), + # 'observed':tidalStatClass.observed.ravel()}) + # seaborn.set(style="darkgrid") + # color = seaborn.color_palette()[2] + # g = seaborn.jointplot("model", "observed", data=df, kind="reg", + # xlim=(df.model.min(), df.model.max()), + # ylim=(df.observed.min(), df.observed.max()), + # color=color, size=7) + # plt.suptitle('Modeled vs. Observed {}: Linear Fit'.format(tidalStatClass.kind)) + # KC: Changed save parameter to be a savepath - making a huge assumption here + # that people are able to enter in the savepath correctly / exists etc. + if savepath.strip() and fname.strip(): + if os.exists(savepath): + fig.savefig(savepath+fname) + fig.clear() + plt.close(fig) + else: + fig.show() + plt.show() + + return fig, ax + +def plotData(tidalStatClass, graph='time', savepath='', fname='', debug=False): + """ + Provides a visualization of the data. + + Takes an option which determines the kind of graph to be made. + time: plots the model data against the observed data over time + scatter : plots the model data vs. observed data + + If a savepath and filename is defined, exports the plot as an image file + to that location. Filenames should include the image file name and extension. + """ + if debug: print "Plotting time-series..." + #define figure frame + fig = plt.figure(figsize=(18,10)) + plt.rc('font',size='22') + ax = fig.add_subplot(111) + + if (graph == 'time'): + ax.plot(tidalStatClass.times, tidalStatClass.model, label='Model Predictions') + ax.plot(tidalStatClass.times, tidalStatClass.observed, color='r', + label='Observed Data') + ax.set_xlabel('Time') + if tidalStatClass.kind == 'elevation': + ax.set_ylabel('Elevation (m)') + if tidalStatClass.kind == 'speed': + ax.set_ylabel('Flow speed (m/s)') + if tidalStatClass.kind == 'direction': + ax.set_ylabel('Flow direction (deg.)') + if tidalStatClass.kind == 'u velocity': + ax.set_ylabel('U velocity (m/s)') + if tidalStatClass.kind == 'v velocity': + ax.set_ylabel('V velocity (m/s)') + if tidalStatClass.kind == 'velocity': + ax.set_ylabel('Signed flow speed (m/s)') + if tidalStatClass.kind == 'cubic speed': + ax.set_ylabel('Cubic speed (m3/s3)') + + fig.suptitle('Predicted and Observed {}'.format(tidalStatClass.kind)) + ax.legend(shadow=True) + + if (graph == 'scatter'): + ax.scatter(tidalStatClass.model, tidalStatClass.observed, c='b', alpha=0.5) + ax.set_xlabel('Predicted Height') + ax.set_ylabel('Observed Height') + fig.suptitle('Predicted vs. Observed {}'.format(tidalStatClass.kind)) + + if savepath.strip() and fname.strip(): + if os.exists(savepath): + fig.savefig(savepath+fname) + fig.clear() + plt.close(fig) + else: + fig.show() + plt.show() + +def taylorDiagram(benchmarks, savepath='', fname='', labels=True, debug=False): + """ + References: + Taylor, K.E.: Summarizing multiple aspects of model performance in a single diagram. + J. Geophys. Res., 106, 7183-7192, 2001 + (also see PCMDI Report 55, http://www-pcmdi.llnl.gov/publications/ab55.html) + + IPCC, 2001: Climate Change 2001: The Scientific Basis, + Contribution of Working Group I to the Third Assessment Report of the Intergovernmental Panel on Climate Change + Houghton, J.T., Y. Ding, D.J. Griggs, M. Noguer, P.J. van der Linden, X. Dai, K. Maskell, and C.A. Johnson (eds.) + Cambridge University Press, Cambridge, United Kingdom and New York, NY, USA, 881 pp. + (see http://www.grida.no/climate/ipcc_tar/wg1/317.htm#fig84) + + Code inspired by Yannick Copin's code (see https://github.com/ycopin) + """ + if debug: print "Plotting time-series..." + + # Setting up graph + tr = PolarAxes.PolarTransform() + # Correlation labels + rlocs = np.concatenate((np.arange(10)/10.,[0.95,0.99])) + tlocs = np.arccos(rlocs) # Conversion to polar angles + gl1 = gf.FixedLocator(tlocs) # Positions + tf1 = gf.DictFormatter(dict(zip(tlocs, map(str,rlocs)))) + # Standard deviation axis extent + smin = 0 + smax = 1.5 + ghelper = fa.GridHelperCurveLinear(tr, extremes=(0,np.pi/2, smin, smax), grid_locator1=gl1, tick_formatter1=tf1) + fig = plt.figure(figsize=(18,10)) + rect=111 + ax = fa.FloatingSubplot(fig, rect, grid_helper=ghelper) + fig.add_subplot(ax) + # Adjust axes + ax.axis["top"].set_axis_direction("bottom") # "Angle axis" + ax.axis["top"].toggle(ticklabels=True, label=True) + ax.axis["top"].major_ticklabels.set_axis_direction("top") + ax.axis["top"].label.set_axis_direction("top") + ax.axis["top"].label.set_text("Correlation") + ax.axis["left"].set_axis_direction("bottom") # "X axis" + ax.axis["left"].label.set_text("Standard deviation") + ax.axis["right"].set_axis_direction("top") # "Y axis" + ax.axis["right"].toggle(ticklabels=True) + ax.axis["right"].major_ticklabels.set_axis_direction("left") + ax.axis["bottom"].set_visible(False) # Useless + # Contours along standard deviations + ax.grid(False) + _ax = ax # Graphical axes + ax = ax.get_aux_axes(tr) # Polar coordinates + # Reference lines + x95 = [0.05, 13.9] # For Prcp, this is for 95th level (r = 0.195) + y95 = [0.0, 71.0] + x99 = [0.05, 19.0] # For Prcp, this is for 99th level (r = 0.254) + y99 = [0.0, 70.0] + ax.plot(x95,y95,color='k') + ax.plot(x99,y99,color='k') + # Add reference point and stddev contour + l, = ax.plot(0.0, 1.0, 'k*', ls='', ms=10, label='Reference') + t = np.linspace(0, np.pi/2) + r = np.zeros_like(t) + 1.0 + ax.plot(t,r, 'k--', label='_') + samplePoints = [l] + + # Plot points + sampleLenght = benchmarks['Type'].shape[0] + colors = plt.matplotlib.cm.jet(np.linspace(0,1,sampleLenght)) + for i in range(sampleLenght): + if labels: + l, = ax.plot(benchmarks['NRMSE'][i]/100.0, benchmarks['r2'][i], + marker='$%d$' % (i+1), ms=10, ls='', mfc=colors[i], mec=colors[i], + label= benchmarks['Type'][i] + " " + benchmarks['gear'][i]) + else: + l, = ax.plot(benchmarks['NRMSE'][i]/100.0, benchmarks['r2'][i], + marker='$%d$' % (i+1), ms=10, ls='', mfc=colors[i], mec=colors[i]) + samplePoints.append(l) + t = np.linspace(0, np.pi/2) + r = np.zeros_like(t) + 1.0 + ax.plot(t,r, 'k--', label='_') + + # Add NRMS contours, and label them + rs, ts = np.meshgrid(np.linspace(smin, smax), np.linspace(0,np.pi/2)) + # Compute centered RMS difference + rms = np.sqrt(1.0 + rs**2 - 2*rs*np.cos(ts)) + contours = ax.contour(ts, rs, rms, 5, colors='0.5') + ax.clabel(contours, inline=1, fontsize=10, fmt='%.1f') + # Add a figure legend + if labels: + lgd = fig.legend(samplePoints,[p.get_label() for p in samplePoints], + numpoints=1, prop=dict(size='small'), loc='upper right') + fig.tight_layout() + + if savepath.strip() and fname.strip(): + if os.exists(savepath): + if labels: + fig.savefig(savepath+fname, bbox_extra_artists=(lgd,), bbox_inches='tight') + else: + fig.savefig(savepath+fname, bbox_inches='tight') + fig.clear() + plt.close(fig) + else: + fig.show() + plt.show() + + return fig, ax + +def benchmarksMap(benchmarks, adcps, fvcom, savepath='', fname='', debug=False): + """ + Plots bathymetric map & model validation benchmarks + + Inputs: + - benchmarks = benchmark attribute from Validation class + - adcps = list or tuple of ADCP objects + - fvcom = FVCOM object + Options: + - savepath = folder path for saving plot, string + - fname = filename for saving plot, string + """ + #if debug: print "Computing flow speed" + #fvcom.Util2D.hori_velo_norm() + #speed = np.mean(fvcom.Variables.hori_velo_norm[:],0) + #if debug: print '...passed' + + # collecting names and locations of adcps + adcpLoc={} + try: + for adcp in adcps: + try: + key = adcp.History[0].split(' ')[-1].split('/')[-1].split('.')[0] + val = benchmarks.loc[key] + indCS = np.where(val['Type'].values == 'cubic_speed')[0][0] + adcpLoc[key] = {'location': [adcp.Variables.lon, adcp.Variables.lat], + 'r2': val['r2'][indCS], + 'NRMSE': val['NRMSE'][indCS], + 'bias': val['bias'][indCS]} + except KeyError: # in case something different than adcp in the list + pass + except TypeError: + key = adcps.History[0].split(' ')[-1].split('/')[-1].split('.')[0] + adcpLoc[key] = [adcps.Variables.lon, adcps.Variables.lat] + val = benchmarks.loc[key] + indCS = np.where(val['Type'].values == 'cubic_speed')[0][0] + adcpLoc[key] = {'location': [adcps.Variables.lon, adcps.Variables.lat], + 'r2': val['r2'][indCS], + 'NRMSE': val['NRMSE'][indCS], + 'bias': val['bias'][indCS]} + + # Plot size and color function of R2 and RMSE + # #background + #cmap=plt.cm.jet + #fvcom.Plots.colormap_var(speed, title='Averaged flow speed', mesh=False, cmap=cmap) + fvcom.Plots.colormap_var(fvcom.Grid.h, title='Bathymetric Map & Model Validation Benchmarks', mesh=False) + + for key in adcpLoc.keys(): + print '...plotting ' + key + '...' + r2 = adcpLoc[key]['r2'] # r2 for cubic velocity + nrmse = adcpLoc[key]['NRMSE'] # nrmse for cubic speed + mk = adcpLoc[key]['bias'] # over or under estimated + if np.sign(mk) == -1.0 : + mk = '_' + else: + mk = '+' + fvcom.Plots._ax.scatter(adcpLoc[key]['location'][0],adcpLoc[key]['location'][1], + marker=mk, lw=2, s=100, color='red') + fvcom.Plots._ax.annotate('r2: '+str(round(r2,2))+' |', + xy=(adcpLoc[key]['location'][0],adcpLoc[key]['location'][1]), + xycoords='data', xytext=(-55, -15), + textcoords='offset points', ha='left', + color='white', fontsize=12) + fvcom.Plots._ax.annotate('nrmse: '+str(round(nrmse,2)), + xy=(adcpLoc[key]['location'][0],adcpLoc[key]['location'][1]), + xycoords='data', xytext=(5, -15), + textcoords='offset points', ha='left', + color='white', fontsize=12) + plt.figtext(.02, .02, + "Notes:\n" + + "'+' represents over-estimation whereas '-' represents under-estimation.\n" + + "r2 and nrmse are respectively based on cubic signed speed and cubic speed. ", + size='x-small') + + if savepath.strip() and fname.strip(): + if os.exists(savepath): + fvcom.Plots._fig.savefig(savepath+fname, bbox_inches='tight') + fvcom.Plots._fig.clear() + plt.close(fvcom.Plots._fig) + else: + fvcom.Plots._fig.show() + plt.show() \ No newline at end of file diff --git a/build/lib/pyseidon_dvt/validationClass/smooth.py b/build/lib/pyseidon_dvt/validationClass/smooth.py new file mode 100644 index 0000000..3de2a50 --- /dev/null +++ b/build/lib/pyseidon_dvt/validationClass/smooth.py @@ -0,0 +1,84 @@ +#!/usr/bin/python2.7 +# encoding: utf-8 +from datetime import timedelta +import numpy as np +import time + +def smooth(data_1, dt_1, data_2, dt_2, delta_t=10, debug=False, debug_plot=False): + ''' + Smooths a dataset by taking the average of all datapoints within + a certain timestep to reduce noise. Lines up two datasets in the + time domain, as well. + Accepts four variables representing the data. data_1 and data_2 are the + data points, dt_1 and dt_2 are the datetimes corresponding to the points. + delta_t is an optional paramter that changes the time_step in minutes. + ''' + if debug: print "smooth..." + + # KC: timestep changed to delta_t, made optional parameter + time_step = timedelta(minutes=delta_t) + + # create POSIX timestamp array corresponding to each dataset + ''' + times_1, times_2 = np.zeros(len(dt_1)), np.zeros(len(dt_2)) + for i in np.arange(times_1.size): + times_1[i] = time.mktime(dt_1[i].timetuple()) + for i in np.arange(times_2.size): + times_2[i] = time.mktime(dt_2[i].timetuple()) + ''' + + make_posix = lambda x: time.mktime(x.timetuple()) + times_1 = map(make_posix, dt_1) + times_2 = map(make_posix, dt_2) + time_1, times_2 = np.array(times_1), np.array(times_2) + + # choose smoothing interval + start = max(times_1[0], times_2[0]) + end = min(times_1[-1], times_2[-1]) + length = end - start + dt_start = max(dt_1[0], dt_2[0]) + + # grab number of steps and timestamp for start time + step_sec = time_step.total_seconds() + steps = int(length / step_sec) + + # sort times into bins + series_1, series_2 = np.zeros(steps - 1), np.zeros(steps - 1) + time_bins = np.arange(steps) * step_sec + start + inds_1 = np.digitize(times_1, time_bins) + inds_2 = np.digitize(times_2, time_bins) + + # identify bin vertices and take means + first_hit_1 = np.searchsorted(inds_1, np.arange(1, steps + 1)) + for j in xrange(steps - 1): + series_1[j] = np.nanmean(data_1[first_hit_1[j]:first_hit_1[j + 1]]) + first_hit_2 = np.searchsorted(inds_2, np.arange(1, steps + 1)) + for j in xrange(steps - 1): + series_2[j] = np.nanmean(data_2[first_hit_2[j]:first_hit_2[j + 1]]) + + ''' + # take averages at each step, create output data + series_1, series_2 = np.zeros(steps), np.zeros(steps) + for i in np.arange(steps): + start_buf = start + step_sec * i + end_buf = start + step_sec * (i + 1) + buf_1 = np.where((times_1 >= start_buf) & (times_1 < end_buf))[0] + buf_2 = np.where((times_2 >= start_buf) & (times_2 < end_buf))[0] + data_buf_1 = data_1[buf_1] + data_buf_2 = data_2[buf_2] + # communicate progress + #if (i % 1000 == 0): + # print 'Currently smoothing at step {} / {}'.format(i, steps) + # calculate mean of data subsets (in the buffers) + if (len(data_buf_1) != 0): + series_1[i] = np.mean(data_buf_1) + else: + series_1[i] = np.nan + if (len(data_buf_2) != 0): + series_2[i] = np.mean(data_buf_2) + else: + series_2[i] = np.nan + ''' + + if debug: print "...smooth done." + return (series_1, series_2, time_step, dt_start) diff --git a/build/lib/pyseidon_dvt/validationClass/tidalStats.py b/build/lib/pyseidon_dvt/validationClass/tidalStats.py new file mode 100644 index 0000000..4caa669 --- /dev/null +++ b/build/lib/pyseidon_dvt/validationClass/tidalStats.py @@ -0,0 +1,715 @@ +#!/usr/bin/python2.7 +# encoding: utf-8 + +from __future__ import division + +import numpy as np +from scipy.stats import t, pearsonr +from datetime import datetime, timedelta +from scipy.interpolate import interp1d +from scipy.signal import correlate +import time +import pandas as pd +from sys import exit + +# local imports +from pyseidon_dvt.utilities.BP_tools import principal_axis + +# Custom error +from pyseidon_dvt.utilities.pyseidon_error import PyseidonError + +class TidalStats: + """ + An object representing a set of statistics on tidal heights used + to determine the skill of a model in comparison to observed data. + Standards are from NOAA's Standard Suite of Statistics. + + Instantiated with two arrays containing predicted and observed + data which have already been interpolated so they line up, the + time step between points, and the start time of the data. + + To remove NaNs in observed data, linear interpolation is performed to + fill gaps. Additionally, NaNs are trimmed from the start and end. + + Functions are used to calculate statistics and to output + visualizations and tables. + """ + def __init__(self, gear, model_data, observed_data, time_step, start_time, + model_u = [], observed_u = [], model_v= [], observed_v = [], + model_time = [], observed_time = [], phase_shift=False, + kind='', debug=False, debug_plot=False): + if debug: print "TidalStats initialisation..." + self._debug = debug + self._debug_plot = debug_plot + self.gear = gear # Type of measurement gear (drifter, adcp,...), str + self.model = np.asarray(model_data) + self.model = self.model.astype(np.float64) + self.observed = np.asarray(observed_data) + self.observed = self.observed.astype(np.float64) + self.model_u = np.asarray(model_u) + self.model_u = self.model_u.astype(np.float64) + self.observed_u = np.asarray(observed_u) + self.observed_u = self.observed_u.astype(np.float64) + self.model_v = np.asarray(model_v) + self.model_v = self.model_v.astype(np.float64) + self.observed_v = np.asarray(observed_v) + self.observed_v = self.observed_v.astype(np.float64) + self.kind = kind + self.step = time_step + self.length = model_data.size + + # TR: pass this step if dealing with Drifter's data + if not self.gear == 'Drifter': + try: + # TR: fix for interpolation pb when 0 index or -1 index array values = nan + if debug: print "...trim nans at start and end of data.." + start_index, end_index = 0, -1 + while np.isnan(self.observed[start_index]) or np.isnan(self.model[start_index]): + start_index += 1 + while np.isnan(self.observed[end_index]) or np.isnan(self.model[end_index]): + end_index -= 1 + except IndexError: # due too nans everywhere, itself due to no obs data at requested depth + raise PyseidonError("-No matching measurement-") + + # Correction for bound index call + if end_index == -1: + end_index = None + else: + end_index += 1 + if debug: print "Start index: ", start_index + if debug: print "End index: ", end_index + + m = self.model[start_index:end_index] + o = self.observed[start_index:end_index] + + setattr(self, 'model', m) + setattr(self, 'observed', o) + + # set up array of datetimes corresponding to the data (and timestamps) + self.times = start_time + np.arange(self.model.size) * time_step + timestamps = np.zeros(len(self.times)) + for j, jj in enumerate(self.times): + timestamps[j] = time.mktime(jj.timetuple()) + + if debug: print "...uses linear interpolation to eliminate any NaNs in the data..." + + if True in np.isnan(self.observed): + time_nonan = timestamps[np.where(~np.isnan(self.observed))[0]] + obs_nonan = self.observed[np.where(~np.isnan(self.observed))[0]] + func = interp1d(time_nonan, obs_nonan) + self.observed = func(timestamps) + if True in np.isnan(self.model): + time_nonan = timestamps[np.where(~np.isnan(self.model))[0]] + mod_nonan = self.model[np.where(~np.isnan(self.model))[0]] + func = interp1d(time_nonan, mod_nonan) + self.model = func(timestamps) + + # TR: pass this step if dealing with Drifter's data + else: + self.times = start_time + np.arange(self.model.size) * time_step # needed for plots, en seconds + #TR: those are not the real times though + + # Applying phase shift correction if needed + if phase_shift: + if debug: print "...applying phase shift..." + try: # Fix for Drifter's data + step_sec = time_step.seconds + except AttributeError: + step_sec = time_step * 24.0 * 60.0 * 60.0 # converts matlabtime to seconds + phase = self.getPhase() + phaseIndex = int(phase * 60.0 / step_sec) + if debug: print "Phase index = "+str(phaseIndex) + # create shifted data) + # if (phaseIndex < 0): + # # left shift + # iM = np.s_[-phaseIndex:] + # iO = np.s_[:self.length + phaseIndex] + # elif (phaseIndex > 0): + # # right shift + # iM = np.s_[:self.length - phaseIndex] + # iO = np.s_[phaseIndex:] + # else: # if (phaseIndex == 0): + # # no shift + # iM = np.s_[:] + # iO = np.s_[:] + # self.model = self.model[iM] + # self.observed = self.observed[iO] + # if not model_u == []: + # self.model_u = self.model_u[iM] + # if not model_v == []: + # self.model_v = self.model_v[iM] + # if not observed_u == []: + # self.observed_u = self.observed_v[iO] + # if not observed_v == []: + # self.observed_v = self.observed_v[iO] + self.model = np.roll(self.model, phaseIndex) + if not model_u == []: + self.model_u = np.roll(self.model_u, phaseIndex) + if not model_v == []: + self.model_v = np.roll(self.model_v, phaseIndex) + if debug: print "...TidalStats initialisation done." + + # Error attributes + if self.kind in ['cubic speed', 'velocity', 'direction']: + # TR: pass this step if dealing with Drifter's data + if not self.gear == 'Drifter': + # interpolate cubic speed, u and v on same time steps + model_timestamps = np.zeros(len(model_time)) + for j, jj in enumerate(model_time): + model_timestamps[j] = time.mktime(jj.timetuple()) + obs_timestamps = np.zeros(len(observed_time)) + for j, jj in enumerate(observed_time): + obs_timestamps[j] = time.mktime(jj.timetuple()) + func_u = interp1d(model_timestamps, model_u) + self.model_u = func_u(timestamps) + func_v = interp1d(model_timestamps, model_v) + self.model_v = func_v(timestamps) + func_u = interp1d(obs_timestamps, observed_u) + self.observed_u = func_u(timestamps) + func_v = interp1d(obs_timestamps, observed_v) + self.observed_v = func_v(timestamps) + + # if self.kind == 'cubic speed': + # # R.Karsten formula + # self.error = ((self.model_u**2.0 + self.model_v**2.0)**(3.0/2.0)) - \ + # ((self.observed_u**2.0 + self.observed_v**2.0)**(3.0/2.0)) + # else: + # self.error = self.observed - self.model + self.error = self.observed - self.model + elif self.kind in ['speed', 'elevation', 'u velocity', 'v velocity', 'Phase']: + self.error = self.observed - self.model + else: + print "---Data kind not supported---" + exit() + + # if debug: print "...establish limits as defined by NOAA standard..." + # if self.kind == 'velocity': + # self.ERROR_BOUND = 0.26 + # elif (self.kind == 'speed' or self.kind == 'velocity'): + # self.ERROR_BOUND = 0.26 + # elif (self.kind == 'elevation'): + # self.ERROR_BOUND = 0.15 + # elif (self.kind == 'direction'): + # self.ERROR_BOUND = 22.5 + # elif (self.kind == 'u velocity' or self.kind == 'v velocity'): + # self.ERROR_BOUND = 0.35 + # elif self.kind == 'cubic speed': + # self.ERROR_BOUND = 0.26**3.0 + # else: + # self.ERROR_BOUND = 0.5 + + # instead of using the NOAA errors, use 10% of the data range + obs_range = 0.1 * (np.nanmax(self.observed) - np.nanmin(self.observed)) + mod_range = 0.1 * (np.nanmax(self.model) - np.nanmin(self.model)) + self.ERROR_BOUND = (obs_range + mod_range) / 2. + + return + + def getRMSE(self, debug=False): + ''' + Returns the root mean squared error of the data. + ''' + if debug or self._debug: print "...getRMSE..." + if self.kind == 'velocity': + # Special definition of rmse - R.Karsten + rmse = np.sqrt(np.nanmean((self.model_u - self.observed_u)**2.0 + (self.model_v - self.observed_v)**2.0)) + else: + rmse = np.sqrt(np.nanmean(self.error**2)) + return rmse + + + def getNRMSE(self, debug=False): + """ + Returns the normalized root mean squared error between the model and + observed data in %. + """ + if debug or self._debug: print "...getNRMSE..." + if self.kind == 'velocity': + # Special definition of rmse - R.Karsten + rmse0 = np.sqrt(np.nanmean((self.observed_u)**2.0 + (self.observed_v)**2.0)) + else: + rmse0 = np.sqrt(np.mean(self.observed**2.0)) + # return 100. * self.getRMSE() / (max(self.observed) - min(self.observed)) + return 100. * self.getRMSE() / rmse0 + + def getSD(self, debug=False): + ''' + Returns the standard deviation of the error. + ''' + if debug or self._debug: print "...getSD..." + return np.sqrt(np.nanmean(abs(self.error - np.nanmean(self.error)**2))) + + def getBias(self, debug=False): + """ + Returns the bias of the model, a measure of over/under-estimation. + """ + if debug or self._debug: print "...getBias..." + return np.nanmean(self.error) + + def getSI(self, debug=False): + """ + Returns the scatter index of the model, a weighted measure of data + scattering. + """ + if debug or self._debug: print "...getSI..." + return self.getRMSE() / np.nanmean(self.observed) + + def getPBIAS(self, debug=False): + """ + Returns the percent bias between the model and the observed data. + + References: + Yapo P. O., Gupta H. V., Sorooshian S., 1996. + Automatic calibration of conceptual rainfall-runoff models: sensitivity to calibration data. + Journal of Hydrology. v181 i1-4. 23-48 + + Sorooshian, S., Q. Duan, and V. K. Gupta. 1993. + Calibration of rainfall-runoff models: Application of global optimization + to the Sacramento Soil Moisture Accounting Model. + Water Resources Research, 29 (4), 1185-1194, doi:10.1029/92WR02617. + """ + if debug or self._debug: print "...getPBIAS..." + + # if self.kind in ['elevation', 'direction', 'u velocity', 'v velocity', 'velocity']: + # norm_error = self.error / self.observed + # pbias = 100. * np.sum(norm_error) / norm_error.size + # else: + # norm_error = self.model - self.observed + # pbias = 100. * (np.sum(norm_error) / np.sum(self.observed)) + # # TR: this formula may lead to overly large values when used with sinusoidal signals + + pbias = 100. * (np.nansum(self.error) / np.nansum(self.observed)) + + return pbias + + + def getNSE(self, debug=False): + """ + Returns the Nash-Sutcliffe Efficiency coefficient of the model vs. + the observed data. Identifies if the model is better for + approximation than the mean of the observed data. + """ + SSE_mod = np.nansum((self.observed - self.model)**2) + SSE_mean = np.nansum((self.observed - np.nanmean(self.observed))**2) + return 1 - SSE_mod / SSE_mean + + def getCORR(self, debug=False): + """ + Returns the Pearson correlation coefficient for the model vs. + the observed data, a number between -1 and 1. -1 implies perfect + negative correlation, 1 implies perfect correlation. + """ + return pearsonr(self.observed, self.model)[0] + + def statsForDirection(self, debug=False): + """ + Special stats for direction + + Outputs: + - err = absolute error + - nerr = absolute error divided by standard deviation in % + """ + if debug: print "Computing special stats for direction..." + pr_axis_mod, pr_ax_var_mod = principal_axis(self.model_u, self.model_v) + pr_axis_obs, pr_ax_var_obs = principal_axis(self.observed_u, self.observed_v) + + # Defines intervals + dir_all_mod = self.model[:] + dir_all_obs = self.observed[:] + ind_mod = np.where(dir_all_mod<0) + ind_obs = np.where(dir_all_obs<0) + dir_all_mod[ind_mod] = dir_all_mod[ind_mod] + 360 + dir_all_obs[ind_obs] = dir_all_obs[ind_obs] + 360 + + # sign speed - eliminating wrap-around + dir_PA_mod = dir_all_mod - pr_axis_mod + dir_PA_mod[dir_PA_mod < -90] += 360 + dir_PA_mod[dir_PA_mod > 270] -= 360 + dir_PA_obs = dir_all_obs - pr_axis_obs + dir_PA_obs[dir_PA_obs < -90] += 360 + dir_PA_obs[dir_PA_obs > 270] -= 360 + + #general direction of flood passed as input argument + floodIndex_mod = np.where((dir_PA_mod >= -90) & (dir_PA_mod<90))[0] + ebbIndex_mod = np.arange(dir_PA_mod.shape[0]) + ebbIndex_mod = np.delete(ebbIndex_mod, floodIndex_mod[:]) + floodIndex_obs = np.where((dir_PA_obs >= -90) & (dir_PA_obs<90))[0] + ebbIndex_obs = np.arange(dir_PA_obs.shape[0]) + ebbIndex_obs = np.delete(ebbIndex_obs, floodIndex_obs[:]) + + # principal axis for ebb and flood + np.delete(floodIndex_mod, np.where(floodIndex_mod >= self.model_u.shape[0])) + np.delete(ebbIndex_mod, np.where(ebbIndex_mod >= self.model_u.shape[0])) + np.delete(floodIndex_obs, np.where(floodIndex_obs >= self.observed_u.shape[0])) + np.delete(ebbIndex_obs, np.where(ebbIndex_obs >= self.observed_u.shape[0])) + + pr_axis_mod_flood, pr_ax_var_mod_flood = principal_axis(self.model_u[floodIndex_mod], + self.model_v[floodIndex_mod]) + pr_axis_mod_ebb, pr_ax_var_mod_ebb = principal_axis(self.model_u[ebbIndex_mod], + self.model_v[ebbIndex_mod]) + pr_axis_obs_flood, pr_ax_var_obs_flood = principal_axis(self.observed_u[floodIndex_obs], + self.observed_v[floodIndex_obs]) + pr_axis_obs_ebb, pr_ax_var_obs_ebb = principal_axis(self.observed_u[ebbIndex_obs], + self.observed_v[ebbIndex_obs]) + err_flood = np.abs(pr_axis_mod_flood - pr_axis_obs_flood) + err_ebb = np.abs(pr_axis_mod_ebb - pr_axis_obs_ebb) + + if debug: print "...ebb direction error: " + str(err_ebb) + " degrees..." + if debug: print "...flood direction error: " + str(err_flood) + " degrees..." + + err = 0.5 * (err_flood + err_ebb) + + #std_flood = np.asarray(dir_all_obs[floodIndex_obs]).std() + #std_ebb = np.asarray(dir_all_obs[ebbIndex_obs]).std() + #std_flood = dir_all_obs[floodIndex_obs].max() - dir_all_obs[floodIndex_obs].min() + #std_ebb = dir_all_obs[ebbIndex_obs].max() - dir_all_obs[ebbIndex_obs].min() + #nerr = 0.5 * (np.abs(err_flood / std_flood) + np.abs(err_ebb / std_ebb)) + nerr = err / 90.0 + + return err, nerr * 100.0 + + + def getCF(self, debug=False): + ''' + Returns the central frequency of the data, i.e. the fraction of + errors that lie within the defined limit. + ''' + central_err = [i for i in self.error if abs(i) < self.ERROR_BOUND] + central_num = len(central_err) + if debug or self._debug: print "...getCF..." + return (float(central_num) / float(self.length)) * 100 + + def getPOF(self, debug=False): + ''' + Returns the positive outlier frequency of the data, i.e. the + fraction of errors that lie above the defined limit. + ''' + upper_err = [i for i in self.error if i > 2 * self.ERROR_BOUND] + upper_num = len(upper_err) + if debug or self._debug: print "...getPOF..." + return (float(upper_num) / float(self.length)) * 100 + + def getNOF(self, debug=False): + ''' + Returns the negative outlier frequency of the data, i.e. the + fraction of errors that lie below the defined limit. + ''' + lower_err = [i for i in self.error if i < -2 * self.ERROR_BOUND] + lower_num = len(lower_err) + if debug or self._debug: print "...getNOF..." + return (float(lower_num) / float(self.length)) * 100 + + def getMDPO(self, debug=False): + ''' + Returns the maximum duration of positive outliers, i.e. the + longest amount of time across the data where the model data + exceeds the observed data by a specified limit. + + Takes one parameter: the number of minutes between consecutive + data points. + ''' + try: #Fix for Drifter's data + timestep = self.step.seconds / 60 + except AttributeError: + timestep = self.step * 24.0 * 60.0 # converts matlabtime (in days) to minutes + + max_duration = 0 + current_duration = 0 + for i in np.arange(self.error.size): + if (self.error[i] > self.ERROR_BOUND): + current_duration += timestep + else: + if (current_duration > max_duration): + max_duration = current_duration + current_duration = 0 + if debug or self._debug: print "...getMDPO..." + return max(max_duration, current_duration) + + def getMDNO(self, debug=False): + ''' + Returns the maximum duration of negative outliers, i.e. the + longest amount of time across the data where the observed + data exceeds the model data by a specified limit. + + Takes one parameter: the number of minutes between consecutive + data points. + ''' + try: #Fix for Drifter's data + timestep = self.step.seconds / 60 + except AttributeError: + timestep = self.step * 24.0 * 60.0 # converts matlabtime (in days) to minutes + + max_duration = 0 + current_duration = 0 + for i in np.arange(self.error.size): + if (self.error[i] < -self.ERROR_BOUND): + current_duration += timestep + else: + if (current_duration > max_duration): + max_duration = current_duration + current_duration = 0 + if debug or self._debug: print "...getMDNO..." + return max(max_duration, current_duration) + + def getMSE(self, debug=False): + """ + Returns the mean square error (float) + """ + mse = np.nanmean(self.error**2.0) + if debug or self._debug: print "...getMSE..." + return mse + + def getNMSE(self, debug=False): + """ + Returns the normalized mean square error in % (float) + """ + mse0 = np.nanmean(self.observed**2.0) + nmse = 100. * self.getMSE() / mse0 + if debug or self._debug: print "...getNMSE..." + return nmse + + + def getWillmott(self, debug=False): + ''' + Returns the Willmott skill statistic. + ''' + + # start by calculating MSE + MSE = self.getMSE() + + # now calculate the rest of it + obs_mean = np.nanmean(self.observed) + skill = 1 - MSE / np.nanmean((abs(self.model - obs_mean) + + abs(self.observed - obs_mean))**2) + if debug or self._debug: print "...getWillmott..." + return skill + + def getPhase(self, max_phase=timedelta(hours=3), debug=False): + ''' + Attempts to find the phase shift between the model data and the + observed data. + + Iteratively tests different phase shifts, and calculates the RMSE + for each one. The shift with the smallest RMSE is returned. + + Argument max_phase is the span of time across which the phase shifts + will be tested. If debug is set to True, a plot of the RMSE for each + phase shift will be shown. + ''' + if debug or self._debug: print "getPhase..." + # grab the length of the timesteps in seconds + max_phase_sec = max_phase.seconds + try: # Fix for Drifter's data + step_sec = self.step.seconds + except AttributeError: + step_sec = self.step * 24.0 * 60.0 * 60.0 # converts matlabtime to seconds + + num_steps = max_phase_sec / step_sec + + if debug or self._debug: print "...iterate through the phase shifts and check RMSE..." + errors = [] + phases = np.arange(-num_steps, num_steps).astype(int) + for i in phases: + # create shifted data + shift_mod = np.roll(self.model, i) + # if (i < 0): + # # left shift + # #shift_mod = self.model[-i:] + # shift_obs = self.observed[:self.length + i] + # if (i > 0): + # # right shift + # shift_mod = self.model[:self.length - i] + # shift_obs = self.observed[i:] + # if (i == 0): + # # no shift + # shift_mod = self.model + # shift_obs = self.observed + + start = self.times[abs(i)] + step = self.times[1] - self.times[0] + + # create TidalStats class for shifted data and get the RMSE + #stats = TidalStats(self.gear, shift_mod, shift_obs, step, start, kind='Phase') + stats = TidalStats(self.gear, shift_mod, self.observed, step, start, kind='Phase') + nrms_error = stats.getNRMSE() + errors.append(nrms_error) + + if debug or self._debug: print "...find the minimum rmse, and thus the minimum phase..." + min_index = errors.index(min(errors)) + best_phase = phases[min_index] + phase_minutes = best_phase * step_sec / 60 + + return phase_minutes + + def altPhase(self, debug=False): + """ + Alternate version of lag detection using scipy's cross correlation function. + """ + if debug or self._debug: print "altPhase..." + # normalize arrays + mod = self.model + mod -= self.model.mean() + mod /= mod.std() + obs = self.observed + obs -= self.observed.mean() + obs /= obs.std() + + if debug or self._debug: print "...get cross correlation and find number of timesteps of shift..." + xcorr = correlate(mod, obs) + samples = np.arange(1 - self.length, self.length) + time_shift = samples[xcorr.argmax()] + + # find number of minutes in time shift + try: #Fix for Drifter's data + step_sec = self.step.seconds + except AttributeError: + step_sec = self.step * 24.0 * 60.0 * 60.0 # converts matlabtime (in days) to seconds + lag = time_shift * step_sec / 60 + + if debug or self._debug: print "...altPhase done." + + return lag + + def getStats(self, phase_shift=False, debug=False): + """ + Returns each of the statistics in a dictionary. + """ + + stats = {} + stats['gear'] = self.gear + stats['RMSE'] = self.getRMSE() + stats['CF'] = self.getCF() + stats['SD'] = self.getSD() + stats['POF'] = self.getPOF() + stats['NOF'] = self.getNOF() + stats['MDPO'] = self.getMDPO() + stats['MDNO'] = self.getMDNO() + stats['skill'] = self.getWillmott() + if not phase_shift: + stats['phase'] = self.getPhase(debug=debug) + else: + stats['phase'] = 0.0 + #stats['phase'] = self.getPhase() + stats['CORR'] = self.getCORR() + stats['NRMSE'] = self.getNRMSE() + stats['NSE'] = self.getNSE() + stats['bias'] = self.getBias() + stats['SI'] = self.getSI() + stats['pbias'] = self.getPBIAS() + stats['MSE'] = self.getMSE() + stats['NMSE'] = self.getNMSE() + + if debug or self._debug: print "...getStats..." + + return stats + + def linReg(self, alpha=0.05, debug=False): + ''' + Does linear regression on the model data vs. recorded data. + + Gives a 100(1-alpha)% confidence interval for the slope + ''' + if debug or self._debug: print "linReg..." + # set stuff up to make the code cleaner + obs = self.observed + mod = self.model + obs_mean = np.mean(obs) + mod_mean = np.mean(mod) + n = mod.size + df = n - 2 + + # calculate square sums + SSxx = np.sum(mod**2) - np.sum(mod)**2 / n + SSyy = np.sum(obs**2) - np.sum(obs)**2 / n + SSxy = np.sum(mod * obs) - np.sum(mod) * np.sum(obs) / n + SSE = SSyy - SSxy**2 / SSxx + MSE = SSE / df + + # estimate parameters + slope = SSxy / SSxx + intercept = obs_mean - slope * mod_mean + sd_slope = np.sqrt(MSE / SSxx) + r_squared = 1 - SSE / SSyy + + # calculate 100(1 - alpha)% CI for slope + width = t.isf(0.5 * alpha, df) * sd_slope + lower_bound = slope - width + upper_bound = slope + width + slope_CI = (lower_bound, upper_bound) + + # calculate 100(1 - alpha)% CI for intercept + lower_intercept = obs_mean - lower_bound * mod_mean + upper_intercept = obs_mean - upper_bound * mod_mean + intercept_CI = (lower_intercept, upper_intercept) + + # estimate 100(1 - alpha)% CI for predictands + predictands = slope * mod + intercept + sd_resid = np.std(obs - predictands) + y_CI_width = t.isf(0.5 * alpha, df) * sd_resid * \ + np.sqrt(1 - 1 / n) + + # return data in a dictionary + data = {} + data['slope'] = slope + data['intercept'] = intercept + data['r_2'] = r_squared + data['slope_CI'] = slope_CI + data['intercept_CI'] = intercept_CI + data['pred_CI_width'] = y_CI_width + data['conf_level'] = 100 * (1 - alpha) + + if debug or self._debug: print "...linReg done." + + return data + + def crossVal(self, alpha=0.05, debug=False): + ''' + Performs leave-one-out cross validation on the linear regression. + + i.e. removes one datum from the set, redoes linreg on the training + set, and uses the results to attempt to predict the missing datum. + ''' + if debug or self._debug: print "crossVal..." + cross_error = np.zeros(self.model.size) + cross_pred = np.zeros(self.model.size) + model_orig = self.model + obs_orig = self.observed + time_orig = self.time + + if debug or self._debug: print "...loop through each element, remove it..." + for i in np.arange(self.model.size): + train_mod = np.delete(model_orig, i) + train_obs = np.delete(obs_orig, i) + train_time = np.delete(time_orig, i) + train_stats = TidalStats(train_mod, train_obs, train_time) + + # redo the linear regression and get parameters + param = train_stats.linReg(alpha) + slope = param['slope'] + intercept = param['intercept'] + + # predict the missing observed value and calculate error + pred_obs = slope * model_orig[i] + intercept + cross_pred[i] = pred_obs + cross_error[i] = abs(pred_obs - obs_orig[i]) + + # calculate PRESS and PRRMSE statistics for predicted data + if debug or self._debug: print "...predicted residual sum of squares and predicted RMSE..." + PRESS = np.sum(cross_error**2) + PRRMSE = np.sqrt(PRESS) / self.model.size + + # return data in a dictionary + data = {} + data['PRESS'] = PRESS + data['PRRMSE'] = PRRMSE + data['cross_pred'] = cross_pred + + if debug or self._debug: print "...crossVal done." + + return data + + def save_data(self, path=''): + df = pd.DataFrame(data={'time': self.times.ravel(), + 'observed':self.observed.ravel(), + 'modeled':self.model.ravel() }) + df.to_csv(path+str(self.kind)+'.csv') diff --git a/build/lib/pyseidon_dvt/validationClass/valReport.py b/build/lib/pyseidon_dvt/validationClass/valReport.py new file mode 100644 index 0000000..0a20da0 --- /dev/null +++ b/build/lib/pyseidon_dvt/validationClass/valReport.py @@ -0,0 +1,402 @@ +#!/usr/bin/python2.7 +# encoding: utf-8 + +import os +import datetime +import matplotlib.cm as cmap +import numpy as np +from decimal import Decimal +from reportlab.lib import colors +from reportlab.lib.enums import TA_JUSTIFY +from reportlab.lib.pagesizes import A4, landscape +from reportlab.platypus import BaseDocTemplate, Frame, Paragraph, PageTemplate +from reportlab.platypus import Spacer, Table, Image, NextPageTemplate, PageBreak +from reportlab.lib.styles import getSampleStyleSheet, ParagraphStyle +from reportlab.lib.units import inch, cm +from reportlab.lib.utils import ImageReader + +# Custom error +from pyseidon_dvt.utilities.pyseidon_error import PyseidonError + + +def write_report(valClass, report_title="validation_report.pdf", debug=False): + """ + Write automatically and dynamically a report based on the validation processed + + Args: + valClass (pyseidon.Validation): Validation object + + Kwargs: + report_title (str): report file name + debug (bool): debug flag + """ + # tests + if not (hasattr(valClass, "Benchmarks") or hasattr(valClass, "HarmonicBenchmarks")): + raise PyseidonError("-Run validation function first-") + + benchflag = False + if hasattr(valClass, "Benchmarks"): + benchflag = True + + harmoflag = False + if hasattr(valClass, "HarmonicBenchmarks"): + harmoflag = True + + # date + now = datetime.date.today() + # initiate doc + doc = BaseDocTemplate(report_title, + pagesize=A4, + title="Validation Report " + now.strftime("- %d %B %Y"), + rightMargin=72,leftMargin=72, + topMargin=72,bottomMargin=18) + + # default style and frames + styles = getSampleStyleSheet() + styles.add(ParagraphStyle(name='Justify', alignment=TA_JUSTIFY)) + pgtemp = [] + imNb = -1 + + #normal frame as for SimpleFlowDocument + frameP = Frame(doc.leftMargin, doc.bottomMargin, doc.width, doc.height, id='normal') # portray frame + frameL = Frame(doc.leftMargin * 0.75, doc.bottomMargin, doc.height, doc.width, id='normal') # landscape frame + + #Two Columns + frame1 = Frame(doc.leftMargin, doc.bottomMargin, doc.width/2-6, doc.height, id='col1') + frame2 = Frame(doc.leftMargin+doc.width/2+6, doc.bottomMargin, doc.width/2-6, doc.height, id='col2') + + # initialize "story" list + story = [] + + # Title + story.append(Spacer(doc.width, A4[1]/3.0)) + story.append(Paragraph("Validation benchmarks and methodologies for FVCOM Hydrodynamic Model results", + styles['Title'])) + story.append(Spacer(doc.width, A4[1]/3.0)) + story.append(Paragraph(now.strftime("%B, %Y"), styles['BodyText'])) + pgtemp.append(PageTemplate(id='OneCol', frames=frameP, onPage=footpagenumber)) + + # Introduction + story.append(NextPageTemplate('OneCol')) + story.append(PageBreak()) + story.append(Paragraph("Introduction", styles['Heading1'])) + story.append(Paragraph("The following report is a summary of validation standards and \ + methodologies usually performed on the FVCOM model output data in comparison \ + to observed data. This collection and analysis of a set of \ + statistics mostly adhere to the benchmarks defined as standards for \ + hydrodynamic model validation by NOAA [1]. Additional statistics have been \ + added to provide additional clarity on the skill of the model [2, 3, 4]." + , styles['Justify'])) # , styles['BodyText'])) + story.append(Paragraph("The present validation set is performed the following variables:" + , styles['BodyText'])) + if benchflag: + story.append(Paragraph("Hydrodynamic quantities", styles['Italic'])) + story.append(Paragraph("el: Elevation (deviation from mean sea level, m)" + , styles['Bullet'], bulletText='-')) + story.append(Paragraph("cubic_speed: Signed cubic flow speed (m/s)" + , styles['Bullet'], bulletText='-')) + story.append(Paragraph("u: normal velocity component (m/s)" + , styles['Bullet'], bulletText='-')) + story.append(Paragraph("v: tangent velocity component (m/s)" + , styles['Bullet'], bulletText='-')) + story.append(Paragraph("vel: signed flow velocity (m/s)" + , styles['Bullet'], bulletText='-')) + story.append(Paragraph("speed: flow speed (m/s)" + , styles['Bullet'], bulletText='-')) + story.append(Paragraph("dir: Current direction (between -180 and 180 degrees)" + , styles['Bullet'], bulletText='-')) + if harmoflag: + story.append(Paragraph("Harmonic coefficients", styles['Italic'])) + story.append(Paragraph("A & A_ci: Amplitude and associated 95% confidence interval" + , styles['Bullet'], bulletText='-')) + story.append(Paragraph("g & g_ci: Greenwich phase lag and associated 95% confidence interval" + , styles['Bullet'], bulletText='-')) + story.append(Paragraph("theta & theta_ci: Current ellipse orientation angle and associated 95% confidence interval" + , styles['Bullet'], bulletText='-')) + story.append(Paragraph("Lsmin & Lsmin_ci: Current ellipse minor axis length and associated 95% confidence interval" + , styles['Bullet'], bulletText='-')) + story.append(Paragraph("Lsmaj & Lsmaj_ci: Current ellipse major axis length and associated 95% confidence interval" + , styles['Bullet'], bulletText='-')) + story.append(Paragraph("constituent: tidal constituent's name" + , styles['Bullet'], bulletText='-')) + story.append(Spacer(1, 12)) + if benchflag and harmoflag: + pgtemp.append(PageTemplate(id='OneCol', frames=frameP, onPage=footpagenumber)) + + # Statistics + if benchflag and harmoflag: + story.append(NextPageTemplate('OneCol')) + story.append(PageBreak()) + + story.append(Paragraph("Statistics", styles['Heading1'])) + if benchflag: + story.append(Paragraph("Following is a list of the statistics used to evaluate model skill, separated \ + into two categories: NOAA's Standard Suite of Statistics (SSS), and those added \ + (Additional)" + , styles['BodyText'])) + story.append(Paragraph("SSS", styles['Heading2'])) + story.append(Paragraph("RMSE: Root Mean Squared Error" + , styles['Bullet'], bulletText='-')) + story.append(Paragraph("NRMSE: Normalized Root Mean Squared Error (in %)" + , styles['Bullet'], bulletText='-')) + story.append(Paragraph("MSE: Mean Square Error" + , styles['Bullet'], bulletText='-')) + story.append(Paragraph("NMSE: Normalized Mean Square Error (in %)" + , styles['Bullet'], bulletText='-')) + story.append(Paragraph("SD: Standard Deviation of Error" + , styles['Bullet'], bulletText='-')) + story.append(Paragraph("CF(X): Central Frequency; percentage of error values that fall within the \ + range (-X, X)" + , styles['Bullet'], bulletText='-')) + story.append(Paragraph("POF(X): Positive Outlier Frequency; percentage of error values that fall \ + above X", + styles['Bullet'], bulletText='-')) + story.append(Paragraph("NOF(X): Negative Outlier Frequency; percentage of error values that fall \ + below -X" + , styles['Bullet'], bulletText='-')) + story.append(Paragraph("MDPO(X): Maximum Duration of Positive Outliers; longest number of \ + minutes during which consecutive errors fall above X" + , styles['Bullet'], bulletText='-')) + story.append(Paragraph("MDNO(X): Maximum Duration of Negative Outliers; longest number of \ + minutes during which consecutive errors fall below X" + , styles['Bullet'], bulletText='-')) + + story.append(Paragraph("Additional", styles['Heading2'])) + story.append(Paragraph("Willmott Skill: A measure of model adherence to observed data between \ + 0 and 1, with 0 being absolutely no adherence, and 1 being perfect" + , styles['Bullet'], bulletText='-')) + story.append(Paragraph("Phase: The phase shift (minutes) of model data that minimizes RMSE \ + across a timespan of +/-3hr" + , styles['Bullet'], bulletText='-')) + story.append(Paragraph("R2 (i.e. coefficient of determination): Measure of the strength of the linear \ + correlation between the model data and the observed data between 0 \ + and 1, with 0 being no correlation, and 1 being perfect correlation" + , styles['Bullet'], bulletText='-')) + story.append(Paragraph("bias: bias of the model, a measure of over/under-estimation" + , styles['Bullet'], bulletText='-')) + story.append(Paragraph("Pbias: percent bias between the model and the observed data" + , styles['Bullet'], bulletText='-')) + + if harmoflag: + story.append(Paragraph("The statistics reported in the 'Harmonic Analysis' section, \ + can be defined as the normalised error (in %) between the observed and simulated \ + harmonic analysis coefficients described in the 'Introduction' section." + , styles['BodyText'])) + + pgtemp.append(PageTemplate(id='OneCol', frames=frameP, onPage=footpagenumber)) + + # Results + story.append(NextPageTemplate('OneCol')) + story.append(PageBreak()) + story.append(Paragraph("Results", styles['Heading1'])) + story.append(Paragraph("The simulated and measured data sets, used in this document to generate the validation \ + benchmarks, cover a " + valClass.History[1].lower() + ".", styles['Justify'])) # , styles['BodyText'])) + story.append(Paragraph("The following map displays the location(s) as well as the type(s) of the measurement(s) \ + used in this validation report.", styles['Justify'])) # , styles['BodyText'])) + story.append(Spacer(1, 12)) + # Map: measurement's locations + imNb += 1 + savename = 'tmp_'+str(imNb)+'_plot.png' + lonmax = -1.0 * np.inf + lonmin = np.inf + latmax = -1.0 * np.inf + latmin = np.inf + for ii, coor in enumerate(valClass._coordinates): + lon = coor[0] + lat = coor[1] + if lon > lonmax: lonmax = lon + if lat > latmax: latmax = lat + if lon < lonmin: lonmin = lon + if lat < latmin: latmin = lat + # redefine colorbar min/max + margin = 0.01 + if valClass._multi_sim: + for sim in valClass._simulated: + try: + if 'fvcom' in sim.__module__: + fvcom = sim + break + except AttributeError: + continue + else: + fvcom = valClass._simulated + indices = np.where(np.logical_and( + np.logical_and(fvcom.Grid.lon[:] < lonmax + margin, + fvcom.Grid.lon[:] > lonmin - margin), + np.logical_and(fvcom.Grid.lat[:] < latmax + margin, + fvcom.Grid.lat[:] > latmin - margin)))[0] + cmax = fvcom.Grid.h[indices].max() + cmin = fvcom.Grid.h[indices].min() + fvcom.Plots.colormap_var(fvcom.Grid.h, + title='Bathymetric Map & Measurement location(s)', + cmax=cmax, cmin=cmin, isoline='var', mesh=False) + # redefine frame + fvcom.Plots._ax.set_xlim([lonmin - margin, lonmax + margin]) + fvcom.Plots._ax.set_ylim([latmin - margin, latmax + margin]) + color = cmap.rainbow(np.linspace(0, 1, len(valClass._coordinates))) + for ii, coor in enumerate(valClass._coordinates): + lon = coor[0] + lat = coor[1] + name = coor[2] + txt = str(ii) + fvcom.Plots._ax.scatter(lon, lat, label=name, + lw=2, s=50, color=color[ii]) + # fvcom.Plots._ax.annotate(txt, (lon, lat), size=20) + fvcom.Plots._ax.legend() + fvcom.Plots._fig.savefig(savename, format='png', bbox_inches='tight') + fvcom.Plots._fig.clear() + # image = Image(savename, width=doc.width, height=doc.height / 1.5) + # story.append(image) + story.append(get_image(savename, width=16*cm)) + + pgtemp.append(PageTemplate(id='OneCol', frames=frameP, onPage=footpagenumber)) + + # table's style + ts = [('ALIGN', (1,1), (-1,-1), 'CENTER'), + ('LINEABOVE', (0,0), (-1,0), 1, colors.black), + ('LINEBELOW', (0,0), (-1,0), 1, colors.black), + ('FONT', (0,0), (-1,0), 'Times-Bold')] + + if benchflag: + df = valClass.Benchmarks + values = df.values.tolist() + # clean up + for ii, l in enumerate(values): + for jj, e in enumerate(l): + if type(e) == float: + values[ii][jj] = float(Decimal("%.2f" % e)) + # Adding id. numbers + for ii in range(len(values)): + values[ii] = [str(ii + 1)] + values[ii] + lista = [['Id.'] + df.columns[:,].values.astype(str).tolist()] + values + nb_pages = -(-len(lista)/20) + if nb_pages == 1: + story.append(NextPageTemplate('landscape')) + story.append(PageBreak()) + story.append(Paragraph("Validation Benchmarks", styles['Heading2'])) + table = Table(lista, style=ts) # , repeatRows=1, rowSplitRange=20) + story.append(table) + pgtemp.append(PageTemplate(id='landscape', frames=frameL, onPage=make_landscape)) + else: + for i in range(nb_pages): + story.append(NextPageTemplate('landscape')) + story.append(PageBreak()) + if i == 0: + story.append(Paragraph("Validation Benchmarks", styles['Heading2'])) + table = Table(lista[i*20:(i+1)*20], style=ts) # , repeatRows=1, rowSplitRange=20) + else: + table = Table([lista[0]]+lista[(i*20)+1:(i+1)*20], style=ts) # , repeatRows=1, rowSplitRange=20) + story.append(table) + pgtemp.append(PageTemplate(id='landscape', frames=frameL, onPage=make_landscape)) + if harmoflag: + story.append(NextPageTemplate('OneCol')) + story.append(PageBreak()) + if type(valClass.HarmonicBenchmarks.elevation) != str: + story.append(Paragraph("Harmonic Analysis - Elevation", styles['Heading2'])) + df = valClass.HarmonicBenchmarks.elevation + values = df.values.tolist() + # clean up + for ii, l in enumerate(values): + for jj, e in enumerate(l): + if type(e) == float: + values[ii][jj] = float(Decimal("%.2f" % e)) + # Adding id. numbers + for ii in range(len(values)): + values[ii] = [str(ii+1)]+values[ii] + lista = [['Id.'] + df.columns[:,].values.astype(str).tolist()] + values + table = Table(lista, style=ts) + story.append(table) + if type(valClass.HarmonicBenchmarks.velocity) != str: + story.append(Paragraph("Harmonic Analysis - Velocity", styles['Heading2'])) + df = valClass.HarmonicBenchmarks.velocity + values = df.values.tolist() + # clean up + for ii, l in enumerate(values): + for jj, e in enumerate(l): + if type(e) == float: + values[ii][jj] = float(Decimal("%.2f" % e)) + lista = [df.columns[:,].values.astype(str).tolist()] + values + table = Table(lista, style=ts) + story.append(table) + pgtemp.append(PageTemplate(id='OneCol', frames=frameP, onPage=footpagenumber)) + + # Taylor diagram + if benchflag: + story.append(NextPageTemplate('OneCol')) + story.append(PageBreak()) + story.append(Paragraph("Taylor Diagram", styles['Heading2'])) + story.append(Paragraph("The Taylor diagram below is a concise statistical summary of how well patterns match each \ + other in terms of their correlation, their root-mean-square difference and the ratio of \ + their variances.", styles['Justify'])) # , styles['BodyText'])) + story.append(Paragraph("Measurements' identification numbers (Id.) \ + can be found in the table above.", styles['Justify'])) # , styles['BodyText'])) + story.append(Spacer(1, 12)) + imNb += 1 + savename = 'tmp_'+str(imNb)+'_plot.png' + valClass.taylor_diagram(savepath="./", fname=savename, labels=False) + # image = Image(savename, width=doc.width , height=doc.height / 2.25) + # story.append(image) + story.append(get_image(savename, width=16*cm)) + + pgtemp.append(PageTemplate(id='OneCol', frames=frameP, onPage=footpagenumber)) + + # References + story.append(NextPageTemplate('OneCol')) + story.append(PageBreak()) + story.append(Paragraph("References", styles['Heading1'])) + story.append(Paragraph("K. W. Hess, T. F. Gross, R. A. Schmalz, J. G. Kelley, F. Aikman and E. Wei, \ + NOS standards for evaluating operational nowcast and forecst hydrodynamic model systems, \ + National Oceanic and Atmospheric Administration, Silver Srping, Maryland, 2003.", + styles['Italic'], bulletText='[1]')) + story.append(Paragraph("K. Gunn and C. Stock-Williams, On validating numerical hydrodynamic models of complex \ + tidal flow, International Journal of Marine Energy, Vols. 3-4, no. Special, pp. 82-97, \ + 2013.", + styles['Italic'], bulletText='[2]')) + story.append(Paragraph("N. Georgas and A. F. Blumberg, Establishing Confidence in Marine Forecast Systems: \ + The Design and Skill Assessment of the New York Harbor Observation and Prediction System, \ + Version 3 (NYHOPS v3), in 11th International Conference on Estuarine and Coastal Modeling, \ + Seattle, Washington, United States, 2010.", + styles['Italic'], bulletText='[3]')) + story.append(Paragraph("Y. Liu, P. MacCready, H. M. Barbara, E. P. Dever, M. Kosro and N. S. Banas, \ + Evaluation of a coastal ocean circulation model for the Columbia River plume in summer \ + 2004, Journal of Geophysical Research, vol. 114, no. C2, p. 1978–2012, 2009.", + styles['Italic'], bulletText='[4]')) + + pgtemp.append(PageTemplate(id='OneCol', frames=frameP, onPage=footpagenumber)) + + # start the construction of the pdf + doc.addPageTemplates(pgtemp) + doc.build(story) + # remove tmp plots + for ii in range(imNb+1): + savename = 'tmp_'+str(ii)+'_plot.png' + os.remove(savename) + + +def footpagenumber(canvas, doc): + """ + Foot note and page number + """ + canvas.saveState() + canvas.setFont('Times-Roman', 9) + canvas.drawString(A4[0]/2.0, 0.75 * inch, "%d" % doc.page) + canvas.restoreState() + +def make_landscape(canvas,doc): + """ + Foot note and page number plus landscape page formatting + """ + canvas.saveState() + canvas.setPageSize(landscape(A4)) + canvas.setFont('Times-Roman', 9) + canvas.drawString(A4[1]/2.0, 0.75 * inch, "%d" % doc.page) + canvas.restoreState() + +def get_image(path, width=1*cm): + """ + Read image from file and fit its size to the given width + """ + img = ImageReader(path) + iw, ih = img.getSize() + aspect = ih / float(iw) + return Image(path, width=width, height=(width * aspect)) \ No newline at end of file diff --git a/build/lib/pyseidon_dvt/validationClass/valTable.py b/build/lib/pyseidon_dvt/validationClass/valTable.py new file mode 100644 index 0000000..09fd58e --- /dev/null +++ b/build/lib/pyseidon_dvt/validationClass/valTable.py @@ -0,0 +1,75 @@ +#!/usr/bin/python2.7 +# encoding: utf-8 +import pandas as pd +# Custom error +from pyseidon_dvt.utilities.pyseidon_error import PyseidonError + +# ALTERNATE VERSION FOR ANDY + +def valTable(struct, suites, save_path, filename, vars, save_csv=False, debug=False, debug_plot=False): + ''' + Takes validation data from the struct and saves it into a .csv file . + + Takes a single argument, a dictionary + ''' + # initialize lists + kind, name, RMSE, CF, SD, POF, NOF, MDPO, MDNO, skill, r2, phase = \ + [], [], [], [], [], [], [], [], [], [], [], [] + bias, pbias, NRMSE, NSE, MSE, NMSE, corr, SI, gear = [], [], [], [], [], [], [], [], [] + + # append to the lists the stats from each site for each variable + for var in vars: + (kind, name, RMSE, CF, SD, POF, NOF, MDPO, MDNO, skill, r2, phase, bias, pbias, NRMSE, NSE, MSE, NMSE, corr, SI, gear) \ + = siteStats(struct, suites, var, kind, name, RMSE, CF, SD, POF, NOF, MDPO, MDNO, skill, r2, phase, + bias, pbias, NRMSE, NSE, MSE, NMSE, corr, SI, gear, debug=False, debug_plot=False) + + # put stats into dict and create dataframe + val_dict = {'Type': kind, 'RMSE': RMSE, 'NRMSE': NRMSE, 'CF': CF, 'SD': SD, 'POF': POF, + 'NOF': NOF, 'MDPO': MDPO, 'MDNO': MDNO, 'skill': skill, 'r2': r2, 'phase': phase, + 'bias': bias, 'pbias': pbias, 'NSE': NSE, 'MSE': MSE, 'NMSE': NMSE, + 'corr': corr, 'SI': SI, 'gear': gear} + + table = pd.DataFrame(data=val_dict, index=name, columns=val_dict.keys()) + + # export as .csv file + if save_csv: + out_file = '{}{}_val.csv'.format(save_path, filename) + table.to_csv(out_file) + return table + +def siteStats(site, suites, variable, type, name, RMSE, CF, SD, POF, NOF, MDPO, MDNO, skill, r2, phase, + bias, pbias, NRMSE, NSE, MSE, NMSE, corr, SI, gear, debug=False, debug_plot=False): + """ + Takes in the run (an array of dictionaries) and the type of the run (a + string). Also takes in the list representing each statistic. + """ + if debug: print "siteStats..." + + stats = suites['{}'.format(variable)] + type.append(variable) + name.append(site['name'].split('/')[-1].split('.')[0]) + + + # add the statistics to the list, round to 2 decimal places + RMSE.append(round(stats['RMSE'], 2)) + CF.append(round(stats['CF'], 2)) + SD.append(round(stats['SD'], 2)) + POF.append(round(stats['POF'], 2)) + NOF.append(round(stats['NOF'], 2)) + MDPO.append(stats['MDPO']) + MDNO.append(stats['MDNO']) + skill.append(round(stats['skill'], 2)) + r2.append(round(stats['r_squared'], 2)) + phase.append(stats['phase']) + bias.append(round(stats['bias'], 2)) + pbias.append(round(stats['pbias'], 2)) + NRMSE.append(round(stats['NRMSE'], 2)) + NSE.append(round(stats['NSE'], 2)) + MSE.append(round(stats['MSE'], 2)) + NMSE.append(round(stats['NMSE'], 2)) + corr.append(round(stats['CORR'], 2)) + SI.append(round(stats['SI'], 2)) + gear.append(site['type']) + if debug: print "...siteStats done." + + return (type, name, RMSE, CF, SD, POF, NOF, MDPO, MDNO, skill, r2, phase, bias, pbias, NRMSE, NSE, MSE, NMSE, corr, SI, gear) diff --git a/build/lib/pyseidon_dvt/validationClass/validationClass.py b/build/lib/pyseidon_dvt/validationClass/validationClass.py new file mode 100644 index 0000000..27d0412 --- /dev/null +++ b/build/lib/pyseidon_dvt/validationClass/validationClass.py @@ -0,0 +1,760 @@ +#!/usr/bin/python2.7 +# encoding: utf-8 + +from __future__ import division + +import numpy as np +import pandas as pd +import cPickle as pkl +from os import makedirs +from os.path import exists + +# Polar Plots +from math import * +from cmath import * + +# Quick fix +from scipy.io import savemat +from utide import solve + +# Local import +from compareData import * +from valTable import valTable +from variablesValidation import _load_validation +from pyseidon_dvt.utilities.interpolation_utils import * + +# Local import +from plotsValidation import taylorDiagram, benchmarksMap +from valReport import write_report +from pyseidon_dvt.utilities.miscellaneous import mattime_to_datetime + +# Custom error +from pyseidon_dvt.utilities.pyseidon_error import PyseidonError + + +class Validation: + """ + **Validation class/structure** + + Class structured as follows: :: + + _History = Quality Control metadata + |_Variables. = observed and simulated variables and quantities + |_validate_data = validation method/function against timeseries + Validation._|_validate_harmonics = validation method/function against + | harmonic coefficients + |_Save_as = "save as" function + + Inputs: + - observed = standalone or list of PySeidon measurement object (i.e. ADCP, TideGauge, Drifter,...) + - simulated = standalone or list of any PySeidon simulation object (i.e. FVCOM or Station) + Option: + - flow = impose flow comparison by surface flow ('sf'), depth-averaged flow ('daf') or at any depth (float) + , if negative = from sea surface downwards, if positive = from sea bottom upwards + - nn = if True then use the nearest location in the grid if the location is outside the grid. + - outpath = specify a path to save validation results (default = './') + """ + def __init__(self, observed, simulated, flow=[], nn=True, outpath='./', debug=False, debug_plot=False): + self._debug = debug + self._flow = flow + self._coordinates = [] + self._fig = None + self._ax = None + if type(observed) in [tuple, list]: + self._multi_meas = True + else: + self._multi_meas = False + if type(simulated) in [tuple, list]: + self._multi_sim = True + else: + self._multi_sim = False + self._debug_plot = debug_plot + if debug: print '-Debug mode on-' + self._nn=nn + if debug and nn: print '-Using nearest neighbour-' + # creates folder to store outputs + outpath=outpath.replace(" ","_") + if outpath[-1] is not '/': + self._outpath = outpath+'/' + else: + self._outpath = outpath + if not self._outpath == './': + while exists(self._outpath): + self._outpath = self._outpath[:-1] + '_bis/' + + #Metadata + if not self._multi_sim: + sim_origin = simulated._origin_file + else: + sim_origin = 'multiple simulation objects' + if not self._multi_meas: + self.History = ['Created from ' + observed._origin_file +\ + ' and ' + sim_origin] + else: + self.History = ['Created from multiple measurement sources' +\ + ' and ' + sim_origin] + if (not self._multi_meas) and (not self._multi_sim): + self._observed = observed + self._simulated = simulated + self.Variables = _load_validation(self._outpath, self._observed, self._simulated, flow=self._flow, nn=self._nn, debug=self._debug) + self._coordinates.append([np.mean(self.Variables.obs.lon), np.mean(self.Variables.obs.lat), self.Variables._obstype]) + + # -Append message to History field + start = mattime_to_datetime(self.Variables.obs.matlabTime[self.Variables._c[0]]) + end = mattime_to_datetime(self.Variables.obs.matlabTime[self.Variables._c[-1]]) + text = 'Temporal domain from ' + str(start) + ' to ' + str(end) + self.History.append(text) + elif (self._multi_meas) and (not self._multi_sim): + self._observed = observed + self._simulated = [simulated] + elif (not self._multi_meas) and (self._multi_sim): + self._observed = [observed] + self._simulated = simulated + else: + self._observed = observed + self._simulated = simulated + + # Creates folder once compatibility test passed + if not self._outpath == './': + makedirs(self._outpath) + if debug: print '-Saving results to {}-'.format(self._outpath) + return + + def _validate_data(self, filename=[], depth=[], slack_velo=0.1, plot=False, save_csv=False, phase_shift=False, + debug=False, debug_plot=False): + """ + This method computes series of standard validation benchmarks. + + Options: + - filename = file name of the .csv file to be saved, string. + - depth = depth at which the validation will be performed, float. + Only applicable for 3D simulations. + - slack_velo = slack water's velocity (m/s), float, everything below will be dumped out + - plot = plot series of validation graphs, boolean. + as well as associated plots in specific folder + - phase_shift = applies phase shift correction to model quantities + + + *References* + - NOAA. NOS standards for evaluating operational nowcast and + forecast hydrodynamic model systems, 2003. + + - K. Gunn, C. Stock-Williams. On validating numerical hydrodynamic + models of complex tidal flow, International Journal of Marine Energy, 2013 + + - N. Georgas, A. Blumberg. Establishing Confidence in Marine Forecast + Systems: The design and skill assessment of the New York Harbor Observation + and Prediction System, version 3 (NYHOPS v3), 2009 + + - Liu, Y., P. MacCready, B. M. Hickey, E. P. Dever, P. M. Kosro, and + N. S. Banas (2009), Evaluation of a coastal ocean circulation model for + the Columbia River plume in summer 2004, J. Geophys. Res., 114 + """ + debug = debug or self._debug + debug_plot = debug_plot or self._debug_plot + # User input + if filename == []: + filename = raw_input('Enter filename for csv file: ') + filename = str(filename) + if type(self._flow) == float: + depth = self._flow + if (depth == [] and self.Variables._3D): + depth = input('Depth from surface at which the validation will be performed: ') + depth = float(depth) + if depth == []: depth = 5.0 + + + # Harmonically reconstruct simulation properties at the original observed matlabTime if harmo is on + if self.Variables.harmo['On']: + observed = self.Variables.harmo['Observed'] + simulated = self.Variables.harmo['Simulated'] + # Harmonic Analysis + if self.Variables._simtype == 'station': + harmo_simEl = simulated.Util2D.Harmonic_analysis_at_point(self.Variables.harmo['nameSite'], elevation=True, velocity=False) + harmo_simVel = simulated.Util2D.Harmonic_analysis_at_point(self.Variables.harmo['nameSite'], elevation=False, velocity=True) + elif self.Variables._simtype == 'fvcom': + harmo_simEl = simulated.Util2D.Harmonic_analysis_at_point(observed.Variables.lon, observed.Variables.lat, elevation=True, velocity=False) + harmo_simVel = simulated.Util2D.Harmonic_analysis_at_point(observed.Variables.lon, observed.Variables.lat, elevation=False, velocity=True) + else: + raise PyseidonError("--Harmonic Analysis not possible with this type of measurement--") + # Reconstruction at recon_time + simEl = simulated.Util2D.Harmonic_reconstruction(harmo_simEl, recon_time=observed.Variables.matlabTime) + simVel = simulated.Util2D.Harmonic_reconstruction(harmo_simVel, recon_time=observed.Variables.matlabTime) + # Pass reconstructed timeseries to variables structure for validation + self.Variables.struct['mod_timeseries']['ua'] = simVel['u'] + self.Variables.struct['mod_timeseries']['va'] = simVel['v'] + self.Variables.struct['mod_timeseries']['el'] = simEl['h'] + self.Variables.struct['mod_time'] = observed.Variables.matlabTime + + #initialisation + vars = [] + self.Suites={} + threeD = self.Variables.sim._3D + if self._flow == 'daf': threeD = False + + if self.Variables.struct['type'] == 'ADCP': + suites = compareOBS(self.Variables.struct, self.Variables._save_path, threeD, + plot=plot, depth=depth, slack_velo=slack_velo, save_csv=save_csv, + phase_shift=phase_shift, debug=debug, debug_plot=debug_plot) + + for key in suites: + self.Suites[key] = suites[key] + vars.append(key) + + elif self.Variables.struct['type'] == 'TideGauge': + suites = compareOBS(self.Variables.struct, self.Variables._save_path, + plot=plot, slack_velo=slack_velo, save_csv=save_csv, + phase_shift=phase_shift, debug=debug, debug_plot=debug_plot) + for key in suites: + self.Suites[key] = suites[key] + vars.append(key) + + elif self.Variables.struct['type'] == 'Drifter': + suites = compareOBS(self.Variables.struct, self.Variables._save_path, self.Variables._3D, + depth=depth, plot=plot, slack_velo=slack_velo, save_csv=save_csv, + phase_shift=phase_shift, debug=debug, debug_plot=debug_plot) + + for key in suites: + self.Suites[key] = suites[key] + vars.append(key) + + else: + raise PyseidonError("-This kind of measurements is not supported yet-") + + # Make csv file + self._Benchmarks = valTable(self.Variables.struct, self.Suites, self.Variables._save_path, filename, vars, + save_csv=save_csv, debug=debug, debug_plot=debug_plot) + + # Display csv + print "---Validation benchmarks---" + pd.set_option('display.max_rows', len(self._Benchmarks)) + print(self._Benchmarks) + pd.reset_option('display.max_rows') + + # Reload _load_validation to return original values + # Is this even needed? -Jeremy + #print '-- Reloading Validation Variables --' + #self.Variables = _load_validation(self._outpath, self._observed, self._simulated, flow=self._flow, nn=self._nn, debug=self._debug) + + def _validate_harmonics(self, filename='', save_csv=False, debug=False, debug_plot=False): + """ + This method computes and store in a csv file the error in % + for each component of the harmonic analysis (i.e. *_error.csv). + + Options: + filename: file name of the .csv file to be saved, string. + save_csv: will save both observed and modeled harmonic + coefficients into *.csv files (i.e. *_harmo_coef.csv) + """ + # check if measurement object is a Drifter + if self.Variables.struct['type'] == 'Drifter': + print "--- Harmonic analysis does not work with Drifter's data ---" + return + # define attributes + if not hasattr(self, "_HarmonicBenchmarks"): + self._HarmonicBenchmarks = HarmonicBenchmarks() + else: + delattr(self, '_HarmonicBenchmarks') + self._HarmonicBenchmarks = HarmonicBenchmarks() + # User input + if filename==[]: + filename = raw_input('Enter filename for csv file: ') + filename = str(filename) + + hasEL=False + hasUV=False + commonlist_data = self.Variables.struct['_commonlist_data'] + if 'el' in commonlist_data: + hasEL=True + + ulist=[var for var in ['ua', 'u'] if var in commonlist_data ] + vlist=[var for var in ['va', 'v'] if var in commonlist_data ] + if len(ulist)>0 and len(vlist)>0: + hasUV=True + + # Harmonic analysis over matching & non-matching time + obs_time = self.Variables.struct['obs_time'] + obs_lat = self.Variables.struct['obs_lat'] + mod_time = self.Variables.struct['mod_time'] + mod_lat = self.Variables.struct['mod_lat'] + if hasEL: + obs_el = self.Variables.struct['obs_timeseries']['el'][:] + mod_el = self.Variables.struct['mod_timeseries']['el'][:] + self.Variables.obs.elCoef = solve(obs_time, obs_el, None, obs_lat, + constit='auto', trend=False, Rayleigh_min=0.95, + method='ols', conf_int='linear') + self.Variables.sim.elCoef = solve(mod_time, mod_el, None, mod_lat, + constit='auto', trend=False, Rayleigh_min=0.95, + method='ols', conf_int='linear') + + if hasUV: + obs_ua = self.Variables.struct['obs_timeseries']['ua'][:] + obs_va = self.Variables.struct['obs_timeseries']['va'][:] + mod_ua = self.Variables.struct['mod_timeseries']['ua'][:] + mod_va = self.Variables.struct['mod_timeseries']['va'][:] + self.Variables.obs.velCoef = solve(obs_time, obs_ua, obs_va, obs_lat, + constit='auto', trend=False, Rayleigh_min=0.95, + method='ols', conf_int='linear') + self.Variables.sim.velCoef = solve(mod_time, mod_ua, mod_va, mod_lat, + constit='auto', trend=False, Rayleigh_min=0.95, + method='ols', conf_int='linear') + + + # find matching and non-matching coef & hold their magnitude and phase for comparison + matchElCoef = [] # Coefficient Names + matchElCoefInd = [] # Coefficient Names' Indexes + matchEl_Adiff = [] # Coefficents' Maginitude Difference + matchEl_gdiff = [] # Coefficients' Phase Difference + for i1, key1 in enumerate(self.Variables.sim.elCoef['name']): + for i2, key2 in enumerate(self.Variables.obs.elCoef['name']): + if key1 == key2: + matchElCoefInd.append((i1,i2)) + matchElCoef.append(key1) + matchEl_Adiff.append(abs(self.Variables.sim.elCoef['A'][i1]*exp(1j*radians(self.Variables.sim.elCoef['g'][i1])) - + self.Variables.obs.elCoef['A'][i2]*exp(1j*radians(self.Variables.obs.elCoef['g'][i2])))) + matchEl_gdiff.append(abs(self.Variables.sim.elCoef['g'][i1] - self.Variables.obs.elCoef['g'][i2])) + matchElCoefInd=np.array(matchElCoefInd) + noMatchElCoef = np.delete(self.Variables.sim.elCoef['name'], matchElCoefInd[:,0]) + np.hstack((noMatchElCoef, np.delete(self.Variables.obs.elCoef['name'], matchElCoefInd[:,1]))) + + # Repeat for Velocity Coefficients + matchVelCoef = [] + matchVelCoefInd = [] + matchVel_LsmajDiff = [] + matchVel_LsminDiff = [] + matchVel_gdiff = [] + try: + for i1, key1 in enumerate(self.Variables.sim.velCoef['name']): + for i2, key2 in enumerate(self.Variables.obs.velCoef['name']): + if key1 == key2: + matchVelCoefInd.append((i1, i2)) + matchVelCoef.append(key1) + matchVel_LsmajDiff.append(abs(self.Variables.sim.velCoef['Lsmaj'][i1]*exp(1j*radians(self.Variables.sim.velCoef['g'][i1])) - + self.Variables.obs.velCoef['Lsmaj'][i2]*exp(1j*radians(self.Variables.obs.velCoef['g'][i2])))) + matchVel_LsminDiff.append(abs(self.Variables.sim.velCoef['Lsmin'][i1]*exp(1j*radians(self.Variables.sim.velCoef['g'][i1])) - + self.Variables.obs.velCoef['Lsmin'][i2]*exp(1j*radians(self.Variables.obs.velCoef['g'][i2])))) + matchVel_gdiff.append(abs(self.Variables.sim.velCoef['g'][i1] - self.Variables.obs.velCoef['g'][i2])) + matchVelCoefInd = np.array(matchVelCoefInd) + noMatchVelCoef = np.delete(self.Variables.sim.velCoef['name'], matchVelCoefInd[:, 0]) + np.hstack((noMatchVelCoef, np.delete(self.Variables.obs.velCoef['name'], matchVelCoefInd[:, 1]))) + except AttributeError: + pass + + # Compare largest ten obs. vs sim. coefficients visually on Polar plots + # I would prefer to have these plots saved to the directory created by validate_harmonics() + top10_el = zip(matchEl_Adiff, matchElCoef, matchEl_gdiff) + top10_el.sort() + top10_el = top10_el[-10:] + top10_el = zip(*top10_el) + plt.axes(polar=True) + plt.plot(top10_el[2], top10_el[0], 'r.') + for i, txt in enumerate(top10_el[1]): + plt.annotate(top10_el[1][i], (str(top10_el[2][i]), str(top10_el[0][i])), size=8) + plt.title('Harmonic Analysis Elevation Coefficient comparison') + plt.savefig('HarmoEl_coeffcompare', format='png') + plt.clf() + + top10_vel_Lsmaj = zip(matchVel_LsmajDiff, matchVelCoef, matchVel_gdiff) + top10_vel_Lsmaj.sort() + top10_vel_Lsmaj = top10_vel_Lsmaj[-10:] + top10_vel_Lsmaj = zip(*top10_vel_Lsmaj) + plt.axes(polar=True) + plt.plot(top10_vel_Lsmaj[2], top10_vel_Lsmaj[0], 'r.') + for i, txt in enumerate(top10_vel_Lsmaj[1]): + plt.annotate(top10_vel_Lsmaj[1][i], (str(top10_vel_Lsmaj[2][i]), str(top10_vel_Lsmaj[0][i])), size=8) + plt.title('Harmonic Analysis Lsmaj Velocity Coefficient comparison') + plt.savefig('HarmoVel_Lsmaj_coeffcompare', format='png') + plt.clf() + + top10_vel_Lsmin = zip(matchVel_LsminDiff, matchVelCoef, matchVel_gdiff) + top10_vel_Lsmin.sort() + top10_vel_Lsmin = top10_vel_Lsmin[-10:] + top10_vel_Lsmin = zip(*top10_vel_Lsmin) + plt.axes(polar=True) + plt.plot(top10_vel_Lsmin[2], top10_vel_Lsmin[0], 'r.') + for i, txt in enumerate(top10_vel_Lsmin[1]): + plt.annotate(top10_vel_Lsmin[1][i], (str(top10_vel_Lsmin[2][i]), str(top10_vel_Lsmin[0][i])), size=8) + plt.title('Harmonic Analysis Lsmin Velocity Coefficient comparison') + plt.savefig('HarmoVel_Lsmin_coeffcompare', format='png') + plt.clf() + + # Compare obs. vs. sim. elevation harmo coef + data = {} + columns = ['A', 'g', 'A_ci', 'g_ci'] + measname = self.Variables.struct['name'].split('/')[-1].split('.')[0] + + # Store harmonics in csv files + if save_csv and hasEL: + # observed elevation coefs + for key in columns: + data[key] = self.Variables.obs.elCoef[key] + data['constituent'] = self.Variables.obs.elCoef['name'] + measlist = [measname] * len(data['constituent']) + table = pd.DataFrame(data=data, index=measlist, + columns=columns + ['constituent']) + # export as .csv file + out_file = '{}{}_obs_el_harmo_coef.csv'.format(self.Variables._save_path, filename) + table.to_csv(out_file) + data = {} + + #modeled elevation coefs + for key in columns: + data[key] = self.Variables.sim.elCoef[key] + data['constituent'] = self.Variables.sim.elCoef['name'] + measlist = [measname] * len(data['constituent']) + table = pd.DataFrame(data=data, index=measlist, + columns=columns + ['constituent']) + # export as .csv file + out_file = '{}{}_sim_el_harmo_coef.csv'.format(self.Variables._save_path, filename) + table.to_csv(out_file) + data = {} + + # error in % + if hasEL: + if not matchElCoef==[]: + for key in columns: + b=self.Variables.sim.elCoef[key][matchElCoefInd[:,0]] + a=self.Variables.obs.elCoef[key][matchElCoefInd[:,1]] + err = abs((a-b)/a) * 100.0 + data[key] = err + data['constituent'] = matchElCoef + measlist = [measname] * len(matchElCoef) + ##create table + table = pd.DataFrame(data=data, index=measlist, columns=columns + ['constituent']) + ##export as .csv file + out_file = '{}{}_el_harmo_error.csv'.format(self.Variables._save_path, filename) + table.to_csv(out_file) + ##print non-matching coefs + if not noMatchElCoef.shape[0]==0: + print "Non-matching harmonic coefficients for elevation: ", noMatchElCoef + else: + print "-No matching harmonic coefficients for elevation-" + + # save dataframe in attribute + self._HarmonicBenchmarks.elevation = table + + #Compare obs. vs. sim. velocity harmo coef + data = {} + columns = ['Lsmaj', 'g', 'theta_ci', 'Lsmin_ci', + 'Lsmaj_ci', 'theta', 'g_ci'] + + #Store harmonics in csv files + if save_csv and hasUV: + #observed elevation coefs + for key in columns: + data[key] = self.Variables.obs.velCoef[key] + data['constituent'] = matchVelCoef + measlist = [measname] * len(data['constituent']) + table = pd.DataFrame(data=data, index=measlist, + columns=columns + ['constituent']) + ##export as .csv file + out_file = '{}{}_obs_velo_harmo_coef.csv'.format(self.Variables._save_path, filename) + table.to_csv(out_file) + data = {} + + #modeled elevation coefs + for key in columns: + data[key] = self.Variables.sim.velCoef[key] + data['constituent'] = matchVelCoef + measlist = [measname] * len(data['constituent']) + table = pd.DataFrame(data=data, index=measlist, + columns=columns + ['constituent']) + ##export as .csv file + out_file = '{}{}_sim_velo_harmo_coef.csv'.format(self.Variables._save_path, filename) + table.to_csv(out_file) + data = {} + + ##error in % + if hasUV: + if not matchVelCoef==[]: + for key in columns: + b=self.Variables.sim.velCoef[key][matchVelCoefInd[:,0]] + a=self.Variables.obs.velCoef[key][matchVelCoefInd[:,1]] + err = abs((a-b)/a) * 100.0 + data[key] = err + data['constituent'] = matchVelCoef + measlist = [measname] * len(matchVelCoef) + ##create table + table = pd.DataFrame(data=data, index=measlist, columns=columns + ['constituent']) + ##export as .csv file + out_file = '{}{}_vel0_harmo_error.csv'.format(self.Variables._save_path, filename) + table.to_csv(out_file) + ##print non-matching coefs + if not noMatchVelCoef.shape[0]==0: + print "Non-matching harmonic coefficients for velocity: ", noMatchVelCoef + else: + print "-No matching harmonic coefficients for velocity-" + + # save dataframe in attribute + self._HarmonicBenchmarks.velocity = table + + def validate_data(self, filename=[], depth=[], slack_velo=0.1, plot=False, save_csv=False, phase_shift=False, + debug=False, debug_plot=False): + """ + This method computes series of standard validation benchmarks. + + Options: + - filename = file name of the .csv file to be saved, string. + - depth = depth at which the validation will be performed, float. + Only applicable for 3D simulations. + If negative = from sea surface downwards, if positive = from sea bottom upwards + - slack_velo = slack water's velocity (m/s), float, everything below will be dumped out + - plot = plot series of validation graphs, boolean. + - save_csv = will save benchmark values into *.csv file + as well as associated plots in specific folder + - phase_shift = applies phase shift correction to model quantities + + *References* + - NOAA. NOS standards for evaluating operational nowcast and + forecast hydrodynamic model systems, 2003. + + - K. Gunn, C. Stock-Williams. On validating numerical hydrodynamic + models of complex tidal flow, International Journal of Marine Energy, 2013 + + - N. Georgas, A. Blumberg. Establishing Confidence in Marine Forecast + Systems: The design and skill assessment of the New York Harbor Observation + and Prediction System, version 3 (NYHOPS v3), 2009 + + - Liu, Y., P. MacCready, B. M. Hickey, E. P. Dever, P. M. Kosro, and + N. S. Banas (2009), Evaluation of a coastal ocean circulation model for + the Columbia River plume in summer 2004, J. Geophys. Res., 114 + """ + if (not self._multi_meas) and (not self._multi_sim): + self._validate_data(filename, depth, slack_velo, plot, save_csv, phase_shift, debug, debug_plot) + self.Benchmarks = self._Benchmarks + else: + I=0 + for sim in self._simulated: + for meas in self._observed: + try: + self.Variables = _load_validation(self._outpath, meas, sim, flow=self._flow, debug=self._debug) + + # -Append message to History field + start = mattime_to_datetime(self.Variables.obs.matlabTime[self.Variables._c[0]]) + end = mattime_to_datetime(self.Variables.obs.matlabTime[self.Variables._c[-1]]) + text = 'Temporal domain from ' + str(start) + ' to ' + str(end) + try: + self.History[1] = text + except IndexError: + self.History.append(text) + + self._validate_data(filename, depth, slack_velo, plot, save_csv, phase_shift, debug, debug_plot) + if I == 0: + self.Benchmarks = self._Benchmarks + I += 1 + else: + self.Benchmarks = pd.concat([self.Benchmarks, self._Benchmarks]) + + self._coordinates.append([np.mean(self.Variables.obs.lon), + np.mean(self.Variables.obs.lat), + self.Variables._obstype]) + except PyseidonError: + #except: # making it even more permissive + print "Error with measurement object "+meas.History[0] + continue + if save_csv: + #if self._multi_meas: + # savepath = self.Variables._save_path[:(self.Variables._save_path[:-1].rfind('/')+1)] + #else: + # savepath = self.Variables._save_path + savepath = self._outpath + #savepath = self.Variables._save_path + try: + out_file = '{}{}_benchmarks.csv'.format(savepath, filename) + self.Benchmarks.to_csv(out_file) + except AttributeError: + raise PyseidonError("-No matching measurement-") + + def validate_harmonics(self, filename=[], save_csv=False, debug=False, debug_plot=False): + """ + This method computes and store in a csv file the error in % + for each component of the harmonic analysis (i.e. *_error.csv). + + Options: + filename: file name of the .csv file to be saved, string. + save_csv: will save both observed and modeled harmonic + coefficients into *.csv files (i.e. *_harmo_coef.csv) + """ + if (not self._multi_meas) and (not self._multi_sim): + self.Variables = _load_validation(self._outpath, self._observed, self._simulated, flow=self._flow, debug=self._debug) + self._validate_harmonics(filename, save_csv, debug, debug_plot) + self.HarmonicBenchmarks = self._HarmonicBenchmarks + else: + I=0 + J=0 + for sim in self._simulated: + for meas in self._observed: + try: + self.Variables = _load_validation(self._outpath, meas, sim, flow=self._flow, debug=self._debug) + self._validate_harmonics(filename, save_csv, debug, debug_plot) + if I == 0 and J == 0: + self.HarmonicBenchmarks = HarmonicBenchmarks() + if I == 0 and type(self._HarmonicBenchmarks.elevation) != str: + self.HarmonicBenchmarks.elevation = self._HarmonicBenchmarks.elevation + I += 1 + elif I != 0 and type(self._HarmonicBenchmarks.elevation) != str: + self.HarmonicBenchmarks.elevation = pd.concat([self.HarmonicBenchmarks.elevation, + self._HarmonicBenchmarks.elevation]) + if J == 0 and type(self._HarmonicBenchmarks.velocity) != str: + self.HarmonicBenchmarks.velocity = self._HarmonicBenchmarks.velocity + J += 1 + elif J != 0 and type(self._HarmonicBenchmarks.velocity) != str: + self.HarmonicBenchmarks.velocity = pd.concat([self.HarmonicBenchmarks.velocity, + self._HarmonicBenchmarks.velocity]) + except PyseidonError: + pass + + # Drop duplicated lines + self.HarmonicBenchmarks.velocity.drop_duplicates(inplace=True) + self.HarmonicBenchmarks.elevation.drop_duplicates(inplace=True) + + if save_csv: + #if self._multi_meas: + # savepath = self.Variables._save_path[:(self.Variables._save_path[:-1].rfind('/')+1)] + #else: + # savepath = self.Variables._save_path + savepath = self._outpath + try: + try: + out_file = '{}{}_elevation_harmonic_benchmarks.csv'.format(savepath, filename) + self.HarmonicBenchmarks.elevation.to_csv(out_file) + except AttributeError: + pass + try: + out_file = '{}{}_velocity_harmonic_benchmarks.csv'.format(savepath, filename) + self.HarmonicBenchmarks.velocity.to_csv(out_file) + except AttributeError: + pass + except AttributeError: + raise PyseidonError("-No matching measurement-") + + def taylor_diagram(self, savepath='', fname="taylor_diagram", labels=True, debug=False): + """ + Plots Taylor diagram based on the results of 'validate_data' + + Options: + - savepath = folder path for saving plot, string + - fname = filename for saving plot, string + - labels = labels and legend, boolean + """ + try: + self._fig, self._ax = taylorDiagram(self.Benchmarks, + savepath=savepath, fname=fname, labels=labels, debug=debug) + except AttributeError: + raise PyseidonError("-validate_data needs to be run first-") + + def benchmarks_map(self, savepath='', fname="benchmarks_map", debug=False): + """ + Plots bathymetric map & model validation benchmarks + + Options: + - savepath = folder path for saving plot, string + - fname = filename for saving plot, string + + Note: this function shall work only if ADCP object(s) and FVCOM object + have been used as inputs + """ + if not self._simulated.__module__.split('.')[-1] == 'fvcomClass': + raise PyseidonError("---work only with a combination ADCP object(s) and FVCOM object---") + try: + benchmarksMap(self.Benchmarks, self._observed, self._simulated, savepath=savepath, fname=fname, debug=debug) + except AttributeError: + raise PyseidonError("---validate_data needs to be run first---") + + def save_as(self, filename, fileformat='pickle', debug=False): + """ + This method saves the current Validation structure as: + - *.p, i.e. python file + - *.mat, i.e. Matlab file + + Inputs: + - filename = path + name of the file to be saved, string + + Options: + - fileformat = format of the file to be saved, i.e. 'pickle' or 'matlab' + """ + debug = debug or self._debug + if debug: print 'Saving file...' + + #Save as different formats + if fileformat=='pickle': + filename = filename + ".p" + f = open(filename, "wb") + data = {} + data['History'] = self.History + try: + data['Benchmarks'] = self.Benchmarks + except AttributeError: + pass + data['Variables'] = self.Variables.__dict__ + #TR: Force caching Variables otherwise error during loading + # with 'netcdf4.Variable' type (see above) + for key in data['Variables']: + listkeys=['Variable', 'ArrayProxy', 'BaseType'] + if any([type(data['Variables'][key]).__name__==x for x in listkeys]): + if debug: + print "Force caching for " + key + data['Variables'][key] = data['Variables'][key][:] + #Save in pickle file + if debug: + print 'Dumping in pickle file...' + try: + pkl.dump(data, f, protocol=pkl.HIGHEST_PROTOCOL) + except (SystemError, MemoryError) as e: + print '---Data too large for machine memory---' + raise + + f.close() + elif fileformat=='matlab': + filename = filename + ".mat" + #TR comment: based on Mitchell O'Flaherty-Sproul's code + dtype = float + data = {} + Grd = {} + Var = {} + Bch = {} + + data['History'] = self.History + Bch = self.Benchmarks + for key in Bch: + data[key] = Bch[key] + Var = self.Variables.__dict__ + #TR: Force caching Variables otherwise error during loading + # with 'netcdf4.Variable' type (see above) + for key in Var: + listkeys=['Variable', 'ArrayProxy', 'BaseType'] + if any([type(Var[key]).__name__ == x for x in listkeys]): + if debug: + print "Force caching for " + key + Var[key] = Var[key][:] + #keyV = key + '-var' + #data[keyV] = Var[key] + data[key] = Var[key] + + #Save in mat file file + if debug: + print 'Dumping in matlab file...' + savemat(filename, data, oned_as='column') + else: + print "---Wrong file format---" + + def write_validation_report(self, report_title="validation_report.pdf", debug=False): + """ + This method writes a report (*.pdf) based on the validation methods' results + + Kwargs: + report_title (str): file name + debug (bool): debug flag + """ + debug = debug or self._debug + write_report(self, report_title=report_title, debug=debug) + +# utility classes +class HarmonicBenchmarks: + """ + Storage for hamonic benchmarks + """ + def __init__(self): + self.elevation = 'No harmonic benchmarks for the elevation yet. Run validate_harmonics' + self.velocity = 'No harmonic benchmarks for the velocity yet. Run validate_harmonics' + return diff --git a/build/lib/pyseidon_dvt/validationClass/variablesValidation.py b/build/lib/pyseidon_dvt/validationClass/variablesValidation.py new file mode 100644 index 0000000..a71e7fe --- /dev/null +++ b/build/lib/pyseidon_dvt/validationClass/variablesValidation.py @@ -0,0 +1,331 @@ +#!/usr/bin/python2.7 +# encoding: utf-8 + +from __future__ import division +import numpy as np +from datetime import datetime, timedelta +from os import makedirs +from os.path import exists + +# Local import +from pyseidon_dvt.utilities.interpolation_utils import * + +# Custom error +from pyseidon_dvt.utilities.pyseidon_error import PyseidonError + +class _load_validation: + """ + **'Variables' subset in Validation class** + + It contains the following items: :: + + _obs. = measurement/observational variables + Validation.Variables._|_sim. = simulated variables + |_struct. = dictionary structure for validation purposes + |_harmo. = dictionary stricture for harmonic analysis validation purposes + + """ + def __init__(self, outpath, observed, simulated, flow='sf', nn=True, debug=False, debug_plot=False): + if debug: print "..variables.." + self.obs = observed.Variables + self.sim = simulated.Variables + self._nn = nn + # Compatibility test + if (observed.__module__.split('.')[-1] == 'drifterClass' and + simulated.__module__.split('.')[-1] == 'stationClass'): + raise PyseidonError("---Station and Drifter are incompatible objects---") + + self.struct = np.array([]) + if flow == 'daf': + self._3D = False + else: + self._3D = simulated.Variables._3D + + try: + # Check if times coincide + obsMax = self.obs.matlabTime[~np.isnan(self.obs.matlabTime)].max() + obsMin = self.obs.matlabTime[~np.isnan(self.obs.matlabTime)].min() + simMax = self.sim.matlabTime.max() + simMin = self.sim.matlabTime.min() + absMin = max(obsMin, simMin) + absMax = min(obsMax, simMax) + A = set(np.where(self.sim.matlabTime[:] >= absMin)[0].tolist()) + B = set(np.where(self.sim.matlabTime[:] <= absMax)[0].tolist()) + C = list(A.intersection(B)) + # -Correction by J.Culina 2014- + C = sorted(C) + # -end- + self._C = np.asarray(C) + + a = set(np.where(self.obs.matlabTime[:] >= absMin)[0].tolist()) + b = set(np.where(self.obs.matlabTime[:] <= absMax)[0].tolist()) + c = list(a.intersection(b)) + # -Correction by J.Culina 2014- + c = sorted(c) + # -end- + self._c = np.asarray(c) + + if len(C) == 0: + print "---Time between simulation and measurement does not match up, only Harmonic Analysis can be done---" + self.harmo = {'On':True, 'Observed':observed, 'Simulated':simulated} + C = np.where(self.sim.matlabTime[:] >= simMin)[0].tolist() + c = np.where(self.obs.matlabTime[:] >= obsMin)[0].tolist() + self._C = np.asarray(C) + self._c = np.asarray(c) + else: + self.harmo = {'On':False} + except AttributeError: + raise PyseidonError("---Observations missing matlabTime comparison is impossible---") + + # Check which variables are available in the observations for comparison + self._obs_vars=dir(self.obs) + if ('lon' and 'lat') not in self._obs_vars: + raise PyseidonError("---Observations missing lon/lat comparison is impossible---") + + # Check what kind of simulated data it is + if simulated.__module__ .split('.')[-1] == 'stationClass': + self._simtype = 'station' + # Find closest point to ADCP + ind = closest_points([self.obs.lon], [self.obs.lat], + simulated.Grid.lon[:], + simulated.Grid.lat[:]) + try: + nameSite = ''.join(simulated.Grid.name[ind,:][0,:]) + except IndexError: # TR: quick fix + nameSite = ''.join(simulated.Grid.name[ind]) + self.harmo['nameSite'] = nameSite + print "Station site: " + nameSite + self.sim.lat = simulated.Grid.lat[ind] + el = self.sim.el[:, ind].ravel() + ua = self.sim.ua[:, ind].ravel() + va = self.sim.va[:, ind].ravel() + if self._3D: + u = np.squeeze(self.sim.u[:, :,ind]) + v = np.squeeze(self.sim.v[:, :,ind]) + sig = np.squeeze(simulated.Grid.siglay[:, ind]) + h = simulated.Grid.h[:] + + # Alternative simulation type + elif simulated.__module__.split('.')[-1] == 'fvcomClass': + self._simtype = 'fvcom' + # Different treatment measurements come from drifter + if not observed.__module__.split('.')[-1] == 'drifterClass': + if debug: print "...Interpolation at measurement location..." + el = simulated.Util2D.interpolation_at_point(self.sim.el, + self.obs.lon, self.obs.lat, nn=self._nn) + ua = simulated.Util2D.interpolation_at_point(self.sim.ua, + self.obs.lon, self.obs.lat, nn=self._nn) + va = simulated.Util2D.interpolation_at_point(self.sim.va, + self.obs.lon, self.obs.lat, nn=self._nn) + if self._3D: + u = simulated.Util3D.interpolation_at_point(self.sim.u, + self.obs.lon, self.obs.lat, nn=self._nn) + v = simulated.Util3D.interpolation_at_point(self.sim.v, + self.obs.lon, self.obs.lat, nn=self._nn) + sig = simulated.Util3D.interpolation_at_point(simulated.Grid.siglay, + self.obs.lon, self.obs.lat, nn=self._nn) + h = simulated.Util3D.interpolation_at_point(simulated.Grid.h[:], + self.obs.lon, self.obs.lat, nn=self._nn) + else: # Interpolation for drifter + if debug: print "...Interpolation at measurement locations & times..." + if self._3D: + lock=True + userInp = flow + while lock: + if userInp == 'daf': + if debug: + print 'flow comparison is depth-averaged' + # TR: temporary fix for proxy access + if self.sim._opendap: + uSim = np.zeros((self._C.shape[0], self.sim.ua.shape[1])) + vSim = np.zeros((self._C.shape[0], self.sim.va.shape[1])) + for i, j in enumerate(self._C): + uSim[i,:] = self.sim.ua[j,:] + vSim[i,:] = self.sim.va[j,:] + else: + uSim = np.squeeze(self.sim.ua[self._C,:]) + vSim = np.squeeze(self.sim.va[self._C,:]) + self._3D = False + lock = False + elif userInp == 'sf': + # Import only the surface velocities + # TR_comment: is surface vertical indice -1 or 0? + # KC : 0 is definitely the surface... + # TR: temporary fix for proxy access + if self.sim._opendap: + uSim = np.zeros((self._C.shape[0], self.sim.u.shape[2])) + vSim = np.zeros((self._C.shape[0], self.sim.v.shape[2])) + for i, j in enumerate(self._C): + uSim[i,:] = self.sim.u[j,0,:] + vSim[i,:] = self.sim.v[j,0,:] + else: + uSim = np.squeeze(self.sim.u[self._C,0,:]) + vSim = np.squeeze(self.sim.v[self._C,0,:]) + self._3D = False + lock = False + if debug: + print 'flow comparison at surface' + elif type(userInp) == float: + if debug: + print 'flow comparison at depth level ', float + # if userInp > 0.0: userInp = userInp*-1.0 + uInterp = simulated.Util3D.interp_at_depth(self.sim.u[:], userInp, debug=debug) + vInterp = simulated.Util3D.interp_at_depth(self.sim.v[:], userInp, debug=debug) + # TR: temporary fix for proxy access + if self.sim._opendap: + uSim = np.zeros((self._C.shape[0], uInterp.shape[1])) + vSim = np.zeros((self._C.shape[0], vInterp.shape[1])) + for i, j in enumerate(self._C): + uSim[i,:] = uInterp[j,:] + vSim[i,:] = vInterp[j,:] + else: + uSim = np.squeeze(uInterp[self._C,:]) + vSim = np.squeeze(vInterp[self._C,:]) + self._3D = False + lock = False + else: + userInp = input("compare flow by 'daf', 'sf' or a float number only!!!") + else: + # TR: temporary fix for proxy access + if self.sim._opendap: + uSim = np.zeros((self._C.shape[0], self.sim.ua.shape[1])) + vSim = np.zeros((self._C.shape[0], self.sim.va.shape[1])) + for i, j in enumerate(self._C): + uSim[i,:] = self.sim.ua[j,:] + vSim[i,:] = self.sim.va[j,:] + else: + uSim = np.squeeze(self.sim.ua[self._C,:]) + vSim = np.squeeze(self.sim.va[self._C,:]) + + # Finding the closest Drifter time to simulated data assuming measurement + # time step way faster than model one + indClosest = [] + for i in self._C: + ind = np.abs(self.obs.matlabTime[:]-self.sim.matlabTime[i]).argmin() + indClosest.append(ind) + # Keep only unique values to avoid sampling in measurement gaps + # TR: this doesn't not guarantee that the unique value kept is indeed + # the closest one among the values relative to the same indice!!! + uniqCloInd, uniqInd = np.unique(indClosest, return_index=True) + + # KC: Convert of matlabTime to datetime, smooth drifter temporally! + self.datetimes = [datetime.fromordinal(int(x))+timedelta(days=x%1) - + timedelta(days=366) for x in self.obs.matlabTime[self._c]] + + uObs = self.obs.u[uniqCloInd] + vObs = self.obs.v[uniqCloInd] + uSim = np.squeeze(uSim[uniqInd[:],:]) + vSim = np.squeeze(vSim[uniqInd[:],:]) + + # print 'uObs: \n', uObs, '\nvObs: \n', vObs + # print 'uSim: \n', vSim.shape, '\nuSim: \n', vSim.shape + + # Interpolation of timeseries at drifter's trajectory points + uSimInterp = np.zeros(len(uniqCloInd)) + vSimInterp = np.zeros(len(uniqCloInd)) + for i in range(len(uniqCloInd)): + uSimInterp[i]=simulated.Util2D.interpolation_at_point(uSim[i,:], + self.obs.lon[uniqCloInd[i]], + self.obs.lat[uniqCloInd[i]]) + vSimInterp[i]=simulated.Util2D.interpolation_at_point(vSim[i,:], + self.obs.lon[uniqCloInd[i]], + self.obs.lat[uniqCloInd[i]]) + # print 'vSimInterp: \n', vSimInterp, '\nuSimInterp: \n', uSimInterp + + else: + raise PyseidonError("-This type of simulations is not supported yet-") + + # Store in dict structure for compatibility purposes (except for drifters) + if not observed.__module__.split('.')[-1] == 'drifterClass': + if not self._3D: + sim_mod = {'ua': ua[C], 'va': va[C], 'el': el[C]} + else: + sim_mod = {'ua': ua[C], 'va': va[C], 'el': el[C], 'u': u[C,:], + 'v': v[C,:], 'siglay': sig[:], 'h': h} + + # Check what kind of observed data it is + if observed.__module__.split('.')[-1] == 'adcpClass' or observed.__module__ == 'adcpClass': + self._obstype = 'adcp' + obstype = 'ADCP' + # Alternative measurement type + elif observed.__module__.split('.')[-1] == 'tidegaugeClass': + self._obstype = 'tidegauge' + obstype = 'TideGauge' + else: + raise PyseidonError("---This type of measurements is not supported yet---") + + # This had to be split into two parts so that a time subset could be used. + # Specify list of data variables from observations that should be used. + dictlist=['ua','va','u','v','el'] + self._commonlist_data = [var for var in self._obs_vars if var in dictlist] + if debug: + print 'Data variables being used' + print self._commonlist_data + obs_mod={} + for key in self._commonlist_data: + obs_mod[key] = getattr(self.obs,key) + obs_mod[key] = obs_mod[key][c,] + + # Specify list of nondata variables that should be used. + dictlist = ['bins','data'] + self._commonlist_nondata = [var for var in self._obs_vars if var in dictlist] + if debug: + print 'Non data variables being used' + print self._commonlist_nondata + for key in self._commonlist_nondata: + obs_mod[key] = getattr(self.obs,key) + + else: + self._obstype = 'drifter' + obstype = 'Drifter' + + #Store in dict structure for compatibility purposes + #Common block for 'struct' + if not observed.__module__.split('.')[-1] == 'drifterClass': + self.struct = {'name': observed.History[0].split(' ')[-1], + 'type': obstype, + 'obs_lat': self.obs.lat, + 'obs_lon': self.obs.lon, + 'mod_lat': self.obs.lat, + 'mod_lon': self.obs.lon, + 'obs_timeseries': obs_mod, + 'mod_timeseries': sim_mod, + 'obs_time': self.obs.matlabTime[c], + 'mod_time': self.sim.matlabTime[C], + '_commonlist_data': self._commonlist_data, + '_commonlist_nondata': self._commonlist_nondata} + else: # Drifter's case + self.struct = {'name': observed.History[0].split(' ')[-1], + 'type': obstype, + 'lat': self.obs.lat[uniqCloInd], + 'lon': self.obs.lon[uniqCloInd], + 'obs_timeseries': {'u': uObs, 'v': vObs}, + 'mod_timeseries': {'u': uSimInterp, 'v': vSimInterp}, + 'obs_time': self.obs.matlabTime[uniqCloInd], + 'mod_time': self.sim.matlabTime[C], + '_commonlist_data': ['u', 'v']} + + if debug: print "..done" + + # find save_path + # self._save_path = '' + # for s in observed.History: + # if 'Create save_path ' not in s: + # continue + # else: + # self._save_path=s.split()[2] + # + # if '' is self._save_path: + # name = self.struct['name'] + # self._save_path = outpath+name.split('/')[-1].split('.')[0]+'/' + # while exists(self._save_path): + # self._save_path = self._save_path[:-1] + '_bis/' + # makedirs(self._save_path) + # observed.History.append('Create save_path {}'.format(self._save_path)) + name = self.struct['name'] + self._save_path = outpath+name.split('/')[-1].split('.')[0]+'/' + while exists(self._save_path): + self._save_path = self._save_path[:-1] + '_bis/' + makedirs(self._save_path) + + return diff --git a/dist/PySeidon_dvt-2.1-py2.7.egg b/dist/PySeidon_dvt-2.1-py2.7.egg new file mode 100644 index 0000000..f6c3d0c Binary files /dev/null and b/dist/PySeidon_dvt-2.1-py2.7.egg differ diff --git a/pyseidon_dvt/adcpClass/functionsAdcp.py b/pyseidon_dvt/adcpClass/functionsAdcp.py index caf5ca4..427d3b3 100644 --- a/pyseidon_dvt/adcpClass/functionsAdcp.py +++ b/pyseidon_dvt/adcpClass/functionsAdcp.py @@ -313,7 +313,7 @@ def speed_histogram(self, t_start=[], t_end=[], time_ind=[], def Harmonic_analysis(self, time_ind=[], t_start=[], t_end=[], elevation=True, velocity=False, - debug=False, **kwargs): + threeD=False, debug=False, **kwargs): ''' This function performs a harmonic analysis on the sea surface elevation time series or the velocity components timeseries. @@ -327,8 +327,10 @@ def Harmonic_analysis(self, or time index as an integer - t_end = end time, as a string ('yyyy-mm-ddThh:mm:ss'), or time index as an integer - - elevation=True means that `solve' will be done for elevation. - - velocity=True means that `solve' will be done for velocity. + - elevation = True means that `solve' will be done for elevation. + - velocity = True means that `solve' will be done for velocity. + - threeD = True means that 'solve' will be done for each layer + of the velocity data instead of on depth-averaged data. Options: Options are the same as for 'solve', which are shown below with @@ -362,16 +364,29 @@ def Harmonic_analysis(self, if velocity: time = self._var.matlabTime[:] - u = self._var.ua[:] - v = self._var.va[:] + lat = self._var.lat + if not threeD: + u = self._var.ua[:] + v = self._var.va[:] - if not argtime==[]: - time = time[argtime[:]] - u = u[argtime[:]] - v = v[argtime[:]] + if not argtime==[]: + time = time[argtime[:]] + u = u[argtime[:]] + v = v[argtime[:]] - lat = self._var.lat - harmo = solve(time, u, v, lat, **kwargs) + harmo = solve(time, u, v, lat, **kwargs) + else: + harmo = {} + for layerIndex in range(self._var.u.data.shape[1]): + u = self._var.u[:,layerIndex] + v = self._var.v[:,layerIndex] + + if not argtime==[]: + time = time[argtime[:]] + u = u[argtime[:]] + v = v[argtime[:]] + + harmo['Layer_'+str(layerIndex)] = solve(time, u, v, lat, **kwargs) if elevation: time = self._var.matlabTime[:] @@ -415,10 +430,37 @@ def Harmonic_reconstruction(self, harmo, time_ind=slice(None), debug=False, **kw debug = (debug or self._debug) time = self._var.matlabTime[time_ind] #TR_comments: Add debug flag in Utide: debug=self._debug - Reconstruct = reconstruct(time,harmo) - + if not 'Layer_0' in harmo: + Reconstruct = reconstruct(time, harmo) + else: + Reconstruct = {} + for layer in harmo: + Reconstruct[layer] = reconstruct(time, harmo[layer]) + return Reconstruct + def velo_stack(self, Reconstruct, debug=False): + """ + This function seperates and stacks u & v into respective matrices + from a 3D harmonically reconstructed dictionary + + Inputs: + - Reconstruct = reconstructed signal, dictionary from Harmonic_reconstruction() + + Outputs: + - u = stacked matrix of u velocity + - v = stacked matrix of v velocity + + """ + debug = (debug or self._debug) + u = Reconstruct['Layer_0']['u'] + v = Reconstruct['Layer_0']['v'] + for i in range(len(Reconstruct))[1:]: + u = np.vstack((u, Reconstruct['Layer_'+str(i)]['u'])) + v = np.vstack((v, Reconstruct['Layer_'+str(i)]['v'])) + + return u, v + def verti_shear(self, t_start=[], t_end=[], time_ind=[], graph=True, dump=False, debug=False, **kwargs): """ diff --git a/pyseidon_dvt/drifterClass/variablesDrifter.py b/pyseidon_dvt/drifterClass/variablesDrifter.py index a2ad69d..1b2a43b 100644 --- a/pyseidon_dvt/drifterClass/variablesDrifter.py +++ b/pyseidon_dvt/drifterClass/variablesDrifter.py @@ -2,10 +2,11 @@ # encoding: utf-8 from __future__ import division -# import numpy as np +import numpy as np # from numpy.ma import MaskError # import h5py from pyseidon_dvt.utilities.miscellaneous import mattime_to_datetime +import sys class _load_drifter: """ @@ -24,22 +25,48 @@ def __init__(self,cls, History, debug=False): print 'Loading variables...' # Pointer to History setattr(self, '_History', History) - - self.matlabTime = cls.Data['velocity'].vel_time[:] - #Sorting values with increasing time step - sortedInd = self.matlabTime.argsort() - self.matlabTime.sort() - self.lat = cls.Data['velocity'].vel_lat[sortedInd] - self.lon = cls.Data['velocity'].vel_lon[sortedInd] - self.u = cls.Data['velocity'].u[sortedInd] - self.v = cls.Data['velocity'].v[sortedInd] + try: + self.matlabTime = cls.Data['velocity'].vel_time[:] + #Sorting values with increasing time step + sortedInd = self.matlabTime.argsort() + self.matlabTime.sort() + self.lat = cls.Data['velocity'].vel_lat[sortedInd] + self.lon = cls.Data['velocity'].vel_lon[sortedInd] + self.u = cls.Data['velocity'].u[sortedInd] + self.v = cls.Data['velocity'].v[sortedInd] + # Luna Ocean Consulting Ltd. new drifter format + except KeyError: + self.matlabTime = cls.Data['time'] + self.original = {} + self.original['lat'] = cls.Data['lat'] + self.original['lon'] = cls.Data['lon'] + self.original['u'] = cls.Data['u'] + self.original['v'] = cls.Data['v'] + self.original['drift_start'] = cls.Data['drift_start'] + self.original['drift_stop'] = cls.Data['drift_stop'] + self.smooth = {} + self.smooth['lat'] = cls.Data['lat_smooth'] + self.smooth['lon'] = cls.Data['lon_smooth'] + self.smooth['u'] = cls.Data['u_smooth'] + self.smooth['v'] = cls.Data['v_smooth'] + self.smooth['drift_start'] = cls.Data['drift_start_smooth'] + self.smooth['drift_stop'] = cls.Data['drift_stop_smooth'] + except: + sys.exit('Drifter file format incompatible') #-Append message to History field - start = mattime_to_datetime(self.matlabTime[0]) - end = mattime_to_datetime(self.matlabTime[-1]) - text = 'Temporal domain from ' + str(start) +\ - ' to ' + str(end) - self._History.append(text) + try: + start = mattime_to_datetime(self.matlabTime[0]) + end = mattime_to_datetime(self.matlabTime[-1]) + text = 'Temporal domain from ' + str(start) +\ + ' to ' + str(end) + self._History.append(text) + except ValueError: + start = mattime_to_datetime(np.nanmin(self.matlabTime)) + end = mattime_to_datetime(np.nanmax(self.matlabTime)) + text = 'Temporal domain from ' + str(start) +\ + ' to ' + str(end) + self._History.append(text) if debug: print '...Passed' diff --git a/pyseidon_dvt/fvcomClass/functionsFvcomThreeD.py b/pyseidon_dvt/fvcomClass/functionsFvcomThreeD.py index b91519e..11c1ca2 100644 --- a/pyseidon_dvt/fvcomClass/functionsFvcomThreeD.py +++ b/pyseidon_dvt/fvcomClass/functionsFvcomThreeD.py @@ -13,6 +13,8 @@ import matplotlib.pyplot as plt from pydap.exceptions import ServerError +from utide import solve, reconstruct + #TR comment: This all routine needs to be tested and debugged class FunctionsFvcomThreeD: """ @@ -1103,3 +1105,158 @@ def _vertical_slice(self, var, start_pt, end_pt, #ax.yaxis.set_major_formatter(ticks) plt.xlabel('Distance along line (m)') plt.ylabel('Depth (m)') + + def Harmonic_analysis_at_point(self, pt_lon, pt_lat, + time_ind=[], t_start=[], t_end=[], + elevation=True, velocity=False, + debug=False, **kwarg): + """ + This function performs a harmonic analysis on the sea surface elevation + time series or the velocity components timeseries. + + Inputs: + - pt_lon = longitude in decimal degrees East, float number + - pt_lat = latitude in decimal degrees North, float number + + Outputs: + - harmo = harmonic coefficients, dictionary + + Options: + - time_ind = time indices to work in, list of integers + - t_start = start time, as a string ('yyyy-mm-ddThh:mm:ss'), or time index as an integer + - t_end = end time, as a string ('yyyy-mm-ddThh:mm:ss'), or time index as an integer + - elevation=True means that 'solve' will be done for elevation. + - velocity=True means that 'solve' will be done for velocity. + + Utide's options: + Options are the same as for 'solve', which are shown below with + their default values: + conf_int=True; cnstit='auto'; notrend=0; prefilt=[]; nodsatlint=0; + nodsatnone=0; gwchlint=0; gwchnone=0; infer=[]; inferaprx=0; + rmin=1; method='cauchy'; tunrdn=1; linci=0; white=0; nrlzn=200; + lsfrqosmp=1; nodiagn=0; diagnplots=0; diagnminsnr=2; + ordercnstit=[]; runtimedisp='yyy' + + *Notes* + For more detailed information about 'solve', please see + https://github.com/wesleybowman/UTide + + """ + debug = (debug or self._debug) + #TR_comments: Add debug flag in Utide: debug=self._debug + index = self.index_finder(pt_lon, pt_lat, debug=False) + argtime = [] + if not time_ind==[]: + argtime = time_ind + elif not t_start==[]: + if type(t_start)==str: + start = datetime.datetime.strptime(t_start, '%Y-%m-%d %H:%M:%S') + end = datetime.datetime.strptime(t_end, '%Y-%m-%d %H:%M:%S') + argtime = time_to_index(start, end, self._var.julianTime[:], debug=debug) + else: + argtime = np.arange(t_start, t_end) + + if velocity == elevation: + raise PyseidonError("---Can only process either velocities or elevation. Change options---") + + if velocity: + harmo = {} + time = self._var.matlabTime[:] + for layerIndex in range(self._var.u.shape[1]): + #if type(self._var.ua).__name__=='Variable': #Fix for netcdf4 lib + # ua = self._var.ua[:] + # va = self._var.va[:] + #else: + U = self._var.u[:,layerIndex] + V = self._var.v[:,layerIndex] + + u = self.interpolation_at_point(U, pt_lon, pt_lat, index=index, debug=debug) + v = self.interpolation_at_point(V, pt_lon, pt_lat, index=index, debug=debug) + if not argtime==[]: + time = time[argtime[:]] + u = u[argtime[:]] + v = v[argtime[:]] + + lat = self._grid.lat[index] + harmo['Layer_'+str(layerIndex)] = solve(time, u, v, lat, **kwarg) + + else: + time = self._var.matlabTime[:] + if type(self._var.el).__name__=='Variable': #Fix for netcdf4 lib + el = self._var.el[:] + else: + el = self._var.el + el = self.interpolation_at_point(el, pt_lon, pt_lat, + index=index, debug=debug) + + if not argtime==[]: + time = time[argtime[:]] + el = el[argtime[:]] + + lat = self._grid.lat[index] + harmo = solve(time, el, None, lat, **kwarg) + #Write meta-data only if computed over all the elements + + return harmo + + def Harmonic_reconstruction(self, harmo, recon_time=[], time_ind=slice(None), debug=False, **kwarg): + """ + This function reconstructs the velocity components or the surface elevation + from harmonic coefficients. + Harmonic_reconstruction calls 'reconstruct'. This function assumes harmonics + ('solve') has already been executed. + + Inputs: + - Harmo = harmonic coefficients from harmo_analysis, dictionary if velocity + - time_ind = time indices to process, list of integers + - recon_time = time you want the harmonic coefficients to be reconstructed at + + Output: + - Reconstruct = reconstructed signal, dictionary + + Options: + Options are the same as for 'reconstruct', which are shown below with + their default values: + cnstit = [], minsnr = 2, minpe = 0 + + *Notes* + For more detailed information about 'reconstruct', please see + https://github.com/wesleybowman/UTide + + """ + debug = (debug or self._debug) + if recon_time == []: + time = self._var.matlabTime[time_ind] + else: + time = recon_time + #TR_comments: Add debug flag in Utide: debug=self._debug + if not 'Layer_0' in harmo: + Reconstruct = reconstruct(time, harmo) + else: + Reconstruct = {} + for layer in harmo: + Reconstruct[layer] = reconstruct(time, harmo[layer]) + + return Reconstruct + + def velo_stack(self, Reconstruct, debug=False): + """ + This function seperates and stacks u & v into respective matrices + from a 3D harmonically reconstructed dictionary + + Inputs: + - Reconstruct = reconstructed signal, dictionary from Harmonic_reconstruction() + + Outputs: + - u = stacked matrix of u velocity + - v = stacked matrix of v velocity + + """ + debug = (debug or self._debug) + u = Reconstruct['Layer_0']['u'] + v = Reconstruct['Layer_0']['v'] + for i in range(len(Reconstruct))[1:]: + u = np.vstack((u, Reconstruct['Layer_'+str(i)]['u'])) + v = np.vstack((v, Reconstruct['Layer_'+str(i)]['v'])) + + return u, v diff --git a/pyseidon_dvt/stationClass/functionsStationThreeD.py b/pyseidon_dvt/stationClass/functionsStationThreeD.py index 9e6120c..177d1ca 100644 --- a/pyseidon_dvt/stationClass/functionsStationThreeD.py +++ b/pyseidon_dvt/stationClass/functionsStationThreeD.py @@ -9,6 +9,8 @@ from pyseidon_dvt.utilities.BP_tools import * import time +from utide import solve, reconstruct + # Custom error from pyseidon_error import PyseidonError @@ -299,6 +301,152 @@ def flow_dir(self, station, t_start=[], t_end=[], time_ind=[], return np.squeeze(dirFlow) + def Harmonic_analysis_at_point(self, station, + time_ind=[], t_start=[], t_end=[], + elevation=True, velocity=False, + debug=False, **kwarg): + """ + This function performs a harmonic analysis on the sea surface elevation + time series or each layer of the velocity components timeseries. + + Inputs: + - station = either station index (interger) or name (string) + + Outputs: + - harmo = harmonic coefficients, dictionary + + Options: + - t_start = start time, as string ('yyyy-mm-ddThh:mm:ss'), or time index as an integer + - t_end = end time, as a string ('yyyy-mm-ddThh:mm:ss'), or time index as an integer + - time_ind = time indices to work in, list of integers + - elevation=True means that 'solve' will be done for elevation. + - velocity=True means that 'solve' will be done for velocity. + + Utide's options: + Options are the same as for 'solve', which are shown below with + their default values: + conf_int=True; cnstit='auto'; notrend=0; prefilt=[]; nodsatlint=0; + nodsatnone=0; gwchlint=0; gwchnone=0; infer=[]; inferaprx=0; + rmin=1; method='cauchy'; tunrdn=1; linci=0; white=0; nrlzn=200; + lsfrqosmp=1; nodiagn=0; diagnplots=0; diagnminsnr=2; + ordercnstit=[]; runtimedisp='yyy' + + *Notes* + For more detailed information about 'solve', please see + https://github.com/wesleybowman/UTide + + """ + debug = (debug or self._debug) + + #Search for the station + index = self.search_index(station) + + argtime = [] + if not time_ind==[]: + argtime = time_ind + elif not t_start==[]: + if type(t_start)==str: + argtime = time_to_index(t_start, t_end, + self._var.matlabTime, + debug=debug) + else: + argtime = np.arange(t_start, t_end) + + if velocity == elevation: + raise PyseidonError("---Can only process either velocities or elevation. Change options---") + + if velocity: + harmo = {} + for layerIndex in range(self._var.u.shape[1]): + time = self._var.matlabTime[:] + u = self._var.u[:,layerIndex,index] + v = self._var.v[:,layerIndex,index] + + if not argtime==[]: + time = time[argtime[:]] + u = u[argtime[:]] + v = v[argtime[:]] + + lat = self._grid.lat[index] + harmo['Layer_'+str(layerIndex)] = solve(time, u, v, lat, **kwarg) + + if elevation: + time = self._var.matlabTime[:] + el = self._var.el[:,index] + + if not argtime==[]: + time = time[argtime[:]] + el = el[argtime[:]] + + lat = self._grid.lat[index] + harmo = solve(time, el, None, lat, **kwarg) + #Write meta-data only if computed over all the elements + + return harmo + + def Harmonic_reconstruction(self, harmo, recon_time=[], time_ind=slice(None), debug=False, **kwarg): + """ + This function reconstructs the velocity components or the surface elevation + from harmonic coefficients. + Harmonic_reconstruction calls 'reconstruct'. This function assumes harmonics + ('solve') has already been executed. + + Inputs: + - Harmo = harmonic coefficient from harmo_analysis + - time_ind = time indices to process, list of integers + - recon_time = time you want the harmonic coefficients to be reconstructed at + + Output: + - Reconstruct = reconstructed signal, dictionary + + Options: + Options are the same as for 'reconstruct', which are shown below with + their default values: + cnstit = [], minsnr = 2, minpe = 0 + + *Notes* + For more detailed information about 'reconstruct', please see + https://github.com/wesleybowman/UTide + + """ + debug = (debug or self._debug) + if recon_time == []: + time = self._var.matlabTime[time_ind] + else: + time = recon_time + + if not 'Layer_0' in harmo: + Reconstruct = reconstruct(time, harmo) + else: + Reconstruct = {} + for layer in harmo: + Reconstruct[layer] = reconstruct(time, harmo[layer]) + + return Reconstruct + + def velo_stack(self, Reconstruct, debug=False): + """ + This function seperates and stacks u & v into respective matrices + from a 3D harmonically reconstructed dictionary + + Inputs: + - Reconstruct = reconstructed signal, dictionary from Harmonic_reconstruction() + + Outputs: + - u = stacked matrix of u velocity + - v = stacked matrix of v velocity + + """ + debug = (debug or self._debug) + u = Reconstruct['Layer_0']['u'] + v = Reconstruct['Layer_0']['v'] + for i in range(len(Reconstruct))[1:]: + u = np.vstack((u, Reconstruct['Layer_'+str(i)]['u'])) + v = np.vstack((v, Reconstruct['Layer_'+str(i)]['v'])) + + return u, v + + #TR_comments: templates # def whatever(self, debug=False): # if debug or self._debug: diff --git a/pyseidon_dvt/stationClass/stationClass.py b/pyseidon_dvt/stationClass/stationClass.py index f22784c..007e6c8 100644 --- a/pyseidon_dvt/stationClass/stationClass.py +++ b/pyseidon_dvt/stationClass/stationClass.py @@ -289,8 +289,47 @@ def __add__(self, StationClass, debug=False): if debug: print "Stacking " + key + "..." except AttributeError: continue + + #New code added by RK Aug 20 + #get unique, sorted time + if debug: + print 'Getting unique, sorted time...' + + #keyword list for unique + + time = getattr(newself.Variables, 'matlabTime') + uniquetime, unique_index=np.unique(time,return_index=True) + sort_index=np.argsort(uniquetime) + index=unique_index[sort_index] + time=time[index] #New time dimension - newself.Grid.ntime = newself.Grid.ntime + StationClass.Grid.ntime + newself.Grid.ntime = time.shape + if debug: print "New time length: " +str(newself.Grid.ntime[0]) + + #keyword list for vstack + kwl=['matlabTime', 'julianTime', 'secondTime'] + for key in kwl: + tmpN = getattr(newself.Variables, key) + setattr(newself.Variables, key, tmpN[index]) + + kwl=['u', 'v', 'w', 'tke', 'gls', 'ua', 'va','el'] + kwl2D=['ua', 'va','el'] + for key in kwl: + try: + if key in kwl2D: + tmpN = getattr(newself.Variables, key)\ + [:,newEle[:]] + setattr(newself.Variables, key,tmpN[index,:]) + if debug: print "Sorting " + key + "..." + else: + tmpN = getattr(newself.Variables, key)\ + [:,:,newEle[:]] + setattr(newself.Variables, key,tmpN[index,:]) + if debug: print "Sorting " + key + "..." + except AttributeError: + continue + #End new code added by RK Aug 20 + #Keep only matching names newself.Grid.name = self.Grid.name[origEle[:]] #Append to new object history