Sync develop to main

training/gordon-group/seed/ActionRuntime/py-gordon-ML_1_0_0.json

@@ -0,0 +1,18 @@
+{
+    "language": "Python",
+    "runtimeVersion": "3.6",
+    "modules": {
+      "conda.scikit-learn":"=0.23.1",
+      "conda.pandas":"=1.0.1",
+      "conda.numpy":"=1.18.1",
+      "conda.scipy":"=1.4.1",
+      "conda.dill":"=0.2.8.2"
+    },
+    "repositories": [
+      "https://repo.continuum.io/pkgs/main",
+      "conda-forge"
+    ],
+    "runtime": "CPython",
+    "name": "py-gordon-ML_1_0_0",
+    "id": "py-gordon-ML_1_0_0"
+}

training/gordon-group/seed/ActionRuntime/py-gordon-nc4-_1_0_0.json

@@ -0,0 +1,15 @@
+{
+    "language": "Python",
+    "runtimeVersion": "3.6",
+    "modules": {
+      "conda.netcdf4": "=1.5.7",
+      "conda.dill":"=0.2.8.2"
+    },
+    "repositories": [
+      "https://repo.continuum.io/pkgs/main",
+      "conda-forge"
+    ],
+    "runtime": "CPython",
+    "name": "py-gordon-nc4_1_0_0",
+    "id": "py-gordon-nc4_1_0_0"
+}

training/gordon-group/seed/ActionRuntime/py-gordon_1_0_0.json

@@ -6,7 +6,9 @@
       "conda.netcdf4": "=1.5.7",
       "conda.numpy": "=1.17.0",
       "conda.pandas": "=1.1.5",
-      "conda.cartopy":"=0.18.0"
+      "conda.cartopy":"=0.18.0",
+      "conda.basemap":"=1.2.2",
+      "conda.iris":"=3.1.0"
     },
     "repositories": [
       "https://repo.continuum.io/pkgs/main",

training/gordon-group/seed/ActionRuntime/py-gordon_2_0_0.json

@@ -2,8 +2,15 @@
     "language": "Python",
     "runtimeVersion": "3.6",
     "modules": {
-      "conda.cartopy":"=0.19.0",
-      "conda.pandas": "=1.0.1"
+      "conda.pip": "=21.2.2",
+      "conda.netcdf4": "=1.5.7",
+      "conda.numpy": "=1.17.0",
+      "conda.pandas": "=1.1.5",
+      "conda.cartopy":"=0.18.0",
+      "conda.basemap":"=1.2.2",
+      "conda.iris":"=3.1.0",
+      "conda.scikit-learn":"=0.23.1",
+      "conda.dill":"=0.2.8.2"
     },
     "repositories": [
       "https://repo.continuum.io/pkgs/main",

training/gordon-group/src/BatchJobs/UpsertAODData/UpsertAODData.c3typ

@@ -0,0 +1,10 @@
+/**
+* UpsertAODData.c3typ
+* Batch job to upsert data from files in {@link SimulationOutputFile}with container=monthly-mean in parallel
+*/
+type UpsertAODData extends BatchJob<UpsertAODData, UpsertAODDataOptions, UpsertAODDataBatch> type key 'PSRT_MNTHMNDT' {
+
+    doStart:        ~ js server
+    processBatch:   ~ js server
+    // allComplete: ~ js server
+}

training/gordon-group/src/BatchJobs/UpsertAODData/UpsertAODData.js

@@ -0,0 +1,41 @@
+/**
+ * UpsertAODData.js
+ * Implementation of UpsertAODData.c3typ
+ * @param {UpsertAODData} job
+ * @param {UpsertAODDataOptions} options
+ */
+ function doStart(job, options) {
+    var batch = [];
+
+    var finalFilter = options.filter.and().eq("container", "monthly-mean");
+
+    var dataset = SimulationOutputFile.fetchObjStream({
+        filter: finalFilter,
+        limit: options.limit,
+        offset: options.offset
+    });
+
+    while(dataset.hasNext()) {
+        batch.push(dataset.next());
+
+        if (batch.length >= options.batchSize || !dataset.hasNext()) {
+            var batchSpec = UpsertAODDataBatch.make({values: batch});
+            job.scheduleBatch(batchSpec);
+
+            batch = [];
+        }
+    }
+}
+
+
+
+/**
+ * @param {UpsertAODDataBatch} batch
+ * @param {UpsertAODData} job
+ * @param {UpsertAODDataOptions} options
+ */
+function processBatch(batch, job, options) {
+    batch.values.forEach(function(file) {
+        file.upsert3HourlyAODAllRefData();
+    });
+}

training/gordon-group/src/BatchJobs/UpsertAODData/UpsertAODDataBatch.c3typ

@@ -0,0 +1,7 @@
+/**
+* UpsertAODDataBatch.c3typ
+* Represents a unit of work (batch) in a {@link UpsertAODData} batch job.
+*/
+type UpsertAODDataBatch {
+    values: [SimulationOutputFile]
+}

training/gordon-group/src/BatchJobs/UpsertAODData/UpsertAODDataOptions.c3typ

@@ -0,0 +1,14 @@
+/**
+* UpsertAODDataOptions.c3typ
+* Represents customization options for a {@link UpsertAODData} batch job
+*/
+type UpsertAODDataOptions {
+    // the number of files that will be processed in each batch
+    batchSize: int = 1
+    // a limit to the number of files that will be processed overall
+    limit: int = -1
+    // offset in the fetch to SimulationOutputFile
+    offset: int = 0
+    // any additional filters
+    filter: Filter
+}

training/gordon-group/src/UpsertData.js → training/gordon-group/src/BatchJobs/UpsertData/UpsertData.js

@@ -15,7 +15,7 @@ function doStart(job, options) {
     while(dataset.hasNext()) {
         batch.push(dataset.next());
 
-        if (dataset.length >= options.batchSize || !dataset.hasNext()) {
+        if (batch.length >= options.batchSize || !dataset.hasNext()) {
             var batchSpec = UpsertDataBatch.make({values: batch});
             job.scheduleBatch(batchSpec);
 

training/gordon-group/src/UpsertObsData.c3typ → training/gordon-group/src/BatchJobs/UpsertObsData/UpsertObsData.c3typ

@@ -1,6 +1,6 @@
 /**
 * UpsertObsData.c3typ
-* Batch job to upsert data from every {@link SimulationSample} in a {@link SimulationEnsemble} in parallel
+* Batch job to upsert data from every {@link ObservationSet} in parallel
 */
 type UpsertObsData extends BatchJob<UpsertObsData, UpsertDataOptions, UpsertObsDataBatch> type key 'UPSERTOBSDATA' {
 

training/gordon-group/src/UpsertObsData.js → training/gordon-group/src/BatchJobs/UpsertObsData/UpsertObsData.js

@@ -15,7 +15,7 @@
     while(dataset.hasNext()) {
         batch.push(dataset.next());
 
-        if (dataset.length >= options.batchSize || !dataset.hasNext()) {
+        if (batch.length >= options.batchSize || !dataset.hasNext()) {
             var batchSpec = UpsertObsDataBatch.make({values: batch});
             job.scheduleBatch(batchSpec);
 

training/gordon-group/src/CustomMLPipeline/machineLearning/GPRegression/GaussianProcessRegressionPipe.c3typ

@@ -0,0 +1,19 @@
+/**
+* GaussianProcessRegressionPipe.c3typ
+* Performs Scikit-Learn's GP Regression for a set of inputs-outputs.
+*/
+entity type GaussianProcessRegressionPipe extends MLLeafPipe<Dataset, Dataset> type key 'GPREG' {
+    // the technique for this regression
+    @db(persistAllFields=true)
+    technique: !GaussianProcessRegressionTechnique
+
+    // train the model
+    @py(env='gordon-ML_1_0_0')
+    train: ~
+    // process data, ie predictions
+    @py(env='gordon-ML_1_0_0')
+    process: ~
+    // guarantee that process() is only allowed after train()
+    @py(env='gordon-ML_1_0_0')
+    isProcessable: ~
+}

training/gordon-group/src/CustomMLPipeline/machineLearning/GPRegression/GaussianProcessRegressionPipe.py

@@ -0,0 +1,44 @@
+def train(this, input, targetOutput, spec):
+    """
+    Performs Scikit-Learn's GaussianProcessRegressor's fit().
+    https://scikit-learn.org/stable/modules/generated/sklearn.gaussian_process.GaussianProcessRegressor.html
+    """
+    from sklearn.gaussian_process import GaussianProcessRegressor
+
+    # get data
+    X = c3.Dataset.toNumpy(dataset=input)
+    y = c3.Dataset.toNumpy(dataset=targetOutput).flatten()
+
+    # get kernel object from c3, make it python again
+    kernel = c3.PythonSerialization.deserialize(serialized=this.technique.kernel.pickledKernel)
+
+    # build and train GPR
+    gp = GaussianProcessRegressor(kernel=kernel)
+    gp.fit(X, y)
+
+    # pickle model
+    this.trainedModel = c3.MLTrainedModelArtifact(model=c3.PythonSerialization.serialize(obj=gp))
+
+    return this
+
+
+def process(this, input, spec):
+    """
+    Performs Scikit-Learn's GaussianProcessRegressor's predict().
+    https://scikit-learn.org/stable/modules/generated/sklearn.gaussian_process.GaussianProcessRegressor.html
+    """
+    # unpickle the model
+    gp = c3.PythonSerialization.deserialize(serialized=this.trainedModel.model)
+
+    # format data
+    X = c3.Dataset.toNumpy(dataset=input)
+
+    return c3.Dataset.fromPython(pythonData=gp.predict(X))
+
+
+def isProcessable(this):
+    """"
+    Guarantees that process() can only be called after train()
+    """
+
+    return this.isTrained()

training/gordon-group/src/CustomMLPipeline/machineLearning/GPRegression/GaussianProcessRegressionTechnique.c3typ

@@ -0,0 +1,12 @@
+/**
+* GaussianProcessRegressionTechnique.c3typ
+* Defines hyper parameters for GP Regression.
+*/
+type GaussianProcessRegressionTechnique mixes MLTechnique {
+    // random seed
+    @ML(hyperParameter=true)
+    randomState: integer
+    // the kernel object
+    @ML(hyperParameter=true)
+    kernel: SklearnGPRKernel
+}

training/gordon-group/src/CustomMLPipeline/machineLearning/GPRegression/kernels/SklearnGPRKernel.c3typ

@@ -0,0 +1,13 @@
+/**
+* SklearnGPRKernel.c3typ
+* Stepping stone for Scikit-Learn Gaussian Process kernels.
+* See: https://scikit-learn.org/stable/modules/classes.html#module-sklearn.gaussian_process
+*/
+type SklearnGPRKernel {
+    // the name of the kernel
+    name: string enum('Constant', 'DotProduct', 'ExpSineSquared', 'Matern', 'RBF', 'RationalQuadratic', 'White')
+    // the kernel hyper parameters 
+    hyperParameters: map<string, double> 
+    // the pickled kernel
+    pickledKernel: string
+}

training/gordon-group/src/CustomMLPipeline/machineLearning/GPRegression/kernels/SklearnGPRKernelConstant.c3typ

@@ -0,0 +1,15 @@
+/**
+* SklearnGPRKernelConstant.c3typ
+* Scikit-Learn Gaussian Process Constant Kernel.
+* See: https://scikit-learn.org/stable/modules/generated/sklearn.gaussian_process.kernels.ConstantKernel.html#sklearn.gaussian_process.kernels.ConstantKernel
+*/
+type SklearnGPRKernelConstant {
+    // Constant value that defines the kernel value
+    constantValue: !double
+    // the SklearnGPRKernel for this object
+    kernel: SklearnGPRKernel
+
+    // constructs the SklearnGPRKernel internal object
+    @py(env='gordon-ML_1_0_0')
+    build: member function(): SklearnGPRKernelConstant
+}

training/gordon-group/src/CustomMLPipeline/machineLearning/GPRegression/kernels/SklearnGPRKernelConstant.py

@@ -0,0 +1,21 @@
+def build(this):
+    """
+    This effectively constructs the type instance, by creating the object to be placed in the kernel field. Ideally this should be replaced by a callback function (beforeMake or afterMake).
+    """
+    from sklearn.gaussian_process.kernels import ConstantKernel
+
+    sklKernel = ConstantKernel(this.constantValue)
+
+    kernel_pickled = c3.PythonSerialization.serialize(obj=sklKernel)
+    kernel_name = 'Constant'
+    kernel_hyperParameters = c3.c3Make(
+        "map<string, double>", {"constantValue": this.constantValue}
+    )
+
+    this.kernel = c3.SklearnGPRKernel(
+        name=kernel_name,
+        hyperParameters=kernel_hyperParameters,
+        pickledKernel=kernel_pickled
+    )
+
+    return this

training/gordon-group/src/CustomMLPipeline/machineLearning/GPRegression/kernels/SklearnGPRKernelDotProduct.c3typ

@@ -0,0 +1,15 @@
+/**
+* SklearnGPRKernelDotProduct.c3typ
+* Scikit-Learn Gaussian Process Dot Product Kernel.
+* See: https://scikit-learn.org/stable/modules/generated/sklearn.gaussian_process.kernels.DotProduct.html#sklearn.gaussian_process.kernels.DotProduct
+*/
+type SklearnGPRKernelDotProduct {
+    // Constant that defines the kernel inhomogenity
+    sigmaZero: !double
+    // the SklearnGPRKernel for this object
+    kernel: SklearnGPRKernel
+
+    // constructs the SklearnGPRKernel internal object
+    @py(env='gordon-ML_1_0_0')
+    build: member function(): SklearnGPRKernelDotProduct
+}

training/gordon-group/src/CustomMLPipeline/machineLearning/GPRegression/kernels/SklearnGPRKernelDotProduct.py

@@ -0,0 +1,21 @@
+def build(this):
+    """
+    This effectively constructs the type instance, by creating the object to be placed in the kernel field. Ideally this should be replaced by a callback function (beforeMake or afterMake).
+    """
+    from sklearn.gaussian_process.kernels import DotProduct
+
+    sklKernel = DotProduct(sigma_0=this.sigmaZero)
+
+    kernel_pickled = c3.PythonSerialization.serialize(obj=sklKernel)
+    kernel_name = 'DotProduct'
+    kernel_hyperParameters = c3.c3Make(
+        "map<string, double>", {"sigmaZero": this.sigmaZero}
+    )
+
+    this.kernel = c3.SklearnGPRKernel(
+        name=kernel_name,
+        hyperParameters=kernel_hyperParameters,
+        pickledKernel=kernel_pickled
+    )
+
+    return this

training/gordon-group/src/CustomMLPipeline/machineLearning/GPRegression/kernels/SklearnGPRKernelExpSineSquared.c3typ

@@ -0,0 +1,17 @@
+/**
+* SklearnGPRKernelExpSineSquared.c3typ
+* Scikit-Learn Gaussian Process Exponential Sine Squared Kernel.
+* See: https://scikit-learn.org/stable/modules/generated/sklearn.gaussian_process.kernels.ExpSineSquared.html#sklearn.gaussian_process.kernels.ExpSineSquared
+*/
+type SklearnGPRKernelExpSineSquared {
+    // Constant that defines the kernel length scale
+    lengthScale: !double
+    // Constant that defines the kernel periodicity
+    periodicity: !double
+    // the SklearnGPRKernel for this object
+    kernel: SklearnGPRKernel
+
+    // constructs the SklearnGPRKernel internal object
+    @py(env='gordon-ML_1_0_0')
+    build: member function(): SklearnGPRKernelExpSineSquared
+}

training/gordon-group/src/CustomMLPipeline/machineLearning/GPRegression/kernels/SklearnGPRKernelExpSineSquared.py

@@ -0,0 +1,23 @@
+def build(this):
+    """
+    This effectively constructs the type instance, by creating the object to be placed in the kernel field. Ideally this should be replaced by a callback function (beforeMake or afterMake).
+    """
+    from sklearn.gaussian_process.kernels import ExpSineSquared
+
+    sklKernel = ExpSineSquared(length_scale=this.lengthScale, periodicity=this.periodicity)
+
+    kernel_pickled = c3.PythonSerialization.serialize(obj=sklKernel)
+    kernel_name = 'ExpSineSquared'
+    kernel_hyperParameters = c3.c3Make(
+        "map<string, double>", {"lengthScale": this.lengthScale,
+            "periodicity": this.periodicity
+        }
+    )
+
+    this.kernel = c3.SklearnGPRKernel(
+        name=kernel_name,
+        hyperParameters=kernel_hyperParameters,
+        pickledKernel=kernel_pickled
+    )
+
+    return this