Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
207 commits
Select commit Hold shift + click to select a range
61f0087
Merge pull request #1 from c3aidti/develop
babreu-ncsa Apr 14, 2022
f285ed7
Update README.md
babreu-ncsa Apr 15, 2022
af73904
Update README.md
babreu-ncsa Apr 15, 2022
f6014b7
Update README.md
babreu-ncsa Apr 15, 2022
094b6c2
Update README.md
babreu-ncsa Apr 15, 2022
97d0de3
Merge pull request #6 from c3aidti/develop
dadamsncsa May 13, 2022
381da25
Merge pull request #8 from c3aidti/develop
dadamsncsa May 25, 2022
cadc89f
Merge pull request #19 from c3aidti/develop
dadamsncsa May 25, 2022
660cf11
Merge pull request #17 from c3aidti/develop
dadamsncsa May 25, 2022
d638489
ignore file
babreu-ncsa Aug 10, 2022
dc50771
bye stress test
babreu-ncsa Aug 10, 2022
1930cc0
leaving just source and transform types
babreu-ncsa Aug 10, 2022
24c6536
bye transform
babreu-ncsa Aug 10, 2022
f21c693
entity - coordinates
babreu-ncsa Aug 10, 2022
374b875
update dependecies
babreu-ncsa Aug 10, 2022
5d428a1
3hourlyaod output
babreu-ncsa Aug 10, 2022
fe40b8c
add sim sample
babreu-ncsa Aug 10, 2022
10fd875
add simulation ensemble
babreu-ncsa Aug 10, 2022
085f475
add model params
babreu-ncsa Aug 10, 2022
bad077b
acure aircraft output
babreu-ncsa Aug 10, 2022
cb434de
acure aircraft series
babreu-ncsa Aug 10, 2022
9b21f70
simulation file table
babreu-ncsa Aug 10, 2022
e60e40f
metafileproc util
babreu-ncsa Aug 10, 2022
ac31353
shrink schema name
babreu-ncsa Aug 10, 2022
38b19fa
ensemble transform
babreu-ncsa Aug 10, 2022
e9824d5
parameters transform
babreu-ncsa Aug 10, 2022
1725196
sample transform
babreu-ncsa Aug 10, 2022
ce2704d
observation set
babreu-ncsa Aug 10, 2022
5efc9f3
atom series
babreu-ncsa Aug 10, 2022
f02ca69
add atom header
babreu-ncsa Aug 10, 2022
3db60ac
oracles set
babreu-ncsa Aug 10, 2022
7e1ca7d
file table
babreu-ncsa Aug 10, 2022
ea871ad
lil fix on atom series
babreu-ncsa Aug 10, 2022
84e1875
transform to series types
babreu-ncsa Aug 10, 2022
e393102
remaining transforms
babreu-ncsa Aug 10, 2022
39f1f72
header
babreu-ncsa Aug 10, 2022
fcc2c0e
add atom seed
babreu-ncsa Aug 10, 2022
26c4d39
netcdf util
babreu-ncsa Aug 10, 2022
9c60ec7
sim sample methods
babreu-ncsa Aug 11, 2022
be73383
sim file methods
babreu-ncsa Aug 11, 2022
8506423
get rid of method to upsert data for all sims
babreu-ncsa Aug 11, 2022
6e1b009
ensemble does not need methods
babreu-ncsa Aug 11, 2022
e7b6cb7
don't set lims directly
babreu-ncsa Aug 11, 2022
f325394
lil fix
babreu-ncsa Aug 11, 2022
5cd5528
something is not working...
babreu-ncsa Aug 11, 2022
e8f243a
another simsam trial
babreu-ncsa Aug 15, 2022
fb9953b
isolating
babreu-ncsa Aug 15, 2022
54312dd
trying again
babreu-ncsa Aug 15, 2022
c51a0c9
?
babreu-ncsa Aug 15, 2022
ac00be0
trying....
babreu-ncsa Aug 15, 2022
a587164
one more
babreu-ncsa Aug 15, 2022
fdd9150
try........
babreu-ncsa Aug 15, 2022
139aad8
oh well
babreu-ncsa Aug 15, 2022
47bb66a
.........
babreu-ncsa Aug 15, 2022
fd4179f
almost there?
babreu-ncsa Aug 15, 2022
7eb3c59
it looks it was the ensemble..
babreu-ncsa Aug 15, 2022
c4e5afd
almost....
babreu-ncsa Aug 15, 2022
e33a2c1
one more
babreu-ncsa Aug 15, 2022
aadcf27
obs set method
babreu-ncsa Aug 17, 2022
a38ea23
legacy netcdf lib
babreu-ncsa Aug 17, 2022
0c02451
try newer python
babreu-ncsa Aug 17, 2022
de60fe3
not that new
babreu-ncsa Aug 17, 2022
63384ed
new netcdf env
babreu-ncsa Aug 17, 2022
1c0ce35
legacy methods
babreu-ncsa Aug 17, 2022
a172ba0
include defs
babreu-ncsa Aug 17, 2022
587c439
close files, use legacy for atom
babreu-ncsa Aug 18, 2022
56ad5fa
grab obsSet object
babreu-ncsa Aug 18, 2022
8c8ff25
fix on time stamps
babreu-ncsa Aug 18, 2022
d9017cd
some buggy version conflict...
babreu-ncsa Aug 18, 2022
41e6fb9
add pandas for test... yes
babreu-ncsa Aug 23, 2022
3593ca5
dump into file
babreu-ncsa Aug 23, 2022
4407b5b
try again
babreu-ncsa Aug 23, 2022
f722977
again....
babreu-ncsa Aug 23, 2022
a0ceb4b
inst()
babreu-ncsa Aug 23, 2022
f7548ba
another kernel
babreu-ncsa Aug 23, 2022
c60f6a4
longshot
babreu-ncsa Aug 23, 2022
569b4dd
atom data is in
babreu-ncsa Aug 23, 2022
423a916
seed atom access nb
babreu-ncsa Aug 24, 2022
4bd02c3
add correct type
babreu-ncsa Aug 25, 2022
b18424c
add nice merge call
babreu-ncsa Aug 25, 2022
f2dcb65
change oracles granularity
babreu-ncsa Sep 27, 2022
3055fbc
back to second
babreu-ncsa Sep 27, 2022
bd28626
works now
babreu-ncsa Oct 20, 2022
6686640
batch job for acure data
babreu-ncsa Oct 24, 2022
7b2d486
implementation was wrong
babreu-ncsa Oct 24, 2022
852ae0a
had the wrong type name
babreu-ncsa Oct 24, 2022
a17e248
bj for aod headers
babreu-ncsa Oct 25, 2022
69d3e2f
job to add data to cassandra
babreu-ncsa Nov 10, 2022
8ce5436
hardware prof id
babreu-ncsa Nov 10, 2022
175c96a
custom ml stuff
babreu-ncsa Nov 11, 2022
6f7442f
fix typo
babreu-ncsa Nov 11, 2022
9b35634
batch job for training
babreu-ncsa Nov 11, 2022
68ca87f
return obj
babreu-ncsa Nov 11, 2022
7f3531f
aod utils
babreu-ncsa Nov 29, 2022
55d7281
add predict util
babreu-ncsa Nov 29, 2022
89ee5de
add correct hwp
babreu-ncsa Nov 29, 2022
b53f675
Merge pull request #20 from c3aidti/smoke-dev
babreu-ncsa Nov 29, 2022
bed347d
add nbformat to kernel
hgordo Jan 20, 2023
bfceec0
undo silly change
hgordo Jan 20, 2023
35fac7c
try again, hopefully correct branch
hgordo Jan 20, 2023
e1d41d4
missing comma
babreu-ncsa Jan 23, 2023
9634103
nbformat v5.1.3
babreu-ncsa Jan 23, 2023
4eca59a
Merge pull request #21 from c3aidti/smoke-dev
babreu-ncsa Jan 23, 2023
bb4c1da
trying MODIS observation
babreu-ncsa Jan 24, 2023
2228b3b
correc key to file
babreu-ncsa Jan 24, 2023
b6456c6
fix typo
babreu-ncsa Jan 24, 2023
c434ab9
another tiny fix
babreu-ncsa Jan 24, 2023
fee9112
correct file paths
babreu-ncsa Jan 24, 2023
8cd58b0
another little fix
babreu-ncsa Jan 24, 2023
a75b355
fix on dates
babreu-ncsa Jan 24, 2023
8ec6434
add gstps
babreu-ncsa Jan 24, 2023
f66e4a4
fix typo
babreu-ncsa Jan 24, 2023
53fc869
metadata for processing
babreu-ncsa Jan 24, 2023
2b55213
small change
babreu-ncsa Jan 30, 2023
f9aa98f
add seed data for SmokePPESims
babreu-ncsa Feb 7, 2023
0be068f
source type for SmokePPE
babreu-ncsa Feb 7, 2023
82e0e72
entity type for SmokePPE parameters
babreu-ncsa Feb 7, 2023
bc191fc
seed ensemble
babreu-ncsa Feb 7, 2023
c35d8c9
transform type for parameters map
babreu-ncsa Feb 7, 2023
9402c4a
type def fix
babreu-ncsa Feb 7, 2023
c106d62
smoke ppe series type
babreu-ncsa Feb 7, 2023
60b655d
raw data ppe type
babreu-ncsa Feb 7, 2023
108112a
annotation
babreu-ncsa Feb 7, 2023
0f282a3
id field
babreu-ncsa Feb 7, 2023
18ae690
shorten schema name
babreu-ncsa Feb 7, 2023
fb71c22
Merge pull request #22 from c3aidti/smoke-dev
babreu-ncsa Feb 7, 2023
7f009a4
transform into SmokePPE sim sam
babreu-ncsa Feb 7, 2023
9eb623a
transform series
babreu-ncsa Feb 7, 2023
f228284
type name fix
babreu-ncsa Feb 7, 2023
93950ea
interpolation python runtime
babreu-ncsa Feb 7, 2023
936f690
change python v
babreu-ncsa Feb 7, 2023
c99e3ab
add dask to runtime
babreu-ncsa Feb 8, 2023
4dce689
from local env
babreu-ncsa Feb 8, 2023
01814b7
change azure blob
hgordo Mar 3, 2023
e463e6b
change back
vasanchez16 Mar 3, 2023
c07625a
Try to generalize for smoke-ppe
vasanchez16 Mar 3, 2023
6fb0ed0
Fixes for recent edits in js and comments in py
vasanchez16 Mar 8, 2023
2bb0a70
this won't be a TS
babreu-ncsa Mar 9, 2023
c2ec22e
another fix here
babreu-ncsa Mar 9, 2023
81e3cc2
create files table for smoke ppe
babreu-ncsa Mar 9, 2023
20a2b79
little fix to find files
babreu-ncsa Mar 9, 2023
27aa7fd
must create a separate type...
babreu-ncsa Mar 9, 2023
43249e0
I did not save the file!
babreu-ncsa Mar 9, 2023
bf01b28
create headers
babreu-ncsa Mar 9, 2023
b0ef8da
make better name
babreu-ncsa Mar 10, 2023
8397624
headers for smoke-ppe
babreu-ncsa Mar 13, 2023
6da7661
just need pseudo-level index
babreu-ncsa Mar 14, 2023
33161a2
persistence order
babreu-ncsa Mar 14, 2023
05662e1
make fields not required
babreu-ncsa Mar 14, 2023
3859185
try again
babreu-ncsa Mar 15, 2023
2d51372
another fix
babreu-ncsa Mar 15, 2023
7d45d78
method to process files for each sample
babreu-ncsa Mar 15, 2023
a7d5770
batch job to upsert smoke ppe data
babreu-ncsa Mar 15, 2023
8d65c62
Merge pull request #23 from c3aidti/smoke-dev
babreu-ncsa Mar 22, 2023
e5100ed
Adding new files for smokePPE
vasanchez16 Mar 23, 2023
95f0680
Adding code to new smokeppe files
vasanchez16 Mar 28, 2023
c56f8a3
CI paper examples
JamesCarzon Mar 29, 2023
662b3aa
Move CI content folder
JamesCarzon Mar 29, 2023
6bd9d0b
Assign file types to postProcessedData
JamesCarzon Mar 29, 2023
b4b2006
Merge branch 'main' into smoke-dev
babreu-ncsa Mar 30, 2023
63fb807
more modifications to smokePPEGaussianML folder
vasanchez16 Apr 10, 2023
642a4ec
capitals
babreu-ncsa Apr 11, 2023
c55be2e
Rename smokePPEGaussianMLTrainingJob.c3typ to SmokePPEGaussianMLTrain…
babreu-ncsa Apr 11, 2023
601a8d5
Rename smokePPEGaussianMLTrainingJob.js to SmokePPEGaussianMLTraining…
babreu-ncsa Apr 11, 2023
c647d4d
Rename smokePPEGaussianMLTrainingJobBatch.c3typ to SmokePPEGaussianML…
babreu-ncsa Apr 11, 2023
cf7da7d
Rename smokePPEGaussianMLTrainingJobOptions.c3typ to SmokePPEGaussian…
babreu-ncsa Apr 11, 2023
031718c
update smoke ppe parameters
babreu-ncsa Apr 12, 2023
38f7078
rm csv from git repo
babreu-ncsa Apr 17, 2023
f12f9a2
add csvs outside app package
babreu-ncsa Apr 17, 2023
b47771d
workaround for nones
babreu-ncsa Apr 17, 2023
8b088b8
train only if sizes are > 0
babreu-ncsa Apr 17, 2023
59e3793
Merge pull request #29 from c3aidti/smoke-dev
babreu-ncsa Apr 17, 2023
6822c77
drop integer targets
babreu-ncsa Apr 19, 2023
402200c
grab entire technique
babreu-ncsa May 2, 2023
d73756f
add full kernel
babreu-ncsa May 2, 2023
bf19532
staging type
babreu-ncsa May 17, 2023
a4a9b57
staging methods
babreu-ncsa May 17, 2023
4033693
send objs
babreu-ncsa May 17, 2023
0239314
pushing PPE predictor
babreu-ncsa May 24, 2023
e283984
fix to get full kernel
babreu-ncsa May 24, 2023
5e0d0f3
desperate attempt
babreu-ncsa May 26, 2023
8034687
extractLearnedParametersJob additions for smokeppe
vasanchez16 Jun 6, 2023
bc38a10
fix to first approach for extract parameters
vasanchez16 Jun 8, 2023
338fe59
second approach to extract learned params job
vasanchez16 Jun 8, 2023
90c5d01
including staged training
babreu-ncsa Jun 14, 2023
e759104
fixed dostart
babreu-ncsa Jun 14, 2023
48fda78
undo first approach to extractParamsJob
vasanchez16 Jun 15, 2023
6519c4c
limit argument test
vasanchez16 Jun 28, 2023
8eabaee
undo limit arg, made no difference
vasanchez16 Jun 29, 2023
a979c23
small fix
babreu-ncsa Jun 29, 2023
ddb681c
Merge branch 'smoke-dev' of https://github.com/c3aidti/smoke into smo…
babreu-ncsa Jun 29, 2023
3a9bfc6
add method to count pipes
babreu-ncsa Jul 12, 2023
180fb09
30min to test ONE line
babreu-ncsa Jul 12, 2023
d3967ea
another line
babreu-ncsa Jul 12, 2023
0bd3787
filter simulation hours of interest
vasanchez16 Jul 15, 2023
a3ae280
Merge branch 'smoke-dev' of https://github.com/c3aidti/smoke into smo…
vasanchez16 Jul 15, 2023
8eb4d26
limit arg test
vasanchez16 Jul 15, 2023
8810dbb
undo sim hours filter, change made outside github
vasanchez16 Jul 15, 2023
021796c
undo second limit arg try
vasanchez16 Jul 15, 2023
5855dc7
include model in predjob output
vasanchez16 Jul 17, 2023
74da7ea
try include gstpId in pred output
vasanchez16 Jul 17, 2023
b9af8e1
add sklearn version
babreu-ncsa Jul 17, 2023
575752d
get all results
babreu-ncsa Sep 7, 2023
de738af
get all results, for old ppe func
vasanchez16 Sep 18, 2023
bc5d2ec
new type with coarse graining
babreu-ncsa Oct 17, 2023
96d418d
Merge pull request #31 from c3aidti/smoke-dev
dadamsncsa Nov 7, 2023
802bb9e
Merge pull request #33 from c3aidti/dev/smokeApp/rc1
dadamsncsa Nov 7, 2023
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions .gitignore
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
.vscode*
.DS_Store*
smoke/.vscode/
24 changes: 21 additions & 3 deletions README.md
Original file line number Diff line number Diff line change
@@ -1,5 +1,23 @@
# app-space-template
# smoke
Hamish Gordon's group space for development and deployment of C3 AI DTI applications.

### Template Repo for new C3ai application
### Main components
The `smoke` space in C3's IDS, which lives in DTI RC01, currently has one application (`smokeApp`) and one branch (`dev01`). The latter corresponds to a C3 tag that can be access via the vanity URL https://dev01-smokeapp.devrc01.c3aids.cloud/static/console/

more
### Typical development workflow
This GitHub repository always contains at least three branches:

- an intrinsic `main` branch
- a C3 `develop` branch that hosts the application code
- a `dev01` branch that represents what is deployed to the tag above

The typical development workflow then goes as follows:

1. Clone the repository to your local environment (we highly encourage using VS Code)
2. Create a new branch **from** `dev01` and give it a meaningful name, e.g. `username-dev`
3. Develop code, regularly commit, and push back to GitHub when you want the code provisioned to the tag
4. Open a pull request **from** `username-dev` **to** `dev01`
5. Github Actions that DTI DevOps have running will then trigger the provisioning of the package to the `dev01` tag

As long as the pull request is open, pushing code to GitHub will always trigger the provisioning workflow.
For that reason, if you have multiple developers in your team, we recommend that you consider more ephemeral pull requests to tackle simple tasks at a time. You are welcome to use GitHub Projects to give them tags and other details, as well as using GitHub Issues to describe the steps of more complex development efforts that can then be broken into several small tasks, each one with its own PR (open, close, next).
135 changes: 135 additions & 0 deletions climateInformatics2023/notebooks/MissingnessMapPlot.ipynb
Original file line number Diff line number Diff line change
@@ -0,0 +1,135 @@
{
"cells": [
{
"cell_type": "code",
"execution_count": null,
"id": "651c359f",
"metadata": {
"ExecuteTime": {
"end_time": "2023-03-29T18:08:35.554369Z",
"start_time": "2023-03-29T18:08:35.166169Z"
}
},
"outputs": [],
"source": [
"import pandas as pd\n",
"import numpy as np\n",
"import matplotlib.pyplot as plt\n",
"from scipy.interpolate import griddata\n",
"\n",
"import sys\n",
"try:\n",
" import cartopy.crs as ccrs\n",
"except ImportError:\n",
" pass\n",
"\n",
"if 'cartopy.crs' not in sys.modules:\n",
" get_cartopy = False\n",
" print('You have not imported the {} module'.format('cartopy.crs'))\n",
"else:\n",
" get_cartopy = True"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "92b0eca9",
"metadata": {
"ExecuteTime": {
"end_time": "2023-03-29T18:09:03.792088Z",
"start_time": "2023-03-29T18:09:03.748109Z"
}
},
"outputs": [],
"source": [
"with open(\"observations\", \"r\") as f:\n",
" obs_df = pd.read_csv(f, index_col=0)\n",
"f.close()\n",
"\n",
"with open(\"outliers\", \"r\") as f:\n",
" outliers = pd.read_csv(f)\n",
"f.close()\n",
"\n",
"my_outliers = outliers[outliers.time<=np.min(outliers.time)]"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "303ba60d",
"metadata": {
"ExecuteTime": {
"end_time": "2023-03-29T18:38:00.271521Z",
"start_time": "2023-03-29T18:37:59.883525Z"
}
},
"outputs": [],
"source": [
"projection = ccrs.PlateCarree(central_longitude=0)\n",
"fig = plt.figure(figsize=(10,10))\n",
"\n",
"# Draw island\n",
"ax = fig.add_subplot(1, 1, 1, projection=projection)\n",
"ax.coastlines()\n",
"\n",
"# Produce gridlines, coordinate labels\n",
"ax.set_extent([-45, 40, -30, 10], ccrs.PlateCarree())\n",
"ax.gridlines(draw_labels=True, crs=projection)\n",
"\n",
"# Add points along flight path and color for altitude\n",
"measurements = ax.scatter(\n",
" my_outliers[(my_outliers.missing==False) & (my_outliers.outlier==False)].longitude,\n",
" my_outliers[(my_outliers.missing==False) & (my_outliers.outlier==False)].latitude,\n",
" zorder=1, alpha=1,\n",
" c=my_outliers[(my_outliers.missing==False) & (my_outliers.outlier==False)].meanResponse, cmap=\"Blues\"\n",
")\n",
"\n",
"cbar = plt.colorbar(measurements, orientation = 'horizontal')\n",
"cbar.set_label('Measured AOD', fontsize = 14)\n",
"\n",
"ax.scatter(\n",
" my_outliers[my_outliers.missing==True].longitude,\n",
" my_outliers[my_outliers.missing==True].latitude,\n",
" zorder=1, alpha=0.5,\n",
" c=\"r\", marker=\"s\", s=50,\n",
" label='Missing measurement'\n",
")\n",
"ax.scatter(\n",
" my_outliers[my_outliers.outlier==True].longitude,\n",
" my_outliers[my_outliers.outlier==True].latitude,\n",
" zorder=1, alpha=0.5,\n",
" c=\"g\", marker=\"s\", s=50,\n",
" label='Outlier'\n",
")\n",
"\n",
"plt.legend(loc = 'upper center', bbox_to_anchor = (0.5, -0.1), ncol = 2, fontsize = 14)\n",
"plt.show()"
]
}
],
"metadata": {
"has_local_update": true,
"is_local": true,
"is_remote": true,
"kernelspec": {
"display_name": "py-gordon_1_0_0",
"language": "Python",
"name": "py-gordon_1_0_0"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.6.13"
},
"last_sync_time": "2023-03-29T19:11:40.222296"
},
"nbformat": 4,
"nbformat_minor": 5
}
1 change: 1 addition & 0 deletions climateInformatics2023/notebooks/classes/Clusterer.ipynb

Large diffs are not rendered by default.

Loading