cmip-tools/process_CMIP_data.py at master · dare-centre/cmip-tools · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
# %% [markdown]
# # Download CMIP data and post-process model results
# Written by Joshua Simmons 10-2021
#
# The goal of this notebook is to enable the download and processing of model ensemble data from the CMIP model outputs.
# %% [markdown]
# ## Imports
import argparse
from functions.f_cmip_data_postprocess import (ensemble_CMIP_variable)

# %% [markdown]
# ## Using Google Cloud
# Here we will use the zarr format data stored on Google's cloud servers.
#
# 1. Get the data list from the cmip6-zarr-consolidated-stores.csv
# 2. Download for a particular model
# 3. Average (+ other stats) across the realisations
# 4. Store the model averaged realisations
#
#  python  process_CMIP_data.py "/Users/jsim3774/projects/CMIP/sample/raw" "/Users/jsim3774/projects/CMIP/sample/processed"
#  --variable tas --experiement historical --frequency Amon

# variable = 'tas'
# experiment = 'historical'
# frequency = 'Amon'

# docker run -v "`pwd`:/code" -v "/Users/jsim3774/projects/CMIP/sample:/sample" -w /code --privileged -it singtest

# singularity exec -B /code:/code -B /sample:/sample ./artemis/testcontainer.simg /bin/bash -c "cd /code && python process_CMIP_data.py '/sample/raw' '/sample/processed --variable tas --experiment historical --frequency Amon --model 'BCC-ESM1' 'CESM2-WACCM' --exclude 'ICON-ESM-LR' --no-download"

# %%
if __name__ == "__main__":
    # setup arguement parsing
    parser = argparse.ArgumentParser(description='Download and processing of model ensemble data from the CMIP model outputs.')
    parser.add_argument('storageDir', metavar='storageDirectory', type=str,
                        help='Directory in which the raw data is stored')
    parser.add_argument('outputDir', metavar='outputDirectory', type=str,
                        help='Directory in which to store the processed data')
    parser.add_argument('--variable', type=str, nargs='*', metavar='modelVariable',
                        required=True,
                        help='The variable(s) to collect from CMIP')
    parser.add_argument('--experiment', type=str, metavar='experimentName',
                        required=True,
                        help='The experiment(s) to collect data for ("historical" or "")')
    parser.add_argument('--frequency', type=str, metavar='outputFrequency',
                        required=True,
                        help='The resolution(s) of the data in time')
    parser.add_argument('--var-check', type=str, nargs='*', dest='varCheck',
                        help='Variables over which to check for model consistency')
    parser.add_argument('--exp-check', type=str, nargs='*', dest='expCheck',
                        help='Experiments over which to check for model consistency')
    parser.add_argument('--no-dask', dest='dask', action='store_false',
                        help='If flag included dask will not be used.')
    parser.add_argument('--force', dest='force', action='store_true',
                        help='If flag included available models will be downloaded even if they already exist in storeDir.')

    # First get the args in
    args = parser.parse_args()

    # now run the actual collection script
    ensemble_CMIP_variable(
        args.storageDir,
        args.outputDir,
        args.variable,
        args.experiment,
        args.frequency,
        force=args.force,
        dask=args.dask,
        varCheck={
            'variable_id': args.varCheck,
            'experiment_id': args.expCheck
        }
    )