Get metadata on available ESGF (CMIP6) datasets and store in json files

In this notebook ESMValTool is used to query available ESGF nodes to find which available climate model runs contain the variables we are interested in. The search_esgf function does a call to online meta-data servers and returns all available climate datasets that match our specified criteria. This notebook is used to make a choice which climate model to use for the impact analyses. In notebook 0a this choice is hard-coded.

Note on names

Confusingly, names like dataset and not uniquely defined. A Dataset object in EMSValTool is a combination of climate model, ensemble member, experiment, etc. However, the ‘climate model’ in a Dataset is identified using the key dataset. This is confusing, but outside of my control. See Juckes et al 2020 for a detailed description of the jargon used in CMIP6.

Note on downloaded data

This notebook only queries for meta-data and does not download any other data. The actual data sits on different servers which may at any time be offline. This function returns a list of data that exists, but it does not guarantee that the data is available at the current time.

Note on eWaterCycle vs ESMValTool

In this notebook we use the search_esgf function we build on top of ESMValTool, but the actual call to the meta data server is made from the ESMValTool package that we have wrapped in eWaterCycle. None of this would be possible without the work of the ESMValTool team.

import ewatercycle.esmvaltool.search
from esmvalcore.config import CFG

from rich import print
import json

/home/mmelotto/.conda/envs/ewatercycle_snellius/lib/python3.12/site-packages/myproxy/__init__.py:1: UserWarning: pkg_resources is deprecated as an API. See https://setuptools.pypa.io/en/latest/pkg_resources.html. The pkg_resources package is slated for removal as early as 2025-11-30. Refrain from using this package or pin to Setuptools<81.
  __import__('pkg_resources').declare_namespace(__name__)

# Parameters
region_id = None
settings_path = "settings.json"

# Parameters
region_id = "camelsgb_76008"
settings_path = "regions/camelsgb_76008/settings.json"

# Setting for ESMValTool to make sure the online esgf resources are always used and
# we don't rely on locally cached information.
CFG['search_esgf'] = 'always'

experiment_of_interest=["historical","ssp126","ssp245","ssp370","ssp585"]
project_of_interest = "CMIP6"
frequency_of_interest="day"
variables_of_interest=["pr", "tas", "rsds"]

# We query the ESGF databases for available datasets. This calls external servers that host ESGF
# metadata which may be down at any moment. This query can take a long time to complete (minutes 
# to half an hour easily)



# valid_datasets = ewatercycle.esmvaltool.search.search_esgf(
#     experiment=experiment_of_interest,
#     project = project_of_interest,
#     frequency = frequency_of_interest,
#     variables = variables_of_interest
# )

# print(valid_datasets["MPI-ESM1-2-HR"])

# # Convert sets to lists for JSON serialization
# serializable_valid_datasets = {key: list(value) for key, value in valid_datasets[0].items()}

# # Write to a JSON file
# with open(f"regions/{region_id}/available_climate_datasets.json", "w") as json_file:
#     json.dump(serializable_valid_datasets, json_file, indent=4)

# Read from the JSON file
with open(f"regions/{region_id}/available_climate_datasets.json", "r") as json_file:
    loaded_dict = json.load(json_file)

# Convert lists back to sets
restored_valid_datasets = {key: set(value) for key, value in loaded_dict.items()}

print("Restored dictionary:", restored_valid_datasets)

Restored dictionary:
{
    'UKESM1-0-LL': {
        'r8i1p1f2',
        'r5i1p1f3',
        'r14i1p1f2',
        'r11i1p1f2',
        'r9i1p1f2',
        'r1i1p1f2',
        'r2i1p1f2',
        'r7i1p1f2',
        'r10i1p1f2',
        'r3i1p1f2',
        'r12i1p1f2',
        'r18i1p1f2',
        'r13i1p1f2',
        'r6i1p1f3',
        'r16i1p1f2',
        'r17i1p1f2',
        'r5i1p1f2',
        'r19i1p1f2',
        'r6i1p1f2',
        'r7i1p1f3',
        'r4i1p1f2',
        'r15i1p1f2'
    },
    'FGOALS-f3-L': {'r1i1p1f1'},
    'CNRM-ESM2-1': {
        'r9i1p1f2',
        'r13i1p1f2',
        'r8i1p1f2',
        'r2i1p1f2',
        'r1i1p1f2',
        'r7i1p1f2',
        'r10i1p1f2',
        'r6i1p1f2',
        'r3i1p1f2',
        'r14i1p1f2',
        'r12i1p1f2',
        'r4i1p1f2',
        'r5i1p1f2',
        'r11i1p1f2',
        'r15i1p1f2'
    },
    'FGOALS-g3': {'r1i1p1f1'},
    'AWI-ESM-1-REcoM': {'r1i1p1f1'},
    'CMCC-CM2-SR5': {
        'r2i1p2f1',
        'r9i1p2f1',
        'r11i1p2f1',
        'r1i1p1f1',
        'r4i1p2f1',
        'r3i1p2f1',
        'r5i1p2f1',
        'r10i1p2f1',
        'r8i1p2f1',
        'r6i1p2f1',
        'r7i1p2f1'
    },
    'ICON-ESM-LR': {'r5i1p1f1', 'r1i1p1f1', 'r3i1p1f1', 'r2i1p1f1', 'r4i1p1f1'},
    'ACCESS-ESM1-5': {'r3i1p1f1', 'r1i1p1f1', 'r2i1p1f1'},
    'IPSL-CM6A-LR': {
        'r5i1p1f1',
        'r27i1p1f1',
        'r7i1p1f1',
        'r21i1p1f1',
        'r30i1p1f1',
        'r8i1p1f1',
        'r31i1p1f1',
        'r4i1p1f1',
        'r18i1p1f1',
        'r13i1p1f1',
        'r32i1p1f1',
        'r17i1p1f1',
        'r1i1p1f1',
        'r6i1p1f1',
        'r15i1p1f1',
        'r10i1p1f1',
        'r14i1p1f1',
        'r20i1p1f1',
        'r33i1p1f1',
        'r9i1p1f1',
        'r16i1p1f1',
        'r29i1p1f1',
        'r25i1p1f1',
        'r12i1p1f1',
        'r19i1p1f1',
        'r11i1p1f1',
        'r28i1p1f1',
        'r3i1p1f1',
        'r24i1p1f1',
        'r26i1p1f1',
        'r22i1p1f1',
        'r2i1p1f1',
        'r23i1p1f1'
    },
    'AWI-ESM-1-1-LR': {'r1i1p1f1'},
    'CanESM5': {
        'r5i1p1f1',
        'r25i1p2f1',
        'r9i1p2f1',
        'r3i1p2f1',
        'r19i1p2f1',
        'r7i1p1f1',
        'r21i1p1f1',
        'r6i1p2f1',
        'r17i1p2f1',
        'r1i1p2f1',
        'r5i1p2f1',
        'r23i1p1f1',
        'r13i1p2f1',
        'r8i1p1f1',
        'r7i1p2f1',
        'r4i1p1f1',
        'r18i1p2f1',
        'r18i1p1f1',
        'r23i1p2f1',
        'r20i1p2f1',
        'r13i1p1f1',
        'r2i1p2f1',
        'r17i1p1f1',
        'r1i1p1f1',
        'r11i1p2f1',
        'r6i1p1f1',
        'r15i1p1f1',
        'r22i1p2f1',
        'r10i1p2f1',
        'r14i1p2f1',
        'r8i1p2f1',
        'r12i1p2f1',
        'r10i1p1f1',
        'r14i1p1f1',
        'r20i1p1f1',
        'r9i1p1f1',
        'r16i1p1f1',
        'r25i1p1f1',
        'r12i1p1f1',
        'r4i1p2f1',
        'r19i1p1f1',
        'r11i1p1f1',
        'r3i1p1f1',
        'r24i1p1f1',
        'r15i1p2f1',
        'r16i1p2f1',
        'r21i1p2f1',
        'r22i1p1f1',
        'r2i1p1f1',
        'r24i1p2f1'
    },
    'MPI-ESM1-2-HR': {
        'r5i1p1f1',
        'r9i1p1f1',
        'r1i1p1f1',
        'r6i1p1f1',
        'r3i1p1f1',
        'r7i1p1f1',
        'r8i1p1f1',
        'r2i1p1f1',
        'r10i1p1f1',
        'r4i1p1f1'
    },
    'NorESM2-MM': {'r1i1p1f1'},
    'CMCC-ESM2': {'r1i1p1f1'},
    'CNRM-CM6-1': {
        'r8i1p1f2',
        'r25i1p1f2',
        'r24i1p1f2',
        'r14i1p1f2',
        'r11i1p1f2',
        'r9i1p1f2',
        'r1i1p1f2',
        'r2i1p1f2',
        'r7i1p1f2',
        'r26i1p1f2',
        'r10i1p1f2',
        'r15i1p1f2',
        'r3i1p1f2',
        'r12i1p1f2',
        'r18i1p1f2',
        'r13i1p1f2',
        'r22i1p1f2',
        'r20i1p1f2',
        'r29i1p1f2',
        'r16i1p1f2',
        'r17i1p1f2',
        'r28i1p1f2',
        'r19i1p1f2',
        'r5i1p1f2',
        'r6i1p1f2',
        'r23i1p1f2',
        'r21i1p1f2',
        'r30i1p1f2',
        'r4i1p1f2',
        'r27i1p1f2'
    },
    'MPI-ESM1-2-LR': {
        'r5i1p1f1',
        'r27i1p1f1',
        'r1i2000p1f1',
        'r48i1p1f1',
        'r37i1p1f1',
        'r46i1p1f1',
        'r38i1p1f1',
        'r34i1p1f1',
        'r43i1p1f1',
        'r40i1p1f1',
        'r7i1p1f1',
        'r21i1p1f1',
        'r30i1p1f1',
        'r50i1p1f1',
        'r47i1p1f1',
        'r8i1p1f1',
        'r31i1p1f1',
        'r4i1p1f1',
        'r18i1p1f1',
        'r13i1p1f1',
        'r32i1p1f1',
        'r17i1p1f1',
        'r1i1p1f1',
        'r6i1p1f1',
        'r15i1p1f1',
        'r35i1p1f1',
        'r39i1p1f1',
        'r41i1p1f1',
        'r42i1p1f1',
        'r49i1p1f1',
        'r36i1p1f1',
        'r10i1p1f1',
        'r14i1p1f1',
        'r20i1p1f1',
        'r44i1p1f1',
        'r33i1p1f1',
        'r9i1p1f1',
        'r45i1p1f1',
        'r16i1p1f1',
        'r29i1p1f1',
        'r25i1p1f1',
        'r12i1p1f1',
        'r19i1p1f1',
        'r11i1p1f1',
        'r28i1p1f1',
        'r3i1p1f1',
        'r24i1p1f1',
        'r26i1p1f1',
        'r22i1p1f1',
        'r2i1p1f1',
        'r23i1p1f1'
    },
    'EC-Earth3': {'r9i1p1f1', 'r13i1p1f1', 'r1i1p1f1', 'r6i1p1f1', 'r15i1p1f1', 'r11i1p1f1', 'r2i1p1f1'},
    'IITM-ESM': {'r1i1p1f1'},
    'BCC-CSM2-MR': {'r1i1p1f1'},
    'CESM2-FV2': {'r1i1p1f1'},
    'EC-Earth3-Veg-LR': {'r1i1p1f1'},
    'NorESM2-LM': {'r3i1p1f1', 'r1i1p1f1', 'r2i1p1f1'},
    'HadGEM3-GC31-MM': {'r4i1p1f3', 'r1i1p1f3', 'r2i1p1f3', 'r3i1p1f3'},
    'KIOST-ESM': {'r1i1p1f1'},
    'HadGEM3-GC31-LL': {
        'r44i1p1f3',
        'r60i1p1f3',
        'r26i1p1f3',
        'r12i1p1f3',
        'r59i1p1f3',
        'r16i1p1f3',
        'r35i1p1f3',
        'r5i1p1f3',
        'r46i1p1f3',
        'r52i1p1f3',
        'r1i1p1f3',
        'r33i1p1f3',
        'r32i1p1f3',
        'r36i1p1f3',
        'r23i1p1f3',
        'r4i1p1f3',
        'r27i1p1f3',
        'r41i1p1f3',
        'r43i1p1f3',
        'r37i1p1f3',
        'r24i1p1f3',
        'r11i1p1f3',
        'r48i1p1f3',
        'r57i1p1f3',
        'r19i1p1f3',
        'r22i1p1f3',
        'r25i1p1f3',
        'r17i1p1f3',
        'r15i1p1f3',
        'r51i1p1f3',
        'r47i1p1f3',
        'r53i1p1f3',
        'r39i1p1f3',
        'r40i1p1f3',
        'r58i1p1f3',
        'r30i1p1f3',
        'r21i1p1f3',
        'r55i1p1f3',
        'r31i1p1f3',
        'r3i1p1f3',
        'r14i1p1f3',
        'r29i1p1f3',
        'r45i1p1f3',
        'r54i1p1f3',
        'r20i1p1f3',
        'r13i1p1f3',
        'r56i1p1f3',
        'r18i1p1f3',
        'r2i1p1f3',
        'r34i1p1f3',
        'r50i1p1f3',
        'r38i1p1f3',
        'r28i1p1f3',
        'r42i1p1f3',
        'r49i1p1f3'
    },
    'CESM2-WACCM': {'r3i1p1f1', 'r1i1p1f1', 'r2i1p1f1'},
    'KACE-1-0-G': {'r3i1p1f1', 'r1i1p1f1', 'r2i1p1f1'},
    'MPI-ESM-1-2-HAM': {'r3i1p1f1', 'r1i1p1f1', 'r2i1p1f1'},
    'GFDL-ESM4': {'r1i1p1f1'},
    'AWI-CM-1-1-MR': {'r5i1p1f1', 'r1i1p1f1', 'r3i1p1f1', 'r2i1p1f1', 'r4i1p1f1'},
    'CESM2-WACCM-FV2': {'r1i1p1f1'},
    'EC-Earth3-Veg': {'r12i1p1f1', 'r14i1p1f1'},
    'EC-Earth3-CC': {'r1i1p1f1'},
    'IPSL-CM5A2-INCA': {'r1i1p1f1'},
    'CESM2': {
        'r5i1p1f1',
        'r9i1p1f1',
        'r1i1p1f1',
        'r6i1p1f1',
        'r3i1p1f1',
        'r7i1p1f1',
        'r8i1p1f1',
        'r2i1p1f1',
        'r10i1p1f1',
        'r4i1p1f1'
    },
    'SAM0-UNICON': {'r1i1p1f1'},
    'MIROC6': {'r3i1p1f1', 'r1i1p1f1', 'r2i1p1f1'},
    'EC-Earth3-AerChem': {'r1i1p1f1'},
    'GFDL-CM4': {'r1i1p1f1'},
    'BCC-ESM1': {'r3i1p1f1', 'r1i1p1f1', 'r2i1p1f1'},
    'TaiESM1': {'r1i1p1f1'},
    'CMCC-CM2-HR4': {'r1i1p1f1'},
    'INM-CM5-0': {
        'r5i1p1f1',
        'r9i1p1f1',
        'r1i1p1f1',
        'r6i1p1f1',
        'r3i1p1f1',
        'r7i1p1f1',
        'r8i1p1f1',
        'r2i1p1f1',
        'r10i1p1f1',
        'r4i1p1f1'
    },
    'INM-CM4-8': {'r1i1p1f1'},
    'NESM3': {'r1i1p1f1', 'r2i1p1f1'},
    'CAMS-CSM1-0': {'r2i1p1f1'},
    'MRI-ESM2-0': {'r5i1p1f1', 'r1i1p1f1', 'r3i1p1f1', 'r1i2p1f1', 'r2i1p1f1', 'r4i1p1f1'},
    'CNRM-CM6-1-HR': {'r1i1p1f2'},
    'NorCPM1': {'r1i1p1f1'},
    'ACCESS-CM2': {'r1i1p1f1'}
}