ERA5 Data generation & merging

Contents

ERA5 Data generation & merging#

ERA5 data will be generated from 1970-2025, loaded and merged into one file

Start-up#

# general python
import warnings
warnings.filterwarnings("ignore", category=UserWarning)

import numpy as np
from pathlib import Path
import pandas as pd
import matplotlib.pyplot as plt
import xarray as xr

#niceties
from rich import print

import yaml

# General eWaterCycle
import ewatercycle
import ewatercycle.models
import ewatercycle.forcing

# Fix ERA5 config error
from esmvalcore.config import CFG   # older: from esmvalcore.experimental import CFG
from pathlib import Path
home = Path.home()
OBS6 = Path('/data/shared/climate-data/obs6')
default = Path('/data/shared/climate-data')
CMIP_path = Path(f'{str(home)}/.esmvaltool/downloads/CMIP6')
CFG['rootpath'] = {'OBS6': [OBS6], 'default': [default], 'CMIP6': [CMIP_path]}
print(CFG['rootpath'])
print(CFG['drs'])

{
    'OBS6': [PosixPath('/data/shared/climate-data/obs6')],
    'default': [PosixPath('/data/shared/climate-data')],
    'CMIP6': [PosixPath('/home/maxime/.esmvaltool/downloads/CMIP6')]
}

{'CMIP3': 'ESGF', 'CMIP5': 'ESGF', 'CMIP6': 'ESGF', 'CORDEX': 'ESGF', 'obs4MIPs': 'ESGF'}

Define range & forcing paths#

# Defining period range
# periods = [
#     [1970, 1975, 1980, 1985, 1990, 1995, 2000, 2005, 2010, 2015, 2020],
#     [1974, 1979, 1984, 1989, 1994, 1999, 2004, 2009, 2014, 2019, 2024]]

# For calibration 1:
# periods = [
#     [1990, 1995, 2000, 2005, 2010],
#     [1994, 1999, 2004, 2009, 2017]]

# For calibration 2:
# periods = [
#     [1999, 2005, 2010, 2018],
#     [2004, 2009, 2017, 2019]]

# For validation 1:
# periods = [
#     [2018, 2022],
#     [2021, 2024]]

# For validation 2:
# periods = [
#     [2019, 2022],
#     [2021, 2024]]

# For CMIP trial:
# periods = [
#     [1995, 2000, 2005, 2010],
#     [1999, 2004, 2009, 2014]]

# For CMIP trial 2
# periods = [
#     [1970, 1975, 1980, 1985, 1990, 1995],
#     [1974, 1979, 1984, 1989, 1994, 1999]]

periods = [
    [1989, 1995, 2000, 2005, 2010],
    [1994, 1999, 2004, 2009, 2014]]

folder_names = []
forcing_paths = {}

# Generate file names & paths
for i in range(len(periods[0])):
    start_year = periods[0][i]
    end_year = periods[1][i]
    
    folder_name = f'ERA5-{start_year}-{end_year}'                         # Add -cal or -val for cal and val folder name
    # forcing_path = Path.home() / "BEP-maxime" / "Workyard" / "forcings" / folder_name
    forcing_path = Path.home() / "BEP-maxime" / "book" / "thesis_projects" / "BSc" / "2026_Q4_MaximedeBekker_CEG" / "Workyard" / "forcings" / "ERA5" / folder_name

    folder_names.append(folder_name)
    forcing_paths[folder_name] = forcing_path

    forcing_path.mkdir(exist_ok=True, parents=True)

# Call path test
print(forcing_paths[folder_names[0]])

start_year = periods[0][0]
end_year = periods[1][-1]

print(start_year)
print(end_year)

/home/maxime/BEP-maxime/book/thesis_projects/BSc/2026_Q4_MaximedeBekker_CEG/Workyard/forcings/ERA5/ERA5-1989-1994

Generate ERA5 data#

# Shapefile path
# shape_file = Path.home() / "BEP-maxime" / "Workyard" / "Shapefiles" / "07DA001_basin.shp"
shape_file = Path.home() / "BEP-maxime" / "book" / "thesis_projects" / "BSc" / "2026_Q4_MaximedeBekker_CEG" / "Workyard" / "Shapefiles" / "07DA001_basin.shp"

# Generate forcing data
# for i in range(1):
for i in range(len(folder_names)):
    # Change start year to start time so ERA5 can interpret it
    start_time_ERA5 = f'{periods[0][i]}-01-01T00:00:00Z'
    end_time_ERA5 = f'{periods[1][i]}-12-31T00:00:00Z'

    # Skip if folder is not empty to prevent errors
    if any(forcing_paths[folder_names[i]].iterdir()):
        print(f"Skipping {folder_names[i]} because the folder is not empty.")
        continue

    # Load forcings
    ERA5_forcing = ewatercycle.forcing.sources['LumpedMakkinkForcing'].generate(
        dataset="ERA5",
        start_time=start_time_ERA5,
        end_time=end_time_ERA5,
        shape=shape_file,
        directory=forcing_paths[folder_names[i]])

    # I need info because my patience is limited
    print(f"Finished {folder_names[i]}")

Finished ERA5-1989-1994

Skipping ERA5-1995-1999 because the folder is not empty.

Skipping ERA5-2000-2004 because the folder is not empty.

Skipping ERA5-2005-2009 because the folder is not empty.

Skipping ERA5-2010-2014 because the folder is not empty.

Load ERA5 data#

ERA5_forcings = {}

for i in range(len(folder_names)):
    # Create new path to find relevant files in the folder
    directory_new = forcing_paths[folder_names[i]] / "work" / "diagnostic" / "script"
    
    # Load forcings
    ERA5_forcings[folder_names[i]] = ewatercycle.forcing.sources["LumpedMakkinkForcing"].load(directory=directory_new)

# Test to see if it works
print(folder_names)
ERA5_forcings[folder_names[0]]

['ERA5-1989-1994', 'ERA5-1995-1999', 'ERA5-2000-2004', 'ERA5-2005-2009', 'ERA5-2010-2014']

LumpedMakkinkForcing(start_time='1989-01-01T00:00:00Z', end_time='1994-12-31T00:00:00Z', directory=PosixPath('/home/maxime/BEP-maxime/book/thesis_projects/BSc/2026_Q4_MaximedeBekker_CEG/Workyard/forcings/ERA5/ERA5-1989-1994/work/diagnostic/script'), shape=PosixPath('/home/maxime/BEP-maxime/book/thesis_projects/BSc/2026_Q4_MaximedeBekker_CEG/Workyard/forcings/ERA5/ERA5-1989-1994/work/diagnostic/script/07DA001_basin.shp'), filenames={'pr': 'OBS6_ERA5_reanaly_1_day_pr_1989-1994.nc', 'tas': 'OBS6_ERA5_reanaly_1_day_tas_1989-1994.nc', 'rsds': 'OBS6_ERA5_reanaly_1_day_rsds_1989-1994.nc', 'evspsblpot': 'Derived_Makkink_evspsblpot.nc'})

Merge ERA5 data#

# Function to seperate files based on file names and combine same file names from different folders
def combine_variable(var_name):
    datasets = []

    for i in range(len(folder_names)):

        if var_name == "evspsblpot":
            file_name = "Derived_Makkink_evspsblpot.nc"
        else:
            file_name = f"OBS6_ERA5_reanaly_1_day_{var_name}_{periods[0][i]}-{periods[1][i]}.nc"

        directory = forcing_paths[folder_names[i]] / "work" / "diagnostic" / "script" / file_name

        print(f"Loading {directory}")

        datasets.append(xr.open_dataset(directory))

    combined = xr.concat(datasets, dim="time")
    return combined

# Running function for various file name types
combined_variables = {}
var_names = ["pr", "tas", "rsds", "evspsblpot"]

# combined_path = Path.home() / "BEP-maxime" / "Workyard" / "forcings" / f'ERA5-{periods[0][0]}-{periods[1][-1]}'
combined_path = Path.home() / "BEP-maxime" / "book" / "thesis_projects" / "BSc" / "2026_Q4_MaximedeBekker_CEG" / "Workyard" / "forcings" / "ERA5" / f'ERA5-{start_year}-{end_year}'
combined_path.mkdir(parents=True, exist_ok=True)

for i in range(len(var_names)):
    combined_variables[var_names[i]] = combine_variable(var_names[i])
    combined_variables[var_names[i]].to_netcdf(combined_path / f'combined_ERA5_{start_year}_{end_year}_{var_names[i]}.nc')

Loading 
/home/maxime/BEP-maxime/book/thesis_projects/BSc/2026_Q4_MaximedeBekker_CEG/Workyard/forcings/ERA5/ERA5-1989-1994/w
ork/diagnostic/script/OBS6_ERA5_reanaly_1_day_pr_1989-1994.nc

Loading 
/home/maxime/BEP-maxime/book/thesis_projects/BSc/2026_Q4_MaximedeBekker_CEG/Workyard/forcings/ERA5/ERA5-1995-1999/w
ork/diagnostic/script/OBS6_ERA5_reanaly_1_day_pr_1995-1999.nc

Loading 
/home/maxime/BEP-maxime/book/thesis_projects/BSc/2026_Q4_MaximedeBekker_CEG/Workyard/forcings/ERA5/ERA5-2000-2004/w
ork/diagnostic/script/OBS6_ERA5_reanaly_1_day_pr_2000-2004.nc

Loading 
/home/maxime/BEP-maxime/book/thesis_projects/BSc/2026_Q4_MaximedeBekker_CEG/Workyard/forcings/ERA5/ERA5-2005-2009/w
ork/diagnostic/script/OBS6_ERA5_reanaly_1_day_pr_2005-2009.nc

Loading 
/home/maxime/BEP-maxime/book/thesis_projects/BSc/2026_Q4_MaximedeBekker_CEG/Workyard/forcings/ERA5/ERA5-2010-2014/w
ork/diagnostic/script/OBS6_ERA5_reanaly_1_day_pr_2010-2014.nc

Loading 
/home/maxime/BEP-maxime/book/thesis_projects/BSc/2026_Q4_MaximedeBekker_CEG/Workyard/forcings/ERA5/ERA5-1989-1994/w
ork/diagnostic/script/OBS6_ERA5_reanaly_1_day_tas_1989-1994.nc

Loading 
/home/maxime/BEP-maxime/book/thesis_projects/BSc/2026_Q4_MaximedeBekker_CEG/Workyard/forcings/ERA5/ERA5-1995-1999/w
ork/diagnostic/script/OBS6_ERA5_reanaly_1_day_tas_1995-1999.nc

Loading 
/home/maxime/BEP-maxime/book/thesis_projects/BSc/2026_Q4_MaximedeBekker_CEG/Workyard/forcings/ERA5/ERA5-2000-2004/w
ork/diagnostic/script/OBS6_ERA5_reanaly_1_day_tas_2000-2004.nc

Loading 
/home/maxime/BEP-maxime/book/thesis_projects/BSc/2026_Q4_MaximedeBekker_CEG/Workyard/forcings/ERA5/ERA5-2005-2009/w
ork/diagnostic/script/OBS6_ERA5_reanaly_1_day_tas_2005-2009.nc

Loading 
/home/maxime/BEP-maxime/book/thesis_projects/BSc/2026_Q4_MaximedeBekker_CEG/Workyard/forcings/ERA5/ERA5-2010-2014/w
ork/diagnostic/script/OBS6_ERA5_reanaly_1_day_tas_2010-2014.nc

Loading 
/home/maxime/BEP-maxime/book/thesis_projects/BSc/2026_Q4_MaximedeBekker_CEG/Workyard/forcings/ERA5/ERA5-1989-1994/w
ork/diagnostic/script/OBS6_ERA5_reanaly_1_day_rsds_1989-1994.nc

Loading 
/home/maxime/BEP-maxime/book/thesis_projects/BSc/2026_Q4_MaximedeBekker_CEG/Workyard/forcings/ERA5/ERA5-1995-1999/w
ork/diagnostic/script/OBS6_ERA5_reanaly_1_day_rsds_1995-1999.nc

Loading 
/home/maxime/BEP-maxime/book/thesis_projects/BSc/2026_Q4_MaximedeBekker_CEG/Workyard/forcings/ERA5/ERA5-2000-2004/w
ork/diagnostic/script/OBS6_ERA5_reanaly_1_day_rsds_2000-2004.nc

Loading 
/home/maxime/BEP-maxime/book/thesis_projects/BSc/2026_Q4_MaximedeBekker_CEG/Workyard/forcings/ERA5/ERA5-2005-2009/w
ork/diagnostic/script/OBS6_ERA5_reanaly_1_day_rsds_2005-2009.nc

Loading 
/home/maxime/BEP-maxime/book/thesis_projects/BSc/2026_Q4_MaximedeBekker_CEG/Workyard/forcings/ERA5/ERA5-2010-2014/w
ork/diagnostic/script/OBS6_ERA5_reanaly_1_day_rsds_2010-2014.nc

Loading 
/home/maxime/BEP-maxime/book/thesis_projects/BSc/2026_Q4_MaximedeBekker_CEG/Workyard/forcings/ERA5/ERA5-1989-1994/w
ork/diagnostic/script/Derived_Makkink_evspsblpot.nc

Loading 
/home/maxime/BEP-maxime/book/thesis_projects/BSc/2026_Q4_MaximedeBekker_CEG/Workyard/forcings/ERA5/ERA5-1995-1999/w
ork/diagnostic/script/Derived_Makkink_evspsblpot.nc

Loading 
/home/maxime/BEP-maxime/book/thesis_projects/BSc/2026_Q4_MaximedeBekker_CEG/Workyard/forcings/ERA5/ERA5-2000-2004/w
ork/diagnostic/script/Derived_Makkink_evspsblpot.nc

Loading 
/home/maxime/BEP-maxime/book/thesis_projects/BSc/2026_Q4_MaximedeBekker_CEG/Workyard/forcings/ERA5/ERA5-2005-2009/w
ork/diagnostic/script/Derived_Makkink_evspsblpot.nc

Loading 
/home/maxime/BEP-maxime/book/thesis_projects/BSc/2026_Q4_MaximedeBekker_CEG/Workyard/forcings/ERA5/ERA5-2010-2014/w
ork/diagnostic/script/Derived_Makkink_evspsblpot.nc

Create YAML file#

# Create yaml
forcing_yaml = {
    "start_time": f"{start_year}-01-01T00:00:00Z",
    "end_time": f"{end_year}-12-31T00:00:00Z",
    "shape": str(shape_file),
    "filenames": {
        "pr": f"combined_ERA5_{start_year}_{end_year}_pr.nc",
        "tas": f"combined_ERA5_{start_year}_{end_year}_tas.nc",
        "rsds": f"combined_ERA5_{start_year}_{end_year}_rsds.nc",
        "evspsblpot": f"combined_ERA5_{start_year}_{end_year}_evspsblpot.nc"}}

# Save the YAML file
yaml_file_path = combined_path / "ewatercycle_forcing.yaml"

with open(yaml_file_path, "w") as yaml_file:
    yaml.dump(forcing_yaml, yaml_file, default_flow_style=False)

ERA5_combined_forcing = ewatercycle.forcing.sources["LumpedMakkinkForcing"].load(directory=combined_path)
print(ERA5_combined_forcing)

LumpedMakkinkForcing(
    start_time='1989-01-01T00:00:00Z',
    end_time='2014-12-31T00:00:00Z',
    directory=PosixPath('/home/maxime/BEP-maxime/book/thesis_projects/BSc/2026_Q4_MaximedeBekker_CEG/Workyard/forci
ngs/ERA5/ERA5-1989-2014'),
    shape=PosixPath('/home/maxime/BEP-maxime/book/thesis_projects/BSc/2026_Q4_MaximedeBekker_CEG/Workyard/Shapefile
s/07DA001_basin.shp'),
    filenames={
        'evspsblpot': 'combined_ERA5_1989_2014_evspsblpot.nc',
        'pr': 'combined_ERA5_1989_2014_pr.nc',
        'rsds': 'combined_ERA5_1989_2014_rsds.nc',
        'tas': 'combined_ERA5_1989_2014_tas.nc'
    }
)