From cce7fa6bc224a8dd5d1fb1cc6cf4c0becc898464 Mon Sep 17 00:00:00 2001 From: Zhang Yunjun Date: Sat, 8 Feb 2025 10:40:43 +0800 Subject: [PATCH] update to the official new CDS (#44) + `autoget.py`: - update `cds_url` link - update to the latest syntax of `cdsapi` for the request dictionary input - use `f'{}'` instead of `.format` syntax + `README.md`: update account setup doc for the official new CDS + `requirements.txt`: the new CDS requires cdsapi>=0.7.2 + update `model.cfg` and `tests/test_dload.py` for the new CDS + `processor.intP2H()`: add whitespace for better readability --------- Co-authored-by: Yuan-Kai Liu <55262505+yuankailiu@users.noreply.github.com> --- README.md | 12 +++---- requirements.txt | 2 +- src/pyaps3/autoget.py | 70 ++++++++++++++++++++++------------------- src/pyaps3/model.cfg | 4 +-- src/pyaps3/processor.py | 12 +++---- tests/test_dload.py | 4 +-- 6 files changed, 54 insertions(+), 50 deletions(-) diff --git a/README.md b/README.md index 36f455b..5eba9fd 100644 --- a/README.md +++ b/README.md @@ -59,19 +59,19 @@ python PyAPS/tests/test_calc.py ### 2. Account setup for [ERA5](https://www.ecmwf.int/en/forecasts/dataset/ecmwf-reanalysis-v5) -ERA5 data set is redistributed over the Copernicus Climate Data Store (CDS)-beta ([migration guide](https://confluence.ecmwf.int/display/CKB/Please+read%3A+CDS+and+ADS+migrating+to+new+infrastructure%3A+Common+Data+Store+%28CDS%29+Engine)). Registration is required for the data access and downloading. +ERA5 data set is redistributed over the Copernicus Climate Data Store (CDS). Registration is required for the data access and downloading. -+ [Create a new account](https://cds-beta.climate.copernicus.eu/) on the CDS-beta website if you don't own a user account yet. Note: the old CDS account won't work. -+ [CDS API setup](https://cds-beta.climate.copernicus.eu/how-to-api#install-the-cds-api-client): Create the local file `$HOME/.cdsapirc` (in your Unix/Linux environment) and add the following two lines: ++ [Create a new account](https://cds.climate.copernicus.eu/) on the CDS website if you don't own a user account yet. Note: the old CDS account won't work ([Goodbye legacy CDS, Hellow New CDS!](https://forum.ecmwf.int/t/goodbye-legacy-climate-data-store-hello-new-climate-data-store-cds/6380)). ++ [CDS API setup](https://cds.climate.copernicus.eu/how-to-api): Create the local file `$HOME/.cdsapirc` (in your Unix/Linux environment) and add the following two lines: ```shell -url: https://cds-beta.climate.copernicus.eu/api +url: https://cds.climate.copernicus.eu/api key: your-personal-access-token ``` -Your Personal Access Token can be found under [Your profile > Personal Access Token](https://cds-beta.climate.copernicus.eu/profile) section or on the [setup guide](https://cds-beta.climate.copernicus.eu/how-to-api#install-the-cds-api-client) page. Alternatively, you could add the token to the `[CDS]` section in `model.cfg` file in the package directory, `site-packages/pyaps3` if installed via conda. Note: using your [old CDS API key](https://cds.climate.copernicus.eu/) will lead to a 401 Client Error and Authentication failed. +Your Personal Access Token can be found under [Your profile > Personal Access Token](https://cds.climate.copernicus.eu/profile) section or on the [setup guide](https://cds.climate.copernicus.eu/how-to-api) page. Alternatively, you could add the token to the `[CDS]` section in `model.cfg` file in the package directory, `site-packages/pyaps3` if installed via conda. Note: using your legacy CDS API key will lead to a 401 Client Error and Authentication failed. -+ **Make sure** that you accept the data license in the Terms of use on ECMWF website: Login, under [Datasets > ERA5 hourly data on pressure levels from 1940 to present > Download > Terms of use](https://cds-beta.climate.copernicus.eu/datasets/reanalysis-era5-pressure-levels?tab=download), click **Accept** to accespt the license to use Copernicus Products. ++ **Make sure** that you accept the data license in the Terms of use on ECMWF website: Login, under [Datasets > ERA5 hourly data on pressure levels from 1940 to present > Download > Terms of use](https://cds.climate.copernicus.eu/datasets/reanalysis-era5-pressure-levels?tab=download), click **Accept** to accespt the license to use Copernicus Products. + Test the account setup by running: diff --git a/requirements.txt b/requirements.txt index cf8525a..cc920d9 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -cdsapi>=0.7.0 +cdsapi>=0.7.2 numpy pygrib scipy diff --git a/src/pyaps3/autoget.py b/src/pyaps3/autoget.py index aeb3aeb..8e42c8f 100644 --- a/src/pyaps3/autoget.py +++ b/src/pyaps3/autoget.py @@ -41,14 +41,14 @@ def ECMWFdload(bdate,hr,filedir,model='ERA5',datatype='fc',humidity='Q',snwe=Non # Initialize # Check data - assert model in ('ERA5', 'ERAINT', 'HRES'), 'Unknown model for ECMWF: {}'.format(model) + assert model in ('ERA5', 'ERAINT', 'HRES'), f'Unknown model for ECMWF: {model}' # Infos for downloading - if model in 'ERAINT': - print('WARNING: you are downloading from the old ECMWF platform. ' + if model == 'ERAINT': + print('WARNING: you are downloading from the legacy ECMWF platform. ' 'ERA-Interim is deprecated, use ERA-5 instead.') - if model in 'ERA5': - cds_url = 'https://cds-beta.climate.copernicus.eu/api' + if model == 'ERA5': + cds_url = 'https://cds.climate.copernicus.eu/api' print('INFO: You are using the latest ECMWF platform for downloading datasets: ', cds_url) #------------------------------------------- @@ -56,17 +56,11 @@ def ECMWFdload(bdate,hr,filedir,model='ERA5',datatype='fc',humidity='Q',snwe=Non # Humidity assert humidity in ('Q','R'), 'Unknown humidity field for ECMWF' - if humidity in 'Q': + if humidity == 'Q': humidparam = 'specific_humidity' if model == 'ERA5' else 133 - elif humidity in 'R': + elif humidity == 'R': humidparam = 'relative_humidity' if model == 'ERA5' else 157 - # Grid size (only for HRES and ERA-Interim) - if model in 'HRES': - gridsize = '0.10/0.10' - elif model in 'ERAINT': - gridsize = '0.75/0.75' - #------------------------------------------- # file name if not flist: @@ -92,7 +86,7 @@ def ECMWFdload(bdate,hr,filedir,model='ERA5',datatype='fc',humidity='Q',snwe=Non #------------------------------------------- # CASE 1: request for CDS API client (new ECMWF platform, for ERA5) - if model in 'ERA5': + if model == 'ERA5': # Contact the server rc_file = os.path.expanduser('~/.cdsapirc') if os.path.isfile(rc_file): @@ -102,39 +96,49 @@ def ECMWFdload(bdate,hr,filedir,model='ERA5',datatype='fc',humidity='Q',snwe=Non key = config.get('CDS', 'key') c = cdsapi.Client(url=url, key=key) - # Pressure levels - pressure_lvls = ['1','2','3','5','7','10','20','30','50', - '70','100','125','150','175','200','225', - '250','300','350','400','450','500','550', - '600','650','700','750','775','800','825', - '850','875','900','925','950','975','1000'] - - # Dictionary - indict = {'product_type' :'reanalysis', - 'format' :'grib', - 'variable' :['geopotential','temperature','{}'.format(humidparam)], - 'pressure_level' : pressure_lvls, - 'year' :'{}'.format(day[0:4]), - 'month' :'{}'.format(day[4:6]), - 'day' :'{}'.format(day[6:8]), - 'time' :'{}:00'.format(hr)} + # request dictionary + pressure_lvls = [ + '1','2','3','5','7','10','20','30','50', + '70','100','125','150','175','200','225', + '250','300','350','400','450','500','550', + '600','650','700','750','775','800','825', + '850','875','900','925','950','975','1000', + ] + + indict = { + 'product_type' : ['reanalysis'], + 'variable' : ['geopotential', 'temperature', f'{humidparam}'], + 'year' : [f'{day[0:4]}'], + 'month' : [f'{day[4:6]}'], + 'day' : [f'{day[6:8]}'], + 'time' : [f'{hr}:00'], + 'pressure_level' : pressure_lvls, + 'data_format' : 'grib', + } # download a geographical area subset if snwe is not None: s, n, w, e = snwe - indict['area'] = '/'.join(['{:.2f}'.format(x) for x in [n, w, s, e]]) + indict['area'] = [n, w, s, e] # Assert grib file not yet downloaded if not os.path.exists(fname): - print('Downloading %d of %d: %s '%(i+1, len(bdate), fname)) + print(f'Downloading {i+1} of {len(bdate)}: {fname} ') print(indict) # Make the request - c.retrieve('reanalysis-{}-pressure-levels'.format(model.lower()),indict,target=fname) + ds_name = f'reanalysis-{model.lower()}-pressure-levels' + c.retrieve(ds_name, indict, target=fname) #------------------------------------------- # CASE 2: request for WEB API client (old ECMWF platform, deprecated, for ERA-Int and HRES) else: + # Grid size (only for HRES and ERA-Interim) + if model == 'HRES': + gridsize = '0.10/0.10' + elif model == 'ERAINT': + gridsize = '0.75/0.75' + # Contact the server from pyaps3.ecmwfapi import ECMWFDataServer url = "https://api.ecmwf.int/v1" diff --git a/src/pyaps3/model.cfg b/src/pyaps3/model.cfg index 6414d8e..e87b46a 100644 --- a/src/pyaps3/model.cfg +++ b/src/pyaps3/model.cfg @@ -1,6 +1,6 @@ # vim: set filetype=cfg: -#####key for ECMWF in Climate Data Store Application Program Interface -#Get it from https://retostauffer.org/code/Download-ERA5/ +#####Personal Access Token for ECMWF in Climate Data Store Application Program Interface +#Get it from https://cds.climate.copernicus.eu/profile #for ERA5 [CDS] key = your-personal-access-token diff --git a/src/pyaps3/processor.py b/src/pyaps3/processor.py index ef6e5d5..c93ba27 100644 --- a/src/pyaps3/processor.py +++ b/src/pyaps3/processor.py @@ -111,12 +111,12 @@ def intP2H(lvls,hgt,gph,tmp,vpr,cdic,verbose=False): #Interpolating pressure values hy = lvls.copy() if (sFlag == True): - val = hy[-1] +(hx[-1] - hx[-2])* (hy[-1] - hy[-2])/(hx[-2]-hx[-3]) + val = hy[-1] + (hx[-1] - hx[-2]) * (hy[-1] - hy[-2]) / (hx[-2] - hx[-3]) #changed from 1 to 0 (-1 should also work), CL hy = np.concatenate((hy,[val]),axis=0) if (eFlag == True): - val = hy[0] - (hx[0] - hx[1]) * (hy[0] - hy[1])/(hx[1]-hx[2]) + val = hy[0] - (hx[0] - hx[1]) * (hy[0] - hy[1]) / (hx[1] - hx[2]) #changed from 1 to 0 (-1 should also work), CL hy = np.concatenate(([val],hy),axis=0) @@ -130,12 +130,12 @@ def intP2H(lvls,hgt,gph,tmp,vpr,cdic,verbose=False): temp = tmp[:,i,j] hy = temp.copy() if (sFlag == True): - val = hy[-1] +(hx[-1] - hx[-2])* (hy[-1] - hy[-2])/(hx[-2]-hx[-3]) + val = hy[-1] + (hx[-1] - hx[-2]) * (hy[-1] - hy[-2]) / (hx[-2] - hx[-3]) #changed from 1 to 0 (-1 should also work), CL hy = np.concatenate((hy,[val]),axis=0) if (eFlag == True): - val = hy[0] - (hx[0] - hx[1]) * (hy[0] - hy[1])/(hx[1]-hx[2]) + val = hy[0] - (hx[0] - hx[1]) * (hy[0] - hy[1]) / (hx[1] - hx[2]) #changed from 1 to 0 (-1 should also work), CL hy = np.concatenate(([val],hy),axis=0) @@ -148,12 +148,12 @@ def intP2H(lvls,hgt,gph,tmp,vpr,cdic,verbose=False): temp = vpr[:,i,j] hy = temp.copy() if (sFlag == True): - val = hy[-1] +(hx[-1] - hx[-2])* (hy[-1] - hy[-2])/(hx[-2]-hx[-3]) + val = hy[-1] + (hx[-1] - hx[-2]) * (hy[-1] - hy[-2]) / (hx[-2] - hx[-3]) #changed from 1 to 0 (-1 should also work), CL hy = np.concatenate((hy,[val]),axis=0) if (eFlag == True): - val = hy[0] - (hx[0] - hx[1]) * (hy[0] - hy[1])/(hx[1]-hx[2]) + val = hy[0] - (hx[0] - hx[1]) * (hy[0] - hy[1]) / (hx[1] - hx[2]) #changed from 1 to 0 (-1 should also work), CL hy = np.concatenate(([val],hy),axis=0) diff --git a/tests/test_dload.py b/tests/test_dload.py index 653e333..d41f585 100644 --- a/tests/test_dload.py +++ b/tests/test_dload.py @@ -8,8 +8,8 @@ print('import pyaps3 from {}'.format(pa.__file__)) print('------------------------------------------------') print('test ERA5 data download') -print('NOTE: Account setup is required on the Copernicus Climate Data Store (CDS)-beta.') -print(' More detailed info can be found on: https://cds-beta.climate.copernicus.eu/how-to-api') +print('NOTE: Account setup is required on the Copernicus Climate Data Store (CDS).') +print(' More detailed info can be found on: https://cds.climate.copernicus.eu/how-to-api') print(' Add your account info to ~/.cdsapirc file.') filedir = os.path.join(os.path.dirname(__file__), 'data', 'ERA5') pa.ECMWFdload(['20200601','20200901'], hr='14', filedir=filedir, model='ERA5', snwe=(30,40,120,140))