RIO-updated

rio-updated/Download_Copernicus_from_ftp.py

@@ -0,0 +1,84 @@
+def Download_fromFTP(configs):
+    import datetime
+    import os
+    import subprocess
+    import tarfile
+    import glob
+    import shutil
+    import ftplib
+    import schedule
+    import time
+    from ftplib import FTP
+    # FTP server details
+    ftp_host = 'nrt.cmems-du.eu'
+    ftp_username = 'xzhang41'
+    ftp_password = 'Zxc_19990115'
+    ftp_directory = '/Core/ARCTIC_ANALYSIS_FORECAST_PHYS_002_001_a/dataset-topaz4-arc-myoceanv2-be'
+    for i in range(len(configs['Dates']['targetdates'])):
+        # Pattern for filenames
+        filename_pattern = configs['Dates']['targetdates'][i]+'_dm-metno-MODEL-topaz4-ARC-b????????-fv02.0.nc'
+
+        # Connect to the FTP server
+        ftp = FTP(ftp_host)
+        ftp.login(user=ftp_username, passwd=ftp_password)
+        ftp.cwd(ftp_directory)
+        if not os.path.exists(configs['output']['output_folder']):
+            # Create the directory
+            os.makedirs(configs['output']['output_folder'])
+        # List files in the directory
+        file_list = ftp.nlst()
+
+        # Download files matching the filename pattern
+        for file_name in file_list:
+            if file_name.endswith('.nc') and file_name.startswith(configs['Dates']['targetdates'][i]+'_dm-metno-MODEL-topaz4-ARC-b'):
+                # Download the file
+                local_path = os.path.join(configs['output']['output_folder'], file_name)
+                with open(local_path, 'wb') as local_file:
+                    ftp.retrbinary('RETR ' + file_name, local_file.write)
+                print(f"Downloaded: {file_name}")
+
+    # Close the FTP connection
+    ftp.quit()
+    heredir = configs['output']['output_folder']
+    os.chdir(heredir)
+    file_paths = glob.glob(f"{heredir}/????????_dm-metno-MODEL-topaz4-ARC-b????????-fv02.0.nc")
+    print(heredir)
+    processed_dir= configs['Dates']['targetdates'][0]+"_processed"
+    if not os.path.exists(processed_dir):
+        os.mkdir(processed_dir)
+    os.chdir(f"{heredir}/{processed_dir}")
+    for ifile in file_paths:
+        subprocess.run(["cdo", "selname,fice,hice", ifile, f"{heredir}/{processed_dir}/icevar_{os.path.basename(ifile)}"])
+    subprocess.run(["cdo", "mergetime", "icevar_*.nc", "Copernicus_alltimes.nc"])
+    os.chdir("..")
+    if not os.path.exists("ready"):
+        os.mkdir("ready")
+    # Specify the source and destination directories
+    source_dir = heredir+"/*_processed"
+    destination_dir = heredir+'/ready'
+
+    # Match the file paths using glob
+    file_path = f"{heredir}/{processed_dir}/Copernicus_alltimes.nc"
+
+    # Create symbolic links in the destination directory
+
+    link_name = os.path.join(destination_dir, os.path.basename(file_path))
+    os.symlink(file_path, link_name)
+    os.chdir(heredir)
+    os.chdir('ready')
+    for td in configs['Dates']['targetdates']:
+        seldate = td
+
+        year = seldate[:4]
+        lastyear = str(int(year) - 1)
+        month = seldate[4:6]
+        day = seldate[6:8]
+
+        print("Selecting date:", seldate)
+        file_pattern = "Copernicus_alltimes.nc"
+        files = glob.glob(file_pattern)
+        for file in files:
+            print(file)
+            outfile = file[:10] + "_" + seldate + "_box.nc"
+            subprocess.run(["cdo", "seldate,{}-{}-{}T00:00:00".format(year, month, day), file,
+                                f"{outfile}"])

rio-updated/Download_S2S_from_ECFS.py

@@ -0,0 +1,11 @@
+from read_config import read_configfile
+from Trim_S2S_ice_data import Download_fromECFS
+
+# Read config file
+#configs=read_configfile() # Reads config_RIOcalc_default.yml
+configs=read_configfile('config_RIOcalc_ECMWF.yml')
+basedate=configs['Dates']['basedate']
+targetdates=configs['Dates']['targetdates']
+directory_path = '/home/zhang/Documents/python/'
+Download_fromECFS(basedate,targetdates,directory_path)
+

rio-updated/README.md

@@ -0,0 +1,46 @@
+Installation
+#############
+
+Software requirements:
+Python 3.9 with following packages:
+  - numpy
+  - xarray (including dask, netCDF4 and bottleneck)
+  - pyyaml
+  - schedule
+  - matplotlib
+
+Use the provided file conda-env_NOCOS-RIO.yml to install a suitable conda environment.
+
+Prepare data
+############
+ECMWF S2S can be accessed through ECMWF's File Storage system(ECFS). One can use teleport SSH access. 
+https://confluence.ecmwf.int/display/UDOC/Teleport+SSH+Access+-+Linux+configuration
+After slected file has been saved to personal folder on ECFS, run Download_S2S_from_ECFS.py(Trim_S2S_ice_data.py should be contained in the same directory) if other source S2S data are avliable, this can be skipped.
+Copernicus can be directly downloaded from ftp server, the download and cropping of the data is already included in RIOengine_NOCOS.py.
+DMI data are obtained from DMI colleague.
+
+Select datatype in engine
+############
+There are currently three data types: ECMWF_S2S, COpernicus, DMI. Set the datatype in use to True in RIOengine_NOCOS.py. If plots are needed, set plot to be true, else set it to false.
+
+Running
+############
+1) Adjust the configuration file config_RIOcalc_default.yml or make a new one and provide its name inside RIOengine_NOCOS.py
+typically for ECMWFS2S data the config filename is config_RIOcalc_ECMWF.yml, for copernicus data is config_RIOcalc_Copernicus.yml and for DMI data is config_RIOcalc_DMI-HYCOM-CICE.yml 
+2) conda activate rioNOCOS
+3) python3 RIOengine_NOCOS.py
+
+Info
+#############
+
+We use variable names as defined by CMIP6/CMOR:
+siconc
+sithick
+sivol
+siitdconc
+siitdthick
+siage
+sisali
+(See explanations in config_RIOcalc_default.yml)
+
+

rio-updated/RIOengine_NOCOS.py

@@ -0,0 +1,43 @@
+# -*- coding: utf-8 -*-
+"""
+
+@author: Andrea Gierisch, DMI
+"""
+# Load RIO functions
+from read_config import read_configfile
+from read_ice_data import read_ice_data
+from save_RIO_toNetCDF import save_toNetcdf
+from calculate_RIO import calcRIO_multicat
+from calculate_RIO import calcRIO_monocat
+from plot_RIO import PLOT
+from Download_Copernicus_from_ftp import Download_fromFTP
+
+
+ECMWF_S2S     = False
+Copernicus    = False
+DMI           = True
+RIOplot       = True
+
+# Read config file
+#configs=read_configfile() # Reads config_RIOcalc_default.yml
+if ECMWF_S2S:
+    configs=read_configfile('config_RIOcalc_ECMWF.yml')
+elif DMI:
+    configs=read_configfile('config_RIOcalc_DMI-HYCOM-CICE.yml')
+else:
+    configs=read_configfile('config_RIOcalc_Copernicus.yml')
+    Download_fromFTP(configs)
+icedata=read_ice_data(configs)
+# Calculate RIO
+if configs['multiCAT']==True:
+    riodata=calcRIO_multicat(icedata,configs)
+elif configs['multiCAT']==False:
+    riodata=calcRIO_monocat(icedata,configs)
+# Save RIO data to netcdf file
+save_toNetcdf(riodata,icedata,configs)
+# plot RIO data
+if RIOplot:
+    PLOT(configs)
+
+
+

rio-updated/Trim_S2S_ice_data.py

@@ -0,0 +1,79 @@
+def Download_fromECFS(basedate,targetdates, directory_path):
+    import datetime
+    import os
+    import subprocess
+    import tarfile
+    import glob
+    import shutil
+    import ftplib
+    import schedule
+    import time
+
+    S2S_filename=f"{basedate}_S2S"
+    directory_file = os.path.join(directory_path, S2S_filename)
+    if not os.path.exists(directory_file):
+        # Create the directory
+        os.makedirs(directory_file)
+    # Specify the source and destination paths
+    year = str(basedate)[:4]
+    month = str(basedate)[4:6]
+    day = str(basedate)[6:8]
+    source_path = f"hpc-login:/ec/res4/scratch/fibl/nemo.{basedate}00.{year}.cf000.tar"
+    destination_path = os.path.join(directory_file, f"nemo.{basedate}00.{year}.cf000.tar")
+    # Execute the SCP command
+    subprocess.call(["scp", source_path, destination_path])
+    with tarfile.open(destination_path, 'r') as tar:
+        tar.extractall(directory_file)
+
+    # Delete the tar file
+    os.remove(destination_path)
+    # Store the current working directory
+    os.chdir(directory_file)
+    for member in os.listdir(directory_file):
+        if "f0" in member:
+            processed_dir = f"{member}_processed"
+            if not os.path.exists(processed_dir):
+                os.mkdir(processed_dir)
+            os.chdir(member)
+
+            for ifile in os.listdir("."):
+                if ifile.startswith("0001_1d_20") and ifile.endswith("_icemod.nc"):
+                    print(f"{member}/{ifile}")
+                    subprocess.run(["cdo", "selname,iicethic,ileadfra", ifile, f"{directory_file}/{processed_dir}/icevar_{ifile}"])
+
+            os.chdir(f"{directory_file}/{processed_dir}")
+            subprocess.run(["cdo", "mergetime", "icevar_0001_1d_20??????_20??????_icemod.nc", "s2s_alltimes.nc"])
+            subprocess.run(["cdo", "selindexbox,1,1442,700,1021", "s2s_alltimes.nc", f"s2s_{basedate}_{member}_alltimes_box.nc"])
+    os.chdir("..")
+    if not os.path.exists("ready"):
+        os.mkdir("ready")
+    # Specify the source and destination directories
+    source_dir = directory_file+"/*_processed"
+    destination_dir = directory_file+'/ready'
+
+    # Match the file paths using glob
+    file_paths = glob.glob(f"{source_dir}/s2s_20??????_?f0??_alltimes_box.nc")
+
+    # Create symbolic links in the destination directory
+    for file_path in file_paths:
+        link_name = os.path.join(destination_dir, os.path.basename(file_path))
+        os.symlink(file_path, link_name)
+    os.chdir(directory_file+'/ready')
+
+    for td in targetdates:
+        seldate = str(td)
+
+        year = seldate[:4]
+        lastyear = str(int(year) - 1)
+        month = seldate[4:6]
+        day = seldate[6:8]
+
+        print("Selecting date:", seldate)
+        for file in os.listdir("."):
+            if file.startswith("s2s_20") and file.endswith("_alltimes_box.nc"):
+                outfile = f"{file[:18]}_{seldate}_box.nc"
+                subprocess.run(["cdo", "seldate,{}-{}-{}T12:00:00".format(year, month, day), file,
+                                f"{outfile}"])
+    os.chdir('..')
+    os.chdir('..')
+