diff --git a/applications/adjoint_tests/Makefile b/applications/adjoint_tests/Makefile index fc5c26056..c6eb1b968 100644 --- a/applications/adjoint_tests/Makefile +++ b/applications/adjoint_tests/Makefile @@ -25,6 +25,10 @@ export PSYCLONE_CONFIG_FILE := $(PROJECT_DIR)/build/psyclone.cfg export BUILD_ADJ_TESTS := TRUE export PSYAD_CONFIG_FILE := $(PSYCLONE_CONFIG_FILE) +export EXTRA_ROSE_META = jules +export META_DIR = $(WORKING_DIR)/../rose-meta +export ROSE_META_DIRS = $(APPS_ROOT_DIR)/rose-meta $(META_DIR) + export INTERNAL_DEPENDENCIES = $(CORE_ROOT_DIR)/infrastructure \ $(CORE_ROOT_DIR)/components/driver \ $(CORE_ROOT_DIR)/components/science \ @@ -123,6 +127,10 @@ build: ALWAYS $(call MESSAGE,========================================) $Q$(MAKE) $(QUIET_ARG) -f $(LFRIC_BUILD)/extract.mk \ SOURCE_DIR=source + $(call MESSAGE,========================================) + $(call MESSAGE,Extracting External Rose Metadata) + $(call MESSAGE,========================================) + $Q$(MAKE) $(QUIET_ARG) -f $(APPS_ROOT_DIR)/build/extract/extract_meta.mk $(call MESSAGE,=========================================================) $(call MESSAGE,Generating $(PROJECT) namelist loaders) $(call MESSAGE,=========================================================) diff --git a/applications/gravity_wave/Makefile b/applications/gravity_wave/Makefile index 62f7c8731..0fceddd6e 100644 --- a/applications/gravity_wave/Makefile +++ b/applications/gravity_wave/Makefile @@ -20,6 +20,10 @@ export PSYCLONE_TRANSFORMATION ?= minimum export PROJECT_DIR := $(realpath $(dir $(lastword $(MAKEFILE_LIST)))) +export EXTRA_ROSE_META = jules +export META_DIR = $(WORKING_DIR)/../rose-meta +export ROSE_META_DIRS = $(APPS_ROOT_DIR)/rose-meta $(META_DIR) + export INTERNAL_DEPENDENCIES = $(CORE_ROOT_DIR)/infrastructure \ $(CORE_ROOT_DIR)/components/driver \ $(CORE_ROOT_DIR)/components/science \ @@ -46,6 +50,7 @@ default: build unit-tests integration-tests documentation doc docs: document-uml document-latex document-api $(Q)echo > /dev/null +include $(APPS_ROOT_DIR)/build/extract/extract_meta.mk include $(CORE_ROOT_DIR)/infrastructure/build/lfric.mk include $(INTERNAL_DEPENDENCIES:=/build/import.mk) @@ -116,6 +121,10 @@ build: ALWAYS $(call MESSAGE,========================================) $Q$(MAKE) $(QUIET_ARG) -f $(LFRIC_BUILD)/extract.mk \ SOURCE_DIR=source + $(call MESSAGE,========================================) + $(call MESSAGE,Extracting External Rose Metadata) + $(call MESSAGE,========================================) + $Q$(MAKE) $(QUIET_ARG) -f $(APPS_ROOT_DIR)/build/extract/extract_meta.mk $(call MESSAGE,=========================================================) $(call MESSAGE,Generating namelist loaders for $(PROJECT)) $(call MESSAGE,=========================================================) @@ -157,8 +166,8 @@ unit-tests/%: export PROJECT = $(PROJECT_NAME) unit-tests/%: export SOURCE_DIR = source unit-tests/%: export TEST_DIR = unit-test unit-tests/%: export WORKING_DIR := $(WORKING_DIR)/unit-test -unit-tests: unit-tests/run - +unit-tests: extract_meta unit-tests/run +unit-tests/build: extract_meta unit-tests/build ############################################################################## # Integration tests diff --git a/applications/gungho_model/Makefile b/applications/gungho_model/Makefile index 8d364462e..b01b6e8c1 100644 --- a/applications/gungho_model/Makefile +++ b/applications/gungho_model/Makefile @@ -19,6 +19,10 @@ export PSYCLONE_TRANSFORMATION ?= minimum export PROJECT_DIR := $(realpath $(dir $(lastword $(MAKEFILE_LIST)))) +export EXTRA_ROSE_META = jules +export META_DIR = $(WORKING_DIR)/../rose-meta +export ROSE_META_DIRS = $(APPS_ROOT_DIR)/rose-meta $(META_DIR) + export INTERNAL_DEPENDENCIES = $(CORE_ROOT_DIR)/infrastructure \ $(CORE_ROOT_DIR)/components/driver \ $(CORE_ROOT_DIR)/components/science \ @@ -87,6 +91,10 @@ build: ALWAYS $(call MESSAGE,========================================) $Q$(MAKE) $(QUIET_ARG) -f $(LFRIC_BUILD)/extract.mk \ SOURCE_DIR=source + $(call MESSAGE,========================================) + $(call MESSAGE,Extracting External Rose Metadata) + $(call MESSAGE,========================================) + $Q$(MAKE) $(QUIET_ARG) -f $(APPS_ROOT_DIR)/build/extract/extract_meta.mk $(call MESSAGE,=========================================================) $(call MESSAGE,Generating $(PROJECT) namelist loaders) $(call MESSAGE,=========================================================) diff --git a/applications/jedi_lfric_tests/Makefile b/applications/jedi_lfric_tests/Makefile index c76919a8f..2324011cc 100644 --- a/applications/jedi_lfric_tests/Makefile +++ b/applications/jedi_lfric_tests/Makefile @@ -20,6 +20,10 @@ export PSYCLONE_TRANSFORMATION ?= minimum export PROJECT_DIR := $(realpath $(dir $(lastword $(MAKEFILE_LIST)))) +export EXTRA_ROSE_META = jules +export META_DIR = $(WORKING_DIR)/../rose-meta +export ROSE_META_DIRS = $(APPS_ROOT_DIR)/rose-meta $(META_DIR) + export INTERNAL_DEPENDENCIES = $(CORE_ROOT_DIR)/infrastructure \ $(CORE_ROOT_DIR)/components/driver \ $(CORE_ROOT_DIR)/components/science \ @@ -49,6 +53,7 @@ default: build integration-tests documentation doc docs: document-uml document-latex document-api $(Q)echo > /dev/null +include $(APPS_ROOT_DIR)/build/extract/extract_meta.mk include $(CORE_ROOT_DIR)/infrastructure/build/lfric.mk include $(INTERNAL_DEPENDENCIES:=/build/import.mk) -include $(PROJECT_DIR)/build/project.mk @@ -123,6 +128,10 @@ build: ALWAYS $Q$(MAKE) $(QUIET_ARG) -f $(LFRIC_BUILD)/extract.mk \ SOURCE_DIR=source \ WORKING_DIR=$(WORKING_DIR) + $(call MESSAGE,========================================) + $(call MESSAGE,Extracting External Rose Metadata) + $(call MESSAGE,========================================) + $Q$(MAKE) $(QUIET_ARG) -f $(APPS_ROOT_DIR)/build/extract/extract_meta.mk $(call MESSAGE,=========================================================) $(call MESSAGE,Generating $(PROJECT) namelist loaders) $(call MESSAGE,=========================================================) @@ -175,8 +184,8 @@ unit-tests/%: export PROJECT = $(PROJECT_NAME) unit-tests/%: export SOURCE_DIR = source unit-tests/%: export TEST_DIR = unit-test unit-tests/%: export WORKING_DIR := $(WORKING_DIR)/unit-test -unit-tests: unit-tests/run - +unit-tests: extract_meta unit-tests/run +unit-tests/build: extract_meta unit-tests/build ############################################################################## # Integration tests @@ -197,8 +206,8 @@ integration-tests/%: export PROJECT = $(PROJECT_NAME) integration-tests/%: export SOURCE_DIR = source integration-tests/%: export TEST_DIR = integration-test integration-tests/%: export WORKING_DIR := $(WORKING_DIR)/integration-tests -integration-tests: integration-tests/run - +integration-tests: extract_meta integration-tests/run +integration-tests/build: extract_meta integration-tests/build ############################################################################## # Clean diff --git a/applications/jules/Makefile b/applications/jules/Makefile index 0a8f311c9..7939fb888 100644 --- a/applications/jules/Makefile +++ b/applications/jules/Makefile @@ -20,6 +20,10 @@ export PSYCLONE_TRANSFORMATION ?= minimum export PROJECT_DIR := $(realpath $(dir $(lastword $(MAKEFILE_LIST)))) +export EXTRA_ROSE_META = jules +export META_DIR = $(WORKING_DIR)/../rose-meta +export ROSE_META_DIRS = $(APPS_ROOT_DIR)/rose-meta $(META_DIR) + # Note - socrates_interface must come before jules_interface as there are dependencies on each other. export INTERNAL_DEPENDENCIES = $(CORE_ROOT_DIR)/infrastructure \ $(CORE_ROOT_DIR)/components/driver \ @@ -100,6 +104,10 @@ build: ALWAYS $Q$(MAKE) $(QUIET_ARG) -f $(LFRIC_BUILD)/extract.mk \ SOURCE_DIR=source $(call MESSAGE,========================================) + $(call MESSAGE,Extracting External Rose Metadata) + $(call MESSAGE,========================================) + $Q$(MAKE) $(QUIET_ARG) -f $(APPS_ROOT_DIR)/build/extract/extract_meta.mk + $(call MESSAGE,========================================) $(call MESSAGE,Generating $(PROJECT) namelist loaders) $(call MESSAGE,========================================) $Q$(MAKE) $(QUIET_ARG) -f $(LFRIC_BUILD)/configuration.mk \ diff --git a/applications/lfric2lfric/Makefile b/applications/lfric2lfric/Makefile index 278036352..f38be413e 100644 --- a/applications/lfric2lfric/Makefile +++ b/applications/lfric2lfric/Makefile @@ -20,6 +20,10 @@ export PSYCLONE_TRANSFORMATION ?= minimum export PROJECT_DIR := $(realpath $(dir $(lastword $(MAKEFILE_LIST)))) +export EXTRA_ROSE_META = jules +export META_DIR = $(WORKING_DIR)/../rose-meta +export ROSE_META_DIRS = $(APPS_ROOT_DIR)/rose-meta $(META_DIR) + export INTERNAL_DEPENDENCIES = $(CORE_ROOT_DIR)/infrastructure \ $(CORE_ROOT_DIR)/components/coupling \ $(CORE_ROOT_DIR)/components/driver \ @@ -47,6 +51,7 @@ default: build unit-tests integration-tests documentation doc docs: document-uml document-latex document-api $(Q)echo > /dev/null +include $(APPS_ROOT_DIR)/build/extract/extract_meta.mk include $(CORE_ROOT_DIR)/infrastructure/build/lfric.mk include $(INTERNAL_DEPENDENCIES:=/build/import.mk) @@ -117,6 +122,10 @@ build: ALWAYS $(call MESSAGE,========================================) $Q$(MAKE) $(QUIET_ARG) -f $(LFRIC_BUILD)/extract.mk \ SOURCE_DIR=source + $(call MESSAGE,========================================) + $(call MESSAGE,Extracting External Rose Metadata) + $(call MESSAGE,========================================) + $Q$(MAKE) $(QUIET_ARG) -f $(APPS_ROOT_DIR)/build/extract/extract_meta.mk $(call MESSAGE,=========================================================) $(call MESSAGE,Generating $(PROJECT) namelist loaders) $(call MESSAGE,=========================================================) @@ -153,8 +162,8 @@ unit-tests/%: export PROJECT = $(PROJECT_NAME) unit-tests/%: export SOURCE_DIR = source/kernel unit-tests/%: export TEST_DIR = unit-test unit-tests/%: export WORKING_DIR := $(WORKING_DIR)/unit-test -unit-tests: unit-tests/run - +unit-tests: extract_meta unit-tests/run +unit-tests/build: extract_meta unit-tests/build ############################################################################## # Integration tests diff --git a/applications/lfric_atm/Makefile b/applications/lfric_atm/Makefile index 7457acec7..a8ab6810d 100644 --- a/applications/lfric_atm/Makefile +++ b/applications/lfric_atm/Makefile @@ -32,6 +32,10 @@ export IGNORE_DEPENDENCIES = c_shum_byteswap \ f_shum_is_denormal_mod +export EXTRA_ROSE_META = jules +export META_DIR = $(WORKING_DIR)/../rose-meta +export ROSE_META_DIRS = $(APPS_ROOT_DIR)/rose-meta $(META_DIR) + # Note - socrates_interface must come before jules_interface as there are dependencies on each other. export INTERNAL_DEPENDENCIES = $(CORE_ROOT_DIR)/infrastructure \ $(CORE_ROOT_DIR)/components/driver \ @@ -115,6 +119,10 @@ build: ALWAYS $Q$(MAKE) $(QUIET_ARG) -f $(LFRIC_BUILD)/extract.mk \ SOURCE_DIR=source $(call MESSAGE,========================================) + $(call MESSAGE,Extracting External Rose Metadata) + $(call MESSAGE,========================================) + $Q$(MAKE) $(QUIET_ARG) -f $(APPS_ROOT_DIR)/build/extract/extract_meta.mk + $(call MESSAGE,========================================) $(call MESSAGE,Generating $(PROJECT) namelist loaders) $(call MESSAGE,========================================) $Q$(MAKE) $(QUIET_ARG) -f $(LFRIC_BUILD)/configuration.mk \ diff --git a/applications/lfric_coupled/Makefile b/applications/lfric_coupled/Makefile index 3954e7075..3a3f647b9 100644 --- a/applications/lfric_coupled/Makefile +++ b/applications/lfric_coupled/Makefile @@ -20,6 +20,10 @@ export PSYCLONE_TRANSFORMATION ?= minimum export PROJECT_DIR := $(realpath $(dir $(lastword $(MAKEFILE_LIST)))) +export EXTRA_ROSE_META = jules +export META_DIR = $(WORKING_DIR)/../rose-meta +export ROSE_META_DIRS = $(APPS_ROOT_DIR)/rose-meta $(META_DIR) + # Note - socrates_interface must come before jules_interface as there are dependencies on each other. export INTERNAL_DEPENDENCIES = $(CORE_ROOT_DIR)/infrastructure \ $(CORE_ROOT_DIR)/components/coupling \ @@ -51,6 +55,7 @@ default: build test-suite documentation doc docs: document-uml document-latex document-api $(Q)echo > /dev/null +include $(APPS_ROOT_DIR)/build/extract/extract_meta.mk include $(CORE_ROOT_DIR)/infrastructure/build/lfric.mk include $(INTERNAL_DEPENDENCIES:=/build/import.mk) include build/project.mk @@ -141,6 +146,10 @@ build: ALWAYS $(call MESSAGE,========================================) $Q$(MAKE) $(QUIET_ARG) -f $(LFRIC_BUILD)/extract.mk \ SOURCE_DIR=source + $(call MESSAGE,========================================) + $(call MESSAGE,Extracting External Rose Metadata) + $(call MESSAGE,========================================) + $Q$(MAKE) $(QUIET_ARG) -f $(APPS_ROOT_DIR)/build/extract/extract_meta.mk $(call MESSAGE,=========================================================) $(call MESSAGE,Generating $(PROJECT) namelist loaders) $(call MESSAGE,=========================================================) @@ -219,7 +228,8 @@ unit-tests/%: export PROJECT = $(PROJECT_NAME) unit-tests/%: export SOURCE_DIR = source unit-tests/%: export TEST_DIR = unit-test unit-tests/%: export WORKING_DIR := $(WORKING_DIR)/unit-test -unit-tests: unit-tests/run +unit-tests: extract_meta unit-tests/run +unit-tests/build: extract_meta unit-tests/build ############################################################################## diff --git a/applications/lfricinputs/Makefile b/applications/lfricinputs/Makefile index 271f4b6b9..e659b9d3b 100644 --- a/applications/lfricinputs/Makefile +++ b/applications/lfricinputs/Makefile @@ -18,6 +18,10 @@ export PSYCLONE_TRANSFORMATION ?= minimum export PROJECT_DIR := $(realpath $(dir $(lastword $(MAKEFILE_LIST)))) +export EXTRA_ROSE_META = jules +export META_DIR = $(WORKING_DIR)/../rose-meta +export ROSE_META_DIRS = $(APPS_ROOT_DIR)/rose-meta $(META_DIR) + export INTERNAL_DEPENDENCIES = $(CORE_ROOT_DIR)/infrastructure \ $(CORE_ROOT_DIR)/components/science \ $(CORE_ROOT_DIR)/components/inventory \ @@ -86,6 +90,10 @@ build: ALWAYS $(call MESSAGE,Extracting $(PROJECT_NAME)) $(call MESSAGE,========================================) $Q$(MAKE) $(QUIET_ARG) -f $(LFRIC_BUILD)/extract.mk + $(call MESSAGE,========================================) + $(call MESSAGE,Extracting External Rose Metadata) + $(call MESSAGE,========================================) + $Q$(MAKE) $(QUIET_ARG) -f $(APPS_ROOT_DIR)/build/extract/extract_meta.mk $(call MESSAGE,=========================================================) $(call MESSAGE,Generating $(PROJECT) namelist loaders) $(call MESSAGE,=========================================================) diff --git a/applications/linear_model/Makefile b/applications/linear_model/Makefile index 5755c6a89..7ad66520d 100644 --- a/applications/linear_model/Makefile +++ b/applications/linear_model/Makefile @@ -19,6 +19,10 @@ export PSYCLONE_TRANSFORMATION ?= minimum export PROJECT_DIR := $(realpath $(dir $(lastword $(MAKEFILE_LIST)))) +export EXTRA_ROSE_META = jules +export META_DIR = $(WORKING_DIR)/../rose-meta +export ROSE_META_DIRS = $(APPS_ROOT_DIR)/rose-meta $(META_DIR) + export INTERNAL_DEPENDENCIES = $(CORE_ROOT_DIR)/infrastructure \ $(CORE_ROOT_DIR)/components/driver \ $(CORE_ROOT_DIR)/components/science \ @@ -88,6 +92,10 @@ build: ALWAYS $(call MESSAGE,========================================) $Q$(MAKE) $(QUIET_ARG) -f $(LFRIC_BUILD)/extract.mk \ SOURCE_DIR=source + $(call MESSAGE,========================================) + $(call MESSAGE,Extracting External Rose Metadata) + $(call MESSAGE,========================================) + $Q$(MAKE) $(QUIET_ARG) -f $(APPS_ROOT_DIR)/build/extract/extract_meta.mk $(call MESSAGE,=========================================================) $(call MESSAGE,Generating $(PROJECT) namelist loaders) $(call MESSAGE,=========================================================) diff --git a/applications/name_transport/Makefile b/applications/name_transport/Makefile index 7ddbee74c..c7ea0be7a 100644 --- a/applications/name_transport/Makefile +++ b/applications/name_transport/Makefile @@ -20,6 +20,10 @@ export PSYCLONE_TRANSFORMATION ?= minimum export PROJECT_DIR := $(realpath $(dir $(lastword $(MAKEFILE_LIST)))) +export EXTRA_ROSE_META = jules +export META_DIR = $(WORKING_DIR)/../rose-meta +export ROSE_META_DIRS = $(APPS_ROOT_DIR)/rose-meta $(META_DIR) + export INTERNAL_DEPENDENCIES = $(CORE_ROOT_DIR)/infrastructure \ $(CORE_ROOT_DIR)/components/driver \ $(CORE_ROOT_DIR)/components/science \ @@ -46,6 +50,7 @@ default: build unit-tests integration-tests documentation doc docs: document-uml document-latex document-api $(Q)echo > /dev/null +include $(APPS_ROOT_DIR)/build/extract/extract_meta.mk include $(CORE_ROOT_DIR)/infrastructure/build/lfric.mk include $(INTERNAL_DEPENDENCIES:=/build/import.mk) -include $(PROJECT_DIR)/build/project.mk @@ -117,6 +122,10 @@ build: ALWAYS $(call MESSAGE,========================================) $Q$(MAKE) $(QUIET_ARG) -f $(LFRIC_BUILD)/extract.mk \ SOURCE_DIR=source + $(call MESSAGE,========================================) + $(call MESSAGE,Extracting External Rose Metadata) + $(call MESSAGE,========================================) + $Q$(MAKE) $(QUIET_ARG) -f $(APPS_ROOT_DIR)/build/extract/extract_meta.mk $(call MESSAGE,=========================================================) $(call MESSAGE,Generating $(PROJECT) namelist loaders) $(call MESSAGE,=========================================================) @@ -160,7 +169,8 @@ unit-tests/%: export SOURCE_DIR = source unit-tests/%: export TEST_DIR = unit-test unit-tests/%: export WORKING_DIR := $(WORKING_DIR)/unit-tests unit-tests/%: export UNIT_TEST_PRE_PROCESS_MACROS = "NO_MPI=no_mpi" -unit-tests: unit-tests/run +unit-tests: extract_meta unit-tests/run +unit-tests/build: extract_meta unit-tests/build ############################################################################## # Integration tests diff --git a/applications/ngarch/Makefile b/applications/ngarch/Makefile index a645cd3d2..fc6718bc5 100644 --- a/applications/ngarch/Makefile +++ b/applications/ngarch/Makefile @@ -20,6 +20,10 @@ export PSYCLONE_TRANSFORMATION ?= minimum export PROJECT_DIR := $(realpath $(dir $(lastword $(MAKEFILE_LIST)))) +export EXTRA_ROSE_META = jules +export META_DIR = $(WORKING_DIR)/../rose-meta +export ROSE_META_DIRS = $(APPS_ROOT_DIR)/rose-meta $(META_DIR) + # Note - socrates_interface must come before jules_interface as there are dependencies on each other. export INTERNAL_DEPENDENCIES = $(CORE_ROOT_DIR)/infrastructure \ $(CORE_ROOT_DIR)/components/driver \ @@ -136,6 +140,10 @@ build: ALWAYS $Q$(MAKE) $(QUIET_ARG) -f $(LFRIC_BUILD)/extract.mk \ SOURCE_DIR=source $(call MESSAGE,========================================) + $(call MESSAGE,Extracting External Rose Metadata) + $(call MESSAGE,========================================) + $Q$(MAKE) $(QUIET_ARG) -f $(APPS_ROOT_DIR)/build/extract/extract_meta.mk + $(call MESSAGE,========================================) $(call MESSAGE,Generating $(PROJECT) namelist loaders) $(call MESSAGE,========================================) $Q$(MAKE) $(QUIET_ARG) -f $(LFRIC_BUILD)/configuration.mk \ diff --git a/applications/shallow_water/Makefile b/applications/shallow_water/Makefile index 13cb94ebd..b52fc5d9f 100644 --- a/applications/shallow_water/Makefile +++ b/applications/shallow_water/Makefile @@ -20,6 +20,10 @@ export PSYCLONE_TRANSFORMATION ?= minimum export PROJECT_DIR := $(realpath $(dir $(lastword $(MAKEFILE_LIST)))) +export EXTRA_ROSE_META = jules +export META_DIR = $(WORKING_DIR)/../rose-meta +export ROSE_META_DIRS = $(APPS_ROOT_DIR)/rose-meta $(META_DIR) + export INTERNAL_DEPENDENCIES = $(CORE_ROOT_DIR)/infrastructure \ $(CORE_ROOT_DIR)/components/driver \ $(CORE_ROOT_DIR)/components/science \ @@ -46,6 +50,7 @@ default: build unit-tests integration-tests documentation doc docs: document-uml document-latex document-api $(Q)echo > /dev/null +include $(APPS_ROOT_DIR)/build/extract/extract_meta.mk include $(CORE_ROOT_DIR)/infrastructure/build/lfric.mk include $(INTERNAL_DEPENDENCIES:=/build/import.mk) -include $(PROJECT_DIR)/build/project.mk @@ -117,6 +122,10 @@ build: ALWAYS $(call MESSAGE,========================================) $Q$(MAKE) $(QUIET_ARG) -f $(LFRIC_BUILD)/extract.mk \ SOURCE_DIR=source + $(call MESSAGE,========================================) + $(call MESSAGE,Extracting External Rose Metadata) + $(call MESSAGE,========================================) + $Q$(MAKE) $(QUIET_ARG) -f $(APPS_ROOT_DIR)/build/extract/extract_meta.mk $(call MESSAGE,=========================================================) $(call MESSAGE,Generating $(PROJECT) namelist loaders) $(call MESSAGE,=========================================================) @@ -159,7 +168,8 @@ unit-tests/%: export PROJECT = $(PROJECT_NAME) unit-tests/%: export SOURCE_DIR = source unit-tests/%: export TEST_DIR = unit-test unit-tests/%: export WORKING_DIR := $(WORKING_DIR)/unit-tests -unit-tests: unit-tests/run +unit-tests: extract_meta unit-tests/run +unit-tests/build: extract_meta unit-tests/build ############################################################################## diff --git a/applications/solver/Makefile b/applications/solver/Makefile index 028af625a..b3c86b49c 100644 --- a/applications/solver/Makefile +++ b/applications/solver/Makefile @@ -20,6 +20,8 @@ export PSYCLONE_TRANSFORMATION ?= minimum export PROJECT_DIR := $(realpath $(dir $(lastword $(MAKEFILE_LIST)))) +export ROSE_META_DIRS = $(APPS_ROOT_DIR)/rose-meta + export INTERNAL_DEPENDENCIES = $(CORE_ROOT_DIR)/infrastructure \ $(CORE_ROOT_DIR)/components/driver \ $(CORE_ROOT_DIR)/components/science \ diff --git a/applications/transport/Makefile b/applications/transport/Makefile index b849928a5..ab8024aad 100644 --- a/applications/transport/Makefile +++ b/applications/transport/Makefile @@ -20,6 +20,10 @@ export PSYCLONE_TRANSFORMATION ?= minimum export PROJECT_DIR := $(realpath $(dir $(lastword $(MAKEFILE_LIST)))) +export EXTRA_ROSE_META = jules +export META_DIR = $(WORKING_DIR)/../rose-meta +export ROSE_META_DIRS = $(APPS_ROOT_DIR)/rose-meta $(META_DIR) + export INTERNAL_DEPENDENCIES = $(CORE_ROOT_DIR)/infrastructure \ $(CORE_ROOT_DIR)/components/driver \ $(CORE_ROOT_DIR)/components/science \ @@ -46,6 +50,7 @@ default: build unit-tests integration-tests documentation doc docs: document-uml document-latex document-api $(Q)echo > /dev/null +include $(APPS_ROOT_DIR)/build/extract/extract_meta.mk include $(CORE_ROOT_DIR)/infrastructure/build/lfric.mk include $(INTERNAL_DEPENDENCIES:=/build/import.mk) -include $(PROJECT_DIR)/build/project.mk @@ -117,6 +122,10 @@ build: ALWAYS $(call MESSAGE,========================================) $Q$(MAKE) $(QUIET_ARG) -f $(LFRIC_BUILD)/extract.mk \ SOURCE_DIR=source + $(call MESSAGE,========================================) + $(call MESSAGE,Extracting External Rose Metadata) + $(call MESSAGE,========================================) + $Q$(MAKE) $(QUIET_ARG) -f $(APPS_ROOT_DIR)/build/extract/extract_meta.mk $(call MESSAGE,=========================================================) $(call MESSAGE,Generating $(PROJECT) namelist loaders) $(call MESSAGE,=========================================================) @@ -160,7 +169,8 @@ unit-tests/%: export SOURCE_DIR = source unit-tests/%: export TEST_DIR = unit-test unit-tests/%: export WORKING_DIR := $(WORKING_DIR)/unit-tests unit-tests/%: export UNIT_TEST_PRE_PROCESS_MACROS = "NO_MPI=no_mpi" -unit-tests: unit-tests/run +unit-tests: extract_meta unit-tests/run +unit-tests/build: extract_meta unit-tests/build ############################################################################## # Integration tests diff --git a/build/extract/extract_meta.mk b/build/extract/extract_meta.mk new file mode 100644 index 000000000..d165f87fb --- /dev/null +++ b/build/extract/extract_meta.mk @@ -0,0 +1,26 @@ +############################################################################## +# (c) Crown copyright 2026 Met Office. All rights reserved. +# The file LICENCE, distributed with this code, contains details of the terms +# under which the code may be used. +############################################################################## +# +# Run this file to extract rose-metadata from external repositories +# +# The following environment variables are used for input: +# WORKING_DIR : Directory to hold working copies of source. +# META_DIR: Directory to save extracted external metadata +# APPS_ROOT_DIR: Location of apps clone +# EXTRA_ROSE_META: List of repos to extract metadata from +# +############################################################################### + +.PHONY: extract_meta + +extract_meta: + $(Q)for REPO in $(EXTRA_ROSE_META) ; do \ + python $(APPS_ROOT_DIR)/build/extract/extract_science.py \ + -r $$REPO \ + -d $(APPS_ROOT_DIR)/dependencies.yaml \ + -w $(WORKING_DIR) \ + -m $(META_DIR) \ + ; done diff --git a/build/extract/extract_science.py b/build/extract/extract_science.py index 845b87b01..8dc8ae7f9 100755 --- a/build/extract/extract_science.py +++ b/build/extract/extract_science.py @@ -1,9 +1,10 @@ import argparse +import logging import os import yaml from pathlib import Path +from shutil import copytree from get_git_sources import clone_and_merge, run_command -import logging def load_yaml(fpath: Path) -> dict: @@ -17,7 +18,42 @@ def load_yaml(fpath: Path) -> dict: return sources -def extract_files(dependencies: dict, extract_lists: dict, working: Path) -> None: +def copy_rose_meta(rose_meta_dest: Path, clone_loc: Path) -> None: + """ + Copy rose-meta contents from extracted dependency to working/../rose-meta + """ + + rose_meta_orig = clone_loc / "rose-meta" + + for directory in rose_meta_orig.iterdir(): + copytree(directory, rose_meta_dest / directory.name, dirs_exist_ok=True) + + +def copy_extracted_files(dependency: str, extract_lists: dict, working: Path, clone_loc: Path) -> None: + """ + Copy extracted files to the working dir based on extract list + """ + + files = extract_lists[dependency] + + # make the working directory location + working_dir = working / dependency + working_dir.mkdir(parents=True, exist_ok=True) + + # rsync extract files from clone loc to the working directory + copy_command = "rsync --include='**/' " + for extract_file in files: + if not extract_file: + continue + if Path(clone_loc / extract_file).is_dir(): + extract_file = extract_file.rstrip("/") + extract_file += "/**" + copy_command += f"--include='{extract_file}' " + copy_command += f"--exclude='*' -avmq {clone_loc}/ {working_dir}" + run_command(copy_command) + + +def extract_files(dependencies: dict, rose_meta: str, extract_lists: dict, working: Path, meta: Path) -> None: """ Clone the dependency to a temporary location Then copy the desired files to the working directory @@ -29,9 +65,8 @@ def extract_files(dependencies: dict, extract_lists: dict, working: Path) -> Non mirror_loc = Path(mirror_loc) for dependency, sources in dependencies.items(): - if dependency not in extract_lists: + if dependency not in extract_lists and dependency != rose_meta: continue - files = extract_lists[dependency] # If the PHYSICS_ROOT environment variable is provided, then use sources there if "PHYSICS_ROOT" in os.environ and Path(os.environ["PHYSICS_ROOT"]).exists(): @@ -40,21 +75,11 @@ def extract_files(dependencies: dict, extract_lists: dict, working: Path) -> Non clone_loc = working.parent / "scratch" / dependency clone_and_merge(dependency, sources, clone_loc, use_mirrors, mirror_loc) - # make the working directory location - working_dir = working / dependency - working_dir.mkdir(parents=True, exist_ok=True) + if rose_meta: + copy_rose_meta(meta, clone_loc) + else: + copy_extracted_files(dependency, extract_lists, working, clone_loc) - # rsync extract files from clone loc to the working directory - copy_command = "rsync --include='**/' " - for extract_file in files: - if not extract_file: - continue - if Path(clone_loc / extract_file).is_dir(): - extract_file = extract_file.rstrip("/") - extract_file += "/**" - copy_command += f"--include='{extract_file}' " - copy_command += f"--exclude='*' -avmq {clone_loc}/ {working_dir}" - run_command(copy_command) def parse_args() -> argparse.Namespace: @@ -70,15 +95,35 @@ def parse_args() -> argparse.Namespace: help="The dependencies file for the apps working copy", ) parser.add_argument("-w", "--working", default=".", help="Build location") + parser.add_argument( + "-m", + "--meta_dir", + default=None, + help="Path to store externally extracted rose-meta. Used if --rose-meta set. " + "Defaults to args.working/../rose-meta" + ) parser.add_argument( "-e", "--extract", default="./extract.yaml", help="Path to file containing extract lists", ) + parser.add_argument( + "-r", + "--rose_meta", + type=str, + default="", + help="Should be a repository in the dependencies file. If set, copy the " + "dependencies rose-meta directory contents to working/../rose-meta. " + "If set, the extract file will be ignored, and just rose-metadata copied" + ) args = parser.parse_args() args.working = Path(args.working) + if args.meta_dir is None: + args.meta_dir = args.working.parent / "rose-meta" + else: + args.meta_dir = Path(args.meta_dir) return args @@ -87,10 +132,12 @@ def main(): logging.basicConfig(level=logging.INFO) - extract_lists: dict = load_yaml(args.extract) dependencies: dict = load_yaml(args.dependencies) - extract_files(dependencies, extract_lists, args.working) + if args.rose_meta: + extract_files(dependencies, args.rose_meta, [], args.working, args.meta_dir) + else: + extract_files(dependencies, args.rose_meta, load_yaml(args.extract), args.working, args.meta_dir) if __name__ == "__main__": diff --git a/build/extract/get_git_sources.py b/build/extract/get_git_sources.py index 4bf5fd632..59afdb006 100644 --- a/build/extract/get_git_sources.py +++ b/build/extract/get_git_sources.py @@ -88,7 +88,11 @@ def datetime_str() -> str: def clone_and_merge( - dependency: str, opts: Union[list, dict], loc: Path, use_mirrors: bool, mirror_loc: Path + dependency: str, + opts: Union[list, dict], + loc: Path, + use_mirrors: bool, + mirror_loc: Path, ) -> None: """ Wrapper script for calling get_source and merge_source for a single dependency diff --git a/dependencies.yaml b/dependencies.yaml index 9bd6b263a..78580c60c 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -30,8 +30,10 @@ lfric_apps: ref: lfric_core: - source: git@github.com:MetOffice/lfric_core.git - ref: 2026.03.2 + # source: git@github.com:james-bruten-mo/lfric_core.git + # ref: lfric_jules_metadata + source: cazld00001l:/var/tmp/persistent/fork_clones/lfric_core + ref: moci: source: git@github.com:MetOffice/moci.git diff --git a/interfaces/coupled_interface/Makefile b/interfaces/coupled_interface/Makefile index bd39248f4..74b920bdf 100644 --- a/interfaces/coupled_interface/Makefile +++ b/interfaces/coupled_interface/Makefile @@ -20,12 +20,17 @@ export PSYCLONE_TRANSFORMATION ?= none export PROJECT_DIR := $(realpath $(dir $(lastword $(MAKEFILE_LIST)))) +export EXTRA_ROSE_META = jules +export META_DIR = $(WORKING_DIR)/../rose-meta +export ROSE_META_DIRS = $(APPS_ROOT_DIR)/rose-meta $(META_DIR) + export INTERNAL_DEPENDENCIES = $(CORE_ROOT_DIR)/infrastructure \ $(CORE_ROOT_DIR)/components/lfric-xios META_VN ?= HEAD META_FILE_DIR = $(PROJECT_DIR)/rose-meta/$(PROJECT_NAME)/$(META_VN) +include $(APPS_ROOT_DIR)/build/extract/extract_meta.mk include $(CORE_ROOT_DIR)/infrastructure/build/lfric.mk include $(PROJECT_DIR)/build/import.mk include $(INTERNAL_DEPENDENCIES:=/build/import.mk) @@ -104,7 +109,8 @@ unit-tests/%: export PROJECT = $(PROJECT_NAME) unit-tests/%: export SOURCE_DIR = source unit-tests/%: export TEST_DIR = unit-test unit-tests/%: export WORKING_DIR := $(WORKING_DIR)/unit-test -unit-tests: unit-tests/run +unit-tests: extract_meta unit-tests/run +unit-tests/build: extract_meta unit-tests/build ############################################################################## diff --git a/interfaces/jedi_lfric_interface/Makefile b/interfaces/jedi_lfric_interface/Makefile index b2f843a18..70120e856 100644 --- a/interfaces/jedi_lfric_interface/Makefile +++ b/interfaces/jedi_lfric_interface/Makefile @@ -35,6 +35,10 @@ export EXTERNAL_DYNAMIC_LIBRARIES = netcdff netcdf yaxt yaxt_c export PROJECT_DIR := $(realpath $(dir $(lastword $(MAKEFILE_LIST)))) +export EXTRA_ROSE_META = jules +export META_DIR = $(WORKING_DIR)/../rose-meta +export ROSE_META_DIRS = $(APPS_ROOT_DIR)/rose-meta $(META_DIR) + export INTERNAL_DEPENDENCIES = $(CORE_ROOT_DIR)/infrastructure \ $(APPS_ROOT_DIR)/science/gungho \ $(CORE_ROOT_DIR)/components/lfric-xios \ @@ -51,6 +55,7 @@ default: unit-tests integration-tests documentation doc docs: document-uml document-api $(Q)echo > /dev/null +include $(APPS_ROOT_DIR)/build/extract/extract_meta.mk include $(CORE_ROOT_DIR)/infrastructure/build/lfric.mk include $(PROJECT_DIR)/build/import.mk include $(INTERNAL_DEPENDENCIES:=/build/import.mk) @@ -92,7 +97,8 @@ unit-tests/%: export PROJECT = jedi_lfric unit-tests/%: export SOURCE_DIR = source unit-tests/%: export TEST_DIR = unit-test unit-tests/%: export WORKING_DIR := $(WORKING_DIR)/unit-tests -unit-tests: unit-tests/run +unit-tests: extract_meta unit-tests/run +unit-tests/build: extract_meta unit-tests/build ############################################################################## @@ -109,7 +115,8 @@ integration-tests/%: export PROJECT = $(PROJECT_NAME) integration-tests/%: export SOURCE_DIR = source integration-tests/%: export TEST_DIR = integration-test integration-tests/%: export WORKING_DIR := $(WORKING_DIR)/integration-tests -integration-tests: integration-tests/run +integration-tests: extract_meta integration-tests/run +integration-tests/build: extract_meta integration-tests/build ############################################################################## diff --git a/interfaces/physics_schemes_interface/Makefile b/interfaces/physics_schemes_interface/Makefile index 98fff3681..92c50a25d 100644 --- a/interfaces/physics_schemes_interface/Makefile +++ b/interfaces/physics_schemes_interface/Makefile @@ -37,6 +37,8 @@ export EXTERNAL_STATIC_LIBRARIES = export PROJECT_DIR := $(realpath $(dir $(lastword $(MAKEFILE_LIST)))) +export ROSE_META_DIRS = $(APPS_ROOT_DIR)/rose-meta + .PHONY: all all: default diff --git a/interfaces/socrates_interface/Makefile b/interfaces/socrates_interface/Makefile index af25a5a47..09cd253d0 100644 --- a/interfaces/socrates_interface/Makefile +++ b/interfaces/socrates_interface/Makefile @@ -36,6 +36,8 @@ export EXTERNAL_DYNAMIC_LIBRARIES = yaxt yaxt_c netcdff netcdf hdf5 export PROJECT_DIR := $(realpath $(dir $(lastword $(MAKEFILE_LIST)))) +export ROSE_META_DIRS = $(APPS_ROOT_DIR)/rose-meta + META_VN ?= HEAD META_FILE_DIR = $(PROJECT_DIR)/rose-meta/$(PROJECT_NAME)-radiation/$(META_VN) diff --git a/rose-stem/flow.cylc b/rose-stem/flow.cylc index f762b6e18..acb509174 100644 --- a/rose-stem/flow.cylc +++ b/rose-stem/flow.cylc @@ -101,7 +101,6 @@ {% do LOG.debug("Not Running Physics Schemes") %} {% for item in ["casim", - "jules", "ukca", "socrates", "socrates-spectral", diff --git a/science/adjoint/Makefile b/science/adjoint/Makefile index 3120ac3e1..1988e6d86 100644 --- a/science/adjoint/Makefile +++ b/science/adjoint/Makefile @@ -9,6 +9,10 @@ export PROFILE ?= fast-debug export PROJECT_DIR := $(realpath $(dir $(lastword $(MAKEFILE_LIST)))) +export EXTRA_ROSE_META = jules +export META_DIR = $(WORKING_DIR)/../rose-meta +export ROSE_META_DIRS = $(APPS_ROOT_DIR)/rose-meta $(META_DIR) + export INTERNAL_DEPENDENCIES = $(CORE_ROOT_DIR)/infrastructure \ $(CORE_ROOT_DIR)/components/driver \ $(CORE_ROOT_DIR)/components/science \ @@ -28,6 +32,7 @@ default: unit-tests integration-tests documentation doc docs: document-api $(Q)echo > /dev/null +include $(APPS_ROOT_DIR)/build/extract/extract_meta.mk include $(CORE_ROOT_DIR)/infrastructure/build/lfric.mk include $(INTERNAL_DEPENDENCIES:=/build/import.mk) @@ -79,8 +84,8 @@ unit-tests/%: export PROJECT = adjoint unit-tests/%: export SOURCE_DIR = source unit-tests/%: export TEST_DIR = unit-test unit-tests/%: export WORKING_DIR := $(WORKING_DIR)/unit-tests -unit-tests: unit-tests/run - +unit-tests: extract_meta unit-tests/run +unit-tests/build: extract_meta unit-tests/build ############################################################################## # Integration tests diff --git a/science/gungho/Makefile b/science/gungho/Makefile index 5a304b0cf..b6f3646f4 100644 --- a/science/gungho/Makefile +++ b/science/gungho/Makefile @@ -33,6 +33,10 @@ export PROFILE ?= fast-debug export PROJECT_DIR := $(realpath $(dir $(lastword $(MAKEFILE_LIST)))) +export EXTRA_ROSE_META = jules +export META_DIR = $(WORKING_DIR)/../rose-meta +export ROSE_META_DIRS = $(APPS_ROOT_DIR)/rose-meta $(META_DIR) + export INTERNAL_DEPENDENCIES = $(CORE_ROOT_DIR)/infrastructure \ $(CORE_ROOT_DIR)/components/driver \ $(CORE_ROOT_DIR)/components/science \ @@ -50,6 +54,7 @@ default: unit-tests integration-tests documentation doc docs: document-api $(Q)echo > /dev/null +include $(APPS_ROOT_DIR)/build/extract/extract_meta.mk include $(CORE_ROOT_DIR)/infrastructure/build/lfric.mk include $(INTERNAL_DEPENDENCIES:=/build/import.mk) @@ -105,7 +110,8 @@ unit-tests/%: export PROJECT = gungho unit-tests/%: export SOURCE_DIR = source unit-tests/%: export TEST_DIR = unit-test unit-tests/%: export WORKING_DIR := $(WORKING_DIR)/unit-tests -unit-tests: unit-tests/run +unit-tests: extract_meta unit-tests/run +unit-tests/build: extract_meta unit-tests/build ############################################################################### # Integration tests @@ -119,7 +125,8 @@ integration-tests/%: export PROJECT = gungho integration-tests/%: export SOURCE_DIR = source integration-tests/%: export TEST_DIR = integration-test integration-tests/%: export WORKING_DIR := $(WORKING_DIR)/integration-tests -integration-tests: integration-tests/run +integration-tests: extrac_meta integration-tests/run +integration-tests/build: extract_meta integration-tests/build ############################################################################### # Clean diff --git a/science/linear/Makefile b/science/linear/Makefile index 91bc86694..61915baa4 100644 --- a/science/linear/Makefile +++ b/science/linear/Makefile @@ -9,6 +9,10 @@ export PROFILE ?= fast-debug export PROJECT_DIR := $(realpath $(dir $(lastword $(MAKEFILE_LIST)))) +export EXTRA_ROSE_META = jules +export META_DIR = $(WORKING_DIR)/../rose-meta +export ROSE_META_DIRS = $(APPS_ROOT_DIR)/rose-meta $(META_DIR) + export INTERNAL_DEPENDENCIES = $(CORE_ROOT_DIR)/infrastructure \ $(CORE_ROOT_DIR)/components/driver \ $(CORE_ROOT_DIR)/components/science \ @@ -27,6 +31,7 @@ default: unit-tests integration-tests documentation doc docs: document-api $(Q)echo > /dev/null +include $(APPS_ROOT_DIR)/build/extract/extract_meta.mk include $(CORE_ROOT_DIR)/infrastructure/build/lfric.mk include $(INTERNAL_DEPENDENCIES:=/build/import.mk) @@ -79,8 +84,8 @@ unit-tests/%: export PROJECT = linear unit-tests/%: export SOURCE_DIR = source unit-tests/%: export TEST_DIR = unit-test unit-tests/%: export WORKING_DIR := $(WORKING_DIR)/unit-tests - -unit-tests: unit-tests/run +unit-tests: extract_meta unit-tests/run +unit-tests/build: extract_meta unit-tests/build ############################################################################## @@ -99,7 +104,8 @@ integration-tests/%: export PROJECT = linear integration-tests/%: export SOURCE_DIR = source integration-tests/%: export TEST_DIR = integration-test integration-tests/%: export WORKING_DIR := $(WORKING_DIR)/integration-tests -integration-tests: integration-tests/run +integration-tests: extract_meta integration-tests/run +integration-tests/build: extract_meta integration-tests/build ############################################################################## # Clean