From d8067798cc84180b8f21e2234683f94ce710229c Mon Sep 17 00:00:00 2001 From: David Huard Date: Mon, 28 Apr 2025 14:44:41 -0400 Subject: [PATCH 1/7] Add :Recharge process. Add `parse` methods for :Reservoir, :HRUs, :SubBasins, :ChannelProfile --- src/ravenpy/config/base.py | 43 +++- src/ravenpy/config/commands.py | 426 ++++++++++++++++++++++++++++---- src/ravenpy/config/processes.py | 12 + tests/conftest.py | 2 +- tests/test_commands.py | 5 +- 5 files changed, 433 insertions(+), 55 deletions(-) diff --git a/src/ravenpy/config/base.py b/src/ravenpy/config/base.py index e39f8691..33d7217e 100644 --- a/src/ravenpy/config/base.py +++ b/src/ravenpy/config/base.py @@ -1,3 +1,4 @@ +import re from collections.abc import Sequence from enum import Enum from textwrap import dedent, indent @@ -107,20 +108,40 @@ def encoder(v: dict) -> dict: class _Record(BaseModel): - """A Record has no nested Command or Record objects.""" - pass class Record(_Record): + """A Record has no nested Command or Record objects. It is typically a list of named + values on a single line. + + For example, SubBasins is a ListCommand, whose root is a list of `SubBasin` Records. + """ + model_config = ConfigDict( extra="forbid", arbitrary_types_allowed=True, populate_by_name=True ) class RootRecord(RootModel, _Record): + """A Record with a root attribute. This is typically used for an unnamed list of + values on a single line. + + For example, the list of HRUs in an HRUGroup is a RootRecord, and the weights of a GridWeights + command are a sequence of records. + """ + model_config = ConfigDict(arbitrary_types_allowed=True, populate_by_name=True) + def __iter__(self): + return iter(self.root) + + def __getitem__(self, item): + return self.root[item] + + def __str__(self): + return " ".join(map(str, self.root)) + class _Command(BaseModel): """Base class for Raven commands.""" @@ -214,7 +235,10 @@ class RootCommand(RootModel, _Command): class FlatCommand(Command): - """Only used to discriminate Commands that should not be nested.""" + """Only used to discriminate Commands that should not be nested. + + HRUGroup, ReadFromNetCDF, Reservoir are examples of FlatCommand. + """ class LineCommand(FlatCommand): @@ -222,6 +246,8 @@ class LineCommand(FlatCommand): A non-nested Command on a single line. :CommandName {field_1} {field_2} ... {field_n}\n + + EvaluationPeriod is a FlatCommand. """ def to_rv(self): @@ -231,9 +257,18 @@ def to_rv(self): return " ".join(out) + "\n" + @classmethod + def parse(cls, s): + """Parse the command and return an instance of LineCommand.""" + pat = rf":{cls.__name__}\s+(?P.+)" + fields = cls.model_fields.keys() + if match := re.match(pat, s): + args = match.group("args").split() + return cls(**dict(zip(fields, args))) + class ListCommand(RootModel, _Command): - """Use so that commands with __root__: Sequence[Command] behave like a list.""" + """Use so that commands with __root__: Sequence[Record] behave like a list.""" root: Sequence[Any] diff --git a/src/ravenpy/config/commands.py b/src/ravenpy/config/commands.py index 0b0d7299..f9b14d87 100644 --- a/src/ravenpy/config/commands.py +++ b/src/ravenpy/config/commands.py @@ -127,6 +127,8 @@ def to_rv(self): class SoilModel(RootCommand): + """:SoilModel SOIL_MULTILAYER 6""" + root: int def to_rv(self): @@ -135,6 +137,8 @@ def to_rv(self): class LinearTransform(Command): + """:LinearTransform 1.0 -273.15""" + scale: float = 1 offset: float = 0 @@ -146,7 +150,14 @@ def to_rv(self): class RainSnowTransition(Command): - """Specify the range of temperatures over which there will be a rain/snow mix when partitioning total precipitation into rain/snow components.""" + """Specify the range of temperatures over which there will be a rain/snow mix when partitioning total precipitation into rain/snow components. + + :RainSnowTransition [temp] [delta] + + # equivalent to (the preferred option) + :GlobalParameter RAINSNOW_TEMP [rainsnow_temp] + :GlobalParameter RAINSNOW_DELTA [rainsnow_delta] + """ temp: Sym """Midpoint of the temperature range [C].""" @@ -159,11 +170,7 @@ def to_rv(self): class EvaluationPeriod(LineCommand): - """ - Evaluation Period. - - :EvaluationPeriod [period_name] [start yyyy-mm-dd] [end yyyy-mm-dd] - """ + """:EvaluationPeriod [period_name] [start yyyy-mm-dd] [end yyyy-mm-dd]""" name: str start: dt.date @@ -171,7 +178,10 @@ class EvaluationPeriod(LineCommand): class CustomOutput(LineCommand): - """Create custom output file to track a single variable, parameter, or forcing function over time at a number of basins, HRUs, or across watershed.""" # noqa: E501 + """Create custom output file to track a single variable, parameter, or forcing function over time at a number of basins, HRUs, or across watershed. + + :CustomOutput DAILY AVERAGE AET BY_HRU + """ # noqa: E501 time_per: Literal["DAILY", "MONTHLY", "YEARLY", "WATER_YEARLY", "CONTINUOUS"] """Time period.""" @@ -209,6 +219,19 @@ def __str__(self): class SoilProfiles(ListCommand): + """SoilProfiles command. + + Example:: + + :SoilProfiles + # name, #horizons, hor1, th1, hor2, th2 + LAKE, 0 + GLACIER, 0 + LOAM_SEQ, 2, LOAM, 0.5, SAND, 1.5 + ALL_SAND, 2, SAND, 0.5, SAND, 1.5 + :EndSoilProfiles + """ + root: Sequence[SoilProfile] @@ -219,13 +242,19 @@ class SubBasin(Record): name: str = "sub_001" downstream_id: int = -1 profile: str = "NONE" - reach_length: float = 0 + reach_length: float | str = 0 gauged: bool = True gauge_id: Optional[str] = "" # This attribute is not rendered to RVH + @classmethod + @field_validator("reach_length") + def check_reach_length(cls, v): + if v == "ZERO-": + return 0 + return v + def __str__(self): d = self.model_dump() - d["reach_length"] = d["reach_length"] # if d["reach_length"] else "ZERO-" d["gauged"] = int(d["gauged"]) del d["gauge_id"] return " ".join(f"{v: <{VALUE_PADDING}}" for v in d.values()) @@ -234,7 +263,17 @@ def __str__(self): class SubBasins(ListCommand): - """SubBasins command (RVH).""" + """SubBasins command (RVH) + + Example:: + + :SubBasins + :Attributes, NAME, DOWNSTREAM_ID, PROFILE, REACH_LENGTH, GAUGED + :Units, none, none, none, km, none + 1, Downstream, -1, DEFAULT, 3.0, 1 + 2, Upstream, 1, DEFAULT, 3.0, 0 + :EndSubBasins + """ root: Sequence[SubBasin] _Attributes: Sequence[str] = PrivateAttr( @@ -242,6 +281,31 @@ class SubBasins(ListCommand): ) _Units: Sequence[str] = PrivateAttr(["none", "none", "none", "none", "km", "none"]) + @classmethod + def parse(cls, s): + """Parse a SubBasins command.""" + pat = r":SubBasins(.+):EndSubBasins" "" + out = [] + keys = [ + "subbasin_id", + "name", + "downstream_id", + "profile", + "reach_length", + "gauged", + ] + + # Remove all commented lines (starting with #) + s = re.sub(r"^\s*#.*$", "", s, flags=re.MULTILINE) + if match := re.search(pat, s.strip(), re.DOTALL): + spat = r"^\s*([^:^\s].+)$" + for line in re.findall(spat, match.groups()[0], re.MULTILINE): + # Split line into fields + values = re.split(r"\s+", line.strip()) + # Convert to SubBasin record + out.append(SubBasin(**dict(zip(keys, values)))) + return cls(root=out) + class HRU(Record): """Record to populate :HRUs command internal table (RVH).""" @@ -351,6 +415,55 @@ def ignore_unrecognized_hrus(cls, values): ) return out + @classmethod + def parse(cls, s): + """Parse a HRUs command.""" + pat = r":HRUs(.+):EndHRUs" + out = [] + keys = [ + "hru_id", + "area", + "elevation", + "latitude", + "longitude", + "subbasin_id", + "land_use_class", + "veg_class", + "soil_profile", + "aquifer_profile", + "terrain_class", + "slope", + "aspect", + ] + # Remove all commented lines (starting with #) + s = re.sub(r"^\s*#.*$", "", s, flags=re.MULTILINE) + + if match := re.search(pat, s.strip(), re.DOTALL): + spat = r"^\s*([^:^\s].+)$" + for line in re.findall(spat, match.groups()[0], re.MULTILINE): + # Split line into fields + values = re.split(r"\s+", line.strip()) + # Convert to HRU record + out.append(HRU(**dict(zip(keys, values)))) + return cls(root=out) + + def rename(self, mapping): + """Rename HRU attributes. + + Parameters + ---------- + mapping : dict + Nested dictionary keyed by HRU attributes, with keys as old names and values as new names. + """ + out = self.copy() + for attr, amap in mapping.items(): + for hru in out.root: + cur = getattr(hru, attr) + for old, new in amap.items(): + if cur == old: + setattr(hru, attr, new) + return out + class HRUGroup(FlatCommand): class _Rec(RootRecord): @@ -380,8 +493,119 @@ class Reservoir(FlatCommand): type: str = Field("RESROUTE_STANDARD", alias="Type") weir_coefficient: float = Field(0, alias="WeirCoefficient") crest_width: float = Field(0, alias="CrestWidth") - max_depth: float = Field(0, alias="MaxDepth") - lake_area: float = Field(0, alias="LakeArea", description="Lake area in m2") + max_depth: float = Field(0, alias="MaxDepth", description="Max depth (m)") + lake_area: float = Field(0, alias="LakeArea", description="Lake area (m2)") + absolute_crest_height: Optional[float] = Field( + None, alias="AbsoluteCrestHeight", description="Absolute crest height (m)" + ) + max_capacity: Optional[float] = Field( + None, alias="MaxCapacity", description="Maximum capacity in m3" + ) + + class SeepageParameters(LineCommand): + """:SeepageParameters [K_seep] [href]""" + + k_seep: float + h_ref: float + + seepage_parameters: Optional[SeepageParameters] = Field( + None, alias="SeepageParameters" + ) + + class StageRelations(RootCommand): + """Stage relations for the reservoir.""" + + class StageRelation(RootRecord): + """Stage relation record. + + h, q, v, a, [u] + """ + + root: ( + tuple[float, float, float, float] + | tuple[float, float, float, float, float] + ) = None + + def __str__(self): + return ", ".join([f"{x:>12}" for x in self.root]) + + root: Sequence[StageRelation] = Field(None) + + stage_relations: Optional[StageRelations] = Field(None, alias="StageRelations") + + class OutflowControlStructure(Command): + """Outflow control structure for the reservoir.""" + + target_subbasin_id: Optional[int] = Field(None, alias="TargetSubBasin") + downstream_reference_elevation: Optional[float] = Field( + None, alias="DownstreamReferenceElevation" + ) + + class StageDischargeTable(RootCommand): + """Stage discharge table for the outflow control structure. + + Example:: + + :StageDischargeTable C1 #one gate open + N + [h,Q]xN + :EndStageDischargeTable + """ + + _n: Optional[int] = None + + class StageDischargeRecord(RootRecord): + """Stage discharge record.""" + + root: tuple[float, float] = None + + root: Sequence[StageDischargeRecord] = Field(None) + + @model_validator(mode="after") + def _set_n(self): + """Set the number of records in the table.""" + if self.root is not None and self.n is None: + self.n = len(self.root) + return self + + @property + def _template(self): + return """ + :StageDischargeTable + {n} + {_records} + :EndStageDischargeTable + """ + + stage_discharge_table: Optional[Sequence[StageDischargeTable]] = Field( + None, alias="StageDischargeTable" + ) + + class BasicWeir(LineCommand): + """Basic weir for the outflow control structure. + + :BasicWeir [curve_name3] [elev] [crestwidth] [coeff] + """ + + curve_name: str + elev: float + crest_width: float + coeff: float + + basic_weir: BasicWeir = Field(None, alias="BasicWeir") + + class OperatingRegime(Command): + """Operating regime for the outflow control structure. + + :OperatingRegime [curve_name] [elev] [coeff] + """ + + name: str + use_curve: str = Field(None, alias="UseCurve") + condition: str = Field(None, alias="Condition") + constraint: str = Field(None, alias="Constraint") + + operating_regime: OperatingRegime = Field(None, alias="OperatingRegime") @property def _template(self): @@ -391,6 +615,48 @@ def _template(self): :EndReservoir """ + @classmethod + def parse(cls, s) -> list: + """Return a list of Reservoir records parsed from an RV file.""" + # Remove all commented lines (starting with #) + s = re.sub(r"^\s*#.*$", "", s, flags=re.MULTILINE) + pat = r":Reservoir\s+(\w+)\s+(.+?):EndReservoir" + out = [] + + for name, content in re.findall(pat, s.strip(), re.DOTALL): + # Extract parameters + subbasin_id = re.search(r":SubBasinID\s+(\d+)", content).group(1) + hru_id = re.search(r":HRUID\s+(\d+)", content).group(1) + type_ = re.search(r":Type\s+(\w+)", content).group(1) + weir_coefficient = re.search( + r":WeirCoefficient\s+([\d.-]+)", content + ).group(1) + crest_width = re.search(r":CrestWidth\s+([\d.-]+)", content).group(1) + max_depth = re.search(r":MaxDepth\s+([\d.-]+)", content).group(1) + lake_area = re.search(r":LakeArea\s+([\d.-]+)", content).group(1) + seepage_parameters = re.search( + r":SeepageParameters\s+([\d.-]+)\s+([\d.-]+)", content + ).groups() + + # Convert to Reservoir record + out.append( + cls( + name=name, + subbasin_id=subbasin_id, + hru_id=hru_id, + type=type_, + weir_coefficient=weir_coefficient, + crest_width=crest_width, + max_depth=max_depth, + lake_area=lake_area, + seepage_parameters=cls.SeepageParameters( + k_seep=seepage_parameters[0], + h_ref=seepage_parameters[1], + ), + ) + ) + return out + class SubBasinGroup(FlatCommand): """SubBasinGroup command (RVH).""" @@ -414,42 +680,103 @@ def to_rv(self): d["sb_ids"] = "\n ".join([", ".join(sbids) for sbids in sbids_lines]) return dedent(template).format(**d) + @classmethod + def parse(cls, s) -> list: + """Parse a SubBasinGroup command and return a list of SubBasinGroup records.""" + # Remove all commented lines (starting with #) + s = re.sub(r"^\s*#.*$", "", s, flags=re.MULTILINE) + pat = r":SubBasinGroup\s+(\w+)\s+(.+?):EndSubBasinGroup" + out = [] + + for name, content in re.findall(pat, s.strip(), re.DOTALL): + + sb_ids = re.split(r"\s+", content.strip()) + + # Convert to SubBasinGroup record + out.append(SubBasinGroup(name=name, sb_ids=sb_ids)) + return out + class SBGroupPropertyMultiplier(LineCommand): + """:SBGroupPropertyMultiplier [group_name] [parameter_name] [mult]""" + group_name: str parameter_name: str - mult: float + mult: Sym -# TODO: Convert to new config -class ChannelProfile(FlatCommand): +class ChannelProfile(Command): """ChannelProfile command (RVP).""" name: str = "chn_XXX" - bed_slope: float = 0 - survey_points: tuple[tuple[float, float], ...] = () - roughness_zones: tuple[tuple[float, float], ...] = () + bed_slope: float = Field(0, alias="BedSlope") - def to_rv(self): - template = """ - :ChannelProfile {name} - :Bedslope {bed_slope} - :SurveyPoints - {survey_points} - :EndSurveyPoints - :RoughnessZones - {roughness_zones} - :EndRoughnessZones - :EndChannelProfile - """ - d = self.model_dump() - d["survey_points"] = "\n".join( - f"{INDENT * 2}{p[0]} {p[1]}" for p in d["survey_points"] - ) - d["roughness_zones"] = "\n".join( - f"{INDENT * 2}{z[0]} {z[1]}" for z in d["roughness_zones"] - ) - return dedent(template).format(**d) + class SurveyPoints(ListCommand): + """SurveyPoints + + [x, bed_elevation] x number of survey points. + """ + + class SurveyPoint(RootRecord): + """SurveyPoint record.""" + + root: tuple[float, float] = () + + root: Sequence[SurveyPoint] = Field((SurveyPoint(),)) + + survey_points: SurveyPoints = Field(SurveyPoints(), alias="SurveyPoints") + + class RoughnessZones(ListCommand): + """RoughnessZones record. + + [x_zone, mannings_n] x number of roughness zones. + """ + + class RoughnessZone(RootRecord): + """RoughnessZone record.""" + + root: tuple[float, float] = () + + root: Sequence[RoughnessZone] = Field((RoughnessZone(),)) + + roughness_zones: RoughnessZones = Field(RoughnessZones(), alias="RoughnessZones") + + @classmethod + def parse(cls, s) -> list: + """Parse ChannelProfile commands and return a list of ChannelProfile records.""" + # Remove all commented lines (starting with #) + s = re.sub(r"^\s*#.*$", "", s, flags=re.MULTILINE) + + pat = r":ChannelProfile\s+(\w+)\s+(.+?):EndChannelProfile" + out = [] + + for name, content in re.findall(pat, s.strip(), re.DOTALL): + bed_slope = re.search(r":Bedslope\s+([\d.-]+)", content).group(1) + survey_points = re.search( + r":SurveyPoints(.+):EndSurveyPoints", content, re.DOTALL + ).group(1) + sp = [ + line.strip().split() + for line in survey_points.splitlines() + if line.strip() + ] + + roughness_zones = re.search( + r":RoughnessZones(.+):EndRoughnessZones", content, re.DOTALL + ).group(1) + rz = [line.split() for line in roughness_zones.splitlines() if line.strip()] + # rz = re.split(r"\s+", roughness_zones.strip()) + + # Convert to ChannelProfile record + out.append( + cls( + name=name, + bed_slope=bed_slope, + survey_points=sp, + roughness_zones=rz, + ) + ) + return out class GridWeights(Command): @@ -471,17 +798,6 @@ class GWRecord(RootRecord): root: tuple[int, int, float] = (1, 0, 1.0) - def __iter__(self): - return iter(self.root) - - def __getitem__(self, item): - return self.root[item] - - def __str__(self): - return " ".join(map(str, self.root)) - - model_config = ConfigDict(arbitrary_types_allowed=True, populate_by_name=True) - data: Sequence[GWRecord] = Field((GWRecord(),)) @classmethod @@ -976,6 +1292,18 @@ def parse(cls, sol): class SoilClasses(ListCommand): + """SoilClasses command. + + Example:: + + :SoilClasses + :Attributes, %SAND, %CLAY, %SILT, %ORGANIC + :Units, none, none, none, none + SAND, 1, 0, 0, 0 + LOAM, 0.5, 0.1, 0.4, 0.4 + :EndSoilClasses + """ + class SoilClass(Record): """SoilClass.""" diff --git a/src/ravenpy/config/processes.py b/src/ravenpy/config/processes.py index 026083e1..936169b1 100644 --- a/src/ravenpy/config/processes.py +++ b/src/ravenpy/config/processes.py @@ -119,6 +119,18 @@ class Interflow(Process): algo: Literal["INTERFLOW_PRMS"] +class Recharge(Process): + algo: Literal[ + "RECHARGE_FROMFILE", + "RECHARGE_CONSTANT", + "RECHARGE_DEFAULT", + "RECHARGE_CONSTANT_OVERLAP", + "RECHARGE_DATA", + "RECHARGE_FLUX", + "RAVEN_DEFAULT", + ] + + class Seepage(Process): algo: Literal["SEEP_LINEAR"] diff --git a/tests/conftest.py b/tests/conftest.py index 145421e0..90dc1543 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -25,7 +25,7 @@ ) RAVEN_TESTING_DATA_BRANCH = os.getenv("RAVEN_TESTING_DATA_BRANCH", "master") -SKIP_TEST_DATA = os.getenv("RAVENPY_SKIP_TEST_DATA") +SKIP_TEST_DATA = True # os.getenv("RAVENPY_SKIP_TEST_DATA") DEFAULT_CACHE = Path(_default_cache_dir) diff --git a/tests/test_commands.py b/tests/test_commands.py index 20971f2a..9c923470 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -17,7 +17,7 @@ ) -def test_custom_ouputs(): +def test_custom_output(): class Test(RV): co: Sequence[rc.CustomOutput] = optfield(alias="CustomOutput") @@ -32,6 +32,9 @@ class Test(RV): == ":CustomOutput YEARLY AVERAGE PRECIP ENTIRE_WATERSHED" ) + new = rc.CustomOutput.parse(str(co)) + assert new == co + def test_evaluation_metrics(): class Test(RV): From 9cc554aca18b1be2cc864a26a96f8fc541b986b8 Mon Sep 17 00:00:00 2001 From: David Huard Date: Thu, 1 May 2025 13:34:18 -0400 Subject: [PATCH 2/7] add check to make sure HRU sum to 1. Tweak to the GridWeight extractor to support data from CLRH. When parsing HRUs, return dict instead of an instance, so users can subclass HRUs --- src/ravenpy/config/commands.py | 27 ++++++++++++++++++++--- src/ravenpy/extractors/routing_product.py | 6 +++++ 2 files changed, 30 insertions(+), 3 deletions(-) diff --git a/src/ravenpy/config/commands.py b/src/ravenpy/config/commands.py index d68ac2a3..c43a84b9 100644 --- a/src/ravenpy/config/commands.py +++ b/src/ravenpy/config/commands.py @@ -444,7 +444,7 @@ def parse(cls, s): # Split line into fields values = re.split(r"\s+", line.strip()) # Convert to HRU record - out.append(HRU(**dict(zip(keys, values)))) + out.append(dict(zip(keys, values))) return cls(root=out) def rename(self, mapping): @@ -705,11 +705,11 @@ class SBGroupPropertyMultiplier(LineCommand): mult: Sym -class ChannelProfile(Command): +class ChannelProfile(FlatCommand): """ChannelProfile command (RVP).""" name: str = "chn_XXX" - bed_slope: float = Field(0, alias="BedSlope") + bed_slope: float = Field(0, alias="Bedslope") class SurveyPoints(ListCommand): """SurveyPoints @@ -741,6 +741,14 @@ class RoughnessZone(RootRecord): roughness_zones: RoughnessZones = Field(RoughnessZones(), alias="RoughnessZones") + @property + def _template(self): + return """ + :{_cmd} {name} + {_commands}{_records} + :End{_cmd} + """ + @classmethod def parse(cls, s) -> list: """Parse ChannelProfile commands and return a list of ChannelProfile records.""" @@ -800,6 +808,18 @@ class GWRecord(RootRecord): data: Sequence[GWRecord] = Field((GWRecord(),)) + @field_validator("data") + @classmethod + def _sum_weights(cls, values): + """Check that for each HRU weights sum to 1.""" + for hru, group in itertools.groupby(values, lambda x: x.root[0]): + total = sum([x.root[2] for x in group]) + if not (0.999 < total < 1.001): + raise ValueError( + f"GridWeights for HRU {hru} do not sum to 1.0: {total:.3f}" + ) + return values + @classmethod def parse(cls, s): pat = r""" @@ -919,6 +939,7 @@ class GriddedForcing(ReadFromNetCDF): def _template(self): return """ :{_cmd} {name} + # HRU GridCell Weight {_commands} :End{_cmd} """ diff --git a/src/ravenpy/extractors/routing_product.py b/src/ravenpy/extractors/routing_product.py index a163b055..b1843203 100644 --- a/src/ravenpy/extractors/routing_product.py +++ b/src/ravenpy/extractors/routing_product.py @@ -362,6 +362,11 @@ def __init__( NetCDF file or shapefile with the data to be weighted. routing_file_path : Path or str Sub-basin delineation. + dim_names : tuple + Names of the dimensions in the NetCDF file. + var_names : tuple + Names of the variables in the NetCDF file. + """ self._dim_names = tuple(dim_names) self._var_names = tuple(var_names) @@ -389,6 +394,7 @@ def __init__( ) self._routing_data = open_shapefile(routing_file_path) + self._routing_data["__INDEX__"] = range(1, len(self._routing_data) + 1) def extract(self) -> dict: """Return dictionary to create a GridWeights command.""" From 5da05098d6d787541ad35e726445515560912217 Mon Sep 17 00:00:00 2001 From: David Huard Date: Thu, 1 May 2025 14:37:08 -0400 Subject: [PATCH 3/7] update changelog --- CHANGELOG.rst | 7 ++++--- src/ravenpy/extractors/routing_product.py | 6 ++++++ 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 6394834d..7d31aef3 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -7,9 +7,10 @@ v0.18.2 (unreleased) New features ^^^^^^^^^^^^ -* Added RelativeHumidityMethod to RVIs. (PR #490) - - +* Added `RelativeHumidityMethod` to RVIs. (PR #490) +* Add `parse` methods to `LineCommand`, `SubBasins`, `HRUs`, `Reservoir`, `SubBasinGroup`, `ChannelProfile`. +* Tweak the `GridWeightExtractor` to support datasets from the Canadian River and Lake Hydrofabric. Allows setting the `routing_id_field` to `__INDEX__` in order to match HRU IDs. +* Add support for `Recharge` process. v0.18.1 (2025-04-15) -------------------- diff --git a/src/ravenpy/extractors/routing_product.py b/src/ravenpy/extractors/routing_product.py index b1843203..2df9dc64 100644 --- a/src/ravenpy/extractors/routing_product.py +++ b/src/ravenpy/extractors/routing_product.py @@ -331,6 +331,9 @@ class GridWeightExtractor: Notes ----- + To use this on HRU GeoJONS created from the Canadian River and Lake Hydrofabric database, set `routing_id_field` + to `__INDEX__`. + The original version of this algorithm can be found at: https://github.com/julemai/GridWeightsGenerator """ @@ -394,6 +397,9 @@ def __init__( ) self._routing_data = open_shapefile(routing_file_path) + + # Add `__INDEX__` column, which is just the index of the dataframe. This is used as the HRUID in + # CLRH datasets. self._routing_data["__INDEX__"] = range(1, len(self._routing_data) + 1) def extract(self) -> dict: From 2e4b224096e092dbdd6ff080806aefb939c86765 Mon Sep 17 00:00:00 2001 From: David Huard Date: Thu, 1 May 2025 16:05:52 -0400 Subject: [PATCH 4/7] added tests for the parsers. New Gridweight validator exposes a problem in the extractor test. --- tests/conftest.py | 3 ++ tests/test_commands.py | 96 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 99 insertions(+) diff --git a/tests/conftest.py b/tests/conftest.py index 90dc1543..20b16c30 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -62,6 +62,9 @@ def populate_testing_data( data_entries.extend( [ "caspar_eccc_hindcasts/geps_watershed.nc", + "clrh/mattawin/06FB002.rvh", + "clrh/mattawin/channel_properties.rvp", + "clrh/mattawin/Lakes.rvh", "eccc_forecasts/geps_watershed.nc", "cmip5/nasa_nex-gddp-1.0_day_inmcm4_historical+rcp45_nex-gddp_1971-1972_subset.nc", "cmip5/nasa_nex-gddp-1.0_day_inmcm4_historical+rcp85_nex-gddp_2070-2071_subset.nc", diff --git a/tests/test_commands.py b/tests/test_commands.py index 33d1c08a..1f296ce5 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -72,6 +72,27 @@ def test_hrus(): assert hrus[0].subbasin_id == 1 +def test_hrus_parse(get_local_testdata): + f = get_local_testdata("clrh/mattawin/06FB002.rvh") + hrus = rc.HRUs.parse(f.read_text()) + assert len(hrus) == 40 + hru = hrus[0] + + assert hru.hru_id == 38 + assert hru.area == 2.1924 + assert hru.elevation == 1440.5898 + assert hru.latitude == 46.6007 + assert hru.longitude == -73.9222 + assert hru.subbasin_id == 23007946 + assert hru.land_use_class == "LAKE" + assert hru.veg_class == "LAKE" + assert hru.soil_profile == "LAKE" + assert hru.aquifer_profile == "[NONE]" + assert hru.terrain_class == "[NONE]" + assert hru.slope == 30.986380 + assert hru.aspect == 190.30589321792877 + + def test_hru_state(): s = rc.HRUState(hru_id=1, data={"SOIL[0]": 1, "SOIL[1]": 2.0}) assert str(s) == "1,1.0,2.0" @@ -467,6 +488,30 @@ class Test(RV): Test().to_rv() +def test_subbasins_parse(get_local_testdata): + f = get_local_testdata("clrh/mattawin/06FB002.rvh") + sb = rc.SubBasins.parse(f.read_text()) + assert len(sb) == 35 + + +def test_reservoir_parse(get_local_testdata): + f = get_local_testdata("clrh/mattawin/Lakes.rvh") + rs = rc.Reservoir.parse(f.read_text()) + assert len(rs) == 5 + r = rs[0] + + assert r.name == "Lake_107056" + assert r.subbasin_id == 23007946 + assert r.hru_id == 38 + assert r.type == "RESROUTE_STANDARD" + assert r.weir_coefficient == 0.6 + assert r.crest_width == 7.7399 + assert r.max_depth == 9.6 + assert r.lake_area == 2192395.4127609455 + assert r.seepage_parameters.k_seep == 0 + assert r.seepage_parameters.h_ref == 0 + + def test_ensemble_mode(): c = rc.EnsembleMode(n=10) s = c.to_rv().strip() @@ -508,3 +553,54 @@ def test_hru_group(): """ ).strip() ) + + +def test_subbasin_group_parse(get_local_testdata): + f = get_local_testdata("clrh/mattawin/06FB002.rvh") + sbgs = rc.SubBasinGroup.parse(f.read_text()) + assert len(sbgs) == 2 + sbg = sbgs[0] + assert sbg.name == "Allsubbasins" + assert len(sbg.sb_ids) == 35 + sbg.sb_ids[0] == 23007946 + + +def test_channel_profile_parse(get_local_testdata): + f = get_local_testdata("clrh/mattawin/channel_properties.rvp") + cps = rc.ChannelProfile.parse(f.read_text()) + assert len(cps) == 20 + + cp = cps[0] + + assert cp.name == "Chn_ZERO_LENGTH" + assert cp.bed_slope == 0.1234500000 + assert len(cp.survey_points) == 8 + assert cp.survey_points[0].root == (0, 1444.5898) + assert cp.survey_points[1].root == (16.0000, 1440.5898) + assert len(cp.roughness_zones) == 3 + assert cp.roughness_zones[0].root == (0, 0.12345000) + + assert str(cp) == dedent( + """ + :ChannelProfile Chn_ZERO_LENGTH + :Bedslope 0.12345 + + :SurveyPoints + 0.0 1444.5898 + 16.0 1440.5898 + 16.2469 1440.5898 + 16.2778 1440.4663 + 16.3395 1440.4663 + 16.3703 1440.5898 + 16.6172 1440.5898 + 32.6173 1444.5898 + :EndSurveyPoints + + :RoughnessZones + 0.0 0.12345 + 16.2469 0.12345 + 16.37035 0.12345 + :EndRoughnessZones + :EndChannelProfile + """ + ) From 840ea71cefb5bfabba826dd6af1c7fcdd24b753d Mon Sep 17 00:00:00 2001 From: David Huard Date: Thu, 1 May 2025 16:07:04 -0400 Subject: [PATCH 5/7] added #PR to changelog --- CHANGELOG.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 7d31aef3..c321b436 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -8,9 +8,9 @@ v0.18.2 (unreleased) New features ^^^^^^^^^^^^ * Added `RelativeHumidityMethod` to RVIs. (PR #490) -* Add `parse` methods to `LineCommand`, `SubBasins`, `HRUs`, `Reservoir`, `SubBasinGroup`, `ChannelProfile`. -* Tweak the `GridWeightExtractor` to support datasets from the Canadian River and Lake Hydrofabric. Allows setting the `routing_id_field` to `__INDEX__` in order to match HRU IDs. -* Add support for `Recharge` process. +* Add `parse` methods to `LineCommand`, `SubBasins`, `HRUs`, `Reservoir`, `SubBasinGroup`, `ChannelProfile`. (PR #492) +* Tweak the `GridWeightExtractor` to support datasets from the Canadian River and Lake Hydrofabric. Allows setting the `routing_id_field` to `__INDEX__` in order to match HRU IDs. (PR #492) +* Add support for `Recharge` process. (PR #492) v0.18.1 (2025-04-15) -------------------- From 94a2003df538b5a75092500cedbfdf916b5878a2 Mon Sep 17 00:00:00 2001 From: David Huard Date: Thu, 1 May 2025 16:23:27 -0400 Subject: [PATCH 6/7] oups --- tests/conftest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/conftest.py b/tests/conftest.py index 20b16c30..13b0c5df 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -25,7 +25,7 @@ ) RAVEN_TESTING_DATA_BRANCH = os.getenv("RAVEN_TESTING_DATA_BRANCH", "master") -SKIP_TEST_DATA = True # os.getenv("RAVENPY_SKIP_TEST_DATA") +SKIP_TEST_DATA = os.getenv("RAVENPY_SKIP_TEST_DATA") DEFAULT_CACHE = Path(_default_cache_dir) From 24dc3049688a1ee70a4d529b2142e9534a242008 Mon Sep 17 00:00:00 2001 From: David Huard Date: Fri, 2 May 2025 15:11:21 -0400 Subject: [PATCH 7/7] add RechargeMethod options. Comment gridweight data validator, as it breaks the cli test suite --- src/ravenpy/config/commands.py | 22 +++++++++++----------- src/ravenpy/config/options.py | 5 +++++ 2 files changed, 16 insertions(+), 11 deletions(-) diff --git a/src/ravenpy/config/commands.py b/src/ravenpy/config/commands.py index c43a84b9..91138c9d 100644 --- a/src/ravenpy/config/commands.py +++ b/src/ravenpy/config/commands.py @@ -808,17 +808,17 @@ class GWRecord(RootRecord): data: Sequence[GWRecord] = Field((GWRecord(),)) - @field_validator("data") - @classmethod - def _sum_weights(cls, values): - """Check that for each HRU weights sum to 1.""" - for hru, group in itertools.groupby(values, lambda x: x.root[0]): - total = sum([x.root[2] for x in group]) - if not (0.999 < total < 1.001): - raise ValueError( - f"GridWeights for HRU {hru} do not sum to 1.0: {total:.3f}" - ) - return values + # @field_validator("data") + # @classmethod + # def _sum_weights(cls, values): + # """Check that for each HRU weights sum to 1.""" + # for hru, group in itertools.groupby(values, lambda x: x.root[0]): + # total = sum([x.root[2] for x in group]) + # if not (0.999 < total < 1.001): + # raise ValueError( + # f"GridWeights for HRU {hru} do not sum to 1.0: {total:.3f}" + # ) + # return values @classmethod def parse(cls, s): diff --git a/src/ravenpy/config/options.py b/src/ravenpy/config/options.py index 4322c362..c7994dfe 100644 --- a/src/ravenpy/config/options.py +++ b/src/ravenpy/config/options.py @@ -406,6 +406,11 @@ class RainSnowFraction(Enum): WANG = "RAINSNOW_WANG" +class RechargeMethod(Enum): + DATA = "RECHARGE_DATA" + NONE = "RECHARGE_NONE" + + class RelativeHumidityMethod(Enum): CONSTANT = "RELHUM_CONSTANT" CORR = "RELHUM_CORR"