diff --git a/gsplatInterface/common.py b/gsplatInterface/common.py index c5dac77..43fd85b 100644 --- a/gsplatInterface/common.py +++ b/gsplatInterface/common.py @@ -3,6 +3,7 @@ import dataclasses from typing import Annotated, Literal, Optional, Tuple +import logging import numpy as np import numpy.typing as npt import torch @@ -107,7 +108,7 @@ class GSplatRenderTabState(RenderTabState): class CameraState(object): c2w: NDArray4x4 K: NDArray3x3 - + @staticmethod def build_K(fov, img_wh: Tuple[int, int]) -> NDArray3x3: W, H = img_wh @@ -120,7 +121,7 @@ def build_K(fov, img_wh: Tuple[int, int]) -> NDArray3x3: ] ) return K - + @classmethod def build(cls, fov, c2w, img_wh): return cls( @@ -133,31 +134,31 @@ def get_K(self, img_wh: Tuple[int, int]) -> NDArray3x3: class ProgressBar: - def __init__(self, items, desc=""): + def __init__(self, items, desc="", useProgressBar=True): self.items = items - self.__progress = ConsoleProgressDisplay(self.length, desc + "\n") - + self.__useProgressBar = useProgressBar + self.__progress = ConsoleProgressDisplay(self.length, desc + "\n") if useProgressBar else None + self.__count = 0 + @property def length(self): return len(self.items) - + def __iter__(self): return self - + def __next__(self): - count = self.__progress.count() - self.__progress += 1 + count = self.__progress.count() if self.__useProgressBar else self.__count + if self.__useProgressBar: + self.__progress += 1 + else: + self.__count += 1 if count < self.length: return self.items[count] raise StopIteration - - @staticmethod - def set_description(message): - # No real solution with boost progress_display so just print is doing the job - print(message) -def createProgressBar(items, desc=""): - return iter(ProgressBar(items, desc)) +def createProgressBar(items, desc="", useIt=True): + return iter(ProgressBar(items, desc, useIt)) def createProgressBarRange(maxValue, desc=""): return iter(ProgressBar(range(maxValue), desc)) diff --git a/gsplatInterface/datasets/sfm/interface.py b/gsplatInterface/datasets/sfm/interface.py index bd9c813..027e0f5 100644 --- a/gsplatInterface/datasets/sfm/interface.py +++ b/gsplatInterface/datasets/sfm/interface.py @@ -12,6 +12,7 @@ import numpy as np from numpy.linalg import inv as invert import json +import logging FORCE_JSON_TYPE = False @@ -111,7 +112,7 @@ def __repr__(self): class SfmFile: def __init__(self, path: Path): - print(f"Read SfmFile {path}") + logging.info(f"Read SfmFile {path}") self.__path = path self.__content = sfmData.SfMData() if not sfmDataIO.load(self.__content, str(path), sfmDataIO.ALL): diff --git a/gsplatInterface/datasets/sfm/sceneManager.py b/gsplatInterface/datasets/sfm/sceneManager.py index ab3cdc1..19ffba0 100644 --- a/gsplatInterface/datasets/sfm/sceneManager.py +++ b/gsplatInterface/datasets/sfm/sceneManager.py @@ -2,6 +2,7 @@ import os import json +import logging from pathlib import Path from typing import Union, List, Dict, Tuple @@ -30,35 +31,35 @@ def __init__(self, path: Union[str, Path]): path = Path(path) self._path = path self._sfmData = SfmReader(path) - + @property def intrinsics(self) -> List[SfmIntrinsic]: """ Intrinsics are the camera parameters """ return self._sfmData.intrinsics - + @property def views(self) -> List[SfmView]: """ Views are the images we feed to the photogrammetry to get poses, landmarks, etc """ return self._sfmData.views - + @property def poses(self) -> Dict[int, SfmPose]: """ Poses are all the viewpoints extracted by the photogrammetry process """ return self._sfmData.poses - + @property def landmarks(self) -> List[SfmLandmark]: """ Landmarks are the reconstructed points in the 3D environment """ return self._sfmData.landmarks - + @property def poseId_to_viewId(self) -> Dict[int, SfmView]: if getattr(self, "__poseId_to_viewId", None) is None: @@ -70,7 +71,7 @@ def poseId_to_viewId(self) -> Dict[int, SfmView]: if poseId not in self.__poseId_to_viewId.keys(): self.__poseId_to_viewId[poseId] = None return self.__poseId_to_viewId - + @property def viewId_to_poseId(self) -> Dict[int, SfmView]: if getattr(self, "__viewId_to_poseId", None) is None: @@ -79,13 +80,13 @@ def viewId_to_poseId(self) -> Dict[int, SfmView]: if poseId not in self.poses: self.__viewId_to_poseId[viewId] = None return self.__viewId_to_poseId - + @property def viewId_to_view(self) -> Dict[int, SfmView]: if getattr(self, "__viewId_to_view", None) is None: self.__viewId_to_view = {view.viewId: view for view in self.views} return self.__viewId_to_view - + @property def name_to_image_id(self) -> Dict[str, int]: if getattr(self, "__name_to_image_id", None) is None: @@ -94,7 +95,7 @@ def name_to_image_id(self) -> Dict[str, int]: imagePath = Path(view.imagePath).stem self.__name_to_image_id[imagePath] = view.frameId return self.__name_to_image_id - + @property def intrinsics_dict(self) -> Dict[int, SfmIntrinsic]: if getattr(self, "__intrinsics_dict", None) is None: @@ -120,14 +121,16 @@ def __init__(self, masksFolder: Optional[str] = None, mesh: Optional[str] = None, metadataFolder: Optional[str] = None, - image_alpha: bool = False): + image_alpha: bool = False, + maskedViewIDs: list = [], + ): self.sfmFile = sfmFile - # TODO : not convienient to use different poses when we have normalization that depend on dataset - self.normalize = False # normalize + # TODO : not convenient to use different poses when we have normalization that depend on dataset + self.normalize = False # normalize self.image_alpha = image_alpha - + manager = SfmSceneManager(self.sfmFile) - + self.masks_exist = masksFolder and os.path.isdir(masksFolder) # Extract extrinsic matrices in world-to-camera format. @@ -136,10 +139,10 @@ def __init__(self, Ks_dict = dict() imsize_dict = dict() # width, height missing_poses = set() - + for k, view in enumerate(manager.views): pose = manager.poses.get(view.poseId, None) - if pose is None: + if pose is None or view.viewId in maskedViewIDs: missing_poses.add(view.poseId) continue # Cam matrix (extrinsic) @@ -152,8 +155,8 @@ def __init__(self, intrinsic = manager.intrinsics_dict[camera_id] K = intrinsic.get_K() Ks_dict[camera_id] = K - - print(f"[Parser] {len(manager.views)-len(missing_poses)}/{len(manager.views)} views, taken by {len(set(camera_ids))} cameras.") + + logging.info(f"[Parser] {len(manager.views)-len(missing_poses)}/{len(manager.views)} views, taken by {len(set(camera_ids))} camera(s).") if len(manager.views)-len(missing_poses) <= 0: raise ValueError("No usable view !") @@ -161,7 +164,7 @@ def __init__(self, # Convert extrinsics to camera-to-world. camtoworlds = np.linalg.inv(w2c_mats) - + # Image names image_names = [v.imageName for v in manager.views if v.poseId not in missing_poses] image_paths = [v.imagePath for v in manager.views if v.poseId not in missing_poses] @@ -171,7 +174,7 @@ def __init__(self, image_paths = [image_paths[i] for i in inds] camtoworlds = camtoworlds[inds] camera_ids = [camera_ids[i] for i in inds] - + # Load external configuration (self.extconf) and bounds (self.bounds that is used in the trainer) self.extconf = { "spiral_radius_scale": 1.0, @@ -180,11 +183,11 @@ def __init__(self, self.bounds = np.array([0.01, 1.0]) if metadataFolder: self.load_extconf(metadataFolder) - + # 3D points and {image_name -> [point_idx]} points = np.array([p.position for p in manager.landmarks]) points_rgb = np.array([p.color for p in manager.landmarks]) - + point_indices = {} for point_id, viewIds in manager.landmarkId_to_viewId.items(): for viewId in viewIds: @@ -222,9 +225,9 @@ def __init__(self, self.points_rgb = points_rgb # np.ndarray, (num_points, 3) self.point_indices = point_indices # Dict[str, np.ndarray], image_name -> [M,] self.transform = transform # np.ndarray, (4, 4) - + self.mesh = Mesh(mesh) - + self.apply_scale() # size of the scene measured by cameras @@ -232,7 +235,7 @@ def __init__(self, scene_center = np.mean(camera_locations, axis=0) dists = np.linalg.norm(camera_locations - scene_center, axis=1) self.scene_scale = np.max(dists) - + def save_cameras(self, path): to_write = "" # Intrinsics @@ -261,10 +264,10 @@ def load_extconf(self, metadataFolder): posefile = os.path.join(metadataFolder, "poses_bounds.npy") if os.path.exists(posefile): self.bounds = np.load(posefile)[:, -2:] - + def apply_scale(self, s_height=1, s_width=1): """ Sometimes the intrinsics corresponds to 2x upsampled images so we need to use this - + s_height = actual_height / colmap_height s_width = actual_width / colmap_width """ @@ -286,11 +289,11 @@ def __init__(self, masksFolder: str = None, metadataFolder: str = None): self.sfmFile = sfmFile - # TODO : not convienient to use different poses when we have normalization that depend on dataset - self.normalize = False # normalize - + # TODO : not convenient to use different poses when we have normalization that depend on dataset + self.normalize = False # normalize + manager = SfmSceneManager(self.sfmFile) - + self.masks_exist = masksFolder and os.path.isdir(masksFolder) # Extract extrinsic matrices in world-to-camera format. @@ -299,7 +302,7 @@ def __init__(self, camera_ids = [] imsize_dict = dict() # width, height intrinsicsDict = dict() - + for poseId, pose in manager.poses.items(): # Cam matrix (extrinsic) pose_ids.append(poseId) @@ -311,27 +314,27 @@ def __init__(self, continue view = manager.viewId_to_view[viewId] camera_ids.append(view.intrinsicId) - + for camera_id, intrinsic in manager.intrinsics_dict.items(): imsize_dict[camera_id] = (int(intrinsic.W), int(intrinsic.H)) K = intrinsic.get_K() intrinsicsDict[camera_id] = K - - print(f"[BaseParser] {len(w2c_mats)} poses, taken by {len(set(camera_ids))} cameras.") + + logging.info(f"[BaseParser] {len(w2c_mats)} poses, taken by {len(set(camera_ids))} camera(s).") if len(w2c_mats) <= 0: raise ValueError("No usable pose !") w2c_mats = np.stack(w2c_mats, axis=0) # Convert extrinsics to camera-to-world. camtoworlds = np.linalg.inv(w2c_mats) - + # Create order from pose ID inds = np.argsort(pose_ids) # Reorder lists pose_ids = [pose_ids[i] for i in inds] camtoworlds = camtoworlds[inds] camera_ids = [camera_ids[i] for i in inds] - + # Find images linked to poses if they exist image_names = [] image_paths = [] @@ -344,7 +347,7 @@ def __init__(self, else: image_names.append(None) image_paths.append(None) - + # Load external configuration (self.extconf) and bounds (self.bounds that is used in the trainer) self.extconf = { "spiral_radius_scale": 1.0, @@ -353,7 +356,7 @@ def __init__(self, self.bounds = np.array([0.01, 1.0]) if metadataFolder: self.load_extconf(metadataFolder) - + self.pose_ids = pose_ids # List[str], (num_poses,) self.image_names = image_names # List[str], (num_poses,) self.image_paths = image_paths # List[str], (num_poses,) @@ -361,7 +364,7 @@ def __init__(self, self.camera_ids = camera_ids # List[int], (num_images,) self.Ks_dict = intrinsicsDict # Dict of camera_id -> K self.imsize_dict = imsize_dict # Dict of camera_id -> (width, height) - + self.apply_scale() @@ -392,25 +395,25 @@ def __init__( # if pre_load_images: # for index in self.indices: # self.get_mask(index) - + def __len__(self): return len(self.indices) - + def get_image(self, index) -> AvImage: return AvImage(self.parser.image_paths[index], alpha=self.parser.image_alpha, open=True) - + def __getitem__(self, item: int) -> Dict[str, Any]: index = self.indices[item] - + image = self.get_image(index) pixels = image.pixels camera_id = self.parser.camera_ids[index] - K = self.parser.Ks_dict[camera_id].copy() + K = self.parser.Ks_dict[camera_id].copy() camtoworlds = self.parser.camtoworlds[index] - + if image.hasAlpha: mask = image.get_alpha_mask() - + image_name = self.parser.image_names[index] if self.patch_size is not None: diff --git a/gsplatInterface/trainer.py b/gsplatInterface/trainer.py index 7d6e7c7..636e9d1 100644 --- a/gsplatInterface/trainer.py +++ b/gsplatInterface/trainer.py @@ -1,15 +1,18 @@ # -*- coding: utf-8 -*- +import os + +os.environ["PYTORCH_CUDA_ALLOC_CONF"] = "expandable_segments:True" + from common import ( - CameraState, - RenderTabState, GSplatRenderTabState, - apply_float_colormap, + CameraState, + RenderTabState, GSplatRenderTabState, + apply_float_colormap, createProgressBar, createProgressBarRange ) import json import math -import os import time from collections import defaultdict from dataclasses import dataclass, field @@ -36,25 +39,37 @@ from typing import Literal, assert_never from utils import AppearanceOptModule, CameraOptModule, knn, rgb_to_sh, set_random_seed +from numpy.core.multiarray import scalar as npscalar +from numpy.dtypes import Float64DType as npFloat64DType +from numpy import dtype as npdtype + from gsplat import export_splats from gsplat.distributed import cli from gsplat.optimizers import SelectiveAdam from gsplat.rendering import rasterization from gsplat.strategy import DefaultStrategy, MCMCStrategy +import logging +loggingFormat = os.environ.get("GSPLAT_LOG_FORMAT", + '[%(asctime)s.%(msecs)03d][%(levelname)s] (%(filename)s:%(lineno)d) %(message)s') +dateFormat = os.environ.get("GSPLAT_LOG_DATEFORMAT", '%H:%M:%S') +logging.basicConfig(level=logging.INFO, format=loggingFormat, datefmt=dateFormat) @dataclass class Config: - # Path to the .pt files. If provide, it will skip training and run evaluation only. + # Path to the .pt files. If provided, it will skip training and run evaluation only. ckpt: Optional[List[str]] = None # Path to the .pt files to load and resume training resume_ckpt: Optional[str] = "" + # Whether to retrieve optimizer state from the checkpoint + retrieve_optimizer_state: bool = True + # Path to the dataset sfm_file: str = "sfm.json" image_alpha: bool = False metadata_folder: str = "" - + # Directory to save results result_dir: str = "results/garden" # Random crop size for training (experimental) @@ -70,13 +85,22 @@ class Config: # Number of training steps max_epochs: int = 600 - + # Steps to save the model save_epochs: List[int] = field( default_factory=lambda: [150, 600], metadata=dict(nargs="*") ) - + + # Views to remove from the training set + mask_views: List[int] = field( + default_factory = lambda: [], + metadata = dict(nargs="*") + ) + + # Use progress bar in the logging + use_progress_bar: bool = False + # Degree of spherical harmonics sh_degree: int = 3 # Turn on another SH degree every this steps @@ -101,7 +125,7 @@ class Config: # Use packed mode for rasterization, this leads to less memory usage but slightly slower. packed: bool = False # Use sparse gradients for optimization. (experimental) - sparse_grad: bool = False + sparse_grad: bool = False # Use visible adam from Taming 3DGS. (experimental) visible_adam: bool = False # Anti-aliasing in rasterization. Might slightly hurt quantitative metrics. @@ -110,6 +134,9 @@ class Config: # Use random background for training to discourage transparency random_bkgd: bool = False + # Whether to use schedulers for the learning rates (only for MCMC) + use_scheduler: bool = True + # LR for 3D point positions means_lr: float = 1.6e-4 # LR for Gaussian scale factors @@ -146,7 +173,7 @@ class Config: # Enable bilateral grid. (experimental) use_bilateral_grid: bool = False - + # Whether use fused-bilateral grid use_fused_bilagrid: bool = False # Shape of the bilateral grid (X, Y, W) @@ -157,13 +184,13 @@ class Config: # Weight for depth loss depth_lambda: float = 1e-2 - # tile size + # tile size tile_size: int = 16 # 3DGUT (uncented transform + eval 3D) with_ut: bool = False with_eval3d: bool = False - + #Strategy default prune_opa: float = 0.005 grow_grad2d: float = 0.0002 @@ -205,6 +232,7 @@ def create_splats_with_optimizers( device: str = "cuda", world_rank: int = 0, world_size: int = 1, + strategy: Union[DefaultStrategy, MCMCStrategy] = DefaultStrategy, ) -> Tuple[torch.nn.ParameterDict, Dict[str, torch.optim.Optimizer]]: #initialize from sfm @@ -225,26 +253,37 @@ def create_splats_with_optimizers( quats = torch.rand((N, 4)) # [N, 4] opacities = torch.logit(torch.full((N,), init_opacity)) # [N,] + global_lr_factor = 1. + + # Adapt means_lr parameter so that a same value has a similar effect for the default strategy and MCMC + means_lr_factor = (1. if isinstance(strategy, MCMCStrategy) else 0.001) * global_lr_factor + logging.info(f"means_lr_factor: {means_lr_factor}") + scales_lr_factor = global_lr_factor + quats_lr_factor = global_lr_factor + opacities_lr_factor = global_lr_factor + sh0_lr_factor = global_lr_factor + shN_lr_factor = global_lr_factor + params = [ # name, value, lr - ("means", torch.nn.Parameter(points), means_lr * scene_scale), - ("scales", torch.nn.Parameter(scales), scales_lr), - ("quats", torch.nn.Parameter(quats), quats_lr), - ("opacities", torch.nn.Parameter(opacities), opacities_lr), + ("means", torch.nn.Parameter(points), means_lr_factor * means_lr * scene_scale), + ("scales", torch.nn.Parameter(scales), scales_lr_factor * scales_lr), + ("quats", torch.nn.Parameter(quats), quats_lr_factor * quats_lr), + ("opacities", torch.nn.Parameter(opacities), opacities_lr_factor * opacities_lr), ] if feature_dim is None: # color is SH coefficients. colors = torch.zeros((N, (sh_degree + 1) ** 2, 3)) # [N, K, 3] colors[:, 0, :] = rgb_to_sh(rgbs) - params.append(("sh0", torch.nn.Parameter(colors[:, :1, :]), sh0_lr)) - params.append(("shN", torch.nn.Parameter(colors[:, 1:, :]), shN_lr)) + params.append(("sh0", torch.nn.Parameter(colors[:, :1, :]), sh0_lr_factor * sh0_lr)) + params.append(("shN", torch.nn.Parameter(colors[:, 1:, :]), shN_lr_factor * shN_lr)) else: # features will be used for appearance and view-dependent shading features = torch.rand(N, feature_dim) # [N, feature_dim] - params.append(("features", torch.nn.Parameter(features), sh0_lr)) + params.append(("features", torch.nn.Parameter(features), sh0_lr_factor * sh0_lr)) colors = torch.logit(rgbs) # [N, 3] - params.append(("colors", torch.nn.Parameter(colors), sh0_lr)) + params.append(("colors", torch.nn.Parameter(colors), sh0_lr_factor * sh0_lr)) splats = torch.nn.ParameterDict({n: v for n, v, _ in params}).to(device) # Scale learning rate based on batch size, reference: @@ -298,6 +337,7 @@ def __init__( normalize=False, metadataFolder=cfg.metadata_folder, image_alpha=cfg.image_alpha, + maskedViewIDs=cfg.mask_views, ) self.trainset = Dataset( @@ -308,7 +348,13 @@ def __init__( self.scene_scale = self.parser.scene_scale * 1.1 * cfg.global_scale - print("Scene scale:", self.scene_scale) + logging.info(f"Scene scale: {self.scene_scale}") + + retrieve_optimizer_state = cfg.retrieve_optimizer_state and cfg.resume_ckpt != "" + if retrieve_optimizer_state and isinstance(cfg.strategy, MCMCStrategy): + # In case of MCMC, the scheduler reduces the effective learning rate by 100 + # This line updates the learning rate accordingly + cfg.means_lr = 0.01 * cfg.means_lr # Model feature_dim = 32 if cfg.app_opt else None @@ -331,11 +377,12 @@ def __init__( device=self.device, world_rank=world_rank, world_size=world_size, + strategy=cfg.strategy, ) - print("Model initialized. Number of GS:", len(self.splats["means"])) + logging.info(f"Model initialized. Number of GS: {len(self.splats['means'])}") # Densification Strategy - + self.cfg.strategy.check_sanity(self.splats, self.optimizers) if isinstance(self.cfg.strategy, DefaultStrategy): @@ -359,11 +406,11 @@ def __init__( elif isinstance(self.cfg.strategy, MCMCStrategy): self.cfg.strategy.cap_max = self.cfg.cap_max self.cfg.strategy.noise_lr = self.cfg.noise_lr - self.cfg.strategy.refine_start_iter = self.cfg.refine_start_iter - self.cfg.strategy.refine_stop_iter = self.cfg.refine_stop_iter - self.cfg.strategy.refine_every = self.cfg.refine_every - self.cfg.strategy.min_opacity = self.cfg.min_opacity - self.cfg.strategy.verbose = True + self.cfg.strategy.refine_start_iter = self.cfg.refine_start_iter + self.cfg.strategy.refine_stop_iter = self.cfg.refine_stop_iter + self.cfg.strategy.refine_every = self.cfg.refine_every + self.cfg.strategy.min_opacity = self.cfg.min_opacity + self.cfg.strategy.verbose = True self.strategy_state = self.cfg.strategy.initialize_state() else: assert_never(self.cfg.strategy) @@ -383,7 +430,6 @@ def __init__( if world_size > 1: self.pose_adjust = DDP(self.pose_adjust) - self.app_optimizers = [] if cfg.app_opt: @@ -500,37 +546,42 @@ def train(self): #Compute effective steps max_steps = cfg.max_epochs * len(self.trainset) - schedulers = [ - # means has a learning rate schedule, that end at 0.01 of the initial value - torch.optim.lr_scheduler.ExponentialLR( - self.optimizers["means"], gamma=0.01 ** (1.0 / max_steps) - ), - ] + schedulers = [] + if isinstance(cfg.strategy, MCMCStrategy) and cfg.use_scheduler: - if cfg.pose_opt: - # pose optimization has a learning rate schedule - schedulers.append( + logging.info("Create schedulers") + + schedulers = [ + # means has a learning rate schedule that ends at 0.01 of the initial value torch.optim.lr_scheduler.ExponentialLR( - self.pose_optimizers[0], gamma=0.01 ** (1.0 / max_steps) + self.optimizers["means"], gamma=0.01 ** (1.0 / max_steps) + ), + ] + + if cfg.pose_opt: + # pose optimization has a learning rate schedule + schedulers.append( + torch.optim.lr_scheduler.ExponentialLR( + self.pose_optimizers[0], gamma=0.01 ** (1.0 / max_steps) + ) ) - ) - if cfg.use_bilateral_grid: - # bilateral grid has a learning rate schedule. Linear warmup for 1000 steps. - schedulers.append( - torch.optim.lr_scheduler.ChainedScheduler( - [ - torch.optim.lr_scheduler.LinearLR( - self.bil_grid_optimizers[0], - start_factor=0.01, - total_iters=1000, - ), - torch.optim.lr_scheduler.ExponentialLR( - self.bil_grid_optimizers[0], gamma=0.01 ** (1.0 / max_steps) - ), - ] + if cfg.use_bilateral_grid: + # bilateral grid has a learning rate schedule. Linear warmup for 1000 steps. + schedulers.append( + torch.optim.lr_scheduler.ChainedScheduler( + [ + torch.optim.lr_scheduler.LinearLR( + self.bil_grid_optimizers[0], + start_factor=0.01, + total_iters=1000, + ), + torch.optim.lr_scheduler.ExponentialLR( + self.bil_grid_optimizers[0], gamma=0.01 ** (1.0 / max_steps) + ), + ] + ) ) - ) trainloader = torch.utils.data.DataLoader( self.trainset, @@ -540,19 +591,52 @@ def train(self): persistent_workers=True, pin_memory=True, ) - + + step = -1 + + if cfg.resume_ckpt != "": + torch.serialization.add_safe_globals([npscalar, npdtype, npFloat64DType]) + logging.info("Loading Gaussians from checkpoint") + ckpt = torch.load(cfg.resume_ckpt, map_location=device, weights_only=True) + for k in self.splats.keys(): + self.splats[k].data = torch.cat([ckpt["splats"][k]]) + if cfg.pose_opt and "pose_adjust" in ckpt: + pose_adjust = self.pose_adjust.module if world_size > 1 else self.pose_adjust + pose_adjust.load_state_dict(ckpt["pose_adjust"]) + if cfg.app_opt and "app_module" in ckpt: + app_module = self.app_module.module if world_size > 1 else self.app_module + app_module.load_state_dict(ckpt["app_module"]) + if cfg.retrieve_optimizer_state: + logging.info("Loading optimizer state from checkpoint") + self.strategy_state = ckpt["strategy_state"] + for name, opt_state_dict in ckpt["optimizer_state_dicts"].items(): + self.optimizers[name].load_state_dict(opt_state_dict) + if cfg.pose_opt and "pose_optimizer_state_dicts" in ckpt: + for optimizer, opt_state_dict in zip(self.pose_optimizers, ckpt["pose_optimizer_state_dicts"]): + optimizer.load_state_dict(opt_state_dict) + if cfg.app_opt and "app_optimizer_state_dicts" in ckpt: + for optimizer, opt_state_dict in zip(self.app_optimizers, ckpt["app_optimizer_state_dicts"]): + optimizer.load_state_dict(opt_state_dict) + if cfg.use_bilateral_grid and "bil_grid_optimizer_state_dicts" in ckpt: + for optimizer, opt_state_dict in zip(self.bil_grid_optimizers, ckpt["bil_grid_optimizer_state_dicts"]): + optimizer.load_state_dict(opt_state_dict) + step = ckpt["step"] - 1 + + if not cfg.use_progress_bar: + logging.info("Training started") # Training loop. dsize = len(self.trainset) - global_tic = time.time() - pbar = createProgressBar(range(0, cfg.max_epochs)) + pbar = createProgressBar(range(0, cfg.max_epochs), "Training progress:", cfg.use_progress_bar) for epoch in pbar: - + trainloader_iter = iter(trainloader) + epochLoss = 0. + for substep in range(0, dsize): - - step = epoch * dsize + substep + + step = step + 1 data = next(trainloader_iter) #Load data sample @@ -564,7 +648,7 @@ def train(self): ) image_ids = data["image_id"].to(device) masks = data["mask"].to(device) if "mask" in data else None # [1, H, W] - + if cfg.depth_loss: points = data["points"].to(device) # [1, M, 2] depths_gt = data["depths"].to(device) # [1, M] @@ -578,6 +662,7 @@ def train(self): # sh schedule sh_degree_to_use = min(step // cfg.sh_degree_interval, cfg.sh_degree) + torch.cuda.empty_cache() # forward renders, alphas, info = self.rasterize_splats( camtoworlds=camtoworlds, @@ -590,6 +675,7 @@ def train(self): image_ids=image_ids, render_mode="RGB+ED" if cfg.depth_loss else "RGB", ) + torch.cuda.empty_cache() if renders.shape[-1] == 4: colors, depths = renders[..., 0:3], renders[..., 3:4] @@ -623,9 +709,7 @@ def train(self): ) if masks is not None: - colors[~masks] = 0 - pixels[~masks] = 0 - + colors[~masks] = pixels[~masks] # loss l1loss = F.l1_loss(colors, pixels) @@ -633,6 +717,7 @@ def train(self): colors.permute(0, 3, 1, 2), pixels.permute(0, 3, 1, 2), padding="valid" ) loss = l1loss * (1.0 - cfg.ssim_lambda) + ssimloss * cfg.ssim_lambda + epochLoss += loss.item() if cfg.depth_loss: # query depths from depth map @@ -666,13 +751,6 @@ def train(self): loss.backward() - desc = f"loss={loss.item():.3f}| " f"sh degree={sh_degree_to_use}| " - if cfg.depth_loss: - desc += f"depth loss={depthloss.item():.6f}| " - - pbar.set_description(desc) - - # Turn Gradients into Sparse Tensor before running optimizer if cfg.sparse_grad: assert cfg.packed, "Sparse gradients only work with packed mode." @@ -727,56 +805,59 @@ def train(self): info=info, packed=cfg.packed, ) - print("Number of GS:", len(self.splats["means"])) elif isinstance(self.cfg.strategy, MCMCStrategy): + means_lr = schedulers[0].get_last_lr()[0] if cfg.use_scheduler else self.optimizers["means"].param_groups[0]["lr"] self.cfg.strategy.step_post_backward( params=self.splats, optimizers=self.optimizers, state=self.strategy_state, step=step, info=info, - lr=schedulers[0].get_last_lr()[0], + lr=means_lr, ) else: assert_never(self.cfg.strategy) - + + if not cfg.use_progress_bar: + logging.info(f"Epoch ({epoch}) Loss: {epochLoss}") + if epoch in [i - 1 for i in cfg.save_epochs] or epoch == cfg.max_epochs - 1: path = f"{self.ckpt_dir}/ckpt_{epoch}_rank{self.world_rank}.pt" - data = {"step": step, "splats": self.splats.state_dict()} - + data = {"step": step, + "splats": self.splats.state_dict(), + "strategy_state": self.strategy_state, + "optimizer_state_dicts": {name: opt.state_dict() for name, opt in self.optimizers.items()}} + if cfg.pose_opt: + data["pose_optimizer_state_dicts"] = [optimizer.state_dict() for optimizer in self.pose_optimizers] if world_size > 1: data["pose_adjust"] = self.pose_adjust.module.state_dict() else: data["pose_adjust"] = self.pose_adjust.state_dict() if cfg.app_opt: + data["app_optimizer_state_dicts"] = [optimizer.state_dict() for optimizer in self.app_optimizers] if world_size > 1: data["app_module"] = self.app_module.module.state_dict() else: data["app_module"] = self.app_module.state_dict() - + if cfg.use_bilateral_grid: + data["bil_grid_optimizer_state_dicts"] = [optimizer.state_dict() for optimizer in self.bil_grid_optimizers] + torch.save(data, path) def main(local_rank: int, world_rank, world_size: int, cfg: Config): - + #Create object for computation runner = Runner(local_rank, world_rank, world_size, cfg) - #if a checkpoint was passed, will load the gaussians from there - if cfg.resume_ckpt != "": - print("Loading from checkpoint") - ckpt = torch.load(cfg.resume_ckpt, map_location=runner.device, weights_only=True) - for k in runner.splats.keys(): - runner.splats[k].data = torch.cat([ckpt["splats"][k]]) - #Launch training runner.train() if __name__ == "__main__": - + import sys from argparse_dataclass import ArgumentParser diff --git a/gsplatInterface/viewer.py b/gsplatInterface/viewer.py index 5db5eea..52569b3 100644 --- a/gsplatInterface/viewer.py +++ b/gsplatInterface/viewer.py @@ -18,16 +18,19 @@ from datasets.sfm.sceneManager import PoseParser # from cameraPosesParser import CameraParser +from numpy.core.multiarray import scalar as npscalar +from numpy.dtypes import Float64DType as npFloat64DType +from numpy import dtype as npdtype def get_cameras_from_sfm(sfmFile): cameras = [] - + psr = PoseParser( sfmFile=sfmFile, normalize=True ) - + # Find a default intrinsic valid_intrinsics = [el for el in psr.camera_ids if el is not None] default_cam_id = None @@ -37,7 +40,7 @@ def get_cameras_from_sfm(sfmFile): else: # Take the first intrinsic we have default_cam_id = list(psr.Ks_dict.keys())[0] - + # Build list of cameras for pose_id, img_name, camId, c2w in zip(psr.pose_ids, psr.image_names, psr.camera_ids, psr.camtoworlds): if camId is None: @@ -49,14 +52,15 @@ def get_cameras_from_sfm(sfmFile): K=K ) cameras.append((pose_id, cam, img_name, imsize)) - + return cameras def main(local_rank: int, world_rank, world_size: int, args): torch.manual_seed(42) device = torch.device("cuda", local_rank) - + torch.serialization.add_safe_globals([npscalar, npdtype, npFloat64DType]) + means, quats, scales, opacities, sh0, shN = [], [], [], [], [], [] for ckpt_path in args.ckpt: ckpt = torch.load(ckpt_path, map_location=device, weights_only=True)["splats"] @@ -74,14 +78,14 @@ def main(local_rank: int, world_rank, world_size: int, args): shN = torch.cat(shN, dim=0) colors = torch.cat([sh0, shN], dim=-2) sh_degree = int(math.sqrt(colors.shape[-2]) - 1) - + print("Number of Gaussians:", len(means)) @torch.no_grad() def viewer_render_fn(camera_state: CameraState, render_tab_state: GSplatRenderTabState): width = render_tab_state.render_width height = render_tab_state.render_height - + c2w = camera_state.c2w K = camera_state.get_K((width, height)) c2w = torch.from_numpy(c2w).float().to(device) @@ -152,10 +156,10 @@ def viewer_render_fn(camera_state: CameraState, render_tab_state: GSplatRenderTa apply_float_colormap(alpha, render_tab_state.colormap).cpu().numpy() ) return renders - + cameras = get_cameras_from_sfm(args.cameras) render_tab_state = GSplatRenderTabState() - + total_render_time = 0. min_render_time, max_render_time = None, 0. for pose_id, camera, imgName, imgSize in createProgressBar(cameras, desc="Rendering images..."): @@ -204,13 +208,13 @@ def viewer_render_fn(camera_state: CameraState, render_tab_state: GSplatRenderTa parser.add_argument( "--output_colorspace", type=str, help="Output colorspace (you can use AUTO, SRGB, ...)" ) - + args = parser.parse_args() - + output_dir = os.path.join(args.output_dir, "renders") if os.path.exists(output_dir): os.rmdir(output_dir) os.makedirs(output_dir) args.output_dir = output_dir - + cli(main, args, verbose=True) diff --git a/meshroom/gaussianSplattingPhotogrammetry.mg b/meshroom/gaussianSplattingPhotogrammetry.mg index ac7212c..a8da21b 100644 --- a/meshroom/gaussianSplattingPhotogrammetry.mg +++ b/meshroom/gaussianSplattingPhotogrammetry.mg @@ -1,20 +1,20 @@ { "header": { "releaseVersion": "2026.1.0+develop", - "fileVersion": "2.0", + "fileVersion": "2.1", "nodesVersions": { - "CameraInit": "12.0", + "CameraInit": "12.1", "ExportImages": "1.1", "FeatureExtraction": "1.3", - "FeatureMatching": "2.0", - "GaussianSplattingOptim": "1.0", + "FeatureMatching": "2.1", + "GaussianSplattingOptim": "1.1", "GaussianSplattingRender": "1.0", "ImageMatching": "2.0", "IntrinsicsTransforming": "1.1", "RelativePoseEstimating": "3.1", - "SfMBootStrapping": "4.1", + "SfMBootStrapping": "4.2", "SfMColorizing": "1.0", - "SfMExpanding": "2.3", + "SfMExpanding": "2.4", "SfMTransform": "3.2", "TracksBuilding": "1.0" }, @@ -118,9 +118,14 @@ ], "inputs": { "sfm_file": "{ExportImages_2.outputSfMData}", - "image_alpha": true, + "image_alpha": "{GaussianSplattingOptim_1.image_alpha}", "resume_ckpt": "{GaussianSplattingOptim_1.model}", - "max_epochs": 100 + "strategy": "{GaussianSplattingOptim_1.strategy}", + "max_epochs": 100, + "refine_start_iter": "{GaussianSplattingOptim_1.refine_start_iter}", + "refine_stop_iter": "{GaussianSplattingOptim_1.refine_stop_iter}", + "reset_every": "{GaussianSplattingOptim_1.reset_every}", + "refine_every": "{GaussianSplattingOptim_1.refine_every}" } }, "GaussianSplattingOptim_3": { @@ -131,9 +136,14 @@ ], "inputs": { "sfm_file": "{ExportImages_3.outputSfMData}", - "image_alpha": true, + "image_alpha": "{GaussianSplattingOptim_2.image_alpha}", "resume_ckpt": "{GaussianSplattingOptim_2.model}", - "max_epochs": 100 + "strategy": "{GaussianSplattingOptim_2.strategy}", + "max_epochs": 100, + "refine_start_iter": "{GaussianSplattingOptim_2.refine_start_iter}", + "refine_stop_iter": "{GaussianSplattingOptim_2.refine_stop_iter}", + "reset_every": "{GaussianSplattingOptim_2.reset_every}", + "refine_every": "{GaussianSplattingOptim_2.refine_every}" } }, "GaussianSplattingRender_1": { @@ -265,4 +275,4 @@ } } } -} \ No newline at end of file +} diff --git a/meshroom/mrGSplat/GaussianSplattingOptim.py b/meshroom/mrGSplat/GaussianSplattingOptim.py index 86933c4..db62854 100755 --- a/meshroom/mrGSplat/GaussianSplattingOptim.py +++ b/meshroom/mrGSplat/GaussianSplattingOptim.py @@ -1,10 +1,10 @@ -__version__ = "1.0" +__version__ = "1.1" from meshroom.core import desc class GaussianSplattingOptim(desc.CommandLineNode): - + commandLine = "gaussianSplattingOptim {allParams}" gpu = desc.Level.INTENSIVE @@ -30,15 +30,19 @@ def buildCommandLine(self, chunk): cmdLine += f" --save_epochs {saveEpochs}" + if node.mask_views.value and len(node.masked_views.value) > 0: + maskedViews = " ".join([str(e.value) for e in node.masked_views.value]) + cmdLine += f" --mask_views {maskedViews}" + if node.patchSize.value > 0: cmdLine += f" --patch_size {node.patchSize.value}" - + if node.image_alpha.value: cmdLine += f" --image_alpha" if node.sparseGradient.value: cmdLine += " --sparse_grad --packed" - + if node.visibleAdam.value: cmdLine += " --visible_adam" @@ -47,31 +51,36 @@ def buildCommandLine(self, chunk): if node.poseOpt.value: cmdLine += " --pose_opt" - + # if node.appOpt.value: # cmdLine += " --app_opt" if node.useBilateralGrid.value: cmdLine += " --use_bilateral_grid" - + if node.useFusedBilagrid.value: cmdLine += " --use_fused_bilagrid" - + if node.useDepthLoss.value: cmdLine += " --depth_loss" - + if node.use3DGut.value: cmdLine += " --with_ut --with_eval3d" - + if node.absGrad.value: cmdLine += " --absgrad" - + if node.revisedOpacity.value: cmdLine += " --revised_opacity" + if not node.retrieveOptimState.value: + cmdLine += " --no-retrieve_optimizer_state" + + if node.useProgressBar.value: + cmdLine += f" --use_progress_bar" + node.nodeDesc.commandLine = cmdLine - return super().buildCommandLine(chunk) inputs = [ @@ -80,7 +89,6 @@ def buildCommandLine(self, chunk): label="sfmData", description="SfMData with the views, poses and intrinsics to use.", value="", - commandLineGroup="allParams" ), desc.BoolParam( name="image_alpha", @@ -94,7 +102,14 @@ def buildCommandLine(self, chunk): label="Resume From Model", description="Resume from Model", value="", - commandLineGroup="allParams" + ), + desc.BoolParam( + name="retrieveOptimState", + label="Retrieve Optimizer State", + description="Whether to retrieve optimizer state from the checkpoint.", + value=True, + commandLineGroup=None, + enabled=lambda node: node.resume_ckpt.value != "" ), desc.ChoiceParam( name='strategy', @@ -111,7 +126,6 @@ def buildCommandLine(self, chunk): description="The number of epochs performed by the optimization.\n" "60 for quick debug, 600 for good quality.", value=600, - commandLineGroup="allParams" ), desc.IntParam( name="patchSize", @@ -127,7 +141,6 @@ def buildCommandLine(self, chunk): description="How many degrees are we using to describe texture in a splat.", value=3, range=[0, 6, 1], - commandLineGroup="allParams" ), desc.IntParam( name="sh_degree_interval", @@ -136,7 +149,6 @@ def buildCommandLine(self, chunk): "We start with 0 degree and activate a new harmonic degree each sh_degree_interval steps.", value=1000, range=[100, 100000, 100], - commandLineGroup="allParams" ), desc.FloatParam( name="init_opa", @@ -144,7 +156,6 @@ def buildCommandLine(self, chunk): description="A gaussian splat is initialized with an opacity between 0 and 1", value=0.1, range=[0.0, 1.0, 0.1], - commandLineGroup="allParams" ), desc.FloatParam( name="init_scale", @@ -152,84 +163,72 @@ def buildCommandLine(self, chunk): description="A gaussian splat is initialized with a scale", value=1.0, range=[0.0, 1.0, 0.1], - commandLineGroup="allParams" ), desc.FloatParam( name="ssim_lambda", label="Weight SSIM", description="Weight of the SSIM loss in the total loss", value=0.2, - commandLineGroup="allParams" ), desc.IntParam( name="tile_size", - label="Tile Size", + label="Tile Size", description="Size of a gpu tile for rendering.", value=16, - commandLineGroup="allParams", ), desc.FloatParam( name="near_plane", label="Rendering Near Plane", description="Near plane used for rendering frustum", value=0.01, - commandLineGroup="allParams" ), desc.FloatParam( name="far_plane", label="Rendering Far Plane", description="Far plane used for rendering frustum", value=1e10, - commandLineGroup="allParams" ), desc.FloatParam( name="means_lr", label="Gaussian Centers LR", description="Gaussian Centers Learning Rate", value=1.6e-4, - commandLineGroup="allParams" ), desc.FloatParam( name="scales_lr", label="Gaussian Scales LR", description="Gaussian Scales Learning Rate", value=5e-3, - commandLineGroup="allParams" ), desc.FloatParam( name="opacities_lr", label="Gaussian Opacities LR", description="Gaussian Opacities Learning Rate", value=5e-2, - commandLineGroup="allParams" ), desc.FloatParam( name="quats_lr", label="Gaussian Orientation LR", description="Gaussian Orientation Learning Rate", value=1e-3, - commandLineGroup="allParams" ), desc.FloatParam( name="sh0_lr", label="Gaussian Brightness LR", description="Gaussian Brightness Learning Rate (Spherical harmonics 1st degree)", value=2.5e-3, - commandLineGroup="allParams" ), desc.FloatParam( name="opacity_reg", label="Opacity regularization", description="Opacity regularization weight", value=0.0, - commandLineGroup="allParams" ), desc.FloatParam( name="scale_reg", label="Scale regularization", description="Scale regularization weight", value=0.0, - commandLineGroup="allParams" ), desc.BoolParam( name="sparseGradient", @@ -264,7 +263,6 @@ def buildCommandLine(self, chunk): label="Pose optimization LR", description="Pose Optimization Learning Rate", value=1e-5, - commandLineGroup="allParams", enabled=lambda node: node.poseOpt.value ), desc.FloatParam( @@ -272,7 +270,6 @@ def buildCommandLine(self, chunk): label="Pose optimization regularization", description="Regularization for camera optimization as weight decay", value=1e-6, - commandLineGroup="allParams", enabled=lambda node: node.poseOpt.value ), # Rendering doesn't work for the moment with that ? @@ -288,7 +285,6 @@ def buildCommandLine(self, chunk): # label="Appearance optimization LR", # description="Appearance Optimization Learning Rate", # value=1e-3, - # commandLineGroup="allParams", # enabled=lambda node: node.appOpt.value # ), # desc.FloatParam( @@ -296,7 +292,6 @@ def buildCommandLine(self, chunk): # label="Appearance optimization regularization", # description="Regularization for Appearance optimization as weight decay", # value=1e-6, - # commandLineGroup="allParams", # enabled=lambda node: node.appOpt.value # ), desc.BoolParam( @@ -325,7 +320,6 @@ def buildCommandLine(self, chunk): label="Depth weight", description="depth weight in optimization", value=1e-2, - commandLineGroup="allParams", enabled=lambda node: node.useDepthLoss.value ), desc.BoolParam( @@ -337,100 +331,88 @@ def buildCommandLine(self, chunk): ), desc.FloatParam( name="prune_opa", - label="prune_opa", + label="prune_opa", description="GSs with opacity below this value will be pruned.", value=0.005, - commandLineGroup="allParams", enabled=lambda node: node.strategy.value == "default" ), desc.FloatParam( name="grow_grad2d", - label="grow_grad2d", + label="grow_grad2d", description="GSs with image plane gradient above this value will be split/duplicated.", value=0.0002, - commandLineGroup="allParams", enabled=lambda node: node.strategy.value == "default" ), desc.FloatParam( name="grow_scale3d", - label="grow_scale3d", + label="grow_scale3d", description="GSs with 3d scale (normalized by scene_scale) below this value will be duplicated. Above will be split.", value=0.01, - commandLineGroup="allParams", enabled=lambda node: node.strategy.value == "default" ), desc.FloatParam( name="grow_scale2d", - label="grow_scale2d", + label="grow_scale2d", description="GSs with 2d scale (normalized by image resolution) above this value will be split", value=0.05, - commandLineGroup="allParams", enabled=lambda node: node.strategy.value == "default" ), desc.FloatParam( name="prune_scale3d", - label="prune_scale3d", + label="prune_scale3d", description="GSs with 3d scale (normalized by scene_scale) above this value will be pruned.", value=0.1, - commandLineGroup="allParams", enabled=lambda node: node.strategy.value == "default" ), desc.FloatParam( name="prune_scale2d", - label="prune_scale2d", + label="prune_scale2d", description="GSs with 2d scale (normalized by image resolution) above this value will be pruned", value=0.15, - commandLineGroup="allParams", enabled=lambda node: node.strategy.value == "default" ), desc.IntParam( name="refine_scale2d_stop_iter", - label="refine_scale2d_stop_iter", + label="refine_scale2d_stop_iter", description="Stop refining GSs based on 2d scale after this iteration.", value=0, - commandLineGroup="allParams", enabled=lambda node: node.strategy.value == "default" ), desc.IntParam( name="refine_start_iter", - label="refine_start_iter", + label="refine_start_iter", description="Start refining GSs after this iteration.", value=500, - commandLineGroup="allParams" ), desc.IntParam( name="refine_stop_iter", - label="refine_stop_iter", + label="refine_stop_iter", description="Stop refining GSs after this iteration.", - commandLineGroup="allParams", value=15000, ), desc.IntParam( name="reset_every", - label="reset_every", + label="reset_every", description="Reset opacities every this steps.", value=3000, - commandLineGroup="allParams", enabled=lambda node: node.strategy.value == "default" ), desc.IntParam( name="refine_every", - label="refine_every", + label="refine_every", description=" Refine GSs every this steps.", value=100, - commandLineGroup="allParams" ), desc.IntParam( name="pause_refine_after_reset", - label="pause_refine_after_reset", + label="pause_refine_after_reset", description="Pause refining GSs until this number of steps after reset, Default is 0 (no pause at all) and one might want to set this number to the number of images in training set.", value=0, - commandLineGroup="allParams", enabled=lambda node: node.strategy.value == "default" ), desc.BoolParam( name="absGrad", - label="absgrad", + label="absgrad", description="Use absolute gradients for GS splitting.", value=False, commandLineGroup=None, @@ -438,7 +420,7 @@ def buildCommandLine(self, chunk): ), desc.BoolParam( name="revisedOpacity", - label="revised_opacity", + label="revised_opacity", description="hether to use revised opacity heuristic from arXiv:2404.06109 (experimental).", value=False, commandLineGroup=None, @@ -446,26 +428,23 @@ def buildCommandLine(self, chunk): ), desc.IntParam( name="cap_max", - label="cap_max", + label="cap_max", description="Maximum number of GS.", value=1000000, - commandLineGroup="allParams", enabled=lambda node: node.strategy.value == "mcmc" ), desc.FloatParam( name="noise_lr", - label="noise_lr", + label="noise_lr", description="MCMC samping noise learning rate.", value=5e5, - commandLineGroup="allParams", enabled=lambda node: node.strategy.value == "mcmc" ), desc.FloatParam( name="min_opacity", - label="min_opacity", + label="min_opacity", description="GSs with opacity below this value will be pruned.", value=0.005, - commandLineGroup="allParams", enabled=lambda node: node.strategy.value == "mcmc" ), desc.BoolParam( @@ -487,6 +466,33 @@ def buildCommandLine(self, chunk): description="All the epochs at which the model will be saved as a check point.", commandLineGroup=None, enabled=lambda node: node.custom_ckpts.value + ), + desc.BoolParam( + name="useProgressBar", + label="Use Progress Bar", + description="Use progress bar in the logging.", + value=False, + commandLineGroup=None, + ), + desc.BoolParam( + name="mask_views", + label="Mask Views", + description="Activate to specify which views to remove from the training set (e.g. to use them for evaluation only).", + value=False, + commandLineGroup=None + ), + desc.ListAttribute( + desc.IntParam( + name="masked_view", + label="Masked View", + description="View to remove from the training set.", + value=0, + ), + name="masked_views", + label="Masked Views", + description="All the views to remove from the training set.", + commandLineGroup=None, + enabled=lambda node: node.mask_views.value ) ] @@ -496,7 +502,6 @@ def buildCommandLine(self, chunk): label="Output", description="Output folder.", value="{nodeCacheFolder}", - commandLineGroup="allParams" ), desc.File( name="model",