diff --git a/Simulations/python/downloadPackages.py b/Simulations/python/downloadPackages.py index 249f5c0..7c2d0ac 100755 --- a/Simulations/python/downloadPackages.py +++ b/Simulations/python/downloadPackages.py @@ -1,7 +1,9 @@ #!/usr/bin/env python import scopesim as sim +from pathlib import Path -sim.download_packages("METIS",release="github:dev_master") -sim.download_packages("Armazones", release="2023-07-11") -sim.download_packages("ELT", release="2024-02-29") +pkgs = ["METIS", "Armazones", "ELT"] + +for pkg in pkgs: + sim.download_packages(pkg,release="stable", save_dir=str(Path.home()) + '/.inst_pkgs' ) \ No newline at end of file diff --git a/Simulations/python/raw_script.py b/Simulations/python/raw_script.py index 4ddef26..c215374 100644 --- a/Simulations/python/raw_script.py +++ b/Simulations/python/raw_script.py @@ -20,8 +20,7 @@ import astropy.units as u from simulationDefinitions import * -from sources import * -#sim.rc.__config__["!SIM.file.local_packages_path"] = DEFAULT_IRDB_LOCATION +import sources logger = get_logger(__file__) @@ -34,7 +33,7 @@ # Current Solution: use the mode field in the YAML file directly. # may want to error trap for invalid combinations in the future -def simulate(fname, mode, kwargs, wcu, source=None, small=False): +def simulate(scopesim_path, fname, mode, kwargs, wcu, source=None, small=False ): """Run main function for this script.""" logger.info("*****************************************") @@ -45,6 +44,10 @@ def simulate(fname, mode, kwargs, wcu, source=None, small=False): logger.debug("output filename: %s", fname) else: logger.warning("Output filename not set, result will not be saved.") + + if scopesim_path is not None: + sources.setup_scopesim_sources(scopesim_path) + SOURCEDICT = sources.get_SOURCEDICT() if isinstance(source, Mapping): src_name = source["name"] @@ -84,9 +87,9 @@ def simulate(fname, mode, kwargs, wcu, source=None, small=False): #set up the simulation if("wavelen" in kwargs["OBS"].keys()): - cmd = sim.UserCommands(use_instrument="metis", set_modes=[mode],properties={"!OBS.wavelen": kwargs["OBS"]['wavelen']}) + cmd = sim.UserCommands(use_instrument="METIS", set_modes=[mode],properties={"!OBS.wavelen": kwargs["OBS"]['wavelen']}) else: - cmd = sim.UserCommands(use_instrument="metis", set_modes=[mode]) + cmd = sim.UserCommands(use_instrument="METIS", set_modes=[mode]) #copy over the OBS settings directly, then set up the optical train diff --git a/Simulations/python/runRecipes.py b/Simulations/python/runRecipes.py index afc979f..2aa3538 100644 --- a/Simulations/python/runRecipes.py +++ b/Simulations/python/runRecipes.py @@ -15,9 +15,7 @@ from datetime import datetime import numpy as np from astropy.io import fits -import astropy -import copy -from multiprocessing import Pool,Process,Manager +from multiprocessing import Pool class runRecipes(): @@ -29,7 +27,8 @@ def __init__(self): self.tDelt = TimeDelta(0, format='sec') self.allFileNames = [] self.allmjd = [] - + self.params = {} + def parseCommandLine(self,args): """ @@ -73,6 +72,9 @@ def parseCommandLine(self,args): parser.add_argument('-n', '--nCores', type=int, default = 1, help='number of cores for parallel processing') + parser.add_argument('-p', '--irdb_path', type=str, + default = './inst_pkgs', + help='Path to the IRDB instrument packages') args = parser.parse_args() @@ -85,16 +87,6 @@ def parseCommandLine(self,args): params['outputDir'] = args.outputDir else: params['outputDir'] = Path(__file__).parents[1] / "output/" - if(args.sequence): - if(args.sequence == "1"): - params['startMJD'] = None - params['sequence'] = True - else: - params['startMJD'] = args.sequence - params['sequence'] = True - else: - params['sequence'] = False - params['startMJD'] = None if(args.doCalib): params['doCalib'] = args.doCalib @@ -120,19 +112,31 @@ def parseCommandLine(self,args): else: params['nCores'] = 1 + if args.irdb_path: + params['irdb_path'] = args.irdb_path + self.params = params + + def setParms(self, **params): + """ + Set parameters directly from keyword arguments or a dictionary. + """ + self.params = params + print(f"Starting Simulations") print(f" input YAML = {params['inputYAML']}, output directory = {params['outputDir']}") - if(params['startMJD'] is not None): - print(f" observation sequence starting at {params['startMJD']}") - elif(params['sequence']): + if(params['sequence'] == "1"): + params['startMJD'] = None + params['sequence'] = True + else: + params['startMJD'] = params['sequence'] + params['sequence'] = True + if(params['sequence']): print(f" Observation sequence will start from first date in YAML file") else: print(f" Observation dates will be taken from YAML file if given") print(f" Automatically generated darks and flats {params['doCalib']}") print(f" Small output option {params['small']}") print(f" Generate External Calibs {params['doCalib']}") - - self.params = params def loadYAML(self): @@ -739,7 +743,7 @@ def _run(self,dorcps): recipe["wcu"] = None #simulate(fname, mode, kwargs,recipe["wcu"], source=recipe["source"], small=self.params['small']) - allArgs.append((fname,mode,kwargs,recipe["wcu"],recipe["source"],self.params["small"])) + allArgs.append((self.params["irdb_path"],fname,mode,kwargs,recipe["wcu"],recipe["source"],self.params["small"])) if(not self.params['testRun']): nCores = self.params['nCores'] diff --git a/Simulations/python/run_recipes.py b/Simulations/python/run_recipes.py index c7d1d7a..3ce1f5f 100755 --- a/Simulations/python/run_recipes.py +++ b/Simulations/python/run_recipes.py @@ -8,42 +8,72 @@ import runRecipes as rr import sys import makeCalibPrototypes - - -def runRecipes(argv): - +from pathlib import Path + +def runRecipes_with_pars(inputYAML=Path(__file__).parent.parent / "YAML" / "allRecipes.yaml", + outputDir="output/", + small=False, + doStatic=True, + catglist=None, + doCalib=1, + sequence='1', + testRun=False, + calibFile=None, + nCores=8, + irdb_path='./inst_pks', + **kwargs): + """ + Run a set of recipes with explicit arguments instead of command line. + Arguments: + inputYAML: Path to the input YAML file. + doCalib: Integer flag for calibration. + calibFile: Path to calibration file. + testRun: Boolean flag for test run. + doStatic: Boolean flag for static calibration. + outputDir: Output directory. + kwargs: Any additional parameters. + """ simulationSet = rr.runRecipes() - #simulationSet.setParms(inputYAML = None) \TODO - set parameters directly rather than commandline - - # get the command line arguments - - simulationSet.parseCommandLine(argv[1:]) + # Set parameters directly + simulationSet.setParms( + irdb_path=irdb_path, + inputYAML=inputYAML, + outputDir=outputDir, + small=small, + doStatic=doStatic, + catglist=catglist, + doCalib=doCalib, + sequence=sequence, + testRun=testRun, + calibFile=calibFile, + nCores=nCores, + **kwargs + ) # read in the YAML simulationSet.loadYAML() # validate the YAML - goodInput = simulationSet.validateYAML() # exit if the YAML entries are not valid - if (not goodInput): - exit + if not goodInput: + return # if requested, get the list of flats and darks - if(simulationSet.params['doCalib'] > 0): + if simulationSet.params['doCalib'] > 0: simulationSet.calculateCalibs() # run the simulations simulationSet.runSimulations() # run the calibrations if requested - if(simulationSet.params['doCalib'] > 0): + if simulationSet.params['doCalib'] > 0: simulationSet.runCalibrations() # if requested, dump the calibration dictionary to a YAML file in the same format as the input YAML - if(simulationSet.params['calibFile'] is not None): + if simulationSet.params['calibFile'] is not None: simulationSet.dumpCalibsToFile() simulationSet.allFileNames.sort() @@ -52,12 +82,16 @@ def runRecipes(argv): # if simulations were done, update the headers - if(not simulationSet.params['testRun']): + if not simulationSet.params['testRun']: simulationSet.updateHeaders() - if(simulationSet.params['doStatic']): + if simulationSet.params['doStatic']: makeCalibPrototypes.generateStaticCalibs(simulationSet.params['outputDir']) + +def runRecpies_with_argv(argv): + simulationSet = rr.runRecipes() + simulationSet.parseCommandLine(argv[1:]) + runRecipes_with_pars(**simulationSet.params) if __name__ == "__main__": - - runRecipes(sys.argv) + runRecpies_with_argv(sys.argv) diff --git a/Simulations/python/sources.py b/Simulations/python/sources.py index d5e7478..b75dc56 100644 --- a/Simulations/python/sources.py +++ b/Simulations/python/sources.py @@ -5,205 +5,189 @@ """ import astropy.units as u -import scopesim as sim -import scopesim_templates as sim_tp -import numpy as np - - import numpy as np import scipy import astropy.io.fits as fits -from astropy import units as u -import matplotlib.pyplot as plt - -import scopesim - - -####################### Definitions used by the sources ######################### - -# import ScopeSim information needed for some of the sources - -imgLM = sim.OpticalTrain(sim.UserCommands(use_instrument="METIS", set_modes=["img_lm"])) -specDictLM = imgLM.cmds['!SIM.spectral'] -imgN = sim.OpticalTrain(sim.UserCommands(use_instrument="METIS", set_modes=["img_n"])) -specDictN = imgN.cmds['!SIM.spectral'] - - - -# a fixed random star field; positions based on image size. We define position, magnitude in the reference -# band and spectral type - - - -starFieldX = np.array([-4.08,3.51,4.01,0.94,3.49,-3.67,0.02, - 2.68,4.20,-0.25,2.29,3.05,-1.00,-4.86, - -1.83,-0.60,-1.01,4.21,-2.84,4.34]) -starFieldY = np.array([4.79,-2.83,3.72,2.09,2.22,0.59,2.83, - -3.45,-0.25,3.02,4.29,4.43,0.37,-0.95, - 2.49,-0.48,4.67,4.71,4.10,-0.52]) - -starFieldM = np.array([14.0,12.4,13.7,13.4,13.4,13.1,13.6, - 12.3,13.0,13.6,13.9,13.9,13.1,12.8,13.5, - 12.9,13.9,13.9,13.8,12.9])*u.ABmag - -starFieldT = ["b0v","a0v","a5v","f0v","f5v", - "g0v","g2v","k0v","k5v",'m0v', - "m5v",'g5iii',"a5v","f0v","f5v", - "g0v","g2v","k0v","k5v",'m0v'] - - -skyStarXa = np.array([-3.72,1.07,4.53,4.94,-0.31,-0.43,5.57,5.00,3.31,-5.48,-1.19,4.64,1.63,5.68,1.97,1.79,-3.87,5.47,-2.71,-3.84,-2.10,2.77,3.77,-1.84,-4.06]) -skyStarYa = np.array([3.70,-5.22,-3.09,3.18,-5.25,-2.19,2.88,-1.55,-0.08,2.38,0.45,5.73,-0.70,-2.50,-2.36,0.75,3.84,-4.26,-3.78,4.68,-3.61,-4.62,0.02,-0.31,-4.50]) -skyStarXb = skyStarXa + 1 -skyStarYb = skyStarYa - 1 - -skyStarM = np.array([14.78,15.82,14.24,14.03,14.26,15.17,14.20,15.17,15.12,15.49,15.86,15.50,15.23,15.33,15.47,14.57,15.53,14.78,14.14,14.73,15.06,14.97,14.36,14.85,15.41])*u.ABmag -skyStarT = ["a0v"]*25 - -################### Setup input Images here (e.g. HEEPS input for coronagraph) ################# - - -hdu = fits.ImageHDU(data=scipy.datasets.face(gray=True).astype('float')) - -# Give the header some proper WCS info -hdu.header.update({"CDELT1": 1, "CUNIT1": "arcsec", "CRPIX1": 0, "CRVAL1": 0, - "CDELT2": 1, "CUNIT2": "arcsec", "CRPIX2": 0, "CRVAL2": 0,}) - +# ScopeSim imports +import scopesim as sim +import scopesim_templates as sim_tp -####################### Dictionary of Sources ######################### - -# Each entry contains a unique name, a scopesim or scopesim_templates source, and -# keywords needed by the source. - - -""" -empty_sky: blank sky, used for twilight flats and darks, or anywhere you don't want a source - -flat_field: lamp based flat field. - -star_field: fixed random star field - -simpleStarNN: a single stellar source at the centre of the field with magnitude NN. The magnitude is - fixed rather than set in the YAML file due to the need to pass the value with astropy units. - -simple_gal: elliptical galaxy - -pinhole_mask: pinhole mask for the WCU - -laser_spectrum_lm: WCU laser spectrum, LM band - -laser_spectrum_n: WCU laser spectrum, N band -""" - - - -SOURCEDICT = { - "empty_sky": (sim_tp.empty_sky, {}), - "flat_field": ( - sim_tp.calibration.flat_field, - { - "temperature": 200, - "amplitude": 0, - "filter_curve": "V", - "extend": 15, - } - ), - "star_field":( - sim_tp.stellar.stars, - { - "amplitudes":starFieldM, - "x":starFieldX, - "y":starFieldY, - "filter_name":"Ks", - "spec_types":starFieldT, - "library":"kurucz", - }), - "star_sky1":( - sim_tp.stellar.stars, - { - "amplitudes":skyStarM, - "x":skyStarXa, - "y":skyStarYa, - "filter_name":"Ks", - "spec_types":skyStarT, - "library":"kurucz", - }), - "star_sky2":( - sim_tp.stellar.stars, - { - "amplitudes":skyStarM, - "x":skyStarXb, - "y":skyStarYb, - "filter_name":"Ks", - "spec_types":skyStarT, - "library":"kurucz", - }), - - "calib_star": ( - sim_tp.stellar.star, - { - "filter_name":"Ks", - "amplitude": 12, - "library": "kurucz", - "spec_type": "k5v" +# Module-level state +_scopesim_state = {} + +def setup_scopesim_sources(local_packages_path): + """Call this once, after setting the ScopeSim path, before using SOURCEDICT or star fields.""" + sim.rc.__config__["!SIM.file.local_packages_path"] = local_packages_path + + imgLM = sim.OpticalTrain(sim.UserCommands(use_instrument="METIS", set_modes=["img_lm"])) + specDictLM = imgLM.cmds['!SIM.spectral'] + imgN = sim.OpticalTrain(sim.UserCommands(use_instrument="METIS", set_modes=["img_n"])) + specDictN = imgN.cmds['!SIM.spectral'] + + hdu = fits.ImageHDU(data=scipy.datasets.face(gray=True).astype('float')) + hdu.header.update({"CDELT1": 1, "CUNIT1": "arcsec", "CRPIX1": 0, "CRVAL1": 0, + "CDELT2": 1, "CUNIT2": "arcsec", "CRPIX2": 0, "CRVAL2": 0,}) + + # Save to module-level state + _scopesim_state.update({ + "imgLM": imgLM, + "specDictLM": specDictLM, + "imgN": imgN, + "specDictN": specDictN, + "hdu": hdu, + }) + + # Precompute star fields etc. + _scopesim_state["starFieldX"] = np.array([-4.08,3.51,4.01,0.94,3.49,-3.67,0.02, + 2.68,4.20,-0.25,2.29,3.05,-1.00,-4.86, + -1.83,-0.60,-1.01,4.21,-2.84,4.34]) + _scopesim_state["starFieldY"] = np.array([4.79,-2.83,3.72,2.09,2.22,0.59,2.83, + -3.45,-0.25,3.02,4.29,4.43,0.37,-0.95, + 2.49,-0.48,4.67,4.71,4.10,-0.52]) + _scopesim_state["starFieldM"] = np.array([14.0,12.4,13.7,13.4,13.4,13.1,13.6, + 12.3,13.0,13.6,13.9,13.9,13.1,12.8,13.5, + 12.9,13.9,13.9,13.8,12.9])*u.ABmag + _scopesim_state["starFieldT"] = ["b0v","a0v","a5v","f0v","f5v", + "g0v","g2v","k0v","k5v",'m0v', + "m5v",'g5iii',"a5v","f0v","f5v", + "g0v","g2v","k0v","k5v",'m0v'] + + # Sky star fields + _scopesim_state["skyStarXa"] = np.array([-3.72,1.07,4.53,4.94,-0.31,-0.43,5.57,5.00,3.31,-5.48,-1.19,4.64,1.63,5.68,1.97,1.79,-3.87,5.47,-2.71,-3.84,-2.10,2.77,3.77,-1.84,-4.06]) + _scopesim_state["skyStarYa"] = np.array([3.70,-5.22,-3.09,3.18,-5.25,-2.19,2.88,-1.55,-0.08,2.38,0.45,5.73,-0.70,-2.50,-2.36,0.75,3.84,-4.26,-3.78,4.68,-3.61,-4.62,0.02,-0.31,-4.50]) + _scopesim_state["skyStarXb"] = _scopesim_state["skyStarXa"] + 1 + _scopesim_state["skyStarYb"] = _scopesim_state["skyStarYa"] - 1 + _scopesim_state["skyStarM"] = np.array([14.78,15.82,14.24,14.03,14.26,15.17,14.20,15.17,15.12,15.49,15.86,15.50,15.23,15.33,15.47,14.57,15.53,14.78,14.14,14.73,15.06,14.97,14.36,14.85,15.41])*u.ABmag + _scopesim_state["skyStarT"] = ["a0v"]*25 + +# Can remove these +def get_starFieldX(): + return _scopesim_state["starFieldX"] +def get_starFieldY(): + return _scopesim_state["starFieldY"] +def get_starFieldM(): + return _scopesim_state["starFieldM"] +def get_starFieldT(): + return _scopesim_state["starFieldT"] + +def get_skyStarXa(): + return _scopesim_state["skyStarXa"] +def get_skyStarYa(): + return _scopesim_state["skyStarYa"] +def get_skyStarXb(): + return _scopesim_state["skyStarXb"] +def get_skyStarYb(): + return _scopesim_state["skyStarYb"] +def get_skyStarM(): + return _scopesim_state["skyStarM"] +def get_skyStarT(): + return _scopesim_state["skyStarT"] + +def get_SOURCEDICT(): + """Return the SOURCEDICT, using the initialized ScopeSim state.""" + s = _scopesim_state + + return { + "empty_sky": (sim_tp.empty_sky, {}), + "flat_field": ( + sim_tp.calibration.flat_field, + { + "temperature": 200, + "amplitude": 0, + "filter_curve": "V", + "extend": 15, + } + ), + "star_field":( + sim_tp.stellar.stars, + { + "amplitudes": s["starFieldM"], + "x": s["starFieldX"], + "y": s["starFieldY"], + "filter_name":"Ks", + "spec_types": s["starFieldT"], + "library":"kurucz", }), - - "simple_gal": ( - sim_tp.extragalactic.elliptical, - { - "sed":"brown/NGC4473", - "z":0, - "amplitude":5, - "filter_name":"Ks", - "pixel_scale":0.1, - "half_light_radius":30, - "n":4, - "ellip":0.5, - "ellipticity":0.5, - "angle":30, - }, + "star_sky1":( + sim_tp.stellar.stars, + { + "amplitudes": s["skyStarM"], + "x": s["skyStarXa"], + "y": s["skyStarYa"], + "filter_name":"Ks", + "spec_types": s["skyStarT"], + "library":"kurucz", + }), + "star_sky2":( + sim_tp.stellar.stars, + { + "amplitudes": s["skyStarM"], + "x": s["skyStarXb"], + "y": s["skyStarYb"], + "filter_name":"Ks", + "spec_types": s["skyStarT"], + "library":"kurucz", + }), + "calib_star": ( + sim_tp.stellar.star, + { + "filter_name":"Ks", + "amplitude": 12, + "library": "kurucz", + "spec_type": "k5v" + }), + "simple_gal": ( + sim_tp.extragalactic.elliptical, + { + "sed":"brown/NGC4473", + "z":0, + "amplitude":5, + "filter_name":"Ks", + "pixel_scale":0.1, + "half_light_radius":30, + "n":4, + "ellip":0.5, + "ellipticity":0.5, + "angle":30, + }, ), - - "simple_gal1": ( - sim_tp.extragalactic.elliptical, - { - "sed":"brown/NGC4473", - "z":0, - "amplitude":0, - "filter_name":"Ks", - "pixel_scale":0.1, - "half_light_radius":30, - "n":4, - "ellip":0.5, - "ellipticity":0.5, - "angle":30, - } + "simple_gal1": ( + sim_tp.extragalactic.elliptical, + { + "sed":"brown/NGC4473", + "z":0, + "amplitude":0, + "filter_name":"Ks", + "pixel_scale":0.1, + "half_light_radius":30, + "n":4, + "ellip":0.5, + "ellipticity":0.5, + "angle":30, + } ), - "pinhole_mask": ( - sim_tp.metis.pinhole_mask, - { - } + "pinhole_mask": ( + sim_tp.metis.pinhole_mask, + {} ), - "laser_spectrum_lm": ( - sim_tp.metis.laser.laser_spectrum_lm, - { - "specdict":specDictLM, - } + "laser_spectrum_lm": ( + sim_tp.metis.laser.laser_spectrum_lm, + { + "specdict": s["specDictLM"], + } ), - "laser_spectrum_n": ( - sim_tp.metis.laser.laser_spectrum_n, - { - "specdict":specDictN, - } + "laser_spectrum_n": ( + sim_tp.metis.laser.laser_spectrum_n, + { + "specdict": s["specDictN"], + } ), - "heeps_image": ( - scopesim.Source, - { - "image_hdu":hdu, - "flux":10*u.ABmag, - } + "heeps_image": ( + sim.Source, + { + "image_hdu": s["hdu"], + "flux": 10*u.ABmag, + } ), - - -} + } diff --git a/Simulations/runESO.sh b/Simulations/runESO.sh index 9aad301..c8f32e7 100755 --- a/Simulations/runESO.sh +++ b/Simulations/runESO.sh @@ -7,5 +7,4 @@ cat YAML/lss.yaml >> YAML/allRecipes.yaml cat YAML/ifu.yaml >> YAML/allRecipes.yaml cat YAML/hci.yaml >> YAML/allRecipes.yaml -python/run_recipes.py --inputYAML=YAML/allRecipes.yaml --outputDir output/ --doCalib=1 --sequence=1 --doStatic --nCores=8 - +python/run_recipes.py --inputYAML=YAML/allRecipes.yaml --outputDir output/ --doCalib=1 --sequence=1 --doStatic --nCores=8 \ No newline at end of file