diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index c1020dd8..ac8c276b 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -180,6 +180,8 @@ jobs: run: python -m pip install --upgrade pip wheel setuptools Sphinx sphinx-rtd-theme - name: Test the documentation run: sphinx-build -W --keep-going -b html doc doc/_build/html + - name: Test building the documentation with "make" + run: cd doc && make html style: name: Style check diff --git a/doc/Makefile b/doc/Makefile index 5044c95b..5a4463bd 100644 --- a/doc/Makefile +++ b/doc/Makefile @@ -2,10 +2,10 @@ # # You can set these variables from the command line. -SPHINXOPTS = +SPHINXOPTS = --keep-going # For macports, use the second line... -#SPHINXBUILD = sphinx-build -SPHINXBUILD = sphinx-build-2.7 +SPHINXBUILD = sphinx-build +#SPHINXBUILD = sphinx-build-2.7 PAPER = BUILDDIR = _build diff --git a/doc/changes.rst b/doc/changes.rst index 81316983..c041882f 100644 --- a/doc/changes.rst +++ b/doc/changes.rst @@ -5,10 +5,19 @@ desisim change log 0.39.0 (unreleased) ------------------- +Features: + +* Support for more data releases on SurveyRelease/Quickquasars (PR `#590`_). + +Test infrastructure: + +* Remove DesiTest (PR `#582`_). * Add numpy/2.x and scipy/1.16.x support. General cleanup. (PR `#589`_). * Add astropy/7.x test support on GitHub (PR `#591`_). +.. _`#582`: https://github.com/desihub/desisim/pull/582 .. _`#589`: https://github.com/desihub/desisim/pull/589 +.. _`#590`: https://github.com/desihub/desisim/pull/590 .. _`#591`: https://github.com/desihub/desisim/pull/591 0.38.2 (2024-12-17) diff --git a/doc/conf.py b/doc/conf.py index 88f8af84..78c1acae 100644 --- a/doc/conf.py +++ b/doc/conf.py @@ -118,8 +118,12 @@ # This value contains a list of modules to be mocked up. This is useful when # some external dependencies are not met at build time and break the # building process. +# Note: desitarget 3.2.0 removed desitarget.mock.mockmaker but this has not yet +# been removed from quickgalaxies and quicktransients, so we mock the submodules here for now. autodoc_mock_imports = [] -for missing in ('astropy', 'desimodel', 'desiutil', 'desispec', 'desisurvey', 'desitarget', 'fitsio', +for missing in ('astropy', 'desimodel', 'desiutil', 'desispec', 'desisurvey', + 'desitarget', 'desitarget.mock', 'desitarget.mock.mockmaker', + 'fitsio', 'healpy', 'matplotlib', 'numpy', 'scipy', 'simqso', 'speclite', 'specsim', 'yaml'): try: foo = import_module(missing) diff --git a/py/desisim/qso_template/desi_qso_templ.py b/py/desisim/qso_template/desi_qso_templ.py index a31006d0..e2aeb72c 100644 --- a/py/desisim/qso_template/desi_qso_templ.py +++ b/py/desisim/qso_template/desi_qso_templ.py @@ -9,7 +9,6 @@ import numpy as np import os -import imp import pdb from scipy.interpolate import interp1d diff --git a/py/desisim/scripts/quickgalaxies.py b/py/desisim/scripts/quickgalaxies.py index 5c113627..91a16dd3 100644 --- a/py/desisim/scripts/quickgalaxies.py +++ b/py/desisim/scripts/quickgalaxies.py @@ -16,7 +16,8 @@ from desisim.templates import BGS from desisim.scripts.quickspectra import sim_spectra -from desitarget.mock.mockmaker import BGSMaker +from desitarget.mock.mockmaker import BGSMaker # requires desitarget<=3.2.0 + from desitarget.cuts import isBGS_colors from desiutil.log import get_logger, DEBUG diff --git a/py/desisim/scripts/quicktransients.py b/py/desisim/scripts/quicktransients.py index 82e52e7d..b5f0a212 100644 --- a/py/desisim/scripts/quicktransients.py +++ b/py/desisim/scripts/quicktransients.py @@ -11,7 +11,7 @@ from desisim.templates import BGS, ELG, LRG from desisim.transients import transients -from desitarget.mock.mockmaker import BGSMaker, ELGMaker, LRGMaker +from desitarget.mock.mockmaker import BGSMaker, ELGMaker, LRGMaker # requires desitarget<=3.2.0 from desitarget.cuts import isBGS_colors, isELG_colors, isLRG_colors from desisim.simexp import reference_conditions diff --git a/py/desisim/spec_qa/redshifts.py b/py/desisim/spec_qa/redshifts.py index 03f10d1b..d38effb8 100644 --- a/py/desisim/spec_qa/redshifts.py +++ b/py/desisim/spec_qa/redshifts.py @@ -1125,7 +1125,7 @@ def dz_summ(simz_tab, outfile=None, pdict=None, min_count=20): if col > 0: plt.setp([axis.get_yticklabels()], visible=False) else: - axis.set_ylabel('Redshift fit residual $\Delta v$ [km/s]') + axis.set_ylabel(r'Redshift fit residual $\Delta v$ [km/s]') #if row < nrows - 1: # plt.setp([axis.get_xticklabels()], visible=False) diff --git a/py/desisim/test/test_batch.py b/py/desisim/test/test_batch.py index b5dec9d4..0731c818 100644 --- a/py/desisim/test/test_batch.py +++ b/py/desisim/test/test_batch.py @@ -1,4 +1,5 @@ import os +import tempfile import unittest from desisim.batch import calc_nodes @@ -7,25 +8,26 @@ class TestBatch(unittest.TestCase): def setUp(self): - self.batchfile = 'batch-d4ae52ada252.sh' - self._PIXPROD = os.getenv('PIXPROD') - self._DESI_SPECTRO_SIM = os.getenv('DESI_SPECTRO_SIM') - self.testdir = 'test-jkhasnqwezzcqhehadzx' - os.environ['PIXPROD'] = 'test' - os.environ['DESI_SPECTRO_SIM'] = self.testdir+'/sim' - os.environ['DESI_SPECTRO_DATA'] = self.testdir+'/sim/test' + self._tempdir = tempfile.TemporaryDirectory(prefix='desi_test_batch-') + self.testdir = self._tempdir.name + self.batchfile = os.path.join(self.testdir, 'batch.sh') + pixprod = os.path.basename(self.testdir) + self._origEnv = dict( + PIXPROD=os.getenv('PIXPROD'), + DESI_SPECTRO_SIM=os.getenv('DESI_SPECTRO_SIM'), + DESI_SPECTRO_DATA=os.getenv('DESI_SPECTRO_DATA'), + ) + os.environ['PIXPROD'] = pixprod + os.environ['DESI_SPECTRO_SIM'] = os.path.join(self.testdir, 'sim') + os.environ['DESI_SPECTRO_DATA'] = os.path.join(self.testdir, 'sim', pixprod) def tearDown(self): - if os.path.exists(self.batchfile): - os.remove(self.batchfile) - if self._PIXPROD is not None: - os.environ['PIXPROD'] = self._PIXPROD - if self._DESI_SPECTRO_SIM is not None: - os.environ['DESI_SPECTRO_SIM'] = os.getenv('DESI_SPECTRO_SIM') - if os.path.isdir(self.testdir): - import shutil - shutil.rmtree(self.testdir) - # os.removedirs(self.testdir) + for key, value in self._origEnv.items(): + if value is None: + os.environ.pop(key, None) + else: + os.environ[key] = value + self._tempdir.cleanup() def test_calc_nodes(self): self.assertEqual(calc_nodes(10, 1.5, 10), 4) diff --git a/py/desisim/test/test_io.py b/py/desisim/test/test_io.py index 1aaff596..137b9338 100644 --- a/py/desisim/test/test_io.py +++ b/py/desisim/test/test_io.py @@ -1,6 +1,6 @@ import unittest, os from uuid import uuid1 -from shutil import rmtree +import tempfile import numpy as np @@ -14,36 +14,29 @@ class TestIO(unittest.TestCase): - #- Create unique test filename in a subdirectory - @classmethod - def setUpClass(cls): - cls.testfile = 'test-{uuid}/test-{uuid}.fits'.format(uuid=uuid1()) - cls.testDir = os.path.join(os.environ['HOME'],'desi_test_io') - cls.origEnv = dict(PIXPROD = None, DESI_SPECTRO_SIM = None) - cls.testEnv = dict( + #- Create a unique test directory and filename for each test method + def setUp(self): + self._tempdir = tempfile.TemporaryDirectory(prefix='desi_test_io-') + self.testDir = self._tempdir.name + self.testfile = os.path.join(self.testDir, 'test-{uuid}.fits'.format(uuid=uuid1())) + self.origEnv = dict(PIXPROD = None, DESI_SPECTRO_SIM = None) + self.testEnv = dict( PIXPROD = 'test', - DESI_SPECTRO_SIM = os.path.join(cls.testDir,'spectro','sim'), + DESI_SPECTRO_SIM = os.path.join(self.testDir,'spectro','sim'), ) - for e in cls.origEnv: + for e in self.origEnv: if e in os.environ: - cls.origEnv[e] = os.environ[e] - os.environ[e] = cls.testEnv[e] - - #- Cleanup test files if they exist - @classmethod - def tearDownClass(cls): - if os.path.exists(cls.testfile): - os.remove(cls.testfile) - testpath = os.path.normpath(os.path.dirname(cls.testfile)) - if testpath != '.': - os.removedirs(testpath) - for e in cls.origEnv: - if cls.origEnv[e] is None: - del os.environ[e] + self.origEnv[e] = os.environ[e] + os.environ[e] = self.testEnv[e] + + #- Cleanup test files and restore environment + def tearDown(self): + for e in self.origEnv: + if self.origEnv[e] is None: + os.environ.pop(e, None) else: - os.environ[e] = cls.origEnv[e] - if os.path.exists(cls.testDir): - rmtree(cls.testDir) + os.environ[e] = self.origEnv[e] + self._tempdir.cleanup() def test_simdir(self): x = io.simdir() diff --git a/py/desisim/test/test_obs.py b/py/desisim/test/test_obs.py index a17aa1e2..9c2c9613 100644 --- a/py/desisim/test/test_obs.py +++ b/py/desisim/test/test_obs.py @@ -1,6 +1,5 @@ import unittest, os -from uuid import uuid1 -from shutil import rmtree +import tempfile from importlib import resources import numpy as np @@ -15,45 +14,29 @@ desi_root_available = 'DESI_ROOT' in os.environ class TestObs(unittest.TestCase): - #- Create test subdirectory - @classmethod - def setUpClass(cls): - cls.night = '20150101' - cls.testfile = 'test-{uuid}/test-{uuid}.fits'.format(uuid=uuid1()) - cls.testDir = os.path.join(os.environ['HOME'],'desi_test_io') - cls.origEnv = dict(PIXPROD = None, DESI_SPECTRO_SIM = None) - cls.testEnv = dict( + #- Create isolated test directory and environment for each test method + def setUp(self): + self.night = '20150101' + self._tempdir = tempfile.TemporaryDirectory(prefix='desi_test_obs-') + self.testDir = self._tempdir.name + self.origEnv = dict(PIXPROD = None, DESI_SPECTRO_SIM = None) + self.testEnv = dict( PIXPROD = 'test', - DESI_SPECTRO_SIM = os.path.join(cls.testDir,'spectro','sim'), + DESI_SPECTRO_SIM = os.path.join(self.testDir,'spectro','sim'), ) - for e in cls.origEnv: + for e in self.origEnv: if e in os.environ: - cls.origEnv[e] = os.environ[e] - os.environ[e] = cls.testEnv[e] + self.origEnv[e] = os.environ[e] + os.environ[e] = self.testEnv[e] - #- Cleanup files but not directories after each test + #- Cleanup and restore environment after each test def tearDown(self): - for expid in range(5): - for filetype in ['simspec', 'simfibermap']: - filename = io.findfile('simspec', self.night, expid) - if os.path.exists(filename): - os.remove(filename) - - #- Cleanup test files if they exist - @classmethod - def tearDownClass(cls): - if os.path.exists(cls.testfile): - os.remove(cls.testfile) - testpath = os.path.normpath(os.path.dirname(cls.testfile)) - if testpath != '.': - os.removedirs(testpath) - for e in cls.origEnv: - if cls.origEnv[e] is None: - del os.environ[e] + for e in self.origEnv: + if self.origEnv[e] is None: + os.environ.pop(e, None) else: - os.environ[e] = cls.origEnv[e] - if os.path.exists(cls.testDir): - rmtree(cls.testDir) + os.environ[e] = self.origEnv[e] + self._tempdir.cleanup() # def new_exposure(program, nspec=5000, night=None, expid=None, tileid=None, \ # airmass=1.0, exptime=None): diff --git a/py/desisim/test/test_pixsim.py b/py/desisim/test/test_pixsim.py index 469ab647..c5fb8717 100755 --- a/py/desisim/test/test_pixsim.py +++ b/py/desisim/test/test_pixsim.py @@ -1,7 +1,5 @@ import unittest, os, sys import tempfile -from uuid import uuid1 -from shutil import rmtree import numpy as np from astropy.io import fits @@ -21,30 +19,9 @@ desi_root_available = 'DESI_ROOT' in os.environ class TestPixsim(unittest.TestCase): - #- Create test subdirectory + #- Class-wide constants that do not write output @classmethod def setUpClass(cls): - global desi_templates_available - cls.testfile = 'test-{uuid}/test-{uuid}.fits'.format(uuid=uuid1()) - cls.testdir = tempfile.mkdtemp() - cls.origEnv = dict( - PIXPROD = None, - SPECPROD = None, - DESI_SPECTRO_SIM = None, - DESI_SPECTRO_DATA = None, - DESI_SPECTRO_REDUX = None, - ) - cls.testEnv = dict( - PIXPROD = 'test', - SPECPROD = 'test', - DESI_SPECTRO_SIM = os.path.join(cls.testdir,'spectro','sim'), - DESI_SPECTRO_DATA = os.path.join(cls.testdir,'spectro','sim', 'test'), - DESI_SPECTRO_REDUX = os.path.join(cls.testdir,'spectro','redux'), - ) - for e in cls.origEnv: - if e in os.environ: - cls.origEnv[e] = os.environ[e] - os.environ[e] = cls.testEnv[e] if desi_templates_available: cls.cosmics = (os.environ['DESI_ROOT'] + '/spectro/templates/cosmics/v0.3/cosmics-bias-r.fits') @@ -54,23 +31,29 @@ def setUpClass(cls): #- to save memory while testing cls.ccdshape = (2000,2000) - #- Cleanup test files if they exist - @classmethod - def tearDownClass(cls): - if os.path.exists(cls.testfile): - os.remove(cls.testfile) - testpath = os.path.normpath(os.path.dirname(cls.testfile)) - if testpath != '.': - os.removedirs(testpath) - for e in cls.origEnv: - if cls.origEnv[e] is None: - del os.environ[e] - else: - os.environ[e] = cls.origEnv[e] - if os.path.exists(cls.testdir): - rmtree(cls.testdir) - def setUp(self): + self._tempdir = tempfile.TemporaryDirectory(prefix='desi_test_pixsim-') + self.testdir = self._tempdir.name + pixprod = 'test-{}'.format(os.path.basename(self.testdir)) + self.origEnv = dict( + PIXPROD = None, + SPECPROD = None, + DESI_SPECTRO_SIM = None, + DESI_SPECTRO_DATA = None, + DESI_SPECTRO_REDUX = None, + ) + self.testEnv = dict( + PIXPROD = pixprod, + SPECPROD = pixprod, + DESI_SPECTRO_SIM = os.path.join(self.testdir,'spectro','sim'), + DESI_SPECTRO_DATA = os.path.join(self.testdir,'spectro','sim', pixprod), + DESI_SPECTRO_REDUX = os.path.join(self.testdir,'spectro','redux'), + ) + for e in self.origEnv: + if e in os.environ: + self.origEnv[e] = os.environ[e] + os.environ[e] = self.testEnv[e] + self.night = '20150105' self.expid = 124 for expid in (self.expid, self.expid+1): @@ -80,22 +63,12 @@ def setUp(self): os.makedirs(pixdir) def tearDown(self): - rawfile = desispec.io.findfile('raw', self.night, self.expid) - if os.path.exists(rawfile): - os.remove(rawfile) - fibermap = desispec.io.findfile('fibermap', self.night, self.expid) - if os.path.exists(fibermap): - os.remove(fibermap) - simspecfile = io.findfile('simspec', self.night, self.expid) - if os.path.exists(simspecfile): - os.remove(simspecfile) - simpixfile = io.findfile('simpix', self.night, self.expid) - if os.path.exists(simpixfile): - os.remove(simpixfile) - for camera in ('b0', 'r0', 'z0'): - pixfile = desispec.io.findfile('preproc', self.night, self.expid, camera=camera) - if os.path.exists(pixfile): - os.remove(pixfile) + for e in self.origEnv: + if self.origEnv[e] is None: + os.environ.pop(e, None) + else: + os.environ[e] = self.origEnv[e] + self._tempdir.cleanup() @unittest.skipUnless(desi_root_available, '$DESI_ROOT not set') diff --git a/py/desisim/test/test_quickcat.py b/py/desisim/test/test_quickcat.py index 629ae7f4..4f669c12 100644 --- a/py/desisim/test/test_quickcat.py +++ b/py/desisim/test/test_quickcat.py @@ -1,4 +1,5 @@ import os +import tempfile import numpy as np import unittest from astropy.table import Table, Column @@ -11,12 +12,15 @@ class TestQuickCat(unittest.TestCase): @classmethod def setUpClass(cls): + cls.origdir = os.getcwd() + cls._tempdir = tempfile.TemporaryDirectory(prefix='desi_test_quickcat-') + cls.testdir = cls._tempdir.name np.random.seed(50) cls.ntiles = 4 tiles = desimodel.io.load_tiles() cls.tileids = tiles['TILEID'][0:cls.ntiles] - cls.tilefiles = ['tile-{:05d}.fits'.format(i) for i in cls.tileids] - cls.tilefiles_multiobs = ['multitile-{:05d}.fits'.format(i) for i in cls.tileids] + cls.tilefiles = [os.path.join(cls.testdir, 'tile-{:05d}.fits'.format(i)) for i in cls.tileids] + cls.tilefiles_multiobs = [os.path.join(cls.testdir, 'multitile-{:05d}.fits'.format(i)) for i in cls.tileids] cls.nspec = n = 5000 targets = Table() @@ -112,9 +116,8 @@ def setUpClass(cls): #- Cleanup test files if they exist @classmethod def tearDownClass(cls): - for filename in cls.tilefiles + cls.tilefiles_multiobs: - if os.path.exists(filename): - os.remove(filename) + os.chdir(cls.origdir) + cls._tempdir.cleanup() def test_quickcat(self): #- First round of obs: perfect input z -> output z diff --git a/py/desisim/test/test_quickgen.py b/py/desisim/test/test_quickgen.py index e15a7ef7..fe66caaf 100644 --- a/py/desisim/test/test_quickgen.py +++ b/py/desisim/test/test_quickgen.py @@ -3,7 +3,7 @@ """ import unittest, os, sys from uuid import uuid1 -from shutil import rmtree +import tempfile import subprocess from astropy.io import fits @@ -55,31 +55,9 @@ def check_env(): log.critical("missing env vars; exiting without running pipeline") sys.exit(1) - #- Create test subdirectory + #- Locate quickgen script once for this test class @classmethod def setUpClass(cls): - global desi_templates_available - cls.testfile = 'test-{uuid}/test-{uuid}.fits'.format(uuid=uuid1()) - cls.testDir = os.path.join(os.environ['HOME'],'desi_test_io') - - cls.origEnv = dict( - PIXPROD = None, - SPECPROD = None, - DESI_SPECTRO_SIM = None, - DESI_SPECTRO_DATA = None, - DESI_SPECTRO_REDUX = None, - ) - cls.testEnv = dict( - PIXPROD = 'test-quickgen', - SPECPROD = 'test-quickgen', - DESI_SPECTRO_SIM = os.path.join(cls.testDir,'spectro','sim'), - DESI_SPECTRO_DATA = os.path.join(cls.testDir,'spectro','sim', 'test-quickgen'), - DESI_SPECTRO_REDUX = os.path.join(cls.testDir,'spectro','redux'), - ) - for e in cls.origEnv: - if e in os.environ: - cls.origEnv[e] = os.environ[e] - os.environ[e] = cls.testEnv[e] if desi_templates_available: cls.cosmics = (os.environ['DESI_ROOT'] + '/spectro/templates/cosmics/v0.3/cosmics-bias-r.fits') @@ -109,46 +87,39 @@ def setUpClass(cls): raise RuntimeError('Unable to auto-locate desisim/bin/quickgen from {}'.format(__file__)) def setUp(self): + self._tempdir = tempfile.TemporaryDirectory(prefix='desi_test_quickgen-') + self.testDir = self._tempdir.name + pixprod = 'test-quickgen-{}'.format(uuid1().hex) + + self.origEnv = dict( + PIXPROD = None, + SPECPROD = None, + DESI_SPECTRO_SIM = None, + DESI_SPECTRO_DATA = None, + DESI_SPECTRO_REDUX = None, + ) + self.testEnv = dict( + PIXPROD = pixprod, + SPECPROD = pixprod, + DESI_SPECTRO_SIM = os.path.join(self.testDir, 'spectro', 'sim'), + DESI_SPECTRO_DATA = os.path.join(self.testDir, 'spectro', 'sim', pixprod), + DESI_SPECTRO_REDUX = os.path.join(self.testDir, 'spectro', 'redux'), + ) + for e in self.origEnv: + if e in os.environ: + self.origEnv[e] = os.environ[e] + os.environ[e] = self.testEnv[e] + self.night = '20150105' self.expid = 124 - #- Cleanup test files if they exist - @classmethod - def tearDownClass(cls): - if os.path.exists(cls.testfile): - os.remove(cls.testfile) - testpath = os.path.normpath(os.path.dirname(cls.testfile)) - if testpath != '.': - os.removedirs(testpath) - for e in cls.origEnv: - if cls.origEnv[e] is None: - del os.environ[e] - else: - os.environ[e] = cls.origEnv[e] - if os.path.exists(cls.testDir): - rmtree(cls.testDir) - def tearDown(self): - for expid in [self.expid, 100, 101, 102]: - fibermap = io.findfile('simfibermap', self.night, expid) - if os.path.exists(fibermap): - os.remove(fibermap) - simspecfile = io.findfile('simspec', self.night, expid) - if os.path.exists(simspecfile): - os.remove(simspecfile) - for camera in ('b0', 'r0', 'z0'): - framefile = desispec.io.findfile('frame', self.night, expid, camera=camera) - if os.path.exists(framefile): - os.remove(framefile) - cframefile = desispec.io.findfile('cframe', self.night, expid, camera=camera) - if os.path.exists(cframefile): - os.remove(cframefile) - skyfile = desispec.io.findfile('sky', self.night, expid, camera=camera) - if os.path.exists(skyfile): - os.remove(skyfile) - fluxcalibfile = desispec.io.findfile('calib', self.night, expid, camera=camera) - if os.path.exists(fluxcalibfile): - os.remove(fluxcalibfile) + for e in self.origEnv: + if self.origEnv[e] is None: + os.environ.pop(e, None) + else: + os.environ[e] = self.origEnv[e] + self._tempdir.cleanup() def test_parse(self): night = self.night