Re-generate test data, and update expectations. (#467)

Author: delucchi-cmu

tests/conftest.py

@@ -116,6 +116,7 @@ def margin_catalog_info(margin_cache_catalog_info_data) -> TableProperties:
 def small_sky_schema() -> pa.Schema:
     return pa.schema(
         [
+            pa.field("_healpix_29", pa.int64()),
             pa.field("id", pa.int64()),
             pa.field("ra", pa.float64()),
             pa.field("dec", pa.float64()),
@@ -124,7 +125,6 @@ def small_sky_schema() -> pa.Schema:
             pa.field("Norder", pa.uint8()),
             pa.field("Dir", pa.uint64()),
             pa.field("Npix", pa.uint64()),
-            pa.field("_healpix_29", pa.int64()),
         ]
     )
 
@@ -133,6 +133,7 @@ def small_sky_schema() -> pa.Schema:
 def small_sky_source_schema() -> pa.Schema:
     return pa.schema(
         [
+            pa.field("_healpix_29", pa.int64()),
             pa.field("source_id", pa.int64()),
             pa.field("source_ra", pa.float64()),
             pa.field("source_dec", pa.float64()),
@@ -145,7 +146,6 @@ def small_sky_source_schema() -> pa.Schema:
             pa.field("Norder", pa.uint8()),
             pa.field("Dir", pa.uint64()),
             pa.field("Npix", pa.uint64()),
-            pa.field("_healpix_29", pa.int64()),
         ]
     )
 
@@ -166,18 +166,18 @@ def association_catalog_schema() -> pa.Schema:
 def margin_catalog_schema() -> pa.Schema:
     return pa.schema(
         [
+            pa.field("_healpix_29", pa.int64()),
             pa.field("id", pa.int64()),
             pa.field("ra", pa.float64()),
             pa.field("dec", pa.float64()),
             pa.field("ra_error", pa.int64()),
             pa.field("dec_error", pa.int64()),
-            pa.field("Norder", pa.uint8()),
-            pa.field("Dir", pa.uint64()),
-            pa.field("Npix", pa.uint64()),
-            pa.field("_healpix_29", pa.int64()),
             pa.field("margin_Norder", pa.uint8()),
             pa.field("margin_Dir", pa.uint64()),
             pa.field("margin_Npix", pa.uint64()),
+            pa.field("Norder", pa.uint8()),
+            pa.field("Dir", pa.uint64()),
+            pa.field("Npix", pa.uint64()),
         ]
     )
 
@@ -196,6 +196,8 @@ def catalog_path(test_data_dir) -> str:
 def margin_catalog_pixels() -> list[HealpixPixel]:
     return [
         HealpixPixel(0, 4),
+        HealpixPixel(0, 7),
+        HealpixPixel(0, 8),
         HealpixPixel(1, 44),
         HealpixPixel(1, 45),
         HealpixPixel(1, 46),

tests/data/generate_data.ipynb

@@ -32,6 +32,7 @@
     "import tempfile\n",
     "from pathlib import Path\n",
     "from dask.distributed import Client\n",
+    "from hats.io.file_io import remove_directory\n",
     "\n",
     "tmp_path = tempfile.TemporaryDirectory()\n",
     "tmp_dir = tmp_path.name\n",
@@ -61,6 +62,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
+    "remove_directory(\"./small_sky\")\n",
     "with tempfile.TemporaryDirectory() as pipeline_tmp:\n",
     "    args = ImportArguments(\n",
     "        input_path=Path(hats_import_dir) / \"small_sky\",\n",
@@ -97,6 +99,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
+    "remove_directory(\"./small_sky_order1\")\n",
     "with tempfile.TemporaryDirectory() as pipeline_tmp:\n",
     "    args = ImportArguments(\n",
     "        input_path=Path(hats_import_dir) / \"small_sky\",\n",
@@ -126,6 +129,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
+    "remove_directory(\"./small_sky_order1_id_index\")\n",
     "with tempfile.TemporaryDirectory() as pipeline_tmp:\n",
     "    args = IndexArguments(\n",
     "        input_catalog_path=\"small_sky\",\n",
@@ -165,6 +169,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
+    "remove_directory(\"./small_sky_order1_margin\")\n",
     "with tempfile.TemporaryDirectory() as pipeline_tmp:\n",
     "    args = MarginCacheArguments(\n",
     "        margin_threshold=7200,\n",
@@ -241,6 +246,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
+    "remove_directory(\"./small_sky_source\")\n",
     "with tempfile.TemporaryDirectory() as pipeline_tmp:\n",
     "    args = ImportArguments(\n",
     "        input_path=Path(hats_import_dir) / \"small_sky_source\",\n",
@@ -284,6 +290,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
+    "remove_directory(\"./small_sky_source_object_index\")\n",
     "with tempfile.TemporaryDirectory() as pipeline_tmp:\n",
     "    args = IndexArguments(\n",
     "        input_catalog_path=\"small_sky_source\",\n",
@@ -321,6 +328,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
+    "remove_directory(\"./square_map\")\n",
     "with tempfile.TemporaryDirectory() as pipeline_tmp:\n",
     "    csv_file = Path(pipeline_tmp) / \"square_map.csv\"\n",
     "    value_frame.to_csv(csv_file, index=False)\n",
@@ -360,7 +368,7 @@
  ],
  "metadata": {
   "kernelspec": {
-   "display_name": "demo",
+   "display_name": "hats",
    "language": "python",
    "name": "python3"
   },

tests/data/small_sky/dataset/Norder=0/Dir=0/Npix=11.parquet

@@ -7,8 +7,8 @@ hats_col_dec=dec
 hats_max_rows=1000000
 hats_order=0
 moc_sky_fraction=0.08333
-hats_builder=hats-import v0.3.6.dev26+g40366b4
-hats_creation_date=2024-10-11T18\:18UTC
-hats_estsize=49177
+hats_builder=hats-import v0.4.6.dev1+gf00cd7a
+hats_creation_date=2025-03-05T16\:16UTC
+hats_estsize=98318
 hats_release_date=2024-09-18
 hats_version=v0.1

tests/data/small_sky/dataset/_common_metadata

@@ -7,8 +7,8 @@ hats_col_dec=dec
 hats_max_rows=1000000
 hats_order=1
 moc_sky_fraction=0.08333
-hats_builder=hats-import v0.3.6.dev26+g40366b4
-hats_creation_date=2024-10-11T18\:18UTC
-hats_estsize=48
+hats_builder=hats-import v0.4.6.dev1+gf00cd7a
+hats_creation_date=2025-03-05T16\:16UTC
+hats_estsize=27
 hats_release_date=2024-09-18
 hats_version=v0.1

tests/data/small_sky/dataset/_metadata

@@ -4,8 +4,8 @@ dataproduct_type=index
 hats_nrows=131
 hats_primary_table_url=small_sky
 hats_index_column=id
-hats_builder=hats-import v0.3.6.dev26+g40366b4
-hats_creation_date=2024-10-11T18\:18UTC
-hats_estsize=8
+hats_builder=hats-import v0.4.6.dev1+gf00cd7a
+hats_creation_date=2025-03-05T16\:16UTC
+hats_estsize=3
 hats_release_date=2024-09-18
 hats_version=v0.1

tests/data/small_sky/point_map.fits

@@ -1,5 +1,7 @@
 Norder,Npix
 0,4
+0,7
+0,8
 1,44
 1,45
 1,46

tests/data/small_sky/properties

@@ -1,15 +1,15 @@
 #HATS catalog
 obs_collection=small_sky_order1_margin
 dataproduct_type=margin
-hats_nrows=28
+hats_nrows=47
 hats_col_ra=ra
 hats_col_dec=dec
 hats_primary_table_url=small_sky_order1
 hats_margin_threshold=7200.0
 hats_order=1
-moc_sky_fraction=0.16667
-hats_builder=hats-import v0.3.6.dev26+g40366b4
-hats_creation_date=2024-10-11T18\:18UTC
-hats_estsize=58
+moc_sky_fraction=0.33333
+hats_builder=hats-import v0.4.6.dev1+gf00cd7a
+hats_creation_date=2025-03-05T16\:16UTC
+hats_estsize=36
 hats_release_date=2024-09-18
 hats_version=v0.1

tests/data/small_sky_order1/dataset/Norder=1/Dir=0/Npix=44.parquet

@@ -7,8 +7,8 @@ hats_col_dec=source_dec
 hats_max_rows=3000
 hats_order=2
 moc_sky_fraction=0.16667
-hats_builder=hats-import v0.3.6.dev26+g40366b4
-hats_creation_date=2024-10-11T18\:19UTC
-hats_estsize=50207
+hats_builder=hats-import v0.4.6.dev1+gf00cd7a
+hats_creation_date=2025-03-05T16\:16UTC
+hats_estsize=99275
 hats_release_date=2024-09-18
 hats_version=v0.1

tests/data/small_sky_order1/dataset/Norder=1/Dir=0/Npix=45.parquet

@@ -4,8 +4,8 @@ dataproduct_type=index
 hats_nrows=148
 hats_primary_table_url=small_sky_source
 hats_index_column=object_id
-hats_builder=hats-import v0.3.6.dev26+g40366b4
-hats_creation_date=2024-10-11T18\:19UTC
-hats_estsize=9
+hats_builder=hats-import v0.4.6.dev1+gf00cd7a
+hats_creation_date=2025-03-05T16\:16UTC
+hats_estsize=3
 hats_release_date=2024-09-18
 hats_version=v0.1

tests/data/small_sky_order1/dataset/Norder=1/Dir=0/Npix=46.parquet

@@ -5,8 +5,8 @@ hats_nrows=12
 hats_max_rows=1000000
 hats_order=1
 moc_sky_fraction=0.25000
-hats_builder=hats-import v0.4.1
-hats_creation_date=2024-11-20T19\:32UTC
-hats_estsize=52
+hats_builder=hats-import v0.4.6.dev1+gf00cd7a
+hats_creation_date=2025-03-05T16\:16UTC
+hats_estsize=34
 hats_release_date=2024-09-18
 hats_version=v0.1

tests/data/small_sky_order1/dataset/Norder=1/Dir=0/Npix=47.parquet

@@ -76,6 +76,7 @@ def test_margin_filter(margin_catalog_info, margin_catalog_pixels):
     pixels = [HealpixPixel(1, 44)]
     filtered_catalog = catalog.filter_from_pixel_list(pixels)
     assert filtered_catalog.get_healpix_pixels() == [
+        HealpixPixel(0, 8),
         HealpixPixel(1, 44),
         HealpixPixel(1, 45),
         HealpixPixel(1, 46),