diff --git a/README.md b/README.md index 3a4020f..0c082d8 100644 --- a/README.md +++ b/README.md @@ -38,6 +38,7 @@ Command line interface for the LCMAP system | lcmap product | --grid --tile --names --years | generate product data for tile | | lcmap product-chip | --grid --cx --cy --names --years | generate product data for chip | | lcmap raster | --grid --tile --names --years | generate tile size map tiff | +| lcmap bundle | --grid --tile --years | generate bundle and store it | ### Parameters @@ -85,8 +86,10 @@ lcmap-cli requires n config file at ~/.usgs/lcmap-cli.edn. :prediction "/prediction" :product "/product" :raster "/raster" + :bundle "/bundle" :segment-instance-count 25 :prediction-instance-count 1 + :bundle-instance-count 1 :prediction-sleep-for 1000 :product-instance-count 1 :raster-instance-count 1 @@ -108,8 +111,10 @@ lcmap-cli requires n config file at ~/.usgs/lcmap-cli.edn. :prediction "/prediction" :product "/product" :raster "/raster" + :bundle "/bundle" :segment-instance-count 1 :prediction-instance-count 1 + :bundle-instance-count 1 :prediction-sleep-for 1000 :product-instance-count 1 :raster-instance-count 1 @@ -131,8 +136,10 @@ lcmap-cli requires n config file at ~/.usgs/lcmap-cli.edn. :prediction "/prediction" :product "/product" :raster "/raster" + :bundle "/bundle" :segment-instance-count 1 :prediction-instance-count 1 + :bundle-instance-count 1 :prediction-sleep-for 1000 :product-instance-count 1 :raster-instance-count 1 @@ -291,6 +298,19 @@ lcmap-cli requires n config file at ~/.usgs/lcmap-cli.edn. # example stderr output: # {"error":"problem processing /maps request: 'helpful_error_message'", "date":"2002-07-01", "tile":"027008", "tx":"111111", "ty":"222222", "product":"change"} - + + + # Creating Tar Files + $ lcmap bundle --grid conus \ + --tile 028008 \ + --years 2001/2017 \ + >> 2001_2017_bundle_success.txt 2>> 2001_2017_bundle_errors.txt + + # example stdout output: + # {"tile":"028008","date":"2017-07-01","grid":"conus","ty":2114805.0,"tx":1634415.0,"resource":"bundle","status":"success","tar":"LCMAP_CU_028008_2017_20190718_CCDC.tar"} + + # example stderr output: + # {"resource":"bundle","http-options":{"timeout":7200000},"grid":"conus","error":{"inputs":{"tile":"028008","date":"2017-07-01","grid":"conus","ty":2114805.0,"tx":1634415.0,"resource":"bundle"}, + # "message":"problem creating data","details":"problem generating tile bundle for tile: 028008 date: 2017-07-01, ...} ``` diff --git a/src/lcmap_cli/config.clj b/src/lcmap_cli/config.clj index 86e264f..0095c34 100644 --- a/src/lcmap_cli/config.clj +++ b/src/lcmap_cli/config.clj @@ -37,6 +37,10 @@ [grid] (:raster-instance-count ((keyword grid) grids))) +(defn bundle-instance-count + [grid] + (:bundle-instance-count ((keyword grid) grids))) + (defn product-mmdd [grid] (let [mm (:product-month ((keyword grid) grids)) diff --git a/src/lcmap_cli/core.clj b/src/lcmap_cli/core.clj index 70f70a2..d1379e8 100644 --- a/src/lcmap_cli/core.clj +++ b/src/lcmap_cli/core.clj @@ -71,6 +71,8 @@ :args (->options [:help :grid :names :cx :cy :years])} :raster {:func #'lcmap-cli.products/raster :args (->options [:help :grid :tile :names :years])} + :bundle {:func #'lcmap-cli.products/bundle + :args (->options [:help :grid :tile :years])} }) (defn usage [action options-summary] diff --git a/src/lcmap_cli/products.clj b/src/lcmap_cli/products.clj index 0a9f25e..bc770d0 100644 --- a/src/lcmap_cli/products.clj +++ b/src/lcmap_cli/products.clj @@ -93,7 +93,7 @@ in-chan (async/chan) out-chan (async/chan) chip_xys (chips (assoc all :dataset "ard")) - {tilex :x tiley :y} (tile-to-xy (assoc all :dataset "ard")) + {tx :x ty :y} (tile-to-xy (assoc all :dataset "ard")) dates (date-range all) products (string/split names #",") products-dates (combo/cartesian-product products dates) @@ -104,8 +104,8 @@ (doseq [pd products-dates] (async/>! in-chan (hash-map :grid grid :tile tile - :tx tilex - :ty tiley + :tx tx + :ty ty :chips chip_xys :date (last pd) :product (first pd) @@ -113,3 +113,23 @@ (doall (map output_fn products-dates)))) +(defn bundle + [{grid :grid tile :tile years :years :as all}] + (let [chunk-size (cfg/bundle-instance-count grid) + in-chan (async/chan) + out-chan (async/chan) + dates (date-range all) + {tx :x ty :y} (tile-to-xy (assoc all :dataset "ard")) + consumers (start-consumers chunk-size in-chan out-chan {:timeout 7200000}) + output_fn (fn [i] (let [result (async/! in-chan (hash-map :grid grid + :tile tile + :tx tx + :ty ty + :date date + :resource "bundle")))) + + (doall (map output_fn dates)))) diff --git a/test/lcmap_cli/products_test.clj b/test/lcmap_cli/products_test.clj index d2c6988..c7d5bb6 100644 --- a/test/lcmap_cli/products_test.clj +++ b/test/lcmap_cli/products_test.clj @@ -87,4 +87,12 @@ (is (= '({:tile "027008", :date "2006--", :grid "conus", :ty 4, :tx 3, :product "tsc", :resource "raster", :http-options {:timeout 7200000}}) (products/raster {:grid "conus" :tile "027008" :names "tsc" :years "2006"}))))) +(deftest bundle-test + (with-redefs [cfg/bundle-instance-count (fn [i] 1) + functions/tile-to-xy (fn [i] {:x 3 :y 4}) + products/handler (fn [a b] b) + products/post-request keys + cfg/http-options {:timeout 9}] + (is (= '({:tile "027008", :date "2006--", :grid "conus", :ty 4, :tx 3, :resource "bundle", :http-options {:timeout 7200000}}) + (products/bundle {:grid "conus" :tile "027008" :tar "foo.tar" :years "2006"})))))