add bundle producing functions

README.md

@@ -38,6 +38,7 @@ Command line interface for the LCMAP system
 | lcmap product               | --grid --tile --names --years                       | generate product data for tile |
 | lcmap product-chip          | --grid --cx --cy --names --years                    | generate product data for chip |
 | lcmap raster                | --grid --tile --names --years                       | generate tile size map tiff    |
+| lcmap bundle                | --grid --tile --years                               | generate bundle and store it   |
 
 ### Parameters
 
@@ -85,8 +86,10 @@ lcmap-cli requires n config file at ~/.usgs/lcmap-cli.edn.
                   :prediction "/prediction"
                   :product "/product"
                   :raster "/raster"
+                  :bundle "/bundle"
                   :segment-instance-count 25
                   :prediction-instance-count 1
+                  :bundle-instance-count 1
                   :prediction-sleep-for 1000
                   :product-instance-count 1
                   :raster-instance-count 1
@@ -108,8 +111,10 @@ lcmap-cli requires n config file at ~/.usgs/lcmap-cli.edn.
                   :prediction "/prediction"
                   :product "/product"
                   :raster "/raster"
+                  :bundle "/bundle"
                   :segment-instance-count 1
                   :prediction-instance-count 1
+                  :bundle-instance-count 1
                   :prediction-sleep-for 1000
                   :product-instance-count 1
                   :raster-instance-count 1
@@ -131,8 +136,10 @@ lcmap-cli requires n config file at ~/.usgs/lcmap-cli.edn.
                   :prediction "/prediction"
                   :product "/product"
                   :raster "/raster"
+                  :bundle "/bundle"
                   :segment-instance-count 1
                   :prediction-instance-count 1
+                  :bundle-instance-count 1
                   :prediction-sleep-for 1000
                   :product-instance-count 1
                   :raster-instance-count 1
@@ -291,6 +298,19 @@ lcmap-cli requires n config file at ~/.usgs/lcmap-cli.edn.
 
    # example stderr output:
    # {"error":"problem processing /maps request: 'helpful_error_message'", "date":"2002-07-01", "tile":"027008", "tx":"111111", "ty":"222222", "product":"change"}
-
+
+
+   # Creating Tar Files
+   $ lcmap bundle --grid conus \
+                  --tile 028008 \
+                  --years 2001/2017 \
+                  >> 2001_2017_bundle_success.txt 2>> 2001_2017_bundle_errors.txt
+
+   # example stdout output:
+   # {"tile":"028008","date":"2017-07-01","grid":"conus","ty":2114805.0,"tx":1634415.0,"resource":"bundle","status":"success","tar":"LCMAP_CU_028008_2017_20190718_CCDC.tar"}
+
+   # example stderr output:
+   # {"resource":"bundle","http-options":{"timeout":7200000},"grid":"conus","error":{"inputs":{"tile":"028008","date":"2017-07-01","grid":"conus","ty":2114805.0,"tx":1634415.0,"resource":"bundle"},
+   #  "message":"problem creating data","details":"problem generating tile bundle for tile: 028008 date: 2017-07-01, ...}
 
 ```

src/lcmap_cli/config.clj

@@ -37,6 +37,10 @@
   [grid]
   (:raster-instance-count ((keyword grid) grids)))
 
+(defn bundle-instance-count
+  [grid]
+  (:bundle-instance-count ((keyword grid) grids)))
+
 (defn product-mmdd
   [grid]
   (let [mm (:product-month ((keyword grid) grids))

src/lcmap_cli/core.clj

@@ -71,6 +71,8 @@
                   :args (->options [:help :grid :names :cx :cy :years])}
    :raster       {:func #'lcmap-cli.products/raster 
                   :args (->options [:help :grid :tile :names :years])}
+   :bundle       {:func #'lcmap-cli.products/bundle
+                  :args (->options [:help :grid :tile :years])}
 })
 
 (defn usage [action options-summary]

src/lcmap_cli/products.clj

@@ -93,7 +93,7 @@
         in-chan    (async/chan)
         out-chan   (async/chan)
         chip_xys   (chips (assoc all :dataset "ard"))
-        {tilex :x tiley :y} (tile-to-xy (assoc all :dataset "ard"))
+        {tx :x ty :y} (tile-to-xy (assoc all :dataset "ard"))
         dates      (date-range all)
         products   (string/split names #",")
         products-dates (combo/cartesian-product products dates)
@@ -104,12 +104,32 @@
      (doseq [pd products-dates]
        (async/>! in-chan (hash-map :grid grid
                                    :tile tile
-                                   :tx tilex
-                                   :ty tiley
+                                   :tx tx
+                                   :ty ty
                                    :chips chip_xys
                                    :date (last pd)
                                    :product (first pd)
                                    :resource "raster"))))
 
     (doall (map output_fn products-dates))))
 
+(defn bundle
+  [{grid :grid tile :tile years :years :as all}]
+  (let [chunk-size (cfg/bundle-instance-count grid)
+        in-chan    (async/chan)
+        out-chan   (async/chan)
+        dates      (date-range all)
+        {tx :x ty :y} (tile-to-xy (assoc all :dataset "ard"))
+        consumers  (start-consumers chunk-size in-chan out-chan {:timeout 7200000})
+        output_fn  (fn [i] (let [result (async/<!! out-chan)] (f/output result) result))]
+
+    (async/go
+     (doseq [date dates]
+       (async/>! in-chan (hash-map :grid grid
+                                   :tile tile
+                                   :tx tx
+                                   :ty ty
+                                   :date date
+                                   :resource "bundle"))))
+
+    (doall (map output_fn dates))))

test/lcmap_cli/products_test.clj

@@ -87,4 +87,12 @@
     (is (= '({:tile "027008", :date "2006--", :grid "conus", :ty 4, :tx 3, :product "tsc", :resource "raster", :http-options {:timeout 7200000}})
        (products/raster {:grid "conus" :tile "027008" :names "tsc" :years "2006"})))))
 
+(deftest bundle-test
+  (with-redefs [cfg/bundle-instance-count (fn [i] 1)
+                functions/tile-to-xy  (fn [i] {:x 3 :y 4})
+                products/handler      (fn [a b] b)
+                products/post-request keys
+                cfg/http-options      {:timeout 9}]
 
+    (is (= '({:tile "027008", :date "2006--", :grid "conus", :ty 4, :tx 3, :resource "bundle", :http-options {:timeout 7200000}})
+       (products/bundle {:grid "conus" :tile "027008" :tar "foo.tar" :years "2006"})))))