Implement audit logging

README.md

@@ -32,6 +32,11 @@ The following are optional configuration parameters
 
 `:http_read_timeout: ` : Timeout in seconds for waiting for a HTTP response (default 10)
 
+`:audit_log_dir:` : If set, write HTTP responses to a directory for audit purpose. The directory will look like:
+
+    ./blobs/fe/625f56522d2820aaed87232d405b7b561e44d4  # gzipped content (HTTP response body), the SHA1 of the uncompressed content is fe625f56522d2820aaed87232d405b7b561e44d4
+    ./requests.log  # request logs, one per line, for example "[2015-05-08 20:15:55 -0700] http://1.2.3.4/foo fe625f56522d2820aaed87232d405b7b561e44d4"
+
 `:cache_timeout: ` : Timeout in seconds for HTTP requests to a same path (default 10)
 
 `:cache_clean_interval: ` : Interval (in secs) to clean the cache (default 3600), set to 0 to disable cache cleaning 

lib/hiera/backend/http_backend.rb

@@ -11,6 +11,8 @@ def initialize
         @http.read_timeout = @config[:http_read_timeout] || 10
         @http.open_timeout = @config[:http_connect_timeout] || 10
 
+        @audit_log_dir = @config[:audit_log_dir]
+
         @cache = {}
         @cache_timeout = @config[:cache_timeout] || 10
         @cache_clean_interval = @config[:cache_clean_interval] || 3600
@@ -146,9 +148,33 @@ def http_get_and_parse(path)
           return
         end
 
-        parse_response httpres.body
+        body = httpres.body
+        begin
+          write_audit_log path, body if @audit_log_dir
+        rescue Exception => e
+          Hiera.warn("[hiera-http]: Failed to write audit log #{e.message}")
+        end
+        parse_response body
       end
 
+      def write_audit_log(http_path, body)
+        require 'digest'
+        require 'fileutils'
+        require 'zlib'
+
+        sha1 = Digest::SHA1.hexdigest(body)
+        blob_path = File.join(@audit_log_dir, 'blobs', sha1[0..1], sha1[2..-1])
+
+        if !File.exist?(blob_path)
+          FileUtils.mkdir_p File.dirname(blob_path)
+          File.open(blob_path, 'wb') do |f|
+            Zlib::GzipWriter::wrap(f) { |gz| gz.write(body); gz.close }
+          end
+        end
+        File.open(File.join(@audit_log_dir, 'requests.log'), 'a') do |f|
+          f.puts "[#{Time.now}] #{http_path} #{sha1}"
+        end
+      end
 
       def periodically_clean_cache(now)
         return if now < @clean_cache_at.to_i