execute amcheck before doing backups

definitions/checks/candlepin/db_index.rb

@@ -0,0 +1,25 @@
+module Checks
+  module Candlepin
+    class DBIndex < ForemanMaintain::Check
+      metadata do
+        description 'Make sure Candlepin DB indexes are OK'
+        label :candlepin_db_index
+        tags :db_index
+        for_feature :candlepin_database
+        confine do
+          feature(:candlepin_database)&.local?
+        end
+      end
+
+      def run
+        status, output = feature(:candlepin_database).amcheck
+
+        if !status.nil?
+          assert(status == 0, "Candlepin DB indexes have issues:\n#{output}")
+        else
+          skip 'amcheck is not available in this setup'
+        end
+      end
+    end
+  end
+end

definitions/checks/foreman/db_index.rb

@@ -0,0 +1,25 @@
+module Checks
+  module Foreman
+    class DBIndex < ForemanMaintain::Check
+      metadata do
+        description 'Make sure Foreman DB indexes are OK'
+        label :foreman_db_index
+        tags :db_index
+        for_feature :foreman_database
+        confine do
+          feature(:foreman_database)&.local?
+        end
+      end
+
+      def run
+        status, output = feature(:foreman_database).amcheck
+
+        if !status.nil?
+          assert(status == 0, "Foreman DB indexes have issues:\n#{output}")
+        else
+          skip 'amcheck is not available in this setup'
+        end
+      end
+    end
+  end
+end

definitions/checks/pulpcore/db_index.rb

@@ -0,0 +1,25 @@
+module Checks
+  module Pulpcore
+    class DBIndex < ForemanMaintain::Check
+      metadata do
+        description 'Make sure Pulpcore DB indexes are OK'
+        label :pulpcore_db_index
+        tags :db_index
+        for_feature :pulpcore_database
+        confine do
+          feature(:pulpcore_database)&.local?
+        end
+      end
+
+      def run
+        status, output = feature(:pulpcore_database).amcheck
+
+        if !status.nil?
+          assert(status == 0, "Pulpcore DB indexes have issues:\n#{output}")
+        else
+          skip 'amcheck is not available in this setup'
+        end
+      end
+    end
+  end
+end

definitions/scenarios/backup.rb

@@ -18,6 +18,9 @@ class Backup < ForemanMaintain::Scenario
 
     def compose
       check_valid_strategy
+      add_step(Checks::Foreman::DBIndex)
+      add_step(Checks::Candlepin::DBIndex)
+      add_step(Checks::Pulpcore::DBIndex)
       add_step(Checks::ForemanTasks::NotRunning.new(:wait_for_tasks => wait_for_tasks?))
       add_step(Checks::Pulpcore::NoRunningTasks.new(:wait_for_tasks => wait_for_tasks?))
       add_step_with_context(Procedures::Backup::AccessibilityConfirmation) if strategy == :offline

lib/foreman_maintain/concerns/base_database.rb

@@ -140,6 +140,40 @@ def raise_psql_missing_error
                 ' Make sure system has psql utility installed.'
       end
 
+      def amcheck
+        # executing the check requires superuser privileges
+        return unless local?
+
+        return unless amcheck_installed?
+
+        psqlcmd = "runuser - postgres -c 'psql --set=ON_ERROR_STOP=on #{configuration['database']}'"
+
+        amcheck_query = <<~SQL
+          SELECT bt_index_check(index => c.oid, heapallindexed => i.indisunique),
+               c.relname,
+               c.relpages
+          FROM pg_index i
+          JOIN pg_opclass op ON i.indclass[0] = op.oid
+          JOIN pg_am am ON op.opcmethod = am.oid
+          JOIN pg_class c ON i.indexrelid = c.oid
+          JOIN pg_namespace n ON c.relnamespace = n.oid
+          WHERE am.amname = 'btree' AND n.nspname = 'public'
+          -- Don't check temp tables, which may be from another session:
+          AND c.relpersistence != 't'
+          -- Function may throw an error when this is omitted:
+          AND c.relkind = 'i' AND i.indisready AND i.indisvalid
+          ORDER BY c.relpages DESC;
+        SQL
+
+        execute_with_status(psqlcmd, :stdin => amcheck_query)
+      end
+
+      def amcheck_installed?
+        sql = "select 'amcheck_installed' from pg_extension where extname='amcheck'"
+        result = query(sql)
+        !result.nil? && !result.empty?
+      end
+
       private
 
       def base_env

test/definitions/scenarios/backup_test.rb

@@ -29,7 +29,11 @@ module Scenarios
     end
 
     let(:checks) do
-      task_checks
+      task_checks + [
+        Checks::Foreman::DBIndex,
+        Checks::Candlepin::DBIndex,
+        Checks::Pulpcore::DBIndex,
+      ]
     end
 
     describe 'offline' do