Limit what we iterate over in providers

lib/puppet/provider/mysql_database/mysql.rb

@@ -6,24 +6,35 @@
 
   commands mysql_raw: 'mysql'
 
-  def self.instances
-    mysql_caller('show databases', 'regular').split("\n").map do |name|
+  def self.instances(managed_databases = nil)
+    db_list = if managed_databases && !managed_databases.empty?
+                managed_databases
+              else
+                mysql_caller('show databases', 'regular').split("\n")
+              end
+    db_list.map { |name|
       attributes = {}
-      mysql_caller(["show variables like '%_database'", name], 'regular').split("\n").each do |line|
-        k, v = line.split(%r{\s})
-        attributes[k] = v
+      begin
+        mysql_caller(["show variables like '%_database'", name], 'regular').split("\n").each do |line|
+          k, v = line.split(%r{\s})
+          attributes[k] = v
+        end
+      rescue Puppet::ExecutionFailure => e
+        raise unless e.message.include?('Unknown database')
+
+        next
       end
       new(name:,
           ensure: :present,
           charset: attributes['character_set_database'],
           collate: attributes['collation_database'])
-    end
+    }.compact
   end
 
   # We iterate over each mysql_database entry in the catalog and compare it against
   # the contents of the property_hash generated by self.instances
   def self.prefetch(resources)
-    databases = instances
+    databases = instances(resources.keys)
     resources.each_key do |database|
       provider = databases.find { |db| db.name == database }
       resources[database].provider = provider if provider

lib/puppet/provider/mysql_grant/mysql.rb

@@ -6,9 +6,10 @@
 
   commands mysql_raw: 'mysql'
 
-  def self.instances
+  def self.instances(managed_users = nil)
     instance_configs = {}
-    users.map do |user|
+    user_list = (managed_users && !managed_users.empty?) ? managed_users : users
+    user_list.map do |user|
       user_string = cmd_user(user)
       query = "SHOW GRANTS FOR #{user_string};"
       begin
@@ -120,7 +121,9 @@ def self.instances
   end
 
   def self.prefetch(resources)
-    users = instances
+    # Extract unique user@host values from grant resource names (format: user@host/table)
+    managed_users = resources.keys.map { |name| name.rpartition('/').first }.uniq
+    users = instances(managed_users)
     resources.each_key do |name|
       if provider = users.find { |user| user.name == name } # rubocop:disable Lint/AssignmentInCondition
         resources[name].provider = provider

lib/puppet/provider/mysql_user/mysql.rb

@@ -7,8 +7,13 @@
 
   # Build a property_hash containing all the discovered information about MySQL
   # users.
-  def self.instances
-    users = mysql_caller("SELECT CONCAT(User, '@',Host) AS User FROM mysql.user where HOST IS NOT NULL AND HOST != ''", 'regular').split("\n")
+  def self.instances(managed_users = nil)
+    if managed_users && !managed_users.empty?
+      user_list = managed_users.map { |u| "'" + u.gsub("'", "''") + "'" }.join(', ')
+      users = mysql_caller("SELECT CONCAT(User, '@',Host) AS User FROM mysql.user WHERE HOST IS NOT NULL AND HOST != '' AND CONCAT(User, '@', Host) IN (#{user_list})", 'regular').split("\n")
+    else
+      users = mysql_caller("SELECT CONCAT(User, '@',Host) AS User FROM mysql.user where HOST IS NOT NULL AND HOST != ''", 'regular').split("\n")
+    end
     # users = users_full.reject { |user| user == 'PUBLIC@' }
     # To reduce the number of calls to MySQL we collect all the properties in
     # one big swoop.
@@ -49,7 +54,7 @@ def self.instances
   # We iterate over each mysql_user entry in the catalog and compare it against
   # the contents of the property_hash generated by self.instances
   def self.prefetch(resources)
-    users = instances
+    users = instances(resources.keys)
     # rubocop:disable Lint/AssignmentInCondition
     resources.each_key do |name|
       if provider = users.find { |user| user.name == name }

spec/acceptance/00_mysql_server_spec.rb

@@ -83,4 +83,112 @@ class { 'mysql::server':
       idempotent_apply(pp)
     end
   end
+
+  describe 'prefetch with managed subset' do
+    let(:bootstrap_pp) do
+      <<-MANIFEST
+        class { 'mysql::server':
+          root_password   => 'test',
+          service_enabled => 'true',
+          service_manage  => 'true',
+        }
+      MANIFEST
+    end
+
+    let(:subset_pp) do
+      <<-MANIFEST
+        class { 'mysql::server':
+          root_password   => 'test',
+          service_enabled => 'true',
+          service_manage  => 'true',
+        }
+
+        mysql_database { 'managed_db':
+          ensure  => 'present',
+          charset => '#{charset}',
+          collate => '#{charset}_general_ci',
+        }
+
+        mysql_user { 'managed_user@localhost':
+          ensure => 'present',
+        }
+
+        mysql_grant { 'managed_user@localhost/managed_db.*':
+          ensure     => 'present',
+          privileges => ['SELECT'],
+          table      => 'managed_db.*',
+          user       => 'managed_user@localhost',
+          require    => [Mysql_user['managed_user@localhost'], Mysql_database['managed_db']],
+        }
+      MANIFEST
+    end
+
+    let(:fixtures_pp) do
+      <<-MANIFEST
+        class { 'mysql::server':
+          root_password   => 'test',
+          service_enabled => 'true',
+          service_manage  => 'true',
+        }
+
+        mysql_database { 'unmanaged_db':
+          ensure  => 'present',
+          charset => '#{charset}',
+          collate => '#{charset}_general_ci',
+        }
+
+        mysql_user { 'managed_user@localhost':
+          ensure => 'present',
+        }
+
+        mysql_user { 'unmanaged_user@localhost':
+          ensure => 'present',
+        }
+
+        mysql_grant { 'managed_user@localhost/unmanaged_db.*':
+          ensure     => 'present',
+          privileges => ['INSERT'],
+          table      => 'unmanaged_db.*',
+          user       => 'managed_user@localhost',
+          require    => [Mysql_user['managed_user@localhost'], Mysql_database['unmanaged_db']],
+        }
+
+        mysql_grant { 'unmanaged_user@localhost/unmanaged_db.*':
+          ensure     => 'present',
+          privileges => ['SELECT'],
+          table      => 'unmanaged_db.*',
+          user       => 'unmanaged_user@localhost',
+          require    => [Mysql_user['unmanaged_user@localhost'], Mysql_database['unmanaged_db']],
+        }
+      MANIFEST
+    end
+
+    it 'applies server setup' do
+      idempotent_apply(bootstrap_pp)
+    end
+
+    it 'creates managed and unmanaged fixture resources' do
+      apply_manifest(fixtures_pp, catch_failures: true)
+    end
+
+    it 'manages only the declared subset idempotently' do
+      idempotent_apply(subset_pp)
+    end
+
+    it 'keeps unmanaged resources present' do
+      probe = run_shell('command -v mysql || command -v mariadb', expect_failures: true)
+      db_client = probe.stdout.to_s.strip
+      skip('No mysql/mariadb client available on acceptance target') if db_client.empty?
+
+      db_query = "SELECT SCHEMA_NAME FROM information_schema.SCHEMATA WHERE SCHEMA_NAME='unmanaged_db'"
+      db_result = run_shell("#{db_client} -NBe \"#{db_query}\"")
+      expect(db_result.exit_code).to eq 0
+      expect(db_result.stdout).to match(%r{^unmanaged_db$})
+
+      user_query = "SELECT CONCAT(user, '@', host) FROM mysql.user WHERE CONCAT(user, '@', host)='unmanaged_user@localhost'"
+      user_result = run_shell("#{db_client} -NBe \"#{user_query}\"")
+      expect(user_result.exit_code).to eq 0
+      expect(user_result.stdout).to match(%r{^unmanaged_user@localhost$})
+    end
+  end
 end

spec/unit/puppet/provider/mysql_database/mysql_spec.rb

@@ -41,13 +41,49 @@
       databases = provider.class.instances.map(&:name)
       expect(parsed_databases).to match_array(databases)
     end
+
+    it 'skips managed databases that no longer exist' do
+      allow(provider.class).to receive(:mysql_caller).with(["show variables like '%_database'", 'managed_existing'], 'regular').and_return("character_set_database latin1\ncollation_database  latin1_swedish_ci\nskip_show_database  OFF") # rubocop:disable Layout/LineLength
+      allow(provider.class).to receive(:mysql_caller)
+        .with(["show variables like '%_database'", 'managed_missing'], 'regular')
+        .and_raise(Puppet::ExecutionFailure, 'ERROR 1049 (42000): Unknown database')
+
+      databases = provider.class.instances(['managed_existing', 'managed_missing']).map(&:name)
+      expect(databases).to eq(['managed_existing'])
+    end
+
+    it 're-raises unexpected database lookup failures' do
+      allow(provider.class).to receive(:mysql_caller).with(["show variables like '%_database'", 'managed_broken'], 'regular').and_raise(Puppet::ExecutionFailure, 'ERROR 1044 (42000): Access denied')
+
+      expect { provider.class.instances(['managed_broken']) }.to raise_error(Puppet::ExecutionFailure, 'ERROR 1044 (42000): Access denied')
+    end
+
+    it 'falls back to full scan when managed database list is empty' do
+      allow(provider.class).to receive(:mysql_caller).with('show databases', 'regular').and_return("fallback_db\n")
+      allow(provider.class).to receive(:mysql_caller).with(["show variables like '%_database'", 'fallback_db'], 'regular').and_return("character_set_database latin1\ncollation_database latin1_swedish_ci\nskip_show_database OFF") # rubocop:disable Layout/LineLength
+
+      databases = provider.class.instances([]).map(&:name)
+      expect(databases).to eq(['fallback_db'])
+    end
   end
 
   describe 'self.prefetch' do
     it 'exists' do
       provider.class.instances
       provider.class.prefetch({})
     end
+
+    it 'only prefetches managed databases from resources' do
+      resources = {
+        'db_one' => instance_double(Puppet::Type.type(:mysql_database)),
+        'db_two' => instance_double(Puppet::Type.type(:mysql_database))
+      }
+
+      expect(provider.class).to receive(:instances).with(['db_one', 'db_two']).and_return([])
+      resources.each_value { |res| allow(res).to receive(:provider=) }
+
+      provider.class.prefetch(resources)
+    end
   end
 
   describe 'create' do

spec/unit/puppet/provider/mysql_grant/mysql_spec.rb

@@ -0,0 +1,52 @@
+# frozen_string_literal: true
+
+require 'spec_helper'
+
+describe Puppet::Type.type(:mysql_grant).provider(:mysql) do
+  describe 'self.instances' do
+    it 'loads grants only for the managed users list' do
+      allow(described_class).to receive(:newer_than).and_return(false)
+      expect(described_class).not_to receive(:users)
+      expect(described_class).to receive(:mysql_caller).with("SHOW GRANTS FOR 'alice'@'localhost';", 'regular').and_return("GRANT SELECT ON `db_one`.* TO 'alice'@'localhost'\n")
+      expect(described_class).not_to receive(:mysql_caller).with("SHOW GRANTS FOR 'bob'@'localhost';", 'regular')
+
+      instances = described_class.instances(['alice@localhost'])
+
+      expect(instances.map(&:name)).to eq(['alice@localhost/db_one.*'])
+    end
+
+    it 'falls back to full user scan when managed user list is empty' do
+      allow(described_class).to receive(:newer_than).and_return(false)
+      expect(described_class).to receive(:users).and_return(['alice@localhost'])
+      allow(described_class).to receive(:mysql_caller).with("SHOW GRANTS FOR 'alice'@'localhost';", 'regular').and_return("GRANT SELECT ON `db_one`.* TO 'alice'@'localhost'\n")
+
+      instances = described_class.instances([])
+
+      expect(instances.map(&:name)).to eq(['alice@localhost/db_one.*'])
+    end
+  end
+
+  describe 'self.prefetch' do
+    it 'only loads grants for managed users from resources' do
+      grant_one = instance_double(Puppet::Type.type(:mysql_grant))
+      grant_two = instance_double(Puppet::Type.type(:mysql_grant))
+      grant_three = instance_double(Puppet::Type.type(:mysql_grant))
+
+      resources = {
+        'alice@localhost/db_one.*' => grant_one,
+        'alice@localhost/db_two.*' => grant_two,
+        'bob@localhost/db_one.*' => grant_three
+      }
+
+      alice_provider = instance_double(described_class, name: 'alice@localhost/db_one.*')
+      bob_provider = instance_double(described_class, name: 'bob@localhost/db_one.*')
+
+      expect(described_class).to receive(:instances).with(['alice@localhost', 'bob@localhost']).and_return([alice_provider, bob_provider])
+      expect(grant_one).to receive(:provider=).with(alice_provider)
+      expect(grant_two).not_to receive(:provider=)
+      expect(grant_three).to receive(:provider=).with(bob_provider)
+
+      described_class.prefetch(resources)
+    end
+  end
+end

spec/unit/puppet/provider/mysql_user/mysql_spec.rb

@@ -168,6 +168,28 @@
       usernames = provider.class.instances.map(&:name)
       expect(parsed_users).to match_array(usernames)
     end
+
+    it 'queries only managed users when user list is provided' do
+      managed_users = ['joe@localhost', "o'reilly@localhost"]
+      user_query = "SELECT CONCAT(User, '@',Host) AS User FROM mysql.user WHERE HOST IS NOT NULL AND HOST != '' AND CONCAT(User, '@', Host) IN ('joe@localhost', 'o''reilly@localhost')"
+
+      allow(provider.class).to receive(:mysql_caller).with(user_query, 'regular').and_return("joe@localhost\no'reilly@localhost")
+      managed_users.each do |user|
+        allow(provider.class).to receive(:mysql_caller).with("SELECT MAX_USER_CONNECTIONS, MAX_CONNECTIONS, MAX_QUESTIONS, MAX_UPDATES, SSL_TYPE, SSL_CIPHER, X509_ISSUER, X509_SUBJECT, PASSWORD /*!50508 , PLUGIN */ FROM mysql.user WHERE CONCAT(user, '@', host) = '#{user}'", 'regular').and_return('10 10 10 10     ') # rubocop:disable Layout/LineLength
+      end
+
+      usernames = provider.class.instances(managed_users).map(&:name)
+      expect(usernames).to match_array(managed_users)
+    end
+
+    it 'falls back to full user scan when managed user list is empty' do
+      full_scan_query = "SELECT CONCAT(User, '@',Host) AS User FROM mysql.user where HOST IS NOT NULL AND HOST != ''"
+      allow(provider.class).to receive(:mysql_caller).with(full_scan_query, 'regular').and_return('joe@localhost')
+      allow(provider.class).to receive(:mysql_caller).with("SELECT MAX_USER_CONNECTIONS, MAX_CONNECTIONS, MAX_QUESTIONS, MAX_UPDATES, SSL_TYPE, SSL_CIPHER, X509_ISSUER, X509_SUBJECT, PASSWORD /*!50508 , PLUGIN */ FROM mysql.user WHERE CONCAT(user, '@', host) = 'joe@localhost'", 'regular').and_return('10 10 10 10     ') # rubocop:disable Layout/LineLength
+
+      usernames = provider.class.instances([]).map(&:name)
+      expect(usernames).to eq(['joe@localhost'])
+    end
   end
 
   describe 'mysql version and type detection' do
@@ -192,6 +214,18 @@
       provider.class.instances
       provider.class.prefetch({})
     end
+
+    it 'only prefetches managed users from resources' do
+      resources = {
+        'joe@localhost' => instance_double(Puppet::Type.type(:mysql_user)),
+        'jane@localhost' => instance_double(Puppet::Type.type(:mysql_user))
+      }
+
+      expect(provider.class).to receive(:instances).with(['joe@localhost', 'jane@localhost']).and_return([])
+      resources.each_value { |res| allow(res).to receive(:provider=) }
+
+      provider.class.prefetch(resources)
+    end
   end
 
   describe 'create' do