feat: persistence of dynamic tuning changes

Currently, any changes made to the tuning via the `instance_*` dbus calls
are lost when tuning is stopped by the service, or when the TuneD service
itself is stopped/restarted, or when the service crashes.

This commit:

* implements a sync of Plugin Instances and Profile Units that is currently missing.
  This way, dynamic instances and device assignments are persistent across stop/start
  dbus calls to TuneD.
* calculates a hash of the current profile after loading it from disk
  (after processing all includes, so we have a "flat" representation)
* creates snapshots of the current profile whenever instances or assigned
  devices change. the snapshot includes the hash of the profile as
  it was initially loaded. for each instance it stores the devices that
  are currently attached.
* restores a snapshot found at startup, if the hashes match (i.e. there
  have been no profile switches and no changes to the profile or any of
  its includes on disk)

  snapshots are restored in case of
  - daemon restarts (systemctl restart/stop/start)
  - daemon crashes
  snapshots are NOT restored in case of
  - reboots (snapshots are stored in /var/run)
  - profile changes (snapshots are explicitly deleted when switching profiles,
    even when "switching" to the same/current profile)

Signed-off-by: Adriaan Schmidt <[email protected]>

Author: adriaan42

tests/unit/profiles/test_merger.py

@@ -1,21 +1,23 @@
 import unittest
 from tuned.profiles.merger import Merger
 from tuned.profiles.profile import Profile
+from tuned.profiles.variables import Variables
 from collections import OrderedDict
 
 class MergerTestCase(unittest.TestCase):
 	def test_merge_without_replace(self):
 		merger = Merger()
+		variables = Variables()
 		config1 = OrderedDict([
 			("main", {"test_option" : "test_value1"}),
 			("net", { "devices": "em0", "custom": "custom_value"}),
 		])
-		profile1 = Profile('test_profile1',config1)
+		profile1 = Profile('test_profile1',config1,variables)
 		config2 = OrderedDict([
 			('main', {'test_option' : 'test_value2'}),
 			('net', { 'devices': 'em1' }),
 		])
-		profile2 = Profile("test_profile2",config2)
+		profile2 = Profile("test_profile2",config2,variables)
 
 		merged_profile = merger.merge([profile1, profile2])
 
@@ -27,16 +29,17 @@ def test_merge_without_replace(self):
 
 	def test_merge_with_replace(self):
 		merger = Merger()
+		variables = Variables()
 		config1 = OrderedDict([
 			("main", {"test_option" : "test_value1"}),
 			("net", { "devices": "em0", "custom": "option"}),
 		])
-		profile1 = Profile('test_profile1',config1)
+		profile1 = Profile('test_profile1',config1,variables)
 		config2 = OrderedDict([
 			("main", {"test_option" : "test_value2"}),
 			("net", { "devices": "em1", "replace": True }),
 		])
-		profile2 = Profile('test_profile2',config2)
+		profile2 = Profile('test_profile2',config2,variables)
 		merged_profile = merger.merge([profile1, profile2])
 
 		self.assertEqual(merged_profile.options["test_option"],"test_value2")
@@ -46,15 +49,16 @@ def test_merge_with_replace(self):
 
 	def test_merge_multiple_order(self):
 		merger = Merger()
+		variables = Variables()
 		config1 = OrderedDict([ ("main", {"test_option" : "test_value1"}),\
 			("net", { "devices": "em0" }) ])
-		profile1 = Profile('test_profile1',config1)
+		profile1 = Profile('test_profile1',config1,variables)
 		config2 = OrderedDict([	("main", {"test_option" : "test_value2"}),\
 			("net", { "devices": "em1" }) ])
-		profile2 = Profile('test_profile2',config2)
+		profile2 = Profile('test_profile2',config2,variables)
 		config3 = OrderedDict([	("main", {"test_option" : "test_value3"}),\
 			("net", { "devices": "em2" }) ])
-		profile3 = Profile('test_profile3',config3)
+		profile3 = Profile('test_profile3',config3,variables)
 		merged_profile = merger.merge([profile1, profile2, profile3])
 
 		self.assertEqual(merged_profile.options["test_option"],"test_value3")

tests/unit/profiles/test_profile.py

@@ -9,33 +9,33 @@ def _create_unit(self, name, config):
 class ProfileTestCase(unittest.TestCase):
 
 	def test_init(self):
-		MockProfile("test", {})
+		MockProfile("test", {}, None)
 
 	def test_create_units(self):
 		profile = MockProfile("test", {
 			"main": { "anything": 10 },
 			"network" : { "type": "net", "devices": "*" },
 			"storage" : { "type": "disk" },
-		})
+		}, None)
 
 		self.assertIs(type(profile.units), collections.OrderedDict)
 		self.assertEqual(len(profile.units), 2)
 		self.assertListEqual(sorted([name_config for name_config in profile.units]), sorted(["network", "storage"]))
 
 	def test_create_units_empty(self):
-		profile = MockProfile("test", {"main":{}})
+		profile = MockProfile("test", {"main":{}}, None)
 
 		self.assertIs(type(profile.units), collections.OrderedDict)
 		self.assertEqual(len(profile.units), 0)
 
 	def test_sets_name(self):
-		profile1 = MockProfile("test_one", {})
-		profile2 = MockProfile("test_two", {})
+		profile1 = MockProfile("test_one", {}, None)
+		profile2 = MockProfile("test_two", {}, None)
 		self.assertEqual(profile1.name, "test_one")
 		self.assertEqual(profile2.name, "test_two")
 
 	def test_change_name(self):
-		profile = MockProfile("oldname", {})
+		profile = MockProfile("oldname", {}, None)
 		self.assertEqual(profile.name, "oldname")
 		profile.name = "newname"
 		self.assertEqual(profile.name, "newname")
@@ -44,15 +44,15 @@ def test_sets_options(self):
 		profile = MockProfile("test", {
 			"main": { "anything": 10 },
 			"network" : { "type": "net", "devices": "*" },
-		})
+		}, None)
 
-		self.assertIs(type(profile.options), dict)
+		self.assertIs(type(profile.options), collections.OrderedDict)
 		self.assertEqual(profile.options["anything"], 10)
 
 	def test_sets_options_empty(self):
 		profile = MockProfile("test", {
 			"storage" : { "type": "disk" },
-		})
+		}, None)
 
 		self.assertIs(type(profile.options), dict)
 		self.assertEqual(len(profile.options), 0)

tuned/consts.py

@@ -19,6 +19,7 @@
 DEFAULT_STORAGE_FILE = "/run/tuned/save.pickle"
 USER_PROFILES_DIR = "/etc/tuned/profiles"
 SYSTEM_PROFILES_DIR = "/usr/lib/tuned/profiles"
+PROFILE_SNAPSHOT_FILE = "/run/tuned/profile-snapshot.conf"
 PERSISTENT_STORAGE_DIR = "/var/lib/tuned"
 PLUGIN_MAIN_UNIT_NAME = "main"
 # Magic section header because ConfigParser does not support "headerless" config

tuned/daemon/controller.py

@@ -407,6 +407,7 @@ def instance_acquire_devices(self, devices, instance_name, caller = None):
 			rets = "Ignoring devices not handled by any instance '%s'." % str(devs)
 			log.info(rets)
 			return (False, rets)
+		self._daemon.sync_instances()
 		return (True, "OK")
 
 	@exports.export("s", "(bsa(ss))")
@@ -472,6 +473,8 @@ def instance_create(self, plugin_name, instance_name, options, caller = None):
 		"""
 		if caller == "":
 			return (False, "Unauthorized")
+		plugin_name = str(plugin_name)
+		instance_name = str(instance_name)
 		if not self._cmd.is_valid_name(plugin_name):
 			return (False, "Invalid plugin_name")
 		if not self._cmd.is_valid_name(instance_name):
@@ -519,6 +522,7 @@ def instance_create(self, plugin_name, instance_name, options, caller = None):
 					other_instance.name, instance.name))
 				plugin._remove_devices_nocheck(other_instance, devs_moving)
 				plugin._add_devices_nocheck(instance, devs_moving)
+		self._daemon.sync_instances()
 		return (True, "OK")
 
 	@exports.export("s", "(bs)")
@@ -561,4 +565,5 @@ def instance_destroy(self, instance_name, caller = None):
 		for device in devices:
 			# _add_device() will find a suitable plugin instance
 			plugin._add_device(device)
+		self._daemon.sync_instances()
 		return (True, "OK")

tuned/daemon/daemon.py

@@ -121,6 +121,11 @@ def _load_profiles(self, profile_names, manual):
 			self._notify_profile_changed(profile_names, False, errstr)
 			raise TunedException(errstr)
 
+		# restore profile snapshot (if there is one)
+		snapshot = self._profile_loader.restore_snapshot(self._profile)
+		if snapshot is not None:
+			self._profile = snapshot
+
 	def set_profile(self, profile_names, manual):
 		if self.is_running():
 			errstr = "Cannot set profile while the daemon is running."
@@ -154,6 +159,40 @@ def set_all_profiles(self, active_profiles, manual, post_loaded_profile,
 			self._save_active_profile(active_profiles, manual)
 			self._save_post_loaded_profile(post_loaded_profile)
 
+	def sync_instances(self):
+		# NOTE: currently, Controller creates the new instances, and here in Daemon
+		#       we discover what happened, and update the profile accordingly.
+		#       a potentially better approach would be to move some of the logic
+		#       from Controller to Daemon, and create/destroy the instances here,
+		#       and at the same time update the profile.
+
+		# remove all units that don't have an instance
+		instance_names = [i.name for i in self._unit_manager.instances]
+		for unit in list(self._profile.units.keys()):
+			if unit in instance_names:
+				continue
+			log.debug("snapshot sync: removing unit '%s'" % unit)
+			del self._profile.units[unit]
+		# create units for new instances
+		for instance in self._unit_manager.instances:
+			if instance.name in self._profile.units:
+				continue
+			log.debug("snapshot sync: creating unit '%s'" % instance.name)
+			config = {
+				"priority": instance.priority,
+				"type": instance._plugin.name,
+				"enabled": instance.active,
+				"devices": instance.devices_expression,
+				"devices_udev_regex": instance.devices_udev_regex,
+				"script_pre": instance.script_pre,
+				"script_post": instance.script_post,
+			}
+			for k, v in instance.options.items():
+				config[k] = v
+			self._profile.units[instance.name] = self._profile._create_unit(instance.name, config)
+		# create profile snapshot
+		self._profile_loader.create_snapshot(self._profile, self._unit_manager.instances)
+
 	@property
 	def profile(self):
 		return self._profile
@@ -200,6 +239,8 @@ def _thread_code(self):
 		self._save_active_profile(" ".join(self._active_profiles),
 					  self._manual)
 		self._save_post_loaded_profile(self._post_loaded_profile)
+		# trigger a profile snapshot
+		self.sync_instances()
 		self._unit_manager.start_tuning()
 		self._profile_applied.set()
 		log.info("static tuning from profile '%s' applied" % self._profile.name)
@@ -368,6 +409,7 @@ def stop(self, profile_switch = False):
 			return False
 		log.info("stopping tuning")
 		if profile_switch:
+			self._profile_loader.remove_snapshot()
 			self._terminate_profile_switch.set()
 		self._terminate.set()
 		self._thread.join()

tuned/plugins/base.py

@@ -164,23 +164,35 @@ def _get_matching_devices(self, instance, devices):
 			udev_devices = self._device_matcher_udev.match_list(instance.devices_udev_regex, udev_devices)
 			return set([x.sys_name for x in udev_devices])
 
+	def restore_devices(self, instance, devices):
+		if not self._devices_supported:
+			return
+
+		log.debug("Restoring devices of instance %s: %s" % (instance.name, " ".join(devices)))
+		for device in devices:
+			if device not in self._free_devices:
+				continue
+			self._free_devices.remove(device)
+			instance.assigned_devices.add(device)
+			self._assigned_devices.add(device)
+
 	def assign_free_devices(self, instance):
 		if not self._devices_supported:
 			return
 
 		log.debug("assigning devices to instance %s" % instance.name)
 		to_assign = self._get_matching_devices(instance, self._free_devices)
-		instance.active = len(to_assign) > 0
-		if not instance.active:
-			log.warning("instance %s: no matching devices available" % instance.name)
-		else:
+		if len(to_assign) > 0:
 			name = instance.name
 			if instance.name != self.name:
 				name += " (%s)" % self.name
 			log.info("instance %s: assigning devices %s" % (name, ", ".join(to_assign)))
 			instance.assigned_devices.update(to_assign) # cannot use |=
 			self._assigned_devices |= to_assign
 			self._free_devices -= to_assign
+		instance.active = len(instance.assigned_devices) > 0
+		if not instance.active:
+			log.warning("instance %s: no matching devices available" % instance.name)
 
 	def release_devices(self, instance):
 		if not self._devices_supported:

tuned/profiles/factory.py

@@ -1,5 +1,5 @@
 import tuned.profiles.profile
 
 class Factory(object):
-	def create(self, name, config):
-		return tuned.profiles.profile.Profile(name, config)
+	def create(self, name, config, variables):
+		return tuned.profiles.profile.Profile(name, config, variables)

tuned/profiles/loader.py

@@ -24,9 +24,6 @@ def __init__(self, profile_locator, profile_factory, profile_merger, global_conf
 		self._global_config = global_config
 		self._variables = variables
 
-	def _create_profile(self, profile_name, config):
-		return tuned.profiles.profile.Profile(profile_name, config)
-
 	@classmethod
 	def safe_name(cls, profile_name):
 		return re.match(r'^[a-zA-Z0-9_.-]+$', profile_name)
@@ -57,22 +54,43 @@ def load(self, profile_names):
 			final_profile = profiles[0]
 
 		final_profile.name = " ".join(profile_names)
-		if "variables" in final_profile.units:
-			self._variables.add_from_cfg(final_profile.units["variables"].options)
-			del(final_profile.units["variables"])
-		# FIXME hack, do all variable expansions in one place
-		self._expand_vars_in_devices(final_profile)
-		self._expand_vars_in_regexes(final_profile)
+		final_profile.process_variables()
+		final_profile.calculate_hash()
 		return final_profile
 
-	def _expand_vars_in_devices(self, profile):
-		for unit in profile.units:
-			profile.units[unit].devices = self._variables.expand(profile.units[unit].devices)
-
-	def _expand_vars_in_regexes(self, profile):
-		for unit in profile.units:
-			profile.units[unit].cpuinfo_regex = self._variables.expand(profile.units[unit].cpuinfo_regex)
-			profile.units[unit].uname_regex = self._variables.expand(profile.units[unit].uname_regex)
+	def create_snapshot(self, profile, instances):
+		snapshot = profile.snapshot(instances)
+		log.debug("Storing profile snapshot in %s:\n%s" % (consts.PROFILE_SNAPSHOT_FILE, snapshot))
+		with open(consts.PROFILE_SNAPSHOT_FILE, "w") as f:
+			f.write(snapshot)
+
+	def restore_snapshot(self, profile):
+		if profile is None:
+			# When tuning is stopped, we are called with profile==None -> skip
+			return None
+		snapshot = None
+		if os.path.isfile(consts.PROFILE_SNAPSHOT_FILE):
+			log.debug("Found profile snapshot '%s'" % consts.PROFILE_SNAPSHOT_FILE)
+			try:
+				config = self._load_config_data(consts.PROFILE_SNAPSHOT_FILE)
+				snapshot_hash = config.get("main", {}).get("profile_base_hash", None)
+				if snapshot_hash == profile._base_hash:
+					snapshot = self._profile_factory.create("restore", config, self._variables)
+					snapshot.name = profile.name
+					snapshot.process_variables()
+					log.info("Restored profile snapshot: %s" % snapshot.name)
+				else:
+					log.debug("Snapshot hash '%s' does not match current base hash '%s'. Not restoring." % (snapshot_hash, profile._base_hash))
+					os.remove(consts.PROFILE_SNAPSHOT_FILE)
+			except InvalidProfileException as e:
+				log.error("Could not process profile snapshot: %s" % e)
+		return snapshot
+
+	def remove_snapshot(self):
+		try:
+			os.remove(consts.PROFILE_SNAPSHOT_FILE)
+		except FileNotFoundError:
+			pass
 
 	def _load_profile(self, profile_names, profiles, processed_files):
 		for name in profile_names:
@@ -84,9 +102,9 @@ def _load_profile(self, profile_names, profiles, processed_files):
 			processed_files.append(filename)
 
 			config = self._load_config_data(filename)
-			profile = self._profile_factory.create(name, config)
+			profile = self._profile_factory.create(name, config, self._variables)
 			if "include" in profile.options:
-				include_names = re.split(r"\s*[,;]\s*", self._variables.expand(profile.options.pop("include")))
+				include_names = re.split(r"\s*[,;]\s*", profile._variables.expand(profile.options.pop("include")))
 				self._load_profile(include_names, profiles, processed_files)
 
 			profiles.append(profile)