Persistence of dynamic tuning changes

tests/unit/profiles/test_merger.py

@@ -1,21 +1,23 @@
 import unittest
 from tuned.profiles.merger import Merger
 from tuned.profiles.profile import Profile
+from tuned.profiles.variables import Variables
 from collections import OrderedDict
 
 class MergerTestCase(unittest.TestCase):
 	def test_merge_without_replace(self):
 		merger = Merger()
+		variables = Variables()
 		config1 = OrderedDict([
 			("main", {"test_option" : "test_value1"}),
 			("net", { "devices": "em0", "custom": "custom_value"}),
 		])
-		profile1 = Profile('test_profile1',config1)
+		profile1 = Profile('test_profile1',config1,variables)
 		config2 = OrderedDict([
 			('main', {'test_option' : 'test_value2'}),
 			('net', { 'devices': 'em1' }),
 		])
-		profile2 = Profile("test_profile2",config2)
+		profile2 = Profile("test_profile2",config2,variables)
 
 		merged_profile = merger.merge([profile1, profile2])
 
@@ -27,16 +29,17 @@ def test_merge_without_replace(self):
 
 	def test_merge_with_replace(self):
 		merger = Merger()
+		variables = Variables()
 		config1 = OrderedDict([
 			("main", {"test_option" : "test_value1"}),
 			("net", { "devices": "em0", "custom": "option"}),
 		])
-		profile1 = Profile('test_profile1',config1)
+		profile1 = Profile('test_profile1',config1,variables)
 		config2 = OrderedDict([
 			("main", {"test_option" : "test_value2"}),
 			("net", { "devices": "em1", "replace": True }),
 		])
-		profile2 = Profile('test_profile2',config2)
+		profile2 = Profile('test_profile2',config2,variables)
 		merged_profile = merger.merge([profile1, profile2])
 
 		self.assertEqual(merged_profile.options["test_option"],"test_value2")
@@ -46,15 +49,16 @@ def test_merge_with_replace(self):
 
 	def test_merge_multiple_order(self):
 		merger = Merger()
+		variables = Variables()
 		config1 = OrderedDict([ ("main", {"test_option" : "test_value1"}),\
 			("net", { "devices": "em0" }) ])
-		profile1 = Profile('test_profile1',config1)
+		profile1 = Profile('test_profile1',config1,variables)
 		config2 = OrderedDict([	("main", {"test_option" : "test_value2"}),\
 			("net", { "devices": "em1" }) ])
-		profile2 = Profile('test_profile2',config2)
+		profile2 = Profile('test_profile2',config2,variables)
 		config3 = OrderedDict([	("main", {"test_option" : "test_value3"}),\
 			("net", { "devices": "em2" }) ])
-		profile3 = Profile('test_profile3',config3)
+		profile3 = Profile('test_profile3',config3,variables)
 		merged_profile = merger.merge([profile1, profile2, profile3])
 
 		self.assertEqual(merged_profile.options["test_option"],"test_value3")

tests/unit/profiles/test_profile.py

@@ -9,33 +9,33 @@ def _create_unit(self, name, config):
 class ProfileTestCase(unittest.TestCase):
 
 	def test_init(self):
-		MockProfile("test", {})
+		MockProfile("test", {}, None)
 
 	def test_create_units(self):
 		profile = MockProfile("test", {
 			"main": { "anything": 10 },
 			"network" : { "type": "net", "devices": "*" },
 			"storage" : { "type": "disk" },
-		})
+		}, None)
 
 		self.assertIs(type(profile.units), collections.OrderedDict)
 		self.assertEqual(len(profile.units), 2)
 		self.assertListEqual(sorted([name_config for name_config in profile.units]), sorted(["network", "storage"]))
 
 	def test_create_units_empty(self):
-		profile = MockProfile("test", {"main":{}})
+		profile = MockProfile("test", {"main":{}}, None)
 
 		self.assertIs(type(profile.units), collections.OrderedDict)
 		self.assertEqual(len(profile.units), 0)
 
 	def test_sets_name(self):
-		profile1 = MockProfile("test_one", {})
-		profile2 = MockProfile("test_two", {})
+		profile1 = MockProfile("test_one", {}, None)
+		profile2 = MockProfile("test_two", {}, None)
 		self.assertEqual(profile1.name, "test_one")
 		self.assertEqual(profile2.name, "test_two")
 
 	def test_change_name(self):
-		profile = MockProfile("oldname", {})
+		profile = MockProfile("oldname", {}, None)
 		self.assertEqual(profile.name, "oldname")
 		profile.name = "newname"
 		self.assertEqual(profile.name, "newname")
@@ -44,15 +44,15 @@ def test_sets_options(self):
 		profile = MockProfile("test", {
 			"main": { "anything": 10 },
 			"network" : { "type": "net", "devices": "*" },
-		})
+		}, None)
 
-		self.assertIs(type(profile.options), dict)
+		self.assertIs(type(profile.options), collections.OrderedDict)
 		self.assertEqual(profile.options["anything"], 10)
 
 	def test_sets_options_empty(self):
 		profile = MockProfile("test", {
 			"storage" : { "type": "disk" },
-		})
+		}, None)
 
 		self.assertIs(type(profile.options), dict)
 		self.assertEqual(len(profile.options), 0)

tuned/consts.py

@@ -19,6 +19,7 @@
 DEFAULT_STORAGE_FILE = "/run/tuned/save.pickle"
 USER_PROFILES_DIR = "/etc/tuned/profiles"
 SYSTEM_PROFILES_DIR = "/usr/lib/tuned/profiles"
+PROFILE_SNAPSHOT_FILE = "/run/tuned/profile-snapshot.conf"
 PERSISTENT_STORAGE_DIR = "/var/lib/tuned"
 PLUGIN_MAIN_UNIT_NAME = "main"
 # Magic section header because ConfigParser does not support "headerless" config

tuned/daemon/controller.py

@@ -407,6 +407,7 @@ def instance_acquire_devices(self, devices, instance_name, caller = None):
 			rets = "Ignoring devices not handled by any instance '%s'." % str(devs)
 			log.info(rets)
 			return (False, rets)
+		self._daemon.sync_instances()
 		return (True, "OK")
 
 	@exports.export("s", "(bsa(ss))")
@@ -472,6 +473,8 @@ def instance_create(self, plugin_name, instance_name, options, caller = None):
 		"""
 		if caller == "":
 			return (False, "Unauthorized")
+		plugin_name = str(plugin_name)
+		instance_name = str(instance_name)
 		if not self._cmd.is_valid_name(plugin_name):
 			return (False, "Invalid plugin_name")
 		if not self._cmd.is_valid_name(instance_name):
@@ -519,6 +522,7 @@ def instance_create(self, plugin_name, instance_name, options, caller = None):
 					other_instance.name, instance.name))
 				plugin._remove_devices_nocheck(other_instance, devs_moving)
 				plugin._add_devices_nocheck(instance, devs_moving)
+		self._daemon.sync_instances()
 		return (True, "OK")
 
 	@exports.export("s", "(bs)")
@@ -561,4 +565,5 @@ def instance_destroy(self, instance_name, caller = None):
 		for device in devices:
 			# _add_device() will find a suitable plugin instance
 			plugin._add_device(device)
+		self._daemon.sync_instances()
 		return (True, "OK")

tuned/daemon/daemon.py

@@ -121,6 +121,11 @@ def _load_profiles(self, profile_names, manual):
 			self._notify_profile_changed(profile_names, False, errstr)
 			raise TunedException(errstr)
 
+		# restore profile snapshot (if there is one)
+		snapshot = self._profile_loader.restore_snapshot(self._profile)
+		if snapshot is not None:
+			self._profile = snapshot
+
 	def set_profile(self, profile_names, manual):
 		if self.is_running():
 			errstr = "Cannot set profile while the daemon is running."
@@ -154,6 +159,40 @@ def set_all_profiles(self, active_profiles, manual, post_loaded_profile,
 			self._save_active_profile(active_profiles, manual)
 			self._save_post_loaded_profile(post_loaded_profile)
 
+	def sync_instances(self):
+		# NOTE: currently, Controller creates the new instances, and here in Daemon
+		#       we discover what happened, and update the profile accordingly.
+		#       a potentially better approach would be to move some of the logic
+		#       from Controller to Daemon, and create/destroy the instances here,
+		#       and at the same time update the profile.
+
+		# remove all units that don't have an instance
+		instance_names = [i.name for i in self._unit_manager.instances]
+		for unit in list(self._profile.units.keys()):
+			if unit in instance_names:
+				continue
+			log.debug("snapshot sync: removing unit '%s'" % unit)
+			del self._profile.units[unit]
+		# create units for new instances
+		for instance in self._unit_manager.instances:
+			if instance.name in self._profile.units:
+				continue
+			log.debug("snapshot sync: creating unit '%s'" % instance.name)
+			config = {
+				"priority": instance.priority,
+				"type": instance._plugin.name,
+				"enabled": instance.active,
+				"devices": instance.devices_expression,
+				"devices_udev_regex": instance.devices_udev_regex,
+				"script_pre": instance.script_pre,
+				"script_post": instance.script_post,
+			}
+			for k, v in instance.options.items():
+				config[k] = v
+			self._profile.units[instance.name] = self._profile._create_unit(instance.name, config)
+		# create profile snapshot
+		self._profile_loader.create_snapshot(self._profile, self._unit_manager.instances)
+
 	@property
 	def profile(self):
 		return self._profile
@@ -200,6 +239,8 @@ def _thread_code(self):
 		self._save_active_profile(" ".join(self._active_profiles),
 					  self._manual)
 		self._save_post_loaded_profile(self._post_loaded_profile)
+		# trigger a profile snapshot
+		self.sync_instances()
 		self._unit_manager.start_tuning()
 		self._profile_applied.set()
 		log.info("static tuning from profile '%s' applied" % self._profile.name)
@@ -368,6 +409,7 @@ def stop(self, profile_switch = False):
 			return False
 		log.info("stopping tuning")
 		if profile_switch:
+			self._profile_loader.remove_snapshot()
 			self._terminate_profile_switch.set()
 		self._terminate.set()
 		self._thread.join()

tuned/plugins/base.py

@@ -164,23 +164,35 @@ def _get_matching_devices(self, instance, devices):
 			udev_devices = self._device_matcher_udev.match_list(instance.devices_udev_regex, udev_devices)
 			return set([x.sys_name for x in udev_devices])
 
+	def restore_devices(self, instance, devices):
+		if not self._devices_supported:
+			return
+
+		log.debug("Restoring devices of instance %s: %s" % (instance.name, " ".join(devices)))
+		for device in devices:
+			if device not in self._free_devices:
+				continue
+			self._free_devices.remove(device)
+			instance.assigned_devices.add(device)
+			self._assigned_devices.add(device)
+
 	def assign_free_devices(self, instance):
 		if not self._devices_supported:
 			return
 
 		log.debug("assigning devices to instance %s" % instance.name)
 		to_assign = self._get_matching_devices(instance, self._free_devices)
-		instance.active = len(to_assign) > 0
-		if not instance.active:
-			log.warning("instance %s: no matching devices available" % instance.name)
-		else:
+		if len(to_assign) > 0:
 			name = instance.name
 			if instance.name != self.name:
 				name += " (%s)" % self.name
 			log.info("instance %s: assigning devices %s" % (name, ", ".join(to_assign)))
 			instance.assigned_devices.update(to_assign) # cannot use |=
 			self._assigned_devices |= to_assign
 			self._free_devices -= to_assign
+		instance.active = len(instance.assigned_devices) > 0
+		if not instance.active:
+			log.warning("instance %s: no matching devices available" % instance.name)
 
 	def release_devices(self, instance):
 		if not self._devices_supported:

tuned/plugins/hotplug.py

@@ -43,6 +43,10 @@ def _add_device_process(self, instance, device_name):
 		self._added_device_apply_tuning(instance, device_name)
 		self._call_device_script(instance, instance.script_post, "apply", [device_name])
 		instance.processed_devices.add(device_name)
+		# This can be a bit racy (we can overcount),
+		# but it shouldn't affect the boolean result
+		instance.active = len(instance.processed_devices) \
+				+ len(instance.assigned_devices) > 0
 
 	def _add_device(self, device_name):
 		if device_name in (self._assigned_devices | self._free_devices):
@@ -63,10 +67,6 @@ def _add_devices_nocheck(self, instance, device_names):
 		"""
 		for dev in device_names:
 			self._add_device_process(instance, dev)
-		# This can be a bit racy (we can overcount),
-		# but it shouldn't affect the boolean result
-		instance.active = len(instance.processed_devices) \
-				+ len(instance.assigned_devices) > 0
 
 	def _remove_device_process(self, instance, device_name):
 		if device_name in instance.processed_devices:

tuned/profiles/factory.py

@@ -1,5 +1,5 @@
 import tuned.profiles.profile
 
 class Factory(object):
-	def create(self, name, config):
-		return tuned.profiles.profile.Profile(name, config)
+	def create(self, name, config, variables):
+		return tuned.profiles.profile.Profile(name, config, variables)

tuned/profiles/loader.py

@@ -24,9 +24,6 @@ def __init__(self, profile_locator, profile_factory, profile_merger, global_conf
 		self._global_config = global_config
 		self._variables = variables
 
-	def _create_profile(self, profile_name, config):
-		return tuned.profiles.profile.Profile(profile_name, config)
-
 	@classmethod
 	def safe_name(cls, profile_name):
 		return re.match(r'^[a-zA-Z0-9_.-]+$', profile_name)
@@ -57,22 +54,43 @@ def load(self, profile_names):
 			final_profile = profiles[0]
 
 		final_profile.name = " ".join(profile_names)
-		if "variables" in final_profile.units:
-			self._variables.add_from_cfg(final_profile.units["variables"].options)
-			del(final_profile.units["variables"])
-		# FIXME hack, do all variable expansions in one place
-		self._expand_vars_in_devices(final_profile)
-		self._expand_vars_in_regexes(final_profile)
+		final_profile.process_variables()
+		final_profile.calculate_hash()
 		return final_profile
 
-	def _expand_vars_in_devices(self, profile):
-		for unit in profile.units:
-			profile.units[unit].devices = self._variables.expand(profile.units[unit].devices)
-
-	def _expand_vars_in_regexes(self, profile):
-		for unit in profile.units:
-			profile.units[unit].cpuinfo_regex = self._variables.expand(profile.units[unit].cpuinfo_regex)
-			profile.units[unit].uname_regex = self._variables.expand(profile.units[unit].uname_regex)
+	def create_snapshot(self, profile, instances):
+		snapshot = profile.snapshot(instances)
+		log.debug("Storing profile snapshot in %s:\n%s" % (consts.PROFILE_SNAPSHOT_FILE, snapshot))
+		with open(consts.PROFILE_SNAPSHOT_FILE, "w") as f:
+			f.write(snapshot)
+
+	def restore_snapshot(self, profile):
+		if profile is None:
+			# When tuning is stopped, we are called with profile==None -> skip
+			return None
+		snapshot = None
+		if os.path.isfile(consts.PROFILE_SNAPSHOT_FILE):
+			log.debug("Found profile snapshot '%s'" % consts.PROFILE_SNAPSHOT_FILE)
+			try:
+				config = self._load_config_data(consts.PROFILE_SNAPSHOT_FILE)
+				snapshot_hash = config.get("main", {}).get("profile_base_hash", None)
+				if snapshot_hash == profile._base_hash:
+					snapshot = self._profile_factory.create("restore", config, self._variables)
+					snapshot.name = profile.name
+					snapshot.process_variables()
+					log.info("Restored profile snapshot: %s" % snapshot.name)
+				else:
+					log.debug("Snapshot hash '%s' does not match current base hash '%s'. Not restoring." % (snapshot_hash, profile._base_hash))
+					os.remove(consts.PROFILE_SNAPSHOT_FILE)
+			except InvalidProfileException as e:
+				log.error("Could not process profile snapshot: %s" % e)
+		return snapshot
+
+	def remove_snapshot(self):
+		try:
+			os.remove(consts.PROFILE_SNAPSHOT_FILE)
+		except FileNotFoundError:
+			pass
 
 	def _load_profile(self, profile_names, profiles, processed_files):
 		for name in profile_names:
@@ -84,9 +102,9 @@ def _load_profile(self, profile_names, profiles, processed_files):
 			processed_files.append(filename)
 
 			config = self._load_config_data(filename)
-			profile = self._profile_factory.create(name, config)
+			profile = self._profile_factory.create(name, config, self._variables)
 			if "include" in profile.options:
-				include_names = re.split(r"\s*[,;]\s*", self._variables.expand(profile.options.pop("include")))
+				include_names = re.split(r"\s*[,;]\s*", profile._variables.expand(profile.options.pop("include")))
 				self._load_profile(include_names, profiles, processed_files)
 
 			profiles.append(profile)