Skip to content

Commit 8b65381

Browse files
prabinovRedhatprabinovRedhat
authored
feat: MTV-4362: add data integrity validation and graceful VM shutdown for copy-offload tests (#420)
Add pre-migration marker file creation on source VM and post-migration verification on the migrated VM to validate data survives the migration. Remove source_vm_power from cold copyoffload configs — MTV handles VM shutdown via ShutdownGuest during cold migration. Fix test ID for TestCopyoffloadThinSnapshotsMigration (MTV-560). Guard shutdown_vm_guest fallback against race condition where guest finishes shutting down between timeout and hard PowerOff call. Made-with: Cursor
1 parent c142a4b commit 8b65381

7 files changed

Lines changed: 265 additions & 36 deletions

File tree

‎libs/providers/vmware.py‎

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -330,6 +330,46 @@ def stop_vm(self, vm):
330330
if vm.runtime.powerState == vm.runtime.powerState.poweredOn:
331331
self.wait_task(task=vm.PowerOff(), action_name=f"Stopping VM {vm.name}")
332332

333+
def shutdown_vm_guest(self, vm: vim.VirtualMachine, timeout: int = 120) -> None:
334+
"""Gracefully shut down a VM's guest OS via VMware Tools.
335+
336+
Requests a clean OS shutdown (ShutdownGuest) which flushes filesystem
337+
buffers and unmounts filesystems before powering off. Falls back to hard
338+
PowerOff if VMware Tools is unavailable or the guest doesn't shut down
339+
within the timeout.
340+
341+
Args:
342+
vm (vim.VirtualMachine): VMware VM object to stop.
343+
timeout (int): Seconds to wait for graceful shutdown before falling back to PowerOff.
344+
"""
345+
if vm.runtime.powerState != vm.runtime.powerState.poweredOn:
346+
return
347+
348+
try:
349+
vm.ShutdownGuest()
350+
LOGGER.info(f"Requested graceful shutdown for VM {vm.name}, waiting up to {timeout}s")
351+
for sample in TimeoutSampler(
352+
wait_timeout=timeout,
353+
sleep=5,
354+
func=lambda: vm.runtime.powerState,
355+
):
356+
if sample != vm.runtime.powerState.poweredOn:
357+
LOGGER.info(f"VM {vm.name} gracefully shut down")
358+
return
359+
except (vim.fault.ToolsUnavailable, vim.fault.InvalidPowerState):
360+
LOGGER.warning(f"VMware Tools unavailable on VM {vm.name}, falling back to hard PowerOff")
361+
except TimeoutExpiredError:
362+
LOGGER.warning(f"Graceful shutdown timed out for VM {vm.name} after {timeout}s, falling back to PowerOff")
363+
364+
if vm.runtime.powerState != vm.runtime.powerState.poweredOn:
365+
LOGGER.info(f"VM {vm.name} already powered off, skipping hard PowerOff")
366+
return
367+
368+
try:
369+
self.wait_task(task=vm.PowerOff(), action_name=f"Stopping VM {vm.name}")
370+
except vim.fault.InvalidPowerState:
371+
LOGGER.info(f"VM {vm.name} powered off during fallback attempt")
372+
333373
@staticmethod
334374
def list_snapshots(vm):
335375
snapshots = []

‎tests/copyoffload/conftest.py‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -369,13 +369,15 @@ def nonpersistent_disk_ready(
369369
to independent_nonpersistent after the VM is powered off.
370370
371371
Args:
372-
vmware_cloud_init_ready (None): Ensures cloud-init has finished and VM is off.
372+
vmware_cloud_init_ready (None): Ensures cloud-init has finished (VM may still be on).
373373
prepared_plan (dict[str, Any]): Processed test plan with VM data.
374374
source_provider (VMWareProvider): The VMware source provider instance.
375375
"""
376376
for vm_data in prepared_plan["virtual_machines"]:
377377
vm_name = vm_data["name"]
378378
provider_vm_api = prepared_plan["source_vms_data"][vm_name]["provider_vm_api"]
379+
if provider_vm_api.runtime.powerState == provider_vm_api.runtime.powerState.poweredOn:
380+
source_provider.shutdown_vm_guest(vm=provider_vm_api)
379381
source_provider.change_disk_mode(
380382
vm=provider_vm_api,
381383
disk_mode="independent_nonpersistent",

‎tests/copyoffload/test_copyoffload_migration.py‎

Lines changed: 63 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@
2020
from ocp_resources.plan import Plan
2121
from ocp_resources.secret import Secret
2222
from ocp_resources.storage_map import StorageMap
23+
from ocp_resources.virtual_machine import VirtualMachine
2324
from pytest_testconfig import config as py_config
2425
from simple_logger.logger import get_logger
2526

@@ -38,7 +39,8 @@
3839
wait_for_migration_complate,
3940
)
4041
from utilities.naming import sanitize_kubernetes_name
41-
from utilities.post_migration import check_vms
42+
from utilities.post_migration import check_vms, verify_data_integrity
43+
from utilities.vmware_guest_operations import create_data_integrity_marker
4244
from utilities.resources import create_and_store_resource
4345
from utilities.ssh_utils import SSHConnectionManager
4446

@@ -220,11 +222,12 @@ def test_check_vms(
220222

221223

222224
class CopyoffloadSnapshotBase:
223-
"""Base class for copy-offload migration tests with snapshots."""
225+
"""Base class for copy-offload snapshot migration tests with data integrity validation."""
224226

225227
storage_map: StorageMap
226228
network_map: NetworkMap
227229
plan_resource: Plan
230+
data_integrity_marker: str
228231

229232
def test_create_storagemap(
230233
self,
@@ -267,8 +270,14 @@ def test_create_storagemap(
267270
f"Expected at least {snapshots_to_create} snapshots, got {len(vm_cfg['snapshots_before_migration'])}"
268271
)
269272

270-
# Cold migration expects VM powered off
271-
source_provider.stop_vm(provider_vm_api)
273+
marker_content = f"mtv-data-integrity-{fixture_store['session_uuid']}"
274+
create_data_integrity_marker(
275+
source_provider=source_provider,
276+
vm=provider_vm_api,
277+
source_provider_data=source_provider_data,
278+
marker_content=marker_content,
279+
)
280+
self.__class__.data_integrity_marker = marker_content
272281

273282
copyoffload_config_data = source_provider_data["copyoffload"]
274283
storage_vendor_product = copyoffload_config_data["storage_vendor_product"]
@@ -405,14 +414,63 @@ def test_check_vms(
405414
destination_provider=destination_provider, plan=prepared_plan, target_namespace=target_namespace
406415
)
407416

417+
def test_check_data_integrity(
418+
self,
419+
prepared_plan: dict[str, Any],
420+
ocp_admin_client: DynamicClient,
421+
source_provider_data: dict[str, Any],
422+
source_provider: BaseProvider,
423+
vm_ssh_connections: SSHConnectionManager,
424+
) -> None:
425+
"""Verify data written before migration survived on the migrated VM.
426+
427+
Starts the migrated VM, reads the marker file via SSH, and confirms
428+
the content matches what was written pre-migration.
429+
430+
Args:
431+
prepared_plan (dict[str, Any]): Processed test plan configuration.
432+
ocp_admin_client (DynamicClient): OpenShift admin client.
433+
source_provider_data (dict[str, Any]): Provider configuration data.
434+
source_provider (BaseProvider): Source provider for VM metadata.
435+
vm_ssh_connections (SSHConnectionManager): SSH connection manager.
436+
"""
437+
vm_cfg = prepared_plan["virtual_machines"][0]
438+
vm_name = vm_cfg["name"]
439+
vm_namespace = prepared_plan["_vm_target_namespace"]
440+
441+
vm = VirtualMachine(
442+
client=ocp_admin_client,
443+
name=vm_name,
444+
namespace=vm_namespace,
445+
)
446+
447+
source_vm = source_provider.vm_dict(name=vm_name)
448+
449+
try:
450+
if not vm.ready:
451+
LOGGER.info(f"Starting migrated VM {vm_name} for data integrity check")
452+
vm.start(wait=True, timeout=300)
453+
else:
454+
LOGGER.info(f"Migrated VM {vm_name} is already running, skipping start")
455+
verify_data_integrity(
456+
vm_name=vm_name,
457+
vm_ssh_connections=vm_ssh_connections,
458+
source_provider_data=source_provider_data,
459+
source_vm_info=source_vm,
460+
expected_marker_content=self.data_integrity_marker,
461+
)
462+
finally:
463+
LOGGER.info(f"Stopping VM {vm_name} after data integrity check")
464+
vm.stop(wait=True, timeout=300)
465+
408466

409467
@pytest.mark.copyoffload
410468
@pytest.mark.incremental
411469
@pytest.mark.parametrize(
412470
"class_plan_config",
413471
[pytest.param(py_config["tests_params"]["test_copyoffload_thin_snapshots_migration"])],
414472
indirect=True,
415-
ids=["copyoffload-thin-snapshots"],
473+
ids=["MTV-560:copyoffload-thin-snapshots"],
416474
)
417475
@pytest.mark.copyoffload_snapshots
418476
@pytest.mark.usefixtures(

‎tests/tests_config/config.py‎

Lines changed: 0 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -65,7 +65,6 @@
6565
"virtual_machines": [
6666
{
6767
"name": "xcopy-template-test",
68-
"source_vm_power": "off",
6968
"guest_agent": True,
7069
"clone": True,
7170
"disk_type": "thin",
@@ -78,7 +77,6 @@
7877
"virtual_machines": [
7978
{
8079
"name": "xcopy-template-test",
81-
"source_vm_power": "off",
8280
"guest_agent": True,
8381
"clone": True,
8482
"disk_type": "thick-lazy",
@@ -91,7 +89,6 @@
9189
"virtual_machines": [
9290
{
9391
"name": "xcopy-template-test",
94-
"source_vm_power": "off",
9592
"guest_agent": True,
9693
"clone": True,
9794
"add_disks": [
@@ -106,7 +103,6 @@
106103
"virtual_machines": [
107104
{
108105
"name": "xcopy-template-test",
109-
"source_vm_power": "off",
110106
"guest_agent": True,
111107
"clone": True,
112108
"add_disks": [
@@ -126,7 +122,6 @@
126122
"virtual_machines": [
127123
{
128124
"name": "xcopy-template-test",
129-
"source_vm_power": "off",
130125
"guest_agent": True,
131126
"clone": True,
132127
"add_disks": [
@@ -141,7 +136,6 @@
141136
"virtual_machines": [
142137
{
143138
"name": "xcopy-template-test",
144-
"source_vm_power": "off",
145139
"guest_agent": True,
146140
"clone": True,
147141
"disk_type": "thin",
@@ -155,7 +149,6 @@
155149
"virtual_machines": [
156150
{
157151
"name": "xcopy-template-test",
158-
"source_vm_power": "off",
159152
"guest_agent": True,
160153
"clone": True,
161154
"disk_type": "thin",
@@ -176,7 +169,6 @@
176169
"virtual_machines": [
177170
{
178171
"name": "xcopy-template-test",
179-
"source_vm_power": "off",
180172
"guest_agent": True,
181173
"clone": True,
182174
"disk_type": "thin",
@@ -196,7 +188,6 @@
196188
"virtual_machines": [
197189
{
198190
"name": "xcopy-template-test",
199-
"source_vm_power": "off",
200191
"guest_agent": True,
201192
"clone": True,
202193
"disk_type": "thin",
@@ -215,7 +206,6 @@
215206
"virtual_machines": [
216207
{
217208
"name": "xcopy-template-test",
218-
"source_vm_power": "off",
219209
"guest_agent": True,
220210
"clone": True,
221211
"disk_type": "thin",
@@ -240,7 +230,6 @@
240230
"virtual_machines": [
241231
{
242232
"name": "xcopy-template-test",
243-
"source_vm_power": "off",
244233
"guest_agent": True,
245234
"clone": True,
246235
"disk_type": "thin",
@@ -273,7 +262,6 @@
273262
"virtual_machines": [
274263
{
275264
"name": "xcopy-template-test",
276-
"source_vm_power": "off",
277265
"guest_agent": True,
278266
"clone": True,
279267
"disk_type": "thin",
@@ -294,7 +282,6 @@
294282
"virtual_machines": [
295283
{
296284
"name": "xcopy-template-test",
297-
"source_vm_power": "off",
298285
"guest_agent": True,
299286
"clone": True,
300287
"disk_type": "thin",
@@ -314,14 +301,12 @@
314301
"virtual_machines": [
315302
{
316303
"name": "xcopy-template-test",
317-
"source_vm_power": "off",
318304
"guest_agent": True,
319305
"clone": True,
320306
"disk_type": "thin",
321307
},
322308
{
323309
"name": "xcopy-template-test",
324-
"source_vm_power": "off",
325310
"guest_agent": True,
326311
"clone": True,
327312
"disk_type": "thin",
@@ -336,7 +321,6 @@
336321
"name": "xcopy-template-test",
337322
"clone_name": "XCopy_Test_VM_CAPS", # Non-conforming name for cloned VM
338323
"preserve_name_format": True, # Don't sanitize the name (keep capitals and underscores)
339-
"source_vm_power": "off",
340324
"guest_agent": True,
341325
"clone": True,
342326
"disk_type": "thin",
@@ -349,7 +333,6 @@
349333
"virtual_machines": [
350334
{
351335
"name": "xcopy-template-test",
352-
"source_vm_power": "off",
353336
"guest_agent": True,
354337
"clone": True,
355338
"target_datastore_id": "non_xcopy_datastore_id",
@@ -370,39 +353,34 @@
370353
"virtual_machines": [
371354
{
372355
"name": "xcopy-template-test",
373-
"source_vm_power": "off",
374356
"guest_agent": True,
375357
"clone": True,
376358
"disk_type": "thick-lazy",
377359
"add_disks": [{"size_gb": 30, "provision_type": "thick-lazy", "disk_mode": "persistent"}],
378360
},
379361
{
380362
"name": "xcopy-template-test",
381-
"source_vm_power": "off",
382363
"guest_agent": True,
383364
"clone": True,
384365
"disk_type": "thick-lazy",
385366
"add_disks": [{"size_gb": 30, "provision_type": "thick-lazy", "disk_mode": "persistent"}],
386367
},
387368
{
388369
"name": "xcopy-template-test",
389-
"source_vm_power": "off",
390370
"guest_agent": True,
391371
"clone": True,
392372
"disk_type": "thick-lazy",
393373
"add_disks": [{"size_gb": 30, "provision_type": "thick-lazy", "disk_mode": "persistent"}],
394374
},
395375
{
396376
"name": "xcopy-template-test",
397-
"source_vm_power": "off",
398377
"guest_agent": True,
399378
"clone": True,
400379
"disk_type": "thick-lazy",
401380
"add_disks": [{"size_gb": 30, "provision_type": "thick-lazy", "disk_mode": "persistent"}],
402381
},
403382
{
404383
"name": "xcopy-template-test",
405-
"source_vm_power": "off",
406384
"guest_agent": True,
407385
"clone": True,
408386
"disk_type": "thick-lazy",
@@ -417,14 +395,12 @@
417395
"virtual_machines": [
418396
{
419397
"name": "xcopy-template-test",
420-
"source_vm_power": "off",
421398
"guest_agent": True,
422399
"clone": True,
423400
"disk_type": "thick-lazy",
424401
},
425402
{
426403
"name": "xcopy-template-test",
427-
"source_vm_power": "off",
428404
"guest_agent": True,
429405
"clone": True,
430406
"disk_type": "thick-lazy",

‎utilities/copyoffload_migration.py‎

Lines changed: 3 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -140,7 +140,8 @@ def wait_for_cloud_init(
140140
provider_vm_api: Provider VM object
141141
file_name: Full path to the file to check for (e.g., "/cloud-init.finish")
142142
timeout: Timeout in seconds (default: 2000)
143-
target_power_state: Desired power state after check ("on" or "off", default: "off")
143+
target_power_state: Expected source VM power state for downstream validation ("on" or "off",
144+
default: "off"). When "off", logs that MTV will handle shutdown. Does not change VM power.
144145
145146
Raises:
146147
TimeoutExpiredError: If cloud-init does not finish within timeout
@@ -205,11 +206,7 @@ def _check_file() -> bool:
205206

206207
finally:
207208
if target_power_state == "off":
208-
LOGGER.info(f"Powering off VM - {vm_name}")
209-
try:
210-
source_provider.stop_vm(provider_vm_api)
211-
except Exception as e:
212-
LOGGER.warning(f"Failed to power off VM '{vm_name}': {type(e).__name__}: {e}")
209+
LOGGER.info(f"VM {vm_name} left powered on — MTV will handle shutdown for cold migration")
213210
else:
214211
LOGGER.info(f"Leaving VM {vm_name} powered on")
215212

0 commit comments

Comments
 (0)