diff --git a/e2e/keywords/node.resource b/e2e/keywords/node.resource index e05856a264..1224a733d2 100644 --- a/e2e/keywords/node.resource +++ b/e2e/keywords/node.resource @@ -200,3 +200,25 @@ Remove dir ${dir_path} on node ${node_id} Wait for Longhorn node ${node_id} down ${node_name} = get_node_by_index ${node_id} wait_for_longhorn_node_down ${node_name} + +Skip test if disk path ${disk_path} is not a PCI BDF + ${is_bdf} = is_bdf_disk_path ${disk_path} + IF not ${is_bdf} + Skip Test requires a block disk provisioned by PCI BDF, got ${disk_path} + END + +Bind device ${disk_path} on node ${node_id} to userspace driver + ${node_name} = get_node_by_index ${node_id} + bind_disk_device_to_userspace_driver ${node_name} ${disk_path} + +Unbind device ${disk_path} on node ${node_id} from userspace driver + ${node_name} = get_node_by_index ${node_id} + unbind_disk_device_from_userspace_driver ${node_name} ${disk_path} + +Wait for device ${disk_path} on node ${node_id} released from userspace driver + ${node_name} = get_node_by_index ${node_id} + wait_for_disk_device_released ${node_name} ${disk_path} + +Wait for disk ${disk_name} on node ${node_id} schedulable + ${node_name} = get_node_by_index ${node_id} + wait_for_disk_schedulable ${node_name} ${disk_name} diff --git a/e2e/libs/keywords/node_keywords.py b/e2e/libs/keywords/node_keywords.py index 28445fe6a1..a9695732f3 100644 --- a/e2e/libs/keywords/node_keywords.py +++ b/e2e/libs/keywords/node_keywords.py @@ -173,3 +173,21 @@ def wait_for_node_disks_schedulable(self, node_name, disk_type=None, expected_sc def remove_dir(self, dir_path, node_name): self.node.remove_dir(dir_path, node_name) + + def is_bdf_disk_path(self, disk_path): + return self.node.is_bdf_disk_path(disk_path) + + def get_disk_device_driver(self, node_name, bdf): + return self.node.get_disk_device_driver(node_name, bdf) + + def bind_disk_device_to_userspace_driver(self, node_name, bdf): + self.node.bind_disk_device_to_userspace_driver(node_name, bdf) + + def unbind_disk_device_from_userspace_driver(self, node_name, bdf): + self.node.unbind_disk_device_from_userspace_driver(node_name, bdf) + + def wait_for_disk_device_released(self, node_name, bdf): + self.node.wait_for_disk_device_released(node_name, bdf) + + def wait_for_disk_schedulable(self, node_name, disk_name): + self.node.wait_for_disk_schedulable(node_name, disk_name) diff --git a/e2e/libs/node/node.py b/e2e/libs/node/node.py index e004d637ab..5f01df3a38 100644 --- a/e2e/libs/node/node.py +++ b/e2e/libs/node/node.py @@ -1,3 +1,4 @@ +import json import time import re import os @@ -14,6 +15,8 @@ from utility.utility import get_retry_count_and_interval from utility.utility import logging from utility.utility import subprocess_exec_cmd +from utility.utility import pod_exec +from workload.pod import list_pods from node_exec import NodeExec class Node: @@ -712,3 +715,80 @@ def remove_dir(self, dir_path, node_name): logging(f"Removing directory {dir_path} on node {node_name}") cmd = f"rm -rf {dir_path}" NodeExec(node_name).issue_cmd(cmd) + + def is_bdf_disk_path(self, disk_path): + return re.match(constant.BDF_PATTERN, str(disk_path)) is not None + + def get_v2_instance_manager_pod_name(self, node_name): + label_selector = f"longhorn.io/component=instance-manager,longhorn.io/data-engine=v2,longhorn.io/node={node_name}" + pods = list_pods(constant.LONGHORN_NAMESPACE, label_selector) + running_pods = [pod for pod in pods if pod.status.phase == "Running" and pod.metadata.deletion_timestamp is None] + assert len(running_pods) > 0, f"Failed to find a running v2 instance manager pod on node {node_name}" + return running_pods[0].metadata.name + + def get_disk_device_driver(self, node_name, bdf): + """Return the PCI driver the device is currently bound to, e.g. nvme or vfio-pci.""" + pod_name = self.get_v2_instance_manager_pod_name(node_name) + output = pod_exec(pod_name, constant.LONGHORN_NAMESPACE, f"{constant.SPDK_SETUP_SCRIPT} disk-status {bdf}") + matched = re.search(r"\{.*\}", output, re.S) + assert matched, f"Failed to get the disk status of {bdf} on node {node_name}: {output}" + return json.loads(matched.group(0)).get("driver", "") + + def is_disk_device_detached_from_kernel_driver(self, node_name, bdf): + """An interrupted bind leaves the device on a userspace driver, or on no + driver at all when the userspace bind failed after the kernel driver was + already released. Both states hide the block device from the kernel.""" + driver = self.get_disk_device_driver(node_name, bdf) + return driver.replace("-", "_") in constant.USERSPACE_PCI_DRIVERS or \ + driver in ("", constant.NO_PCI_DRIVER) + + def bind_disk_device_to_userspace_driver(self, node_name, bdf): + """Simulate a disk creation that was interrupted after binding the device.""" + pod_name = self.get_v2_instance_manager_pod_name(node_name) + logging(f"Binding device {bdf} on node {node_name} to a userspace PCI driver") + # The bind mode only rebinds the device, unlike the default config mode which + # also reallocates hugepages underneath the running SPDK target. + pod_exec(pod_name, constant.LONGHORN_NAMESPACE, + f"PCI_ALLOWED={bdf} DRIVER_OVERRIDE=vfio-pci {constant.SPDK_SETUP_SCRIPT} bind") + self.wait_for_disk_device_detached_from_kernel_driver(node_name, bdf) + + def unbind_disk_device_from_userspace_driver(self, node_name, bdf): + pod_name = self.get_v2_instance_manager_pod_name(node_name) + logging(f"Unbinding device {bdf} on node {node_name} from its userspace PCI driver") + pod_exec(pod_name, constant.LONGHORN_NAMESPACE, f"{constant.SPDK_SETUP_SCRIPT} unbind {bdf}") + + def wait_for_disk_device_detached_from_kernel_driver(self, node_name, bdf): + for i in range(self.retry_count): + logging(f"Waiting for device {bdf} on node {node_name} detached from its kernel driver ... ({i})") + if self.is_disk_device_detached_from_kernel_driver(node_name, bdf): + return + time.sleep(self.retry_interval) + assert False, f"Device {bdf} on node {node_name} is still driven by the kernel: {self.get_disk_device_driver(node_name, bdf)}" + + def wait_for_disk_device_released(self, node_name, bdf): + for i in range(self.retry_count): + driver = self.get_disk_device_driver(node_name, bdf) + logging(f"Waiting for device {bdf} on node {node_name} released back to its kernel driver, current driver = {driver} ... ({i})") + if not self.is_disk_device_detached_from_kernel_driver(node_name, bdf): + return + time.sleep(self.retry_interval) + assert False, f"Device {bdf} on node {node_name} is not driven by the kernel, current driver {self.get_disk_device_driver(node_name, bdf)}" + + def is_disk_schedulable(self, node_name, disk_name): + node = get_longhorn_client().by_id_node(node_name) + disk = node["disks"].get(disk_name) + if disk is None: + return False + conditions = disk["conditions"] + return conditions["Ready"]["status"] == "True" and conditions["Schedulable"]["status"] == "True" + + def wait_for_disk_schedulable(self, node_name, disk_name): + for i in range(self.retry_count): + logging(f"Waiting for disk {disk_name} on node {node_name} ready and schedulable ... ({i})") + try: + if self.is_disk_schedulable(node_name, disk_name): + return + except Exception as e: + logging(f"Getting disk {disk_name} on node {node_name} failed: {e}") + time.sleep(self.retry_interval) + assert False, f"Disk {disk_name} on node {node_name} is not ready and schedulable: {get_longhorn_client().by_id_node(node_name)['disks']}" diff --git a/e2e/libs/utility/constant.py b/e2e/libs/utility/constant.py index 2a2eb251ce..f78fe9a632 100644 --- a/e2e/libs/utility/constant.py +++ b/e2e/libs/utility/constant.py @@ -49,3 +49,9 @@ DEFAULT_BLOCK_DISK_NAME = "block-disk" DEFAULT_FILESYSTEM_DISK_NAME_PREFIX = "default-disk" + +SPDK_SETUP_SCRIPT = "/usr/src/spdk/scripts/setup.sh" +BDF_PATTERN = r"^[a-fA-F0-9]{4}:[a-fA-F0-9]{2}:[a-fA-F0-9]{2}\.[a-fA-F0-9]$" +USERSPACE_PCI_DRIVERS = ["vfio_pci", "uio_pci_generic"] +# What the SPDK setup script reports for a device that is bound to no driver. +NO_PCI_DRIVER = "-" diff --git a/e2e/tests/negative/v2_block_disk_device_recovery.robot b/e2e/tests/negative/v2_block_disk_device_recovery.robot new file mode 100644 index 0000000000..a183b4557d --- /dev/null +++ b/e2e/tests/negative/v2_block_disk_device_recovery.robot @@ -0,0 +1,75 @@ +*** Settings *** +Documentation Negative Test Cases for v2 block disk device recovery +... Ref: https://github.com/longhorn/longhorn/issues/13893 + +Test Tags negative v2 node-disk-mgmt + +Resource ../keywords/variables.resource +Resource ../keywords/common.resource +Resource ../keywords/node.resource +Resource ../keywords/longhorn.resource +Resource ../keywords/volume.resource + +Test Setup Set up v2 block disk test environment +Test Teardown Cleanup test resources + +*** Keywords *** +Set up v2 block disk test environment + Set up test environment + Enable v2 data engine and add block disks + Skip test if disk path ${DISK_PATH} is not a PCI BDF + +*** Test Cases *** +V2 Block Disk Should Become Ready When Its Device Is Left Bound To Userspace Driver + [Documentation] An interrupted disk creation can leave the NVMe device bound to + ... vfio-pci. With diskDriver auto, the device must still be resolved back to the + ... nvme driver, otherwise the disk stays Ready=False and Schedulable=False forever. + IF '${DATA_ENGINE}' == 'v1' + Skip Test only validate on v2 data engine + END + + Given Disable disk ${DEFAULT_BLOCK_DISK_NAME} scheduling without ready check on node 0 + And Delete disk ${DEFAULT_BLOCK_DISK_NAME} on node 0 + And Wait for device ${DISK_PATH} on node 0 released from userspace driver + + When Bind device ${DISK_PATH} on node 0 to userspace driver + And Add block disk ${DEFAULT_BLOCK_DISK_NAME} to node 0 with path ${DISK_PATH} + + Then Wait for disk ${DEFAULT_BLOCK_DISK_NAME} on node 0 schedulable + +V2 Block Disk Device Should Be Released After The Disk Is Removed + [Documentation] Removing a block disk must hand the device back to the kernel. + ... Leaving it bound to a userspace driver makes the disk unusable on re-provisioning. + IF '${DATA_ENGINE}' == 'v1' + Skip Test only validate on v2 data engine + END + + Given Wait for disk ${DEFAULT_BLOCK_DISK_NAME} on node 0 schedulable + + When Disable disk ${DEFAULT_BLOCK_DISK_NAME} scheduling without ready check on node 0 + And Delete disk ${DEFAULT_BLOCK_DISK_NAME} on node 0 + + Then Wait for device ${DISK_PATH} on node 0 released from userspace driver + And Add block disk ${DEFAULT_BLOCK_DISK_NAME} to node 0 with path ${DISK_PATH} + And Wait for disk ${DEFAULT_BLOCK_DISK_NAME} on node 0 schedulable + +V2 Block Disk Device Should Be Released After Instance Manager Restart + [Documentation] The disk record only lives in the instance manager memory. After a + ... restart the disk deletion must still release the device instead of reporting + ... success and leaking it. + IF '${DATA_ENGINE}' == 'v1' + Skip Test only validate on v2 data engine + END + + Given Wait for disk ${DEFAULT_BLOCK_DISK_NAME} on node 0 schedulable + + # Deleting the disk while the instance manager has no record of it forces the + # deletion through the orphan device release path. + When Disable disk ${DEFAULT_BLOCK_DISK_NAME} scheduling without ready check on node 0 + And Delete v2 instance manager on node 0 + And Wait for node 0 block disk unschedulable + And Delete disk ${DEFAULT_BLOCK_DISK_NAME} on node 0 + + Then Wait for device ${DISK_PATH} on node 0 released from userspace driver + And Add block disk ${DEFAULT_BLOCK_DISK_NAME} to node 0 with path ${DISK_PATH} + And Wait for disk ${DEFAULT_BLOCK_DISK_NAME} on node 0 schedulable diff --git a/e2e/tests/regression/test_v2.robot b/e2e/tests/regression/test_v2.robot index 940e2b0f9f..6130083806 100644 --- a/e2e/tests/regression/test_v2.robot +++ b/e2e/tests/regression/test_v2.robot @@ -642,3 +642,37 @@ Test CPU Manager Policy And Data Engine Number Of CPU Cores And Run command in pod ${LONGHORN_NAMESPACE}/${im_pod} and wait for output ... awk '/^Cpus_allowed_list:/ {print $2}' /proc/self/status ... ^[0-9]+-[0-9]+$ + +Test V2 Block Disk Recovery After Interrupted Provisioning + [Tags] node-disk-mgmt + [Documentation] Regression for longhorn/longhorn#13893. + ... A disk creation interrupted after the device was bound leaves the NVMe device + ... on vfio-pci and records neither diskDriver, diskPath nor diskUUID. Removing and + ... re-adding the disk must still recover it instead of keeping it Ready=False and + ... Schedulable=False with "unsupported disk driver vfio-pci for disk path". + IF '${DATA_ENGINE}' == 'v1' + Skip Test only validate on v2 data engine + END + Skip test if disk path ${DISK_PATH} is not a PCI BDF + + Given Disable disk ${DEFAULT_BLOCK_DISK_NAME} scheduling without ready check on node 0 + And Delete disk ${DEFAULT_BLOCK_DISK_NAME} on node 0 + And Wait for device ${DISK_PATH} on node 0 released from userspace driver + + # Simulate a provisioning that failed after binding the device. + When Bind device ${DISK_PATH} on node 0 to userspace driver + And Add block disk ${DEFAULT_BLOCK_DISK_NAME} to node 0 with path ${DISK_PATH} + Then Wait for disk ${DEFAULT_BLOCK_DISK_NAME} on node 0 schedulable + + # Unprovisioning must hand the device back so that it can be provisioned again. + When Disable disk ${DEFAULT_BLOCK_DISK_NAME} scheduling without ready check on node 0 + And Delete disk ${DEFAULT_BLOCK_DISK_NAME} on node 0 + Then Wait for device ${DISK_PATH} on node 0 released from userspace driver + + When Add block disk ${DEFAULT_BLOCK_DISK_NAME} to node 0 with path ${DISK_PATH} + Then Wait for disk ${DEFAULT_BLOCK_DISK_NAME} on node 0 schedulable + And Create volume 0 with dataEngine=v2 + And Attach volume 0 to node 0 + And Wait for volume 0 healthy + And Write data to volume 0 + And Check volume 0 data is intact