test(v2): add e2e cases for v2 block disk device recovery

e2e/keywords/node.resource

@@ -200,3 +200,25 @@ Remove dir ${dir_path} on node ${node_id}
 Wait for Longhorn node ${node_id} down
     ${node_name} =    get_node_by_index    ${node_id}
     wait_for_longhorn_node_down    ${node_name}
+
+Skip test if disk path ${disk_path} is not a PCI BDF
+    ${is_bdf} =    is_bdf_disk_path    ${disk_path}
+    IF    not ${is_bdf}
+        Skip    Test requires a block disk provisioned by PCI BDF, got ${disk_path}
+    END
+
+Bind device ${disk_path} on node ${node_id} to userspace driver
+    ${node_name} =    get_node_by_index    ${node_id}
+    bind_disk_device_to_userspace_driver    ${node_name}    ${disk_path}
+
+Unbind device ${disk_path} on node ${node_id} from userspace driver
+    ${node_name} =    get_node_by_index    ${node_id}
+    unbind_disk_device_from_userspace_driver    ${node_name}    ${disk_path}
+
+Wait for device ${disk_path} on node ${node_id} released from userspace driver
+    ${node_name} =    get_node_by_index    ${node_id}
+    wait_for_disk_device_released    ${node_name}    ${disk_path}
+
+Wait for disk ${disk_name} on node ${node_id} schedulable
+    ${node_name} =    get_node_by_index    ${node_id}
+    wait_for_disk_schedulable    ${node_name}    ${disk_name}

e2e/libs/keywords/node_keywords.py

@@ -173,3 +173,21 @@ def wait_for_node_disks_schedulable(self, node_name, disk_type=None, expected_sc
 
     def remove_dir(self, dir_path, node_name):
         self.node.remove_dir(dir_path, node_name)
+
+    def is_bdf_disk_path(self, disk_path):
+        return self.node.is_bdf_disk_path(disk_path)
+
+    def get_disk_device_driver(self, node_name, bdf):
+        return self.node.get_disk_device_driver(node_name, bdf)
+
+    def bind_disk_device_to_userspace_driver(self, node_name, bdf):
+        self.node.bind_disk_device_to_userspace_driver(node_name, bdf)
+
+    def unbind_disk_device_from_userspace_driver(self, node_name, bdf):
+        self.node.unbind_disk_device_from_userspace_driver(node_name, bdf)
+
+    def wait_for_disk_device_released(self, node_name, bdf):
+        self.node.wait_for_disk_device_released(node_name, bdf)
+
+    def wait_for_disk_schedulable(self, node_name, disk_name):
+        self.node.wait_for_disk_schedulable(node_name, disk_name)

e2e/libs/node/node.py

@@ -1,3 +1,4 @@
+import json
 import time
 import re
 import os
@@ -14,6 +15,8 @@
 from utility.utility import get_retry_count_and_interval
 from utility.utility import logging
 from utility.utility import subprocess_exec_cmd
+from utility.utility import pod_exec
+from workload.pod import list_pods
 from node_exec import NodeExec
 
 class Node:
@@ -712,3 +715,80 @@ def remove_dir(self, dir_path, node_name):
         logging(f"Removing directory {dir_path} on node {node_name}")
         cmd = f"rm -rf {dir_path}"
         NodeExec(node_name).issue_cmd(cmd)
+
+    def is_bdf_disk_path(self, disk_path):
+        return re.match(constant.BDF_PATTERN, str(disk_path)) is not None
+
+    def get_v2_instance_manager_pod_name(self, node_name):
+        label_selector = f"longhorn.io/component=instance-manager,longhorn.io/data-engine=v2,longhorn.io/node={node_name}"
+        pods = list_pods(constant.LONGHORN_NAMESPACE, label_selector)
+        running_pods = [pod for pod in pods if pod.status.phase == "Running" and pod.metadata.deletion_timestamp is None]
+        assert len(running_pods) > 0, f"Failed to find a running v2 instance manager pod on node {node_name}"
+        return running_pods[0].metadata.name
+
+    def get_disk_device_driver(self, node_name, bdf):
+        """Return the PCI driver the device is currently bound to, e.g. nvme or vfio-pci."""
+        pod_name = self.get_v2_instance_manager_pod_name(node_name)
+        output = pod_exec(pod_name, constant.LONGHORN_NAMESPACE, f"{constant.SPDK_SETUP_SCRIPT} disk-status {bdf}")
+        matched = re.search(r"\{.*\}", output, re.S)
+        assert matched, f"Failed to get the disk status of {bdf} on node {node_name}: {output}"
+        return json.loads(matched.group(0)).get("driver", "")
+
+    def is_disk_device_detached_from_kernel_driver(self, node_name, bdf):
+        """An interrupted bind leaves the device on a userspace driver, or on no
+        driver at all when the userspace bind failed after the kernel driver was
+        already released. Both states hide the block device from the kernel."""
+        driver = self.get_disk_device_driver(node_name, bdf)
+        return driver.replace("-", "_") in constant.USERSPACE_PCI_DRIVERS or \
+            driver in ("", constant.NO_PCI_DRIVER)
+
+    def bind_disk_device_to_userspace_driver(self, node_name, bdf):
+        """Simulate a disk creation that was interrupted after binding the device."""
+        pod_name = self.get_v2_instance_manager_pod_name(node_name)
+        logging(f"Binding device {bdf} on node {node_name} to a userspace PCI driver")
+        # The bind mode only rebinds the device, unlike the default config mode which
+        # also reallocates hugepages underneath the running SPDK target.
+        pod_exec(pod_name, constant.LONGHORN_NAMESPACE,
+                 f"PCI_ALLOWED={bdf} DRIVER_OVERRIDE=vfio-pci {constant.SPDK_SETUP_SCRIPT} bind")
+        self.wait_for_disk_device_detached_from_kernel_driver(node_name, bdf)
+
+    def unbind_disk_device_from_userspace_driver(self, node_name, bdf):
+        pod_name = self.get_v2_instance_manager_pod_name(node_name)
+        logging(f"Unbinding device {bdf} on node {node_name} from its userspace PCI driver")
+        pod_exec(pod_name, constant.LONGHORN_NAMESPACE, f"{constant.SPDK_SETUP_SCRIPT} unbind {bdf}")
+
+    def wait_for_disk_device_detached_from_kernel_driver(self, node_name, bdf):
+        for i in range(self.retry_count):
+            logging(f"Waiting for device {bdf} on node {node_name} detached from its kernel driver ... ({i})")
+            if self.is_disk_device_detached_from_kernel_driver(node_name, bdf):
+                return
+            time.sleep(self.retry_interval)
+        assert False, f"Device {bdf} on node {node_name} is still driven by the kernel: {self.get_disk_device_driver(node_name, bdf)}"
+
+    def wait_for_disk_device_released(self, node_name, bdf):
+        for i in range(self.retry_count):
+            driver = self.get_disk_device_driver(node_name, bdf)
+            logging(f"Waiting for device {bdf} on node {node_name} released back to its kernel driver, current driver = {driver} ... ({i})")
+            if not self.is_disk_device_detached_from_kernel_driver(node_name, bdf):
+                return
+            time.sleep(self.retry_interval)
+        assert False, f"Device {bdf} on node {node_name} is not driven by the kernel, current driver {self.get_disk_device_driver(node_name, bdf)}"
+
+    def is_disk_schedulable(self, node_name, disk_name):
+        node = get_longhorn_client().by_id_node(node_name)
+        disk = node["disks"].get(disk_name)
+        if disk is None:
+            return False
+        conditions = disk["conditions"]
+        return conditions["Ready"]["status"] == "True" and conditions["Schedulable"]["status"] == "True"
+
+    def wait_for_disk_schedulable(self, node_name, disk_name):
+        for i in range(self.retry_count):
+            logging(f"Waiting for disk {disk_name} on node {node_name} ready and schedulable ... ({i})")
+            try:
+                if self.is_disk_schedulable(node_name, disk_name):
+                    return
+            except Exception as e:
+                logging(f"Getting disk {disk_name} on node {node_name} failed: {e}")
+            time.sleep(self.retry_interval)
+        assert False, f"Disk {disk_name} on node {node_name} is not ready and schedulable: {get_longhorn_client().by_id_node(node_name)['disks']}"

e2e/libs/utility/constant.py

@@ -49,3 +49,9 @@
 
 DEFAULT_BLOCK_DISK_NAME = "block-disk"
 DEFAULT_FILESYSTEM_DISK_NAME_PREFIX = "default-disk"
+
+SPDK_SETUP_SCRIPT = "/usr/src/spdk/scripts/setup.sh"
+BDF_PATTERN = r"^[a-fA-F0-9]{4}:[a-fA-F0-9]{2}:[a-fA-F0-9]{2}\.[a-fA-F0-9]$"
+USERSPACE_PCI_DRIVERS = ["vfio_pci", "uio_pci_generic"]
+# What the SPDK setup script reports for a device that is bound to no driver.
+NO_PCI_DRIVER = "-"

e2e/tests/negative/v2_block_disk_device_recovery.robot

@@ -0,0 +1,75 @@
+*** Settings ***
+Documentation    Negative Test Cases for v2 block disk device recovery
+...    Ref: https://github.com/longhorn/longhorn/issues/13893
+
+Test Tags    negative    v2    node-disk-mgmt
+
+Resource    ../keywords/variables.resource
+Resource    ../keywords/common.resource
+Resource    ../keywords/node.resource
+Resource    ../keywords/longhorn.resource
+Resource    ../keywords/volume.resource
+
+Test Setup    Set up v2 block disk test environment
+Test Teardown    Cleanup test resources
+
+*** Keywords ***
+Set up v2 block disk test environment
+    Set up test environment
+    Enable v2 data engine and add block disks
+    Skip test if disk path ${DISK_PATH} is not a PCI BDF
+
+*** Test Cases ***
+V2 Block Disk Should Become Ready When Its Device Is Left Bound To Userspace Driver
+    [Documentation]    An interrupted disk creation can leave the NVMe device bound to
+    ...    vfio-pci. With diskDriver auto, the device must still be resolved back to the
+    ...    nvme driver, otherwise the disk stays Ready=False and Schedulable=False forever.
+    IF    '${DATA_ENGINE}' == 'v1'
+        Skip    Test only validate on v2 data engine
+    END
+
+    Given Disable disk ${DEFAULT_BLOCK_DISK_NAME} scheduling without ready check on node 0
+    And Delete disk ${DEFAULT_BLOCK_DISK_NAME} on node 0
+    And Wait for device ${DISK_PATH} on node 0 released from userspace driver
+
+    When Bind device ${DISK_PATH} on node 0 to userspace driver
+    And Add block disk ${DEFAULT_BLOCK_DISK_NAME} to node 0 with path ${DISK_PATH}
+
+    Then Wait for disk ${DEFAULT_BLOCK_DISK_NAME} on node 0 schedulable
+
+V2 Block Disk Device Should Be Released After The Disk Is Removed
+    [Documentation]    Removing a block disk must hand the device back to the kernel.
+    ...    Leaving it bound to a userspace driver makes the disk unusable on re-provisioning.
+    IF    '${DATA_ENGINE}' == 'v1'
+        Skip    Test only validate on v2 data engine
+    END
+
+    Given Wait for disk ${DEFAULT_BLOCK_DISK_NAME} on node 0 schedulable
+
+    When Disable disk ${DEFAULT_BLOCK_DISK_NAME} scheduling without ready check on node 0
+    And Delete disk ${DEFAULT_BLOCK_DISK_NAME} on node 0
+
+    Then Wait for device ${DISK_PATH} on node 0 released from userspace driver
+    And Add block disk ${DEFAULT_BLOCK_DISK_NAME} to node 0 with path ${DISK_PATH}
+    And Wait for disk ${DEFAULT_BLOCK_DISK_NAME} on node 0 schedulable
+
+V2 Block Disk Device Should Be Released After Instance Manager Restart
+    [Documentation]    The disk record only lives in the instance manager memory. After a
+    ...    restart the disk deletion must still release the device instead of reporting
+    ...    success and leaking it.
+    IF    '${DATA_ENGINE}' == 'v1'
+        Skip    Test only validate on v2 data engine
+    END
+
+    Given Wait for disk ${DEFAULT_BLOCK_DISK_NAME} on node 0 schedulable
+
+    # Deleting the disk while the instance manager has no record of it forces the
+    # deletion through the orphan device release path.
+    When Disable disk ${DEFAULT_BLOCK_DISK_NAME} scheduling without ready check on node 0
+    And Delete v2 instance manager on node 0
+    And Wait for node 0 block disk unschedulable
+    And Delete disk ${DEFAULT_BLOCK_DISK_NAME} on node 0
+
+    Then Wait for device ${DISK_PATH} on node 0 released from userspace driver
+    And Add block disk ${DEFAULT_BLOCK_DISK_NAME} to node 0 with path ${DISK_PATH}
+    And Wait for disk ${DEFAULT_BLOCK_DISK_NAME} on node 0 schedulable

e2e/tests/regression/test_v2.robot

@@ -642,3 +642,37 @@ Test CPU Manager Policy And Data Engine Number Of CPU Cores
     And Run command in pod ${LONGHORN_NAMESPACE}/${im_pod} and wait for output
     ...    awk '/^Cpus_allowed_list:/ {print $2}' /proc/self/status
     ...    ^[0-9]+-[0-9]+$
+
+Test V2 Block Disk Recovery After Interrupted Provisioning
+    [Tags]    node-disk-mgmt
+    [Documentation]    Regression for longhorn/longhorn#13893.
+    ...    A disk creation interrupted after the device was bound leaves the NVMe device
+    ...    on vfio-pci and records neither diskDriver, diskPath nor diskUUID. Removing and
+    ...    re-adding the disk must still recover it instead of keeping it Ready=False and
+    ...    Schedulable=False with "unsupported disk driver vfio-pci for disk path".
+    IF    '${DATA_ENGINE}' == 'v1'
+        Skip    Test only validate on v2 data engine
+    END
+    Skip test if disk path ${DISK_PATH} is not a PCI BDF
+
+    Given Disable disk ${DEFAULT_BLOCK_DISK_NAME} scheduling without ready check on node 0
+    And Delete disk ${DEFAULT_BLOCK_DISK_NAME} on node 0
+    And Wait for device ${DISK_PATH} on node 0 released from userspace driver
+
+    # Simulate a provisioning that failed after binding the device.
+    When Bind device ${DISK_PATH} on node 0 to userspace driver
+    And Add block disk ${DEFAULT_BLOCK_DISK_NAME} to node 0 with path ${DISK_PATH}
+    Then Wait for disk ${DEFAULT_BLOCK_DISK_NAME} on node 0 schedulable
+
+    # Unprovisioning must hand the device back so that it can be provisioned again.
+    When Disable disk ${DEFAULT_BLOCK_DISK_NAME} scheduling without ready check on node 0
+    And Delete disk ${DEFAULT_BLOCK_DISK_NAME} on node 0
+    Then Wait for device ${DISK_PATH} on node 0 released from userspace driver
+
+    When Add block disk ${DEFAULT_BLOCK_DISK_NAME} to node 0 with path ${DISK_PATH}
+    Then Wait for disk ${DEFAULT_BLOCK_DISK_NAME} on node 0 schedulable
+    And Create volume 0 with    dataEngine=v2
+    And Attach volume 0 to node 0
+    And Wait for volume 0 healthy
+    And Write data to volume 0
+    And Check volume 0 data is intact