feat(linux): network isolated cluster install package via cached first

e2e/scenario_test.go

@@ -1035,6 +1035,7 @@ func Test_Ubuntu2204Gen2_Containerd_NetworkIsolatedCluster_NoneCached(t *testing
 					PrivateEgress: &datamodel.PrivateEgress{
 						Enabled:                 true,
 						ContainerRegistryServer: fmt.Sprintf("%s.azurecr.io/aks-managed-repository", config.PrivateACRName(config.Config.DefaultLocation)),
+						TestMode:                true,
 					},
 				}
 				nbc.AgentPoolProfile.LocalDNSProfile = nil

parts/linux/cloud-init/artifacts/cse_cmd.sh

@@ -178,6 +178,7 @@ SYSCTL_CONTENT="{{GetSysctlContent}}"
 PRIVATE_EGRESS_PROXY_ADDRESS="{{GetPrivateEgressProxyAddress}}"
 BOOTSTRAP_PROFILE_CONTAINER_REGISTRY_SERVER="{{GetBootstrapProfileContainerRegistryServer}}"
 MCR_REPOSITORY_BASE="{{GetMCRRepositoryBase}}"
+NETWORK_ISOLATED_CLUSTER_TEST_MODE="{{GetNetworkIsolatedClusterTestMode}}" # for ab e2e only for local ab test to download from acr regardless of cached
 ENABLE_IMDS_RESTRICTION="{{EnableIMDSRestriction}}"
 INSERT_IMDS_RESTRICTION_RULE_TO_MANGLE_TABLE="{{InsertIMDSRestrictionRuleToMangleTable}}"
 SHOULD_ENABLE_LOCALDNS="{{ShouldEnableLocalDNS}}"

parts/linux/cloud-init/artifacts/cse_install.sh

@@ -524,6 +524,18 @@ installToolFromBootstrapProfileRegistry() {
     # Try to pull distro-specific packages (e.g., .deb for Ubuntu) from registry
     local download_root="/tmp/kubernetes/downloads" # /opt folder will return permission error
 
+    if [ "${NETWORK_ISOLATED_CLUSTER_TEST_MODE}" = "true" ]; then
+        echo "NETWORK_ISOLATED_CLUSTER_TEST_MODE=true, skipping installPackageFromCache for ${tool_name}"
+    else
+        if installPackageFromCache "$tool_name" "$version"; then
+            if [ -n "$install_path" ]; then
+                mv "$(which "$tool_name")" "$install_path"
+            fi
+            return 0
+        fi
+    fi
+    echo "install from cache failed for ${tool_name}, start to pull from registry"
+
     version_tag="${version}"
     if [ "${version}" != "v*" ]; then
         version_tag="v${version_tag}"

parts/linux/cloud-init/artifacts/mariner/cse_helpers_mariner.sh

@@ -78,4 +78,46 @@ dnf_download() {
     done
     echo Executed dnf download --resolve --downloaddir="\"${downloadDir}\"" "$@" $i times;
 }
+
+findCachedRpmFileName() {
+    local packageName="${1}"
+    local desiredVersion="${2}"
+    local downloadDir="${3}"
+
+    local rpmFile=""
+    rpmFile=$(ls "${downloadDir}" | grep "${packageName}" | grep "${desiredVersion}" | sort -V | tail -n 1) || rpmFile=""
+    echo "${rpmFile}"
+}
+
+buildRpmInstallArgsFromCache() {
+    local packageName="${1}"
+    local desiredVersion="${2}"
+    local rpmFile="${3}"
+    local downloadDir="${4}"
+
+    local -a rpmArgs=("${rpmFile}")
+    local -a cachedRpmFiles=()
+    mapfile -t cachedRpmFiles < <(find "${downloadDir}" -maxdepth 1 -type f -name "*.rpm" -print 2>/dev/null | sort)
+
+    # selecting the correct version of dependency rpms from the cache
+    for cachedRpm in "${cachedRpmFiles[@]}"; do
+      if [ "${cachedRpm}" = "${rpmFile}" ]; then
+        continue
+      fi
+
+      local cachedBaseName
+      cachedBaseName=$(basename "${cachedRpm}")
+
+      case "${cachedBaseName}" in
+        *${packageName}*)
+          echo "Skipping cached ${packageName} rpm ${cachedBaseName} because it does not match desired version ${desiredVersion}" >&2
+          continue
+          ;;
+      esac
+
+      rpmArgs+=("${cachedRpm}")
+    done
+
+    printf '%s\n' "${rpmArgs[@]}"
+}
 #EOF

parts/linux/cloud-init/artifacts/mariner/cse_install_mariner.sh

@@ -411,7 +411,9 @@ installRPMPackageFromFile() {
     downloadDir="$(getPackageDownloadDir "${packageName}")"
 
     # check cached rpms for matching filename
-    rpmFile=$(ls "${downloadDir}" | grep "${packageName}" | grep "${desiredVersion}" | sort -V | tail -n 1) || rpmFile=""
+    local rpmFile
+    local -a rpmArgs=()
+    rpmFile="$(findCachedRpmFileName "${packageName}" "${desiredVersion}" "${downloadDir}")"
     if [ -z "${rpmFile}" ]; then
         if fallbackToKubeBinaryInstall "${packageName}" "${desiredVersion}"; then
             echo "Successfully installed ${packageName} version ${desiredVersion} from binary fallback"
@@ -428,36 +430,15 @@ installRPMPackageFromFile() {
         fi
         echo "Did not find cached rpm file, downloading ${packageName} version ${fullPackageVersion}"
         downloadPkgFromVersion "${packageName}" ${fullPackageVersion} "${downloadDir}"
-        rpmFile=$(ls "${downloadDir}" | grep "${packageName}" | grep "${desiredVersion}" | sort -V | tail -n 1) || rpmFile=""
+        rpmFile="$(findCachedRpmFileName "${packageName}" "${desiredVersion}" "${downloadDir}")"
     fi
     if [ -z "${rpmFile}" ]; then
         echo "Failed to locate ${packageName} rpm"
         return 1
     fi
 
     rpmFile="${downloadDir}/${rpmFile}"
-    local rpmArgs=("${rpmFile}")
-    local -a cachedRpmFiles=()
-    mapfile -t cachedRpmFiles < <(find "${downloadDir}" -maxdepth 1 -type f -name "*.rpm" -print 2>/dev/null | sort)
-
-    # selecting the correct version of dependency rpms from the cache
-    for cachedRpm in "${cachedRpmFiles[@]}"; do
-      if [ "${cachedRpm}" = "${rpmFile}" ]; then
-        continue
-      fi
-
-      local cachedBaseName
-      cachedBaseName=$(basename "${cachedRpm}")
-
-      case "${cachedBaseName}" in
-        *${packageName}*)
-          echo "Skipping cached ${packageName} rpm ${cachedBaseName} because it does not match desired version ${desiredVersion}"
-          continue
-          ;;
-      esac
-
-      rpmArgs+=("${cachedRpm}")
-    done
+    mapfile -t rpmArgs < <(buildRpmInstallArgsFromCache "${packageName}" "${desiredVersion}" "${rpmFile}" "${downloadDir}")
 
     if [ ${#rpmArgs[@]} -gt 1 ]; then
         echo "Installing ${packageName} with cached dependency RPMs: ${rpmArgs[*]}"
@@ -474,6 +455,56 @@ installRPMPackageFromFile() {
     rm -rf "${downloadDir}"
 }
 
+installPackageFromCache() {
+    local packageName="${1}"
+    local desiredVersion="${2}"
+    echo "installing ${packageName} version ${desiredVersion} (cache only)"
+    local downloadDir
+    downloadDir="$(getPackageDownloadDir "${packageName}")"
+
+    # check cached rpms for matching filename
+    local rpmFile=""
+    local -a rpmArgs=()
+    rpmFile="$(findCachedRpmFileName "${packageName}" "${desiredVersion}" "${downloadDir}")"
+    if [ -z "${rpmFile}" ]; then
+        echo "Failed to locate cached ${packageName} rpm for version ${desiredVersion}"
+        return 1
+    fi
+
+    rpmFile="${downloadDir}/${rpmFile}"
+    mapfile -t rpmArgs < <(buildRpmInstallArgsFromCache "${packageName}" "${desiredVersion}" "${rpmFile}" "${downloadDir}")
+
+    if [ ${#rpmArgs[@]} -gt 1 ]; then
+        echo "Installing ${packageName} with cached dependency RPMs: ${rpmArgs[*]}"
+    fi
+
+    # check if additional dependencies are required
+    local dnfPlanOutput=""
+    # Use --assumeno to inspect transaction plan without installing.
+    # dnf can still return non-zero on --assumeno, so rely on output parsing.
+    dnfPlanOutput=$(dnf install -y --assumeno --disablerepo='*' "${rpmArgs[@]}" 2>&1 || true)
+    if echo "${dnfPlanOutput}" | grep -q "Installing dependencies"; then
+      echo "Additional dependencies are required for ${packageName}; cache-only install is not allowed"
+      return 1
+    fi
+    if echo "${dnfPlanOutput}" | grep -Eq "nothing provides|requires .* but none of the providers can be installed|Problem:"; then
+      echo "Dependency resolution failed during precheck for ${packageName}"
+      return 1
+    fi
+
+    # Cache-only install: if required dependencies are not available in cache/repo, return 1.
+    # Cannot implement dnf_install because it will attempt to run makecache
+    if ! dnf install -y --disablerepo='*' "${rpmArgs[@]}"; then
+        echo "Failed to install ${packageName} from cache-only RPMs"
+        return 1
+    fi
+
+    mkdir -p /opt/bin
+    ln -snf "/usr/bin/${packageName}" "/opt/bin/${packageName}"
+    rm -rf "${downloadDir}"
+    return 0
+}
+
 downloadPkgFromVersion() {
     packageName="${1:-}"
     packageVersion="${2:-}"

parts/linux/cloud-init/artifacts/ubuntu/cse_install_ubuntu.sh

@@ -328,6 +328,55 @@ installPkgWithAptGet() {
     rm -rf ${downloadDir}
 }
 
+installPackageFromCache() {
+    local packageName="${1:-}"
+    local packageVersion="${2}"
+    local downloadDir="/opt/${packageName}/downloads"
+    local debFile=""
+    local aptPlanOutput=""
+    local plannedPkg=""
+    local hasAdditionalDependencies="false"
+
+    echo "installing ${packageName} version ${packageVersion} (cache only)"
+
+    debFile=$(ls "${downloadDir}" | grep "${packageName}" | grep "${packageVersion}" | sort -V | tail -n 1) || debFile=""
+    if [ -z "${debFile}" ]; then
+        echo "Failed to locate cached ${packageName} deb for version ${packageVersion}"
+        return 1
+    fi
+
+    debFile="${downloadDir}/${debFile}"
+
+    # Simulate install first to detect whether apt would pull additional dependencies.
+    aptPlanOutput=$(apt-get -s install "${debFile}" 2>&1 || true)
+    if echo "${aptPlanOutput}" | grep -Eqi "unmet dependencies|depends:|unable to correct problems|but it is not installable"; then
+        echo "Dependency resolution failed during precheck for ${packageName}"
+        return 1
+    fi
+
+    while read -r plannedPkg; do
+        if [ -n "${plannedPkg}" ] && [ "${plannedPkg}" != "${packageName}" ]; then
+            hasAdditionalDependencies="true"
+            break
+        fi
+    done < <(echo "${aptPlanOutput}" | awk '/^Inst / {print $2}')
+
+    if [ "${hasAdditionalDependencies}" = "true" ]; then
+        echo "Additional dependencies are required for ${packageName}; cache-only install is not allowed"
+        return 1
+    fi
+
+    if ! logs_to_events "AKS.CSE.install${packageName}.installDebPackageFromFile" "installDebPackageFromFile ${debFile}"; then
+        echo "Failed to install ${packageName} from cache-only deb ${debFile}"
+        return 1
+    fi
+
+    mkdir -p /opt/bin
+    ln -snf "/usr/bin/${packageName}" "/opt/bin/${packageName}"
+    rm -rf "${downloadDir}"
+    return 0
+}
+
 downloadPkgFromVersion() {
     packageName="${1:-}"
     packageVersion="${2:-}"