[WIP] Merged apps + Cache

bin/command/build/baked/export.sh

@@ -81,6 +81,22 @@ function Command::export() {
                 Images::push "${tag}"
             fi
 
+            if [ -z "${pushDestination}" ]; then
+                Images::printAll "${tag}"
+            fi
+            ;;
+        app-image | app-images)
+            Console::verbose "${INFO}Build and export images${NC}"
+            Images::buildApplication --force
+            Images::tagApplications "${tag}"
+            Assets::build
+            Images::buildFrontend --force
+            Images::tagFrontend "${tag}"
+
+            if [ -n "${pushDestination}" ]; then
+                Images::push "${tag}"
+            fi
+
             if [ -z "${pushDestination}" ]; then
                 Images::printAll "${tag}"
             fi

bin/sdk/assets/baked.sh

@@ -54,6 +54,40 @@ function Assets::getImageTag() {
     echo -n "${SPRYKER_DOCKER_PREFIX}_builder_assets:${SPRYKER_DOCKER_TAG}-${SPRYKER_REPOSITORY_HASH}"
 }
 
+function Assets::_packageLockHash() {
+    md5sum package-lock.json 2>/dev/null | cut -d' ' -f1 || md5 -q package-lock.json 2>/dev/null
+}
+
+function Assets::_buildFromEcrImage() {
+    local ecrImage=${1}
+    local builderAssetsImage=${2}
+    local currentHash="${SPRYKER_BUILD_HASH:-current}"
+
+    # Pull if not available locally
+    if ! docker image inspect "${ecrImage}" >/dev/null 2>&1; then
+        docker pull "${ecrImage}" >/dev/null 2>&1 || true
+    fi
+
+    if docker image inspect "${ecrImage}" >/dev/null 2>&1; then
+        Console::start "Building from ECR image ${ecrImage}..."
+        if echo "FROM busybox
+COPY --from=${ecrImage} /data/public /data/public
+RUN cd /data/public/Yves/assets && \
+    for dir in */; do \
+        [ -d \"\${dir}\" ] && [ \"\${dir%/}\" != '${currentHash}' ] && mv \"\${dir%/}\" '${currentHash}' && break; \
+    done; true" | \
+            docker build -t "${builderAssetsImage}" -f - .; then
+            Console::end "[BUILT]"
+            return "${TRUE}"
+        else
+            Console::error "Failed to build image from ECR"
+            return "${FALSE}"
+        fi
+    fi
+
+    return "${FALSE}"
+}
+
 function Assets::areBuilt() {
     Console::start "Checking assets are built..."
 
@@ -64,6 +98,22 @@ function Assets::areBuilt() {
         return "${TRUE}"
     fi
 
+    if [ -n "${AWS_ACCOUNT_ID}" ] && [ -n "${AWS_REGION}" ] && [ -n "${SPRYKER_PROJECT_NAME}" ]; then
+        local ecr_base="${AWS_ACCOUNT_ID}.dkr.ecr.${AWS_REGION}.amazonaws.com"
+        local lockHash=$(Assets::_packageLockHash)
+
+        if [ -n "${lockHash}" ]; then
+            local hashImage="${ecr_base}/${SPRYKER_PROJECT_NAME}-builder_assets:${lockHash}"
+            Console::start "Trying builder_assets:${lockHash}..."
+            if Assets::_buildFromEcrImage "${hashImage}" "${builderAssetsImage}"; then
+                return "${TRUE}"
+            fi
+            Console::start "[NOT FOUND] ${hashImage}"
+        fi
+    else
+        echo "AWS variables not set - AWS_ACCOUNT_ID: '${AWS_ACCOUNT_ID}', AWS_REGION: '${AWS_REGION}', SPRYKER_PROJECT_NAME: '${SPRYKER_PROJECT_NAME}'"
+    fi
+
     return "${FALSE}"
 }
 

bin/sdk/images/baked/ecr2.sh

@@ -0,0 +1,143 @@
+#!/usr/bin/env bash
+
+import sdk/images/baked.sh
+
+function Images::_composerLockHash() {
+    md5sum composer.lock 2>/dev/null | cut -d' ' -f1 || md5 -q composer.lock 2>/dev/null
+}
+
+function Images::pullComposerCache() {
+    local ecr_base="${AWS_ACCOUNT_ID}.dkr.ecr.${AWS_REGION}.amazonaws.com"
+    local repository_name="${SPRYKER_PROJECT_NAME}-composer_cache"
+    local lockHash=$(Images::_composerLockHash)
+
+    Console::start "Pulling composer cache from ECR..."
+
+    if [ -n "${lockHash}" ]; then
+        local hashImage="${ecr_base}/${repository_name}:${lockHash}"
+        if docker pull "${hashImage}" >/dev/null 2>&1; then
+            export SPRYKER_COMPOSER_CACHE_IMAGE="${hashImage}"
+            Console::end "[FOUND] ${lockHash}"
+            return "${TRUE}"
+        fi
+    fi
+
+    Console::end "[NOT FOUND]"
+    return "${FALSE}"
+}
+
+function Images::buildApplication() {
+    Console::verbose "${INFO}Building application images for AWS ECR${NC}"
+
+    Images::pullComposerCache || true
+    Images::_buildApp baked "${TRUE}" "${TRUE}"
+}
+
+function Images::buildFrontend() {
+    Console::verbose "${INFO}Building Frontend image for AWS ECR${NC}"
+
+    Images::_buildFrontend baked
+    Images::_buildGateway
+}
+
+function Images::tagApplications() {
+    local tag=${1:-${SPRYKER_DOCKER_TAG}}
+
+    Console::verbose "${INFO}Tag images for AWS ECR${NC}"
+    for application in "${SPRYKER_APPLICATIONS[@]}"; do
+        local application="$(echo "$application" | tr '[:upper:]' '[:lower:]')"
+        docker tag "${SPRYKER_DOCKER_PREFIX}_app:${SPRYKER_DOCKER_TAG}" "${AWS_ACCOUNT_ID}.dkr.ecr.${AWS_REGION}.amazonaws.com/${SPRYKER_PROJECT_NAME}-${application}:${tag}"
+        docker tag "${SPRYKER_DOCKER_PREFIX}_app:${SPRYKER_DOCKER_TAG}" "${AWS_ACCOUNT_ID}.dkr.ecr.${AWS_REGION}.amazonaws.com/${SPRYKER_PROJECT_NAME}-${application}:latest"
+    done
+
+    docker tag "${SPRYKER_DOCKER_PREFIX}_jenkins:${SPRYKER_DOCKER_TAG}" "${AWS_ACCOUNT_ID}.dkr.ecr.${AWS_REGION}.amazonaws.com/${SPRYKER_PROJECT_NAME}-jenkins:${tag}"
+    docker tag "${SPRYKER_DOCKER_PREFIX}_jenkins:${SPRYKER_DOCKER_TAG}" "${AWS_ACCOUNT_ID}.dkr.ecr.${AWS_REGION}.amazonaws.com/${SPRYKER_PROJECT_NAME}-jenkins:latest"
+}
+
+function Images::tagFrontend() {
+    Console::verbose "${INFO}Tagging Frontend for AWS ECR${NC}"
+
+    local tag=${1:-${SPRYKER_DOCKER_TAG}}
+    local ecr_base="${AWS_ACCOUNT_ID}.dkr.ecr.${AWS_REGION}.amazonaws.com"
+
+    docker tag "${SPRYKER_DOCKER_PREFIX}_frontend:${SPRYKER_DOCKER_TAG}" "${ecr_base}/${SPRYKER_PROJECT_NAME}-frontend:${tag}"
+    docker tag "${SPRYKER_DOCKER_PREFIX}_frontend:${SPRYKER_DOCKER_TAG}" "${ecr_base}/${SPRYKER_PROJECT_NAME}-frontend:latest"
+
+    local source_builder_assets_image="$(Assets::getImageTag)"
+    local builder_assets_image="${ecr_base}/${SPRYKER_PROJECT_NAME}-builder_assets:${tag}"
+    local builder_assets_latest="${ecr_base}/${SPRYKER_PROJECT_NAME}-builder_assets:latest"
+    local repository_name="${SPRYKER_PROJECT_NAME}-builder_assets"
+    local minimal_assets_image="${SPRYKER_DOCKER_PREFIX}_builder_assets_minimal:${SPRYKER_DOCKER_TAG}"
+
+    if ! docker image inspect "${minimal_assets_image}" >/dev/null 2>&1; then
+        Console::verbose "${INFO}Creating minimal builder_assets image (only /data/public)${NC}"
+        echo "FROM scratch
+COPY --from=${source_builder_assets_image} /data/public /data/public" | \
+            docker build -t "${minimal_assets_image}" -f - . >/dev/null 2>&1
+    fi
+
+    local assetsLockHash=$(Assets::_packageLockHash)
+    local assetsRepository="${SPRYKER_PROJECT_NAME}-builder_assets"
+
+    if [ -n "${assetsLockHash}" ] && ! aws ecr describe-images --repository-name "${assetsRepository}" --image-ids imageTag="${assetsLockHash}" --region "${AWS_REGION}" &>/dev/null; then
+        docker tag "${minimal_assets_image}" "${ecr_base}/${assetsRepository}:${assetsLockHash}"
+        export SPRYKER_PUSH_ASSETS_CACHE="${TRUE}"
+        Console::verbose "${INFO}Assets cache tagged: ${assetsLockHash} [NEW]${NC}"
+    else
+        Console::verbose "${INFO}Assets cache already exists for ${assetsLockHash} [SKIP]${NC}"
+    fi
+
+    local composerLockHash=$(Images::_composerLockHash)
+    local composerRepository="${SPRYKER_PROJECT_NAME}-composer_cache"
+
+    if [ -n "${composerLockHash}" ] && ! aws ecr describe-images --repository-name "${composerRepository}" --image-ids imageTag="${composerLockHash}" --region "${AWS_REGION}" &>/dev/null; then
+        local composerCacheHash="${ecr_base}/${composerRepository}:${composerLockHash}"
+
+        Console::start "Creating composer cache image..."
+        if echo "FROM ${SPRYKER_PLATFORM_IMAGE} AS export
+RUN --mount=type=cache,id=composer,sharing=locked,target=/composer-cache,uid=1000 \
+    mkdir -p /export/cache && cp -a /composer-cache/. /export/cache/ 2>/dev/null || true
+
+FROM scratch
+COPY --from=export /export /
+" | docker build -t "${composerCacheHash}" -f - . 2>&1; then
+            export SPRYKER_PUSH_COMPOSER_CACHE="${TRUE}"
+            Console::end "[DONE] ${composerLockHash} [NEW]"
+        else
+            Console::end "[SKIPPED] No composer cache found"
+        fi
+    else
+        Console::verbose "${INFO}Composer cache already exists for ${composerLockHash} [SKIP]${NC}"
+    fi
+}
+
+function Images::push() {
+    Console::verbose "${INFO}Pushing images to AWS ECR${NC}"
+    local tag=${1:-${SPRYKER_DOCKER_TAG}}
+    local ecr_base="${AWS_ACCOUNT_ID}.dkr.ecr.${AWS_REGION}.amazonaws.com"
+
+    local all_images=(boffice)
+    all_images+=(frontend jenkins)
+
+    for app in "${all_images[@]}"; do
+        echo "${ecr_base}/${SPRYKER_PROJECT_NAME}-${app}:${tag}"
+        docker push "${ecr_base}/${SPRYKER_PROJECT_NAME}-${app}:${tag}" &
+        docker push "${ecr_base}/${SPRYKER_PROJECT_NAME}-${app}:latest" &
+    done
+
+    if [ "${SPRYKER_PUSH_ASSETS_CACHE}" == "${TRUE}" ]; then
+        local assetsLockHash=$(Assets::_packageLockHash)
+        local builderAssetsHash="${ecr_base}/${SPRYKER_PROJECT_NAME}-builder_assets:${assetsLockHash}"
+        echo "${builderAssetsHash}"
+        docker push "${builderAssetsHash}" &
+    fi
+
+    if [ "${SPRYKER_PUSH_COMPOSER_CACHE}" == "${TRUE}" ]; then
+        local composerLockHash=$(Images::_composerLockHash)
+        local composerCacheHash="${ecr_base}/${SPRYKER_PROJECT_NAME}-composer_cache:${composerLockHash}"
+        echo "${composerCacheHash}"
+        docker push "${composerCacheHash}" &
+    fi
+
+    wait
+}

bin/sdk/images/common.sh

@@ -32,6 +32,7 @@ function Images::_buildApp() {
     local -a sshArgument=()
     local folder=${1}
     local withPushImages=${2:-${FALSE}}
+    local withCache=${3:-${FALSE}}
     local baseAppImage="${SPRYKER_DOCKER_PREFIX}_base_app:${SPRYKER_DOCKER_TAG}"
     local appImage="${SPRYKER_DOCKER_PREFIX}_app:${SPRYKER_DOCKER_TAG}"
     local localAppImage="${SPRYKER_DOCKER_PREFIX}_local_app:${SPRYKER_DOCKER_TAG}"
@@ -67,6 +68,11 @@ function Images::_buildApp() {
         --build-arg "SPRYKER_NPM_VERSION=${SPRYKER_NPM_VERSION}" \
         "${DEPLOYMENT_PATH}/context" 1>&2
 
+    local -a composerCacheArg=()
+    if [ -n "${SPRYKER_COMPOSER_CACHE_IMAGE}" ]; then
+        composerCacheArg=(--build-arg "SPRYKER_COMPOSER_CACHE_IMAGE=${SPRYKER_COMPOSER_CACHE_IMAGE}")
+    fi
+
     docker build \
         -t "${appImage}" \
         -f "${DEPLOYMENT_PATH}/images/${folder}/application/Dockerfile" \
@@ -86,15 +92,18 @@ function Images::_buildApp() {
         --build-arg "SPRYKER_COMPOSER_VERBOSE=${SPRYKER_COMPOSER_VERBOSE}" \
         --build-arg "SPRYKER_BUILD_HASH=${SPRYKER_BUILD_HASH:-"current"}" \
         --build-arg "SPRYKER_BUILD_STAMP=${SPRYKER_BUILD_STAMP:-""}" \
+        "${composerCacheArg[@]}" \
         . 1>&2
 
-    docker build \
-        -t "${localAppImage}" \
-        -t "${runtimeImage}" \
-        -f "${DEPLOYMENT_PATH}/images/common/application-local/Dockerfile" \
-        --progress="${PROGRESS_TYPE}" \
-        --build-arg "SPRYKER_PARENT_IMAGE=${appImage}" \
-        "${DEPLOYMENT_PATH}/context" 1>&2
+    if [ "${withPushImages}" == "${FALSE}" ]; then
+        docker build \
+            -t "${localAppImage}" \
+            -t "${runtimeImage}" \
+            -f "${DEPLOYMENT_PATH}/images/common/application-local/Dockerfile" \
+            --progress="${PROGRESS_TYPE}" \
+            --build-arg "SPRYKER_PARENT_IMAGE=${appImage}" \
+            "${DEPLOYMENT_PATH}/context" 1>&2
+    fi
 
     if [ -n "${SPRYKER_XDEBUG_MODE_ENABLE}" ]; then
         docker build \
@@ -105,38 +114,46 @@ function Images::_buildApp() {
             "${DEPLOYMENT_PATH}/context" 1>&2
     fi
 
-    Console::verbose "${INFO}Building CLI images${NC}"
-    docker build \
-        -t "${baseCliImage}" \
-        -t "${pipelineImage}" \
-        -f "${DEPLOYMENT_PATH}/images/common/cli/Dockerfile" \
-        --progress="${PROGRESS_TYPE}" \
-        --build-arg "SPRYKER_PARENT_IMAGE=${localAppImage}" \
-        --build-arg "BLACKFIRE_EXTENSION_ENABLED=$(Bool::normalizeBashBool ${BLACKFIRE_EXTENSION_ENABLED:-${FALSE}})" \
-        "${DEPLOYMENT_PATH}/context" 1>&2
+    if ([ "${withCache}" == "${TRUE}" ] && ! Assets::areBuilt) || [ "${withCache}" == "${FALSE}" ]; then
+        Console::verbose "${INFO}Building CLI images${NC}"
 
-    docker build \
-        -t "${cliImage}" \
-        -t "${runtimeCliImage}" \
-        -f "${DEPLOYMENT_PATH}/images/${folder}/cli/Dockerfile" \
-        "${sshArgument[@]}" \
-        --secret "id=secrets-env,src=$SECRETS_FILE_PATH" \
-        --progress="${PROGRESS_TYPE}" \
-        --build-arg "SPRYKER_PARENT_IMAGE=${baseCliImage}" \
-        --build-arg "DEPLOYMENT_PATH=${DEPLOYMENT_PATH}" \
-        --build-arg "SPRYKER_PIPELINE=${SPRYKER_PIPELINE}" \
-        --build-arg "SPRYKER_BUILD_HASH=${SPRYKER_BUILD_HASH:-"current"}" \
-        --build-arg "SPRYKER_BUILD_STAMP=${SPRYKER_BUILD_STAMP:-""}" \
-        --build-arg "SPRYKER_NPM_TOKEN=${SPRYKER_NPM_TOKEN:-""}" \
-        .  1>&2
+        local cliParentImage="${localAppImage}"
+        if [ "${withPushImages}" == "${TRUE}" ]; then
+            cliParentImage="${appImage}"
+        fi
 
-    if [ -n "${SPRYKER_XDEBUG_MODE_ENABLE}" ]; then
         docker build \
-            -t "${runtimeCliImage}" \
-            -f "${DEPLOYMENT_PATH}/images/debug/cli/Dockerfile" \
+            -t "${baseCliImage}" \
+            -t "${pipelineImage}" \
+            -f "${DEPLOYMENT_PATH}/images/common/cli/Dockerfile" \
             --progress="${PROGRESS_TYPE}" \
-            --build-arg "SPRYKER_PARENT_IMAGE=${cliImage}" \
+            --build-arg "SPRYKER_PARENT_IMAGE=${cliParentImage}" \
+            --build-arg "BLACKFIRE_EXTENSION_ENABLED=$(Bool::normalizeBashBool ${BLACKFIRE_EXTENSION_ENABLED:-${FALSE}})" \
             "${DEPLOYMENT_PATH}/context" 1>&2
+
+        docker build \
+            -t "${cliImage}" \
+            -t "${runtimeCliImage}" \
+            -f "${DEPLOYMENT_PATH}/images/${folder}/cli/Dockerfile" \
+            "${sshArgument[@]}" \
+            --secret "id=secrets-env,src=$SECRETS_FILE_PATH" \
+            --progress="${PROGRESS_TYPE}" \
+            --build-arg "SPRYKER_PARENT_IMAGE=${baseCliImage}" \
+            --build-arg "DEPLOYMENT_PATH=${DEPLOYMENT_PATH}" \
+            --build-arg "SPRYKER_PIPELINE=${SPRYKER_PIPELINE}" \
+            --build-arg "SPRYKER_BUILD_HASH=${SPRYKER_BUILD_HASH:-"current"}" \
+            --build-arg "SPRYKER_BUILD_STAMP=${SPRYKER_BUILD_STAMP:-""}" \
+            --build-arg "SPRYKER_NPM_TOKEN=${SPRYKER_NPM_TOKEN:-""}" \
+            .  1>&2
+
+        if [ -n "${SPRYKER_XDEBUG_MODE_ENABLE}" ]; then
+            docker build \
+                -t "${runtimeCliImage}" \
+                -f "${DEPLOYMENT_PATH}/images/debug/cli/Dockerfile" \
+                --progress="${PROGRESS_TYPE}" \
+                --build-arg "SPRYKER_PARENT_IMAGE=${cliImage}" \
+                "${DEPLOYMENT_PATH}/context" 1>&2
+        fi
     fi
 
     if [ "${withPushImages}" == "${TRUE}" ]; then
@@ -242,5 +259,4 @@ function Images::printAll() {
     done
 
     printf "%s %s_frontend:%s\n" "frontend" "${SPRYKER_DOCKER_PREFIX}" "${tag}-frontend"
-    printf "%s %s_pipeline:%s\n" "pipeline" "${SPRYKER_DOCKER_PREFIX}" "${tag}-pipeline"
 }

generator/src/templates/nginx/conf.d/frontend.default.conf.twig

@@ -34,6 +34,17 @@ server {
 {% for applicationName, applicationData in groupData['applications'] %}
 {% for endpoint, endpointData in applicationData['endpoints'] %}
 {% set zedHost = _endpointMap[endpointData['identifier']]['backend-gateway'] | default('') %}
+{% if not zedHost %}
+    {% for groupDataInner in groups %}
+        {% for appNameInner, appDataInner in groupDataInner['applications'] %}
+            {% for epInner, epDataInner in appDataInner['endpoints'] %}
+                {% if epDataInner['identifier'] == endpointData['identifier'] and epDataInner['entry-point'] == 'BackendGateway' %}
+                    {% set zedHost = epInner %}
+                {% endif %}
+            {% endfor %}
+        {% endfor %}
+    {% endfor %}
+{% endif %}
 {% include "nginx/http/#{applicationData['application']}.server.conf.twig" with {
     endpoint: endpoint,
     endpointData: endpointData,

images/baked/application/Dockerfile

@@ -1,6 +1,8 @@
 # syntax = docker/dockerfile:experimental
 ARG SPRYKER_PARENT_IMAGE
+ARG SPRYKER_COMPOSER_CACHE_IMAGE=scratch
 
+FROM ${SPRYKER_COMPOSER_CACHE_IMAGE} AS composer-cache
 FROM ${SPRYKER_PARENT_IMAGE} AS application-production-dependencies
 
 USER spryker
@@ -10,8 +12,13 @@ COPY --chown=spryker:spryker composer.json composer.lock ${srcRoot}/
 ARG SPRYKER_COMPOSER_MODE
 ARG SPRYKER_COMPOSER_VERBOSE
 RUN --mount=type=cache,id=composer,sharing=locked,target=/home/spryker/.composer/cache,uid=1000 \
-  --mount=type=ssh,uid=1000 --mount=type=secret,id=secrets-env,uid=1000 \
-  set -o allexport && . /run/secrets/secrets-env && set +o allexport \
+  --mount=type=bind,from=composer-cache,source=/,target=/tmp/composer-cache \
+  --mount=type=ssh,uid=1000 \
+  --mount=type=secret,id=secrets-env,uid=1000 \
+  echo "=== ECR bind mount ===" && ls /tmp/composer-cache/ 2>/dev/null && ls /tmp/composer-cache/cache/ 2>/dev/null | head -5 || echo "[empty or no cache dir]" \
+  && cp -a /tmp/composer-cache/cache/. /home/spryker/.composer/cache/ 2>/dev/null || true \
+  && echo "=== Cache after import ===" && du -sh /home/spryker/.composer/cache/ && ls /home/spryker/.composer/cache/ \
+  && set -o allexport && . /run/secrets/secrets-env && set +o allexport \
   && composer install --no-scripts --no-interaction ${SPRYKER_COMPOSER_MODE} ${SPRYKER_COMPOSER_VERBOSE}
 
 FROM application-production-dependencies AS application-production-codebase