From 3e26ccbea5930e295cff13c1ff64e258cbce0281 Mon Sep 17 00:00:00 2001 From: GiladShapira94 Date: Wed, 15 Apr 2026 17:39:17 +0300 Subject: [PATCH 01/22] first commit --- .github/workflows/pr-validation.yml | 105 ++++++++++++++++++++++++++++ .github/workflows/release.yml | 54 +++++++++++--- cliff.toml | 67 ++++++++++++++++++ 3 files changed, 217 insertions(+), 9 deletions(-) create mode 100644 .github/workflows/pr-validation.yml create mode 100644 cliff.toml diff --git a/.github/workflows/pr-validation.yml b/.github/workflows/pr-validation.yml new file mode 100644 index 00000000..88c4172e --- /dev/null +++ b/.github/workflows/pr-validation.yml @@ -0,0 +1,105 @@ +name: PR Validation + +on: + pull_request: + types: [opened, edited, reopened] + +jobs: + validate-and-label: + runs-on: ubuntu-latest + permissions: + pull-requests: write + issues: write + + steps: + - name: Validate PR title and assign label + uses: actions/github-script@v7 + with: + script: | + const title = context.payload.pull_request.title; + const prNumber = context.payload.pull_request.number; + + const allowedScopes = ['feature', 'fix', 'docs', 'perf', 'improvement', 'revert', 'breaking', 'ci']; + + const scopeToLabel = { + feature: { name: 'feature', color: '0075ca' }, + ci: { name: 'ci', color: '0075ca' }, + fix: { name: 'bug', color: 'd73a4a' }, + docs: { name: 'documentation', color: '0052cc' }, + improvement: { name: 'improvement', color: 'bfd4f2'}, + revert: { name: 'revert', color: 'fef2c0' }, + breaking: { name: 'breaking-change',color: 'b60205' }, + }; + + const match = title.match(/^\[([^\]]+)\]\s+\S+/); + + if (!match) { + core.setFailed( + `PR title must follow the format: [scope] description\n` + + `Example: [Feat] Add SeaweedFS bucket auto-creation\n` + + `Allowed scopes: ${allowedScopes.join(', ')}` + ); + return; + } + + const scope = match[1].toLowerCase(); + + if (!allowedScopes.includes(scope)) { + core.setFailed( + `Invalid scope "[${match[1]}]".\n` + + `Allowed scopes: ${allowedScopes.join(', ')}\n` + + `Example: [Feat] Add SeaweedFS bucket auto-creation` + ); + return; + } + + const { name: labelName, color: labelColor } = scopeToLabel[scope]; + + // Ensure label exists in the repo, create it if not + try { + await github.rest.issues.getLabel({ + owner: context.repo.owner, + repo: context.repo.repo, + name: labelName, + }); + } catch (e) { + if (e.status === 404) { + await github.rest.issues.createLabel({ + owner: context.repo.owner, + repo: context.repo.repo, + name: labelName, + color: labelColor, + }); + core.info(`Created label: ${labelName}`); + } else { + throw e; + } + } + + // Remove any stale scope labels from a previous title edit + const allScopeLabels = Object.values(scopeToLabel).map(l => l.name); + const currentLabels = context.payload.pull_request.labels.map(l => l.name); + + for (const stale of currentLabels) { + if (allScopeLabels.includes(stale) && stale !== labelName) { + await github.rest.issues.removeLabel({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: prNumber, + name: stale, + }); + core.info(`Removed stale label: ${stale}`); + } + } + + // Apply the correct label if not already present + if (!currentLabels.includes(labelName)) { + await github.rest.issues.addLabels({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: prNumber, + labels: [labelName], + }); + } + + core.info(`PR title valid — scope: [${scope}] → label: ${labelName}`); diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index a2cfb4f2..b673f917 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -36,6 +36,21 @@ jobs: git config user.name "$GITHUB_ACTOR" git config user.email "$GITHUB_ACTOR@users.noreply.github.com" + - name: Extract Chart Version and check RC + id: version_check + run: | + CHART_VERSION=$(grep '^version:' charts/mlrun-ce/Chart.yaml | awk '{print $2}') + if [[ -z "$CHART_VERSION" ]]; then + echo "Error: Failed to extract version from Chart.yaml" >&2 + exit 1 + fi + echo "version=$CHART_VERSION" >> $GITHUB_OUTPUT + if [[ "$CHART_VERSION" =~ -rc ]]; then + echo "is_rc=true" >> $GITHUB_OUTPUT + else + echo "is_rc=false" >> $GITHUB_OUTPUT + fi + - name: Add Helm Repos run: | helm repo add stable https://charts.helm.sh/stable @@ -52,18 +67,39 @@ jobs: env: CR_TOKEN: "${{ secrets.GITHUB_TOKEN }}" - - name: Extract Chart Version from Chart.yaml - id: extract_version + - name: Find first RC tag for this version + if: steps.version_check.outputs.is_rc == 'false' + id: first_rc run: | - CHART_VERSION=$(grep '^version:' charts/mlrun-ce/Chart.yaml | awk '{print $2}') - if [[ -z "$CHART_VERSION" ]]; then - echo "Error: Failed to extract version from Chart.yaml" >&2 - exit 1 - fi - echo "version=$CHART_VERSION" >> $GITHUB_OUTPUT + VERSION="${{ steps.version_check.outputs.version }}" + FIRST_RC=$(git tag --sort=version:refname \ + | grep "^mlrun-ce-${VERSION}-rc." \ + | head -1) + echo "tag=${FIRST_RC}" >> $GITHUB_OUTPUT + echo "First RC tag: ${FIRST_RC}" + + - name: Generate release notes with git-cliff + if: steps.version_check.outputs.is_rc == 'false' + uses: orhun/git-cliff-action@v4 + with: + config: cliff.toml + args: >- + ${{ steps.first_rc.outputs.tag }}^..HEAD + --tag mlrun-ce-${{ steps.version_check.outputs.version }} + --ignore-tags "mlrun-ce-${{ steps.version_check.outputs.version }}-rc.*" + env: + OUTPUT: RELEASE_NOTES.md + + - name: Update GitHub Release with release notes + if: steps.version_check.outputs.is_rc == 'false' + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + gh release edit "mlrun-ce-${{ steps.version_check.outputs.version }}" \ + --notes-file RELEASE_NOTES.md outputs: - version: ${{ steps.extract_version.outputs.version }} + version: ${{ steps.version_check.outputs.version }} deploy_ce_onprem: needs: release diff --git a/cliff.toml b/cliff.toml new file mode 100644 index 00000000..d68d65f8 --- /dev/null +++ b/cliff.toml @@ -0,0 +1,67 @@ +[changelog] +header = "" +body = """ +{% for group, commits in commits | sort(attribute="group") | group_by(attribute="group") %}\ + ### {{ group | striptags | trim | upper_first }} + {% for commit in commits %}\ + - {% if commit.scope %}*({{ commit.scope }})* {% endif %}\ + {% if commit.breaking %}[**breaking**] {% endif %}\ + {{ commit.message | split(pat="\n") | first | upper_first }} \ + ([{{ commit.id | truncate(length=7, end="") }}](https://github.com/mlrun/ce/commit/{{ commit.id }}))\ + \n \ + {% endfor %} +{% endfor %}\n +""" +trim = true +footer = "" + +[git] +conventional_commits = false +filter_unconventional = false +split_commits = false +commit_preprocessors = [] +commit_parsers = [ + # skip merge commits + { message = "(?i)^merge", skip = true }, + + # new PR title format (enforced going forward): [scope] ... + { message = "(?i)^\\[feat\\]", group = "Features" }, + { message = "(?i)^\\[fix\\]", group = "Bug Fixes" }, + { message = "(?i)^\\[perf\\]", group = "Performance" }, + { message = "(?i)^\\[refactor\\]", group = "Refactor" }, + { message = "(?i)^\\[docs?\\]", group = "Documentation" }, + { message = "(?i)^\\[chore\\]", group = "Miscellaneous" }, + { message = "(?i)^\\[revert\\]", group = "Reverts" }, + { message = "(?i)^\\[breaking\\]", group = "Breaking Changes" }, + + # conventional commits format: feat: / fix: ... + { message = "(?i)^feat", group = "Features" }, + { message = "(?i)^fix", group = "Bug Fixes" }, + { message = "(?i)^perf", group = "Performance" }, + { message = "(?i)^refactor", group = "Refactor" }, + { message = "(?i)^docs?", group = "Documentation" }, + { message = "(?i)^chore\\(deps\\)", group = "Dependencies" }, + { message = "(?i)^chore", group = "Miscellaneous" }, + { message = "(?i)^revert", group = "Reverts" }, + + # historical [ComponentName] format — infer type from the verb in the message + { message = "(?i)^\\[[^\\]]+\\].*(fix|bug|broken|regression)", group = "Bug Fixes" }, + { message = "(?i)^\\[[^\\]]+\\].*(add|support|enable|upgrade|update|migrate|connect|expose|allow)", group = "Features" }, + { message = "(?i)^\\[[^\\]]+\\].*(disable|remove|clean|deprecat)", group = "Miscellaneous" }, + { message = "(?i)^\\[[^\\]]+\\]", group = "Changes" }, + + # plain English fallback + { message = "(?i)^(fix|bug)", group = "Bug Fixes" }, + { message = "(?i)^(add|update|upgrade|support|enable|migrate|expose|allow)", group = "Features" }, + { message = "(?i)^(remove|disable|clean|deprecat)", group = "Miscellaneous" }, + + # catch-all — anything that didn't match above + { message = ".*", group = "Other" }, +] +protect_breaking_commits = false +filter_commits = false +tag_pattern = "mlrun-ce-[0-9].*" +skip_tags = "" +ignore_tags = "" +topo_order = false +sort_commits = "newest" From 91aa535fe398ebef6b4c4a1272bcb47210539fc4 Mon Sep 17 00:00:00 2001 From: GiladShapira94 Date: Wed, 15 Apr 2026 17:47:58 +0300 Subject: [PATCH 02/22] fix --- .github/workflows/pr-validation.yml | 39 +++++++---------------------- 1 file changed, 9 insertions(+), 30 deletions(-) diff --git a/.github/workflows/pr-validation.yml b/.github/workflows/pr-validation.yml index 88c4172e..4c66fa91 100644 --- a/.github/workflows/pr-validation.yml +++ b/.github/workflows/pr-validation.yml @@ -22,13 +22,13 @@ jobs: const allowedScopes = ['feature', 'fix', 'docs', 'perf', 'improvement', 'revert', 'breaking', 'ci']; const scopeToLabel = { - feature: { name: 'feature', color: '0075ca' }, - ci: { name: 'ci', color: '0075ca' }, - fix: { name: 'bug', color: 'd73a4a' }, - docs: { name: 'documentation', color: '0052cc' }, - improvement: { name: 'improvement', color: 'bfd4f2'}, - revert: { name: 'revert', color: 'fef2c0' }, - breaking: { name: 'breaking-change',color: 'b60205' }, + feature: 'feature', + ci: 'ci', + fix: 'bug', + docs: 'documentation', + improvement: 'improvement', + revert: 'revert', + breaking: 'breaking-change', }; const match = title.match(/^\[([^\]]+)\]\s+\S+/); @@ -53,31 +53,10 @@ jobs: return; } - const { name: labelName, color: labelColor } = scopeToLabel[scope]; - - // Ensure label exists in the repo, create it if not - try { - await github.rest.issues.getLabel({ - owner: context.repo.owner, - repo: context.repo.repo, - name: labelName, - }); - } catch (e) { - if (e.status === 404) { - await github.rest.issues.createLabel({ - owner: context.repo.owner, - repo: context.repo.repo, - name: labelName, - color: labelColor, - }); - core.info(`Created label: ${labelName}`); - } else { - throw e; - } - } + const labelName = scopeToLabel[scope]; // Remove any stale scope labels from a previous title edit - const allScopeLabels = Object.values(scopeToLabel).map(l => l.name); + const allScopeLabels = Object.values(scopeToLabel); const currentLabels = context.payload.pull_request.labels.map(l => l.name); for (const stale of currentLabels) { From 355728a81fce02a233323b76cccdc2dc0aced72b Mon Sep 17 00:00:00 2001 From: GiladShapira94 Date: Thu, 16 Apr 2026 10:47:22 +0300 Subject: [PATCH 03/22] fix run issue --- .github/workflows/pr-validation.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pr-validation.yml b/.github/workflows/pr-validation.yml index 4c66fa91..4dde66c2 100644 --- a/.github/workflows/pr-validation.yml +++ b/.github/workflows/pr-validation.yml @@ -2,19 +2,19 @@ name: PR Validation on: pull_request: - types: [opened, edited, reopened] + types: [opened, edited, reopened, synchronize] jobs: validate-and-label: runs-on: ubuntu-latest permissions: pull-requests: write - issues: write steps: - name: Validate PR title and assign label uses: actions/github-script@v7 with: + repo-token: "${{ secrets.GITHUB_TOKEN }}" script: | const title = context.payload.pull_request.title; const prNumber = context.payload.pull_request.number; From ba88c03624e19e0a409f6723d2976da982099578 Mon Sep 17 00:00:00 2001 From: GiladShapira94 Date: Thu, 16 Apr 2026 10:49:43 +0300 Subject: [PATCH 04/22] fix run issue --- .github/workflows/pr-validation.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pr-validation.yml b/.github/workflows/pr-validation.yml index 4dde66c2..59271c39 100644 --- a/.github/workflows/pr-validation.yml +++ b/.github/workflows/pr-validation.yml @@ -14,7 +14,7 @@ jobs: - name: Validate PR title and assign label uses: actions/github-script@v7 with: - repo-token: "${{ secrets.GITHUB_TOKEN }}" + github-token: "${{ secrets.GITHUB_TOKEN }}" script: | const title = context.payload.pull_request.title; const prNumber = context.payload.pull_request.number; From 5ed1f8353c10004be1a9cd0acd0a19cbf0bc5797 Mon Sep 17 00:00:00 2001 From: GiladShapira94 Date: Thu, 16 Apr 2026 11:19:17 +0300 Subject: [PATCH 05/22] fix run issue --- .github/workflows/pr-validation.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pr-validation.yml b/.github/workflows/pr-validation.yml index 59271c39..d61ee441 100644 --- a/.github/workflows/pr-validation.yml +++ b/.github/workflows/pr-validation.yml @@ -1,14 +1,14 @@ name: PR Validation on: - pull_request: + pull_request_target: types: [opened, edited, reopened, synchronize] jobs: validate-and-label: runs-on: ubuntu-latest permissions: - pull-requests: write + issues: write steps: - name: Validate PR title and assign label From 9640e9cc5955baabcde303f00928687777855840 Mon Sep 17 00:00:00 2001 From: GiladShapira94 Date: Thu, 16 Apr 2026 11:45:07 +0300 Subject: [PATCH 06/22] remove label --- .github/workflows/pr-validation.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pr-validation.yml b/.github/workflows/pr-validation.yml index d61ee441..8b9aa3d9 100644 --- a/.github/workflows/pr-validation.yml +++ b/.github/workflows/pr-validation.yml @@ -19,7 +19,7 @@ jobs: const title = context.payload.pull_request.title; const prNumber = context.payload.pull_request.number; - const allowedScopes = ['feature', 'fix', 'docs', 'perf', 'improvement', 'revert', 'breaking', 'ci']; + const allowedScopes = ['feature', 'fix', 'docs', 'improvement', 'revert', 'breaking', 'ci']; const scopeToLabel = { feature: 'feature', From cbe71fc2b18506ea817982271e4d884502023c3d Mon Sep 17 00:00:00 2001 From: GiladShapira94 Date: Mon, 20 Apr 2026 15:02:22 +0300 Subject: [PATCH 07/22] fix after review --- .github/workflows/pr-validation.yml | 8 ++++---- .github/workflows/release.yml | 28 +++++++++++++++++++++++----- cliff.toml | 23 ++++++++++------------- 3 files changed, 37 insertions(+), 22 deletions(-) diff --git a/.github/workflows/pr-validation.yml b/.github/workflows/pr-validation.yml index 8b9aa3d9..6a9e24b5 100644 --- a/.github/workflows/pr-validation.yml +++ b/.github/workflows/pr-validation.yml @@ -36,8 +36,8 @@ jobs: if (!match) { core.setFailed( `PR title must follow the format: [scope] description\n` + - `Example: [Feat] Add SeaweedFS bucket auto-creation\n` + - `Allowed scopes: ${allowedScopes.join(', ')}` + `Allowed scopes (case-insensitive): [Feature], [Fix], [Docs], [Improvement], [Revert], [Breaking], [CI]\n` + + `Example: [Feature] Add SeaweedFS bucket auto-creation` ); return; } @@ -47,8 +47,8 @@ jobs: if (!allowedScopes.includes(scope)) { core.setFailed( `Invalid scope "[${match[1]}]".\n` + - `Allowed scopes: ${allowedScopes.join(', ')}\n` + - `Example: [Feat] Add SeaweedFS bucket auto-creation` + `Allowed scopes (case-insensitive): [Feature], [Fix], [Docs], [Improvement], [Revert], [Breaking], [CI]\n` + + `Example: [Fix] Resolve crash on startup` ); return; } diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index b673f917..5de4bdfd 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -72,11 +72,29 @@ jobs: id: first_rc run: | VERSION="${{ steps.version_check.outputs.version }}" + FIRST_RC=$(git tag --sort=version:refname \ - | grep "^mlrun-ce-${VERSION}-rc." \ + | grep "^mlrun-ce-${VERSION}-rc\." \ | head -1) - echo "tag=${FIRST_RC}" >> $GITHUB_OUTPUT - echo "First RC tag: ${FIRST_RC}" + + if [[ -n "$FIRST_RC" ]]; then + echo "range=${FIRST_RC}^..HEAD" >> $GITHUB_OUTPUT + echo "Range start: first RC tag ${FIRST_RC}" + else + # Hotfix with no RC — use previous stable tag so notes cover only this version + PREV_STABLE=$(git tag --sort=version:refname \ + | grep "^mlrun-ce-[0-9]" \ + | grep -v "\-rc\." \ + | grep -v "^mlrun-ce-${VERSION}$" \ + | tail -1) + if [[ -n "$PREV_STABLE" ]]; then + echo "range=${PREV_STABLE}..HEAD" >> $GITHUB_OUTPUT + echo "Range start: previous stable tag ${PREV_STABLE}" + else + echo "range=HEAD" >> $GITHUB_OUTPUT + echo "Range start: none (first ever release)" + fi + fi - name: Generate release notes with git-cliff if: steps.version_check.outputs.is_rc == 'false' @@ -84,7 +102,7 @@ jobs: with: config: cliff.toml args: >- - ${{ steps.first_rc.outputs.tag }}^..HEAD + ${{ steps.first_rc.outputs.range }} --tag mlrun-ce-${{ steps.version_check.outputs.version }} --ignore-tags "mlrun-ce-${{ steps.version_check.outputs.version }}-rc.*" env: @@ -93,7 +111,7 @@ jobs: - name: Update GitHub Release with release notes if: steps.version_check.outputs.is_rc == 'false' env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + CR_TOKEN: "${{ secrets.GITHUB_TOKEN }}" run: | gh release edit "mlrun-ce-${{ steps.version_check.outputs.version }}" \ --notes-file RELEASE_NOTES.md diff --git a/cliff.toml b/cliff.toml index d68d65f8..aa741b03 100644 --- a/cliff.toml +++ b/cliff.toml @@ -24,36 +24,33 @@ commit_parsers = [ # skip merge commits { message = "(?i)^merge", skip = true }, - # new PR title format (enforced going forward): [scope] ... - { message = "(?i)^\\[feat\\]", group = "Features" }, + # new PR title format (enforced by pr-validation.yml): [scope] description + # allowed scopes: feature, fix, docs, improvement, revert, breaking, ci + { message = "(?i)^\\[feature\\]", group = "Features" }, { message = "(?i)^\\[fix\\]", group = "Bug Fixes" }, - { message = "(?i)^\\[perf\\]", group = "Performance" }, - { message = "(?i)^\\[refactor\\]", group = "Refactor" }, - { message = "(?i)^\\[docs?\\]", group = "Documentation" }, - { message = "(?i)^\\[chore\\]", group = "Miscellaneous" }, + { message = "(?i)^\\[docs\\]", group = "Documentation" }, + { message = "(?i)^\\[improvement\\]", group = "Improvements" }, { message = "(?i)^\\[revert\\]", group = "Reverts" }, { message = "(?i)^\\[breaking\\]", group = "Breaking Changes" }, + { message = "(?i)^\\[ci\\]", group = "CI/CD" }, # conventional commits format: feat: / fix: ... { message = "(?i)^feat", group = "Features" }, { message = "(?i)^fix", group = "Bug Fixes" }, - { message = "(?i)^perf", group = "Performance" }, - { message = "(?i)^refactor", group = "Refactor" }, + { message = "(?i)^refactor", group = "Improvements" }, { message = "(?i)^docs?", group = "Documentation" }, - { message = "(?i)^chore\\(deps\\)", group = "Dependencies" }, - { message = "(?i)^chore", group = "Miscellaneous" }, + { message = "(?i)^chore\\(deps\\)", group = "Improvements" }, { message = "(?i)^revert", group = "Reverts" }, # historical [ComponentName] format — infer type from the verb in the message { message = "(?i)^\\[[^\\]]+\\].*(fix|bug|broken|regression)", group = "Bug Fixes" }, { message = "(?i)^\\[[^\\]]+\\].*(add|support|enable|upgrade|update|migrate|connect|expose|allow)", group = "Features" }, - { message = "(?i)^\\[[^\\]]+\\].*(disable|remove|clean|deprecat)", group = "Miscellaneous" }, - { message = "(?i)^\\[[^\\]]+\\]", group = "Changes" }, + { message = "(?i)^\\[[^\\]]+\\]", group = "Other" }, # plain English fallback { message = "(?i)^(fix|bug)", group = "Bug Fixes" }, { message = "(?i)^(add|update|upgrade|support|enable|migrate|expose|allow)", group = "Features" }, - { message = "(?i)^(remove|disable|clean|deprecat)", group = "Miscellaneous" }, + { message = "(?i)^(remove|disable|clean|deprecat)", group = "Other" }, # catch-all — anything that didn't match above { message = ".*", group = "Other" }, From 3413976c63b7f124c3147faf42d0dabe5f855051 Mon Sep 17 00:00:00 2001 From: GiladShapira94 Date: Sun, 26 Apr 2026 17:59:11 +0300 Subject: [PATCH 08/22] fix after review --- .github/workflows/pr-validation.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/pr-validation.yml b/.github/workflows/pr-validation.yml index 6a9e24b5..2ec97d5c 100644 --- a/.github/workflows/pr-validation.yml +++ b/.github/workflows/pr-validation.yml @@ -2,7 +2,10 @@ name: PR Validation on: pull_request_target: - types: [opened, edited, reopened, synchronize] + types: [opened, edited, reopened] +concurrency: + group: pr-validation-${{ github.event.pull_request.number }} + cancel-in-progress: true jobs: validate-and-label: From 5364f3c6d06c591282d5df54881741bdfceffa3a Mon Sep 17 00:00:00 2001 From: GiladShapira94 Date: Mon, 27 Apr 2026 17:04:33 +0300 Subject: [PATCH 09/22] first commit --- charts/mlrun-ce/values.yaml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/charts/mlrun-ce/values.yaml b/charts/mlrun-ce/values.yaml index 5be67e10..94337589 100644 --- a/charts/mlrun-ce/values.yaml +++ b/charts/mlrun-ce/values.yaml @@ -328,6 +328,9 @@ seaweedfs: enabled: false volume: enabled: false + dataDirs: + - name: data1 + maxVolumes: 12 filer: enabled: false @@ -342,6 +345,8 @@ seaweedfs: # Reduces from 4 component pods down to 1, cutting CPU/memory footprint significantly. allInOne: enabled: true + extraEnvironmentVars: + WEED_MASTER_VOLUME_GROWTH_COPY_1: "1" s3: enabled: true port: 8333 From 61988fb2882b455a1fe62d86ac7b8d0624735fa0 Mon Sep 17 00:00:00 2001 From: GiladShapira94 Date: Mon, 27 Apr 2026 18:13:34 +0300 Subject: [PATCH 10/22] fix installation issue --- charts/mlrun-ce/values.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/charts/mlrun-ce/values.yaml b/charts/mlrun-ce/values.yaml index 94337589..58815271 100644 --- a/charts/mlrun-ce/values.yaml +++ b/charts/mlrun-ce/values.yaml @@ -322,6 +322,8 @@ seaweedfs: # delete other pods during data migration (e.g., multi-node rebalancing). # Enabling this creates a ClusterRole, which conflicts in multi-NS deployments. createClusterRole: false + extraEnvironmentVars: + WEED_MASTER_VOLUME_GROWTH_COPY_1: "1" # Disable individual component pods - allInOne runs everything in a single deployment master: @@ -345,8 +347,6 @@ seaweedfs: # Reduces from 4 component pods down to 1, cutting CPU/memory footprint significantly. allInOne: enabled: true - extraEnvironmentVars: - WEED_MASTER_VOLUME_GROWTH_COPY_1: "1" s3: enabled: true port: 8333 From 3ac86b329c19e4665af0a59a73f5fcd340914ae4 Mon Sep 17 00:00:00 2001 From: GiladShapira94 <100074049+GiladShapira94@users.noreply.github.com> Date: Mon, 27 Apr 2026 18:32:00 +0300 Subject: [PATCH 11/22] Update release.yml --- .github/workflows/release.yml | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 5de4bdfd..bdabc31b 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -51,21 +51,21 @@ jobs: echo "is_rc=false" >> $GITHUB_OUTPUT fi - - name: Add Helm Repos - run: | - helm repo add stable https://charts.helm.sh/stable - helm repo add nuclio https://nuclio.github.io/nuclio/charts - helm repo add v3io-stable https://v3io.github.io/helm-charts/stable - helm repo add minio https://charts.min.io/ - helm repo add spark-operator https://kubeflow.github.io/spark-operator - helm repo add prometheus-community https://prometheus-community.github.io/helm-charts - helm repo add strimzi https://strimzi.io/charts/ - helm repo add seaweedfs https://seaweedfs.github.io/seaweedfs/helm - - - name: Run chart-releaser - uses: helm/chart-releaser-action@cae68fefc6b5f367a0275617c9f83181ba54714f - env: - CR_TOKEN: "${{ secrets.GITHUB_TOKEN }}" + # - name: Add Helm Repos + # run: | + # helm repo add stable https://charts.helm.sh/stable + # helm repo add nuclio https://nuclio.github.io/nuclio/charts + # helm repo add v3io-stable https://v3io.github.io/helm-charts/stable + # helm repo add minio https://charts.min.io/ + # helm repo add spark-operator https://kubeflow.github.io/spark-operator + # helm repo add prometheus-community https://prometheus-community.github.io/helm-charts + # helm repo add strimzi https://strimzi.io/charts/ + # helm repo add seaweedfs https://seaweedfs.github.io/seaweedfs/helm + + # - name: Run chart-releaser + # uses: helm/chart-releaser-action@cae68fefc6b5f367a0275617c9f83181ba54714f + # env: + # CR_TOKEN: "${{ secrets.GITHUB_TOKEN }}" - name: Find first RC tag for this version if: steps.version_check.outputs.is_rc == 'false' From 961888866d18df47bd2269ab202909cd4c5ab3cb Mon Sep 17 00:00:00 2001 From: GiladShapira94 Date: Tue, 28 Apr 2026 10:47:46 +0300 Subject: [PATCH 12/22] change chart version --- charts/mlrun-ce/Chart.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/charts/mlrun-ce/Chart.yaml b/charts/mlrun-ce/Chart.yaml index 04562d73..7f066992 100644 --- a/charts/mlrun-ce/Chart.yaml +++ b/charts/mlrun-ce/Chart.yaml @@ -1,6 +1,6 @@ apiVersion: v1 name: mlrun-ce -version: 0.11.0-rc.33 +version: 0.11.0-rc.34 description: MLRun Open Source Stack home: https://iguazio.com icon: https://www.iguazio.com/wp-content/uploads/2019/10/Iguazio-Logo.png From c95a662b0881087d374cf0309a2385e61a7cc6c8 Mon Sep 17 00:00:00 2001 From: GiladShapira94 <100074049+GiladShapira94@users.noreply.github.com> Date: Tue, 28 Apr 2026 14:21:00 +0300 Subject: [PATCH 13/22] Update pr-validation.yml --- .github/workflows/pr-validation.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/pr-validation.yml b/.github/workflows/pr-validation.yml index 2ec97d5c..c9a32903 100644 --- a/.github/workflows/pr-validation.yml +++ b/.github/workflows/pr-validation.yml @@ -12,6 +12,7 @@ jobs: runs-on: ubuntu-latest permissions: issues: write + pull-requests: write steps: - name: Validate PR title and assign label From 59655d21dddc7877bc1e610b18dc1ed95e51e390 Mon Sep 17 00:00:00 2001 From: GiladShapira94 <100074049+GiladShapira94@users.noreply.github.com> Date: Tue, 28 Apr 2026 14:25:47 +0300 Subject: [PATCH 14/22] Update pr-validation.yml --- .github/workflows/pr-validation.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/pr-validation.yml b/.github/workflows/pr-validation.yml index c9a32903..28673fa7 100644 --- a/.github/workflows/pr-validation.yml +++ b/.github/workflows/pr-validation.yml @@ -11,7 +11,6 @@ jobs: validate-and-label: runs-on: ubuntu-latest permissions: - issues: write pull-requests: write steps: From 55691a6032568f6ebc5f5e445352c7ba442eaf95 Mon Sep 17 00:00:00 2001 From: GiladShapira94 <100074049+GiladShapira94@users.noreply.github.com> Date: Tue, 28 Apr 2026 14:32:39 +0300 Subject: [PATCH 15/22] Update release.yml --- .github/workflows/release.yml | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index bdabc31b..2dacb409 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -119,13 +119,13 @@ jobs: outputs: version: ${{ steps.version_check.outputs.version }} - deploy_ce_onprem: - needs: release - uses: ./.github/workflows/deploy_ce_onprem_public.yaml - with: - version: ${{ needs.release.outputs.version }} - secrets: - GH_APP_ID: ${{ secrets.GH_APP_ID }} - GH_APP_PRIVATE_KEY: ${{ secrets.GH_APP_PRIVATE_KEY }} - DEPLOYMENT_REPO: ${{ secrets.DEPLOYMENT_REPO }} - SYSTEM_ID: ${{ secrets.SYSTEM_ID }} + # deploy_ce_onprem: + # needs: release + # uses: ./.github/workflows/deploy_ce_onprem_public.yaml + # with: + # version: ${{ needs.release.outputs.version }} + # secrets: + # GH_APP_ID: ${{ secrets.GH_APP_ID }} + # GH_APP_PRIVATE_KEY: ${{ secrets.GH_APP_PRIVATE_KEY }} + # DEPLOYMENT_REPO: ${{ secrets.DEPLOYMENT_REPO }} + # SYSTEM_ID: ${{ secrets.SYSTEM_ID }} From efa5752ac5de99f7ada071af5231028e0c667f37 Mon Sep 17 00:00:00 2001 From: GiladShapira94 Date: Tue, 28 Apr 2026 14:49:37 +0300 Subject: [PATCH 16/22] [Fix] testing fix --- charts/mlrun-ce/README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/charts/mlrun-ce/README.md b/charts/mlrun-ce/README.md index e3707b48..1283f5bf 100644 --- a/charts/mlrun-ce/README.md +++ b/charts/mlrun-ce/README.md @@ -271,6 +271,7 @@ $ rm -rf my-mlrun-mlrun-ce-mlrun ... ``` + ### Using Kubeflow Pipelines MLRun enables you to run your functions while saving outputs and artifacts in a way that is visible to Kubeflow Pipelines. From 252af2c7aeef0c64161ee3881c389928daaab611 Mon Sep 17 00:00:00 2001 From: GiladShapira94 Date: Tue, 28 Apr 2026 15:03:58 +0300 Subject: [PATCH 17/22] print the release rc --- charts/mlrun-ce/Chart.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/charts/mlrun-ce/Chart.yaml b/charts/mlrun-ce/Chart.yaml index 7f066992..fd6d41c1 100644 --- a/charts/mlrun-ce/Chart.yaml +++ b/charts/mlrun-ce/Chart.yaml @@ -1,6 +1,6 @@ apiVersion: v1 name: mlrun-ce -version: 0.11.0-rc.34 +version: 0.11.0 description: MLRun Open Source Stack home: https://iguazio.com icon: https://www.iguazio.com/wp-content/uploads/2019/10/Iguazio-Logo.png From f46516cae73cc51f478ed541a13a84ec225d3276 Mon Sep 17 00:00:00 2001 From: GiladShapira94 Date: Mon, 4 May 2026 13:17:31 +0300 Subject: [PATCH 18/22] Add Claude and Cursor instructions files and skills --- .cursorrules | 65 ++++++++++++++++++++++++ .gitignore | 3 ++ AGENTS.md | 72 ++++++++++++++++++++++++-- CLAUDE.md | 13 +++++ CONTRIBUTING.md | 131 ++++++++++++++++++++++++++++++++++++++++++++++++ Makefile | 2 +- 6 files changed, 281 insertions(+), 5 deletions(-) create mode 100644 .cursorrules create mode 100644 CONTRIBUTING.md diff --git a/.cursorrules b/.cursorrules new file mode 100644 index 00000000..40ac1dec --- /dev/null +++ b/.cursorrules @@ -0,0 +1,65 @@ +# MLRun Community Edition — Cursor Rules + +## Source of Truth + +`AGENTS.md` is the authoritative reference for this project. Read it before making any suggestions. It covers architecture, design patterns, template conventions, component dependencies, how to add new components, and common debugging scenarios. + +`CONTRIBUTING.md` covers the development workflow, commit format, and PR process. + +## Preferred Response Patterns + + +- Helm install commands: always include `--namespace mlrun --wait` +- Values changes: show `--set` flags or a patch values file overlay, not edits to `values.yaml` directly +- New templates: show the complete file including the `{{- if .Values..enabled }}` guard and `include "mlrun-ce.common.labels"` call +- Service references within templates: use `{{ .Release.Namespace }}`, never hardcode namespace strings +- After any `requirements.yaml` change: remind the user to run `make helm-update-dependencies` and commit `requirements.lock` +- If a change affects the default installation, remind the user to update all three values files (`admin_installation_values.yaml`, `non_admin_installation_values.yaml`, `non_admin_cluster_ip_installation_values.yaml`) with the appropriate default +- If a change adds a new component, changes a component version, or changes the installation process, remind the user to update `charts/mlrun-ce/README.md` + +## Common Tasks (Claude Code has `/render`, `/bump`, `/pr` skills for these) + +When a user asks you to help with the following tasks, use the commands below — these are the manual equivalents of the Claude Code skills defined in `.claude/commands/`. + +**Render chart templates** (`/render` in Claude Code) +```bash +# Full chart +helm template mlrun charts/mlrun-ce -f charts/mlrun-ce/values.yaml + +# Single template or directory +helm template mlrun charts/mlrun-ce -f charts/mlrun-ce/values.yaml \ + --show-only templates/ + +# With a values overlay (e.g. non-admin install) +helm template mlrun charts/mlrun-ce \ + -f charts/mlrun-ce/values.yaml \ + -f charts/mlrun-ce/non_admin_installation_values.yaml +``` + +**Bump Chart version** (`/bump` in Claude Code) +Read the current version from `charts/mlrun-ce/Chart.yaml` and increment: +- `patch` — `0.11.0` → `0.11.1` +- `minor` — `0.11.3` → `0.12.0` +- `rc` — `0.11.0-rc.34` → `0.11.0-rc.35` (or append `-rc.1` if no suffix) + +Always show the old → new version before writing and confirm with the user. + +**Generate PR description** (`/pr` in Claude Code) +Run `git diff upstream/development...HEAD`, `git log upstream/development..HEAD --oneline`, and `git diff upstream/development...HEAD --name-only`, then fill in `.github/pull_request_template.md` based on the changes. Check `[x]` on checklist items confirmable from the diff; leave `[ ]` on items requiring human action. Flag missing version bumps, unsynced values files, and breaking changes. + +## Do Not Suggest + +- `helm upgrade --install` without running `make helm-update-dependencies` first +- Adding a new sub-chart to `requirements.yaml` for custom resources — add templates to `charts/mlrun-ce/templates//` instead +- `kubectl apply` for resources managed by this chart +- `Chart.yaml` apiVersion v2 dependency blocks (this chart uses apiVersion v1 + `requirements.yaml`) +- Creating a second credentials Secret — mount the existing `storage-credentials` Secret via `envFrom` +- Hardcoding namespace names in templates — use `{{ .Release.Namespace }}` +- Using `kafka.enabled + strimzi-kafka-operator.enabled` as a combined condition — the template guard is only `kafka.enabled`; Strimzi is a prerequisite, not a co-guard +- Treating `seaweedfs-s3-config` as a SeaweedFS dependency — SeaweedFS *creates* it; Pipelines and MLRun *consume* it + +## Workflow (from CONTRIBUTING.md) + +- Fork-based workflow: PRs target `upstream/development`, not `origin/development` +- Branch naming: `/` — e.g. `feature/add-redis-support`, `fix/CE-111` +- Always bump `charts/mlrun-ce/Chart.yaml` version before opening a PR diff --git a/.gitignore b/.gitignore index cc8f86f3..23624f03 100644 --- a/.gitignore +++ b/.gitignore @@ -4,3 +4,6 @@ charts/mlrun-ce/charts/* **/.DS_Store *.DS_Store **/__pycache__ + +# Claude Code local settings (machine-specific, not for commit) +.claude/settings.local.json diff --git a/AGENTS.md b/AGENTS.md index 3960b950..da5f5f30 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -39,23 +39,22 @@ CLEANUP_ON_EXIT=true ./tests/kind-test.sh # Auto-cleanup after test ### Template Organization (`charts/mlrun-ce/templates/`) -- `config/` — ConfigMaps and Secrets shared across components: MLRun env config, Jupyter env config, S3 credentials secret, Pipelines config, Spark config, Grafana dashboards +- `config/` — ConfigMaps and Secrets shared across components: MLRun env config, Jupyter env config, storage credentials secret, Pipelines config, Spark config, Grafana dashboards - `seaweedfs/` — SeaweedFS-specific resources: S3 IAM config secret, bucket init job, admin UI NodePort service, ingress - `kafka/` — Kafka Strimzi custom resources: KafkaNodePool, Kafka cluster CR, bootstrap alias Service, RBAC, NetworkPolicy - `timescaledb/` — TimescaleDB Deployment, Service, PVC - `jupyter-notebook/` — Jupyter Deployment and supporting resources - `pipelines/` — Kubeflow Pipelines resources +- `spark-operator/` — Spark controller RBAC - `persistency/` — PVC definitions - `aws/` — AWS-specific resources ### Key Design Patterns -**S3 credentials propagation**: The top-level `s3.accessKey`/`s3.secretKey`/`s3.bucket` values flow into a `s3-credentials` Secret (created by `templates/config/s3-credentials-secret.yaml`), which is then mounted via `envFrom` in MLRun API and Jupyter pods. SeaweedFS uses the same credentials via the `seaweedfs-s3-config` Secret. +**S3 credentials propagation**: The top-level `storage.s3.accessKey`/`storage.s3.secretKey`/`storage.s3.bucket` values flow into a `storage-credentials` Secret (created by `templates/config/storage-secret.yaml`), which is then mounted via `envFrom` in MLRun API and Jupyter pods. SeaweedFS uses the same credentials via the `seaweedfs-s3-config` Secret. **Global registry anchor**: `global.registry: &userRegistry` in `values.yaml` uses YAML anchors to multiplex the same docker registry config to both `nuclio.global.registry` and `mlrun.global.registry`. -**SeaweedFS as S3 backend**: SeaweedFS replaced MinIO. The helpers in `_helpers.tpl` (`mlrun-ce.s3.*`) generate the SeaweedFS service URL. Legacy `mlrun-ce.minio.*` helpers are kept as aliases pointing to the SeaweedFS helpers. - **Component enable/disable**: Most components can be disabled via `.enabled: false`. The Kafka setup requires the Strimzi operator (deployed as a sub-chart via `strimzi-kafka-operator`) and custom Strimzi CRs in `templates/kafka/`. ### Values Files @@ -63,3 +62,68 @@ CLEANUP_ON_EXIT=true ./tests/kind-test.sh # Auto-cleanup after test - `charts/mlrun-ce/admin_installation_values.yaml` — admin install - `charts/mlrun-ce/non_admin_installation_values.yaml` — non-admin install - `charts/mlrun-ce/non_admin_cluster_ip_installation_values.yaml` — non-admin with ClusterIP + +## Quick-Start Dev Workflow + +From a fresh clone to a linted chart: + +1. `make helm-repo-add` — adds all external repos (reads `requirements.yaml`; idempotent) +2. `make helm-update-dependencies` — downloads sub-chart tarballs into `charts/mlrun-ce/charts/` (must run before any lint or template render) +3. `make helm-lint` — runs `helm lint charts/mlrun-ce` + `ct lint --target-branch development` + - `ct` only lints charts with changes relative to the target branch; always run from a feature branch, not directly on `development` +4. Render all templates locally (no cluster needed): + ```bash + helm template mlrun charts/mlrun-ce -f charts/mlrun-ce/values.yaml + ``` +5. Render a single template file: + ```bash + helm template mlrun charts/mlrun-ce -f charts/mlrun-ce/values.yaml --show-only templates/kafka/kafka-cluster.yaml + ``` +6. Schema-validate without a cluster: + ```bash + helm template mlrun charts/mlrun-ce -f charts/mlrun-ce/values.yaml | kubectl apply --dry-run=client -f - + ``` + +## Component Dependency Map + +| Component | Enabled by | Runtime dependencies | Key templates / notes | +|---|---|---|---| +| MLRun API + UI + DB | `mlrun.enabled` | `storage-credentials` Secret, `mlrun-common-env` ConfigMap; mlrun-db (MySQL) is bundled inside the mlrun sub-chart | `templates/config/mlrun-env-configmap.yaml`; rest is in the `mlrun` sub-chart | +| Jupyter | `jupyterNotebook.enabled` | `storage-credentials` Secret, `jupyter-common-env` ConfigMap | `templates/jupyter-notebook/` | +| Nuclio | always on (no `enabled` guard in umbrella) | `global.registry` must be set | sub-chart only — no custom templates | +| MPI Operator | always on (no `enabled` guard in umbrella) | none | sub-chart only — no custom templates | +| SeaweedFS | `seaweedfs.enabled` | PVC for data storage; creates `seaweedfs-s3-config` Secret consumed by Pipelines and MLRun | `templates/seaweedfs/`; `seaweedfs.s3.enableAuth: true` must be set or the Secret is skipped | +| Spark Operator | `spark-operator.enabled` | none | sub-chart + `templates/spark-operator/spark-controller-rbac.yaml` | +| Kafka | `kafka.enabled` | Strimzi CRDs — `strimzi-kafka-operator` sub-chart must also be enabled as a prerequisite; CRs use post-install hooks to wait for CRDs | `templates/kafka/` | +| Pipelines | `pipelines.enabled` | SeaweedFS (`seaweedfs.enabled` checked at render time; adds init container to wait for it), `mlrun-pipelines-config` ConfigMap | `templates/pipelines/`, `templates/config/mlrun-pipelines-config.yaml` | +| TimescaleDB | `timescaledb.enabled` | none; uses its own `-timescaledb-secret` for the DB password | `templates/timescaledb/` — custom StatefulSet, not a sub-chart | +| Prometheus + Grafana | `kube-prometheus-stack.enabled` | none at runtime; model monitoring dashboards are pre-loaded as static JSON ConfigMaps | sub-chart config in `values.yaml`; dashboards in `templates/config/model-monitoring-*.yml` | + +## How to Add a New Component + +1. Add a top-level block to `values.yaml`: + ```yaml + myComponent: + enabled: true + ``` +2. Create `charts/mlrun-ce/templates/myComponent/`. +3. Every template file must open with `{{- if .Values.myComponent.enabled }}` and close with `{{- end }}`. +4. Add label helpers to `_helpers.tpl` following the `mlrun-ce..labels` / `mlrun-ce..selectorLabels` pattern. +5. NodePort selection — avoid all currently occupied ports: + - 30010 Grafana, 30020 Prometheus, 30040 Jupyter, 30050 Nuclio + - 30060 MLRun UI, 30070 MLRun API, 30093 SeaweedFS Admin, 30094 SeaweedFS S3 + - 30100 Pipelines, 30110 TimescaleDB +6. NodePort services must be optional and only created when the component is enabled. +7. Must create a NodePort service if the component exposes a user-facing UI or API that should be accessible outside the cluster. If the component is internal-only, use a ClusterIP service instead. +8. Storage credentials — mount the existing `storage-credentials` Secret via `envFrom.secretRef`; do not create a second credentials secret. +8. CRD dependencies — if the component depends on CRDs from a sub-chart, use `helm.sh/hook: post-install,post-upgrade` with an appropriate `hook-weight` on the CRs (see `templates/kafka/` for the established pattern). +10. Update all three values files to explicitly set `myComponent.enabled: true/false` as appropriate for each install mode. +11. Add the component's service URL to `templates/NOTES.txt` using the existing conditional pattern. +12. Update `charts/mlrun-ce/README.md` if a new NodePort is exposed. +13. Bump the version in `charts/mlrun-ce/Chart.yaml`. +14. Keep secrets and ENV's naming consistent with existing patterns (`storage-credentials` Secret, `mlrun-common-env` ConfigMap, etc.). +15. Add a section to this AGENTS.md file describing the component's architecture, dependencies, and any special design patterns used. +16. Try to reuse existing patterns and templates as much as possible — for example, if the component needs a ConfigMap of environment variables, add them to `templates/config/` and follow the same pattern as `mlrun-common-env` or `jupyter-common-env`. +17. Try and customize the component's configuration via `values.yaml` rather than hardcoding values in the templates. For example, if the component needs a port number, add a `myComponent.port` value and reference it in the template, rather than hardcoding a port. +18. Each k8s that support limit and request should be added to the values file and template or use the default values from the sub-chart if it already supports it. +15. Run `make helm-lint` and fix any lint errors before opening a PR. diff --git a/CLAUDE.md b/CLAUDE.md index 43c994c2..e9f35cb7 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -1 +1,14 @@ @AGENTS.md +@CONTRIBUTING.md + +## Preferred Response Patterns + + +- Values changes: show `--set` flags or a patch values file overlay, not edits to `values.yaml` directly, unless there is a change with the default value that should be reflected in `values.yaml` (e.g. a new component's `enabled` flag) +- New templates: show the complete file including the `{{- if .Values..enabled }}` guard and `include "mlrun-ce.common.labels"` call +- Service references within templates: use `{{ .Release.Namespace }}`, never hardcode namespace strings +- After any `requirements.yaml` change: remind the user to run `make helm-update-dependencies` and commit `requirements.lock` +- PRs target `upstream/development` — the repo uses a fork-based workflow; always reference `upstream/development` as the base branch, not `origin/development` +- Branch names follow `/` — e.g. `feature/add-redis-support` or `fix/CE-111` +- Please make sure to update all values files in `charts/mlrun-ce/` if your change affects the default installation (admin, non-admin, cluster IP) — e.g. if you add a new component with an `enabled` flag, add it to all three values files with the appropriate default value +- Update README docs if your change adds a new component, changes component version or changes the installation process. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 00000000..f5cf8d2b --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,131 @@ +# Contributing to MLRun Community Edition + +## Prerequisites + +| Tool | Minimum version | Purpose | +|---|---|---| +| helm | 3.6 | Chart rendering, linting, install | +| kubectl | 1.24 | Cluster interaction | + +For Kubernetes storage class setup and cluster prerequisites, see [charts/mlrun-ce/README.md](charts/mlrun-ce/README.md#prerequisites). + +## First-Time Setup + +```bash +# 1. Add all external Helm repos (reads requirements.yaml; idempotent) +make helm-repo-add + +# 2. Download sub-chart tarballs into charts/mlrun-ce/charts/ +make helm-update-dependencies + +# 3. Lint the chart +make helm-lint +``` + +## Development Workflow + +1. Fork your own copy of [MLRun CE repository](https://github.com/mlrun/ce). +2. Clone your fork and set the upstream remote to the main repository: + +```bash +git clone +cd ce +git remote add upstream https://github.com/mlrun/ce +``` +2. Branch from `development` — the default branch is `development`, not `main` or `master`. +3. Branch naming convention: `/`, e.g. `feature/add-redis-support` or `fix/CE-111`. +2. All PRs target `upstream/development`. +3. Bump the version in `charts/mlrun-ce/Chart.yaml` before opening a PR (format: `major.minor.patch` or `major.minor.patch-rc.N`, e.g. `0.11.0-rc.1`). +4. If you change `requirements.yaml`, run `make helm-update-dependencies` and commit the updated `requirements.lock` alongside it. +6. Each change should support Local Mac installations and native k8s as explained in [prerequisites documentation](https://docs.mlrun.org/en/stable/install-mlrun-ce/kubernetes-install.html#prerequisites). + +## Commit and PR Title Format + +All PR titles (and squash-merge commit messages) must match: + +``` +[Scope] Short description +``` + +Allowed scopes (case-insensitive): `Feature`, `Fix`, `Docs`, `Improvement`, `Revert`, `Breaking`, `CI` + +Examples: +- `[Feature] Add Redis sub-chart support` +- `[Fix] Correct SeaweedFS S3 endpoint in mlrun-common-env` +- `[Breaking] Remove legacy minio helper aliases` + +The format is enforced by `.github/workflows/pr-validation.yml` and auto-applies a label on the PR. Squash-merge titles become changelog entries via `git-cliff` and `cliff.toml`. + +## Testing + +Three levels, run them in order: + +### 1. Lint (no cluster required) + +```bash +make helm-lint +``` + +### 2. Schema validation (no cluster required) + +Renders the chart and validates resource manifests against the Kubernetes API schema without a running cluster: + +```bash +helm template mlrun charts/mlrun-ce -f charts/mlrun-ce/values.yaml \ + | kubectl apply --dry-run=client -f - +``` + +Use a specific values overlay to test a particular install mode: + +```bash +helm template mlrun charts/mlrun-ce \ + -f charts/mlrun-ce/values.yaml \ + -f charts/mlrun-ce/non_admin_installation_values.yaml \ + | kubectl apply --dry-run=client -f - +``` + +### 3. End-to-end on Kind (full install) + +```bash +./tests/kind-test.sh full # Create Kind cluster + install chart + verify +./tests/kind-test.sh create # Create cluster only +./tests/kind-test.sh install # Install chart (assumes cluster exists) +./tests/kind-test.sh verify # Verify installation +./tests/kind-test.sh delete # Delete Kind cluster +CLEANUP_ON_EXIT=true ./tests/kind-test.sh full # Auto-cleanup after test +``` + +Requires docker, kind, kubectl, helm in `$PATH`. + +## Adding a New Component + +For the full step-by-step guide see [AGENTS.md — How to Add a New Component](AGENTS.md#how-to-add-a-new-component). The summary: + +1. Add an `enabled` flag block to `values.yaml`. +2. Create `charts/mlrun-ce/templates//`. +3. Wrap every template file with `{{- if .Values..enabled }}` / `{{- end }}`. +4. Mount the existing `storage-credentials` Secret for storage access; don't create a new one. +5. Use `helm.sh/hook: post-install,post-upgrade` for resources that depend on CRDs. +6. Update all three install-mode values files: + - `charts/mlrun-ce/values.yaml` + - `charts/mlrun-ce/admin_installation_values.yaml` + - `charts/mlrun-ce/non_admin_installation_values.yaml` +7. Update `templates/NOTES.txt` and `charts/mlrun-ce/README.md`. +8. Bump `charts/mlrun-ce/Chart.yaml` version. +9. Run `make helm-lint`. + +## PR Checklist + +When you open a PR, `.github/pull_request_template.md` is automatically populated with a checklist. Every item in that checklist must be addressed before the PR can merge. Key items include: + +- Testing against a real cluster (not just lint) +- Version bump in `Chart.yaml` +- Multi-namespace values files kept in sync +- Documentation updated +- Breaking changes disclosed + +## Release Process + +Versions follow `major.minor.patch[-rc.N]` (e.g. `0.11.0-rc.1` in `Chart.yaml`). RC builds are published automatically by the `release.yml` workflow on tag push. Non-RC (stable) releases use a separate workflow path. + +The changelog is generated by `git-cliff` using `cliff.toml`. Entries are grouped by scope: Features, Bug Fixes, Improvements, Breaking Changes, CI/CD, etc. The squash-merge PR title becomes the changelog entry — keep it descriptive. diff --git a/Makefile b/Makefile index 2cba3c0b..7e76d317 100644 --- a/Makefile +++ b/Makefile @@ -36,7 +36,7 @@ package: ## Package the application .PHONY: helm-lint helm-lint: helm-repo-add ## Lint Helm Chart @helm lint charts/mlrun-ce - @ct lint --target-branch $(HELM_LINT_DEFAULT_BRANCH) + @ct lint --target-branch $(HELM_LINT_DEFAULT_BRANCH) --chart-dirs charts/mlrun-ce --validate-maintainers=false --helm-extra-args "--timeout 600s" .PHONY: helm-update-dependencies helm-update-dependencies: ## Update Helm Chart dependencies From d69ac44e6b17a4bf2488739218c6dc38d1314931 Mon Sep 17 00:00:00 2001 From: GiladShapira94 Date: Mon, 4 May 2026 13:33:54 +0300 Subject: [PATCH 19/22] fix make helm-line --- Makefile | 2 +- charts/mlrun-ce/Chart.yaml | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index 7e76d317..6415679a 100644 --- a/Makefile +++ b/Makefile @@ -36,7 +36,7 @@ package: ## Package the application .PHONY: helm-lint helm-lint: helm-repo-add ## Lint Helm Chart @helm lint charts/mlrun-ce - @ct lint --target-branch $(HELM_LINT_DEFAULT_BRANCH) --chart-dirs charts/mlrun-ce --validate-maintainers=false --helm-extra-args "--timeout 600s" + @ct lint --target-branch $(HELM_LINT_DEFAULT_BRANCH) --validate-maintainers=false --helm-extra-args "--timeout 600s" .PHONY: helm-update-dependencies helm-update-dependencies: ## Update Helm Chart dependencies diff --git a/charts/mlrun-ce/Chart.yaml b/charts/mlrun-ce/Chart.yaml index 16ed4151..06d2db3f 100644 --- a/charts/mlrun-ce/Chart.yaml +++ b/charts/mlrun-ce/Chart.yaml @@ -1,6 +1,6 @@ apiVersion: v1 name: mlrun-ce -version: 0.11.0-rc.34 +version: 0.11.0-rc.35 description: MLRun Open Source Stack home: https://iguazio.com icon: https://www.iguazio.com/wp-content/uploads/2019/10/Iguazio-Logo.png @@ -8,4 +8,4 @@ sources: [] maintainers: - name: MLRun email: mlrun@iguazio.com - url: https://www.mlrun.org/ \ No newline at end of file + url: https://www.mlrun.org/ From eaad8673f0687f84f5c07638e3ba79d8bc9dd614 Mon Sep 17 00:00:00 2001 From: GiladShapira94 Date: Mon, 4 May 2026 13:58:19 +0300 Subject: [PATCH 20/22] commit --- .claude/skills/bump/SKILL.md | 23 ++++++++ .claude/skills/pr/SKILL.md | 107 +++++++++++++++++++++++++++++++++++ CONTRIBUTING.md | 13 ----- 3 files changed, 130 insertions(+), 13 deletions(-) create mode 100644 .claude/skills/bump/SKILL.md create mode 100644 .claude/skills/pr/SKILL.md diff --git a/.claude/skills/bump/SKILL.md b/.claude/skills/bump/SKILL.md new file mode 100644 index 00000000..6e553d56 --- /dev/null +++ b/.claude/skills/bump/SKILL.md @@ -0,0 +1,23 @@ +--- +name: bump +description: Bump the chart version in charts/mlrun-ce/Chart.yaml (patch, minor, or rc) +allowed-tools: Read(charts/mlrun-ce/Chart.yaml) Edit(charts/mlrun-ce/Chart.yaml) +--- + +Bump the version in `charts/mlrun-ce/Chart.yaml`. + +Usage: /bump + +- `patch` — increment the patch digit: `0.11.0` → `0.11.1` +- `minor` — increment the minor digit and reset patch: `0.11.3` → `0.12.0` +- `rc` — increment the RC counter on the current version: `0.11.0-rc.34` → `0.11.0-rc.35` + - If the current version has no RC suffix, add `-rc.1`: `0.11.0` → `0.11.0-rc.1` + +Steps: +1. Read the current version from `charts/mlrun-ce/Chart.yaml` (the `version:` field). +2. Compute the new version according to the argument above. +3. Show the user: "Bumping `` → ``" and ask for confirmation before writing. +4. On confirmation, update the `version:` field in `charts/mlrun-ce/Chart.yaml` in-place. +5. Remind the user: version bumps must be committed before opening a PR, and the PR title must follow `[Scope] description` format. + +If no argument is given, show the current version and list the three options with the resulting version for each, then ask which to apply. diff --git a/.claude/skills/pr/SKILL.md b/.claude/skills/pr/SKILL.md new file mode 100644 index 00000000..3cafec80 --- /dev/null +++ b/.claude/skills/pr/SKILL.md @@ -0,0 +1,107 @@ +--- +name: pr +description: Analyze branch changes and generate a fully filled PR description ready to paste into GitHub +allowed-tools: Bash(git diff*) Bash(git log*) +--- + +Analyze the current branch changes and generate a fully filled PR description ready to paste into GitHub. + +## Steps + +1. **Gather context** — run these in parallel: + - `git diff upstream/development...HEAD` — full diff against the base branch + - `git log upstream/development..HEAD --oneline` — commit list + - `git diff upstream/development...HEAD --name-only` — changed files + +2. **Analyze the diff** carefully: + - What components or templates were changed? (check which `templates/` subdirs, `values.yaml` sections, `requirements.yaml`, `Chart.yaml`) + - Were any new values keys added? Do they need to be reflected in the three install-mode values files? + - Were any Secrets, ConfigMaps, or port numbers changed? (potential breaking changes) + - Was `Chart.yaml` version bumped? If not, flag it. + - Were `requirements.yaml` or `requirements.lock` changed? + - Does `charts/mlrun-ce/README.md` need updating (new NodePort, new component, new install step)? + +3. **Detect breaking changes** — flag as breaking if any of: + - A value key was renamed or removed + - A Secret or ConfigMap name changed + - A NodePort number changed + - A sub-chart was upgraded with a major version bump + - The storage credentials structure changed + - Any hook annotation or hook-weight changed in a way that affects upgrade order + +5. Provide an option PR title following the `[Scope] description` format, where Scope is one of: `['feature', 'fix', 'docs', 'improvement', 'revert', 'breaking', 'ci']`. For example: `[Feature] Add Redis support to mlrun-ce`. +6. **Fill the PR template** — produce the complete filled template below. Be specific and concrete; do not use placeholder text. + +--- + +Apply these checklist rules before writing the output: +- `[x]` — you can confirm this item is satisfied from the diff alone +- `[ ]` — requires human action, judgment, or external system access + +Specific rules: +- "tested" → always `[ ]` +- "documentation PR" → always `[ ]` +- "QA tests / Jira ticket" → always `[ ]` +- "installation verified" → always `[ ]` +- `Chart.yaml` version bump → `[x]` if diff shows version changed, otherwise `[ ]` and add to Warnings +- Multi-namespace values files → `[x]` if all three are in the diff OR the change has no effect on install-mode values; `[ ]` with a note if a new value was added only to `values.yaml` +- README update → `[x]` if `charts/mlrun-ce/README.md` is in the diff OR no new NodePorts/components were added; otherwise `[ ]` + +Output exactly this structure with real content (no placeholder text): + +```markdown +### 📝 Description +<2-4 sentences: what changed, why, and what it affects> + +--- + +### 🛠️ Changes Made + + +--- + +### ✅ Checklist +- [ ] I have tested the changes in this PR +- [ ] I confirmed whether my changes require a change in documentation and if so, I created another PR in MLRun for the relevant documentation. +- [ ] I confirmed whether my changes require a changes in QA tests, for example: credentials changes, resources naming change and if so, I updated the relevant Jira ticket for QA. +- [ ] I increased the Chart version in `charts/mlrun-ce/Chart.yaml`. +- [ ] I confirmed that the installation works both on a local Docker Desktop environment and on a real cluster when using the required [prerequisites](https://docs.mlrun.org/en/stable/install-mlrun-ce/kubernetes-install.html#prerequisites). + - [ ] If installation issues were found, I updated the relevant Jira ticket with the issue and steps to reproduce, or updated the prerequisites documentation if the issue is related to missing or outdated prerequisites. +- [ ] If needed, update https://github.com/mlrun/ce/blob/development/charts/mlrun-ce/README.md with the relevant installation instructions and version Matrix. +- [ ] If needed, update the following values files for multi namespace support: + - [ ] [Admin values](https://github.com/mlrun/ce/blob/development/charts/mlrun-ce/admin_installation_values.yaml) + - [ ] [User values Node Port](https://github.com/mlrun/ce/blob/development/charts/mlrun-ce/non_admin_installation_values.yaml) + - [ ] [User values ClusterIP](https://github.com/mlrun/ce/blob/development/charts/mlrun-ce/non_admin_cluster_ip_installation_values.yaml) + +--- + +### 🧪 Testing + + +--- + +### 🔗 References +- Ticket link: +- External links: +- Design docs links (Optional): + +--- + +### 🚨 Breaking Changes? + +- [ ] Yes (explain below) +- [ ] No + + + +--- + +### 🔍️ Additional Notes + +``` + +Then replace each `[ ]` with `[x]` on items you can confirm from the diff, following the rules above. + +After outputting the filled template, add a short **"Warnings"** section (outside the template) listing anything that needs human attention before opening the PR (missing version bump, unsynced values files, potential breaking changes, etc.). + +Between every sentence that end with a `.` add a two new lines to make it more readable. \ No newline at end of file diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index f5cf8d2b..9fcd62f9 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -84,19 +84,6 @@ helm template mlrun charts/mlrun-ce \ | kubectl apply --dry-run=client -f - ``` -### 3. End-to-end on Kind (full install) - -```bash -./tests/kind-test.sh full # Create Kind cluster + install chart + verify -./tests/kind-test.sh create # Create cluster only -./tests/kind-test.sh install # Install chart (assumes cluster exists) -./tests/kind-test.sh verify # Verify installation -./tests/kind-test.sh delete # Delete Kind cluster -CLEANUP_ON_EXIT=true ./tests/kind-test.sh full # Auto-cleanup after test -``` - -Requires docker, kind, kubectl, helm in `$PATH`. - ## Adding a New Component For the full step-by-step guide see [AGENTS.md — How to Add a New Component](AGENTS.md#how-to-add-a-new-component). The summary: From b6083e33bf438f43412c460d5eef24c5e6f6b059 Mon Sep 17 00:00:00 2001 From: GiladShapira94 Date: Mon, 4 May 2026 14:29:24 +0300 Subject: [PATCH 21/22] fix small changes --- AGENTS.md | 8 -------- 1 file changed, 8 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index da5f5f30..025b262b 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -18,14 +18,6 @@ make helm-repo-add # Package the chart as a tarball make package - -# Run full local end-to-end test on a Kind cluster (requires docker, kind, kubectl, helm) -./tests/kind-test.sh full # Create Kind cluster + install chart -./tests/kind-test.sh create # Create cluster only -./tests/kind-test.sh install # Install chart (assumes cluster exists) -./tests/kind-test.sh verify # Verify installation -./tests/kind-test.sh delete # Delete Kind cluster -CLEANUP_ON_EXIT=true ./tests/kind-test.sh # Auto-cleanup after test ``` ## Architecture From f0b314ad67e010aad2e7eeec27a20cabfa400d6c Mon Sep 17 00:00:00 2001 From: GiladShapira94 Date: Mon, 4 May 2026 15:15:17 +0300 Subject: [PATCH 22/22] fix small changes --- .claude/skills/bump/SKILL.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.claude/skills/bump/SKILL.md b/.claude/skills/bump/SKILL.md index 6e553d56..a2ab3138 100644 --- a/.claude/skills/bump/SKILL.md +++ b/.claude/skills/bump/SKILL.md @@ -1,7 +1,7 @@ --- name: bump description: Bump the chart version in charts/mlrun-ce/Chart.yaml (patch, minor, or rc) -allowed-tools: Read(charts/mlrun-ce/Chart.yaml) Edit(charts/mlrun-ce/Chart.yaml) +allowed-tools: Read(charts/mlrun-ce/Chart.yaml) Edit(charts/mlrun-ce/Chart.yaml) Read(charts/mlrun-ce/README.md) Edit(charts/mlrun-ce/README.md) --- Bump the version in `charts/mlrun-ce/Chart.yaml`. @@ -19,5 +19,6 @@ Steps: 3. Show the user: "Bumping `` → ``" and ask for confirmation before writing. 4. On confirmation, update the `version:` field in `charts/mlrun-ce/Chart.yaml` in-place. 5. Remind the user: version bumps must be committed before opening a PR, and the PR title must follow `[Scope] description` format. +6. Update the MLRun CE version under Version Matrix in `charts/mlrun-ce/README.md`. If no argument is given, show the current version and list the three options with the resulting version for each, then ask which to apply.