From fa78d7ad61be55533c027bd61dbec0c96853389f Mon Sep 17 00:00:00 2001 From: serg Date: Wed, 29 Apr 2026 20:59:37 +0300 Subject: [PATCH 01/18] prepare llms files Signed-off-by: serg --- config.toml | 16 +++++++-- layouts/_default/home.llmstxt.txt | 29 ++++++++++++++++ layouts/_default/section.llmsfulltxt.txt | 11 ++++++ layouts/_default/section.llmstxt.txt | 25 ++++++++++++++ layouts/_default/single.llmsfulltxt.txt | 9 +++++ layouts/partials/llms-decode-entities.txt | 24 +++++++++++++ layouts/partials/llms-full-content.txt | 21 ++++++++++++ layouts/partials/llms-is-excluded.txt | 22 ++++++++++++ layouts/partials/llms-list-pages.txt | 40 +++++++++++++++++++++ layouts/partials/llms-page-count.txt | 14 ++++++++ layouts/partials/meta.html | 42 +++++++++++++++++++++++ layouts/robots.txt | 1 + 12 files changed, 252 insertions(+), 2 deletions(-) create mode 100644 layouts/_default/home.llmstxt.txt create mode 100644 layouts/_default/section.llmsfulltxt.txt create mode 100644 layouts/_default/section.llmstxt.txt create mode 100644 layouts/_default/single.llmsfulltxt.txt create mode 100644 layouts/partials/llms-decode-entities.txt create mode 100644 layouts/partials/llms-full-content.txt create mode 100644 layouts/partials/llms-is-excluded.txt create mode 100644 layouts/partials/llms-list-pages.txt create mode 100644 layouts/partials/llms-page-count.txt create mode 100644 layouts/partials/meta.html diff --git a/config.toml b/config.toml index b169f8434..1e5107393 100644 --- a/config.toml +++ b/config.toml @@ -5,6 +5,7 @@ title = "Kubermatic Docs" disableKinds = ["taxonomy"] ignoreErrors = ["error-disable-taxonomy"] enableRobotsTXT = true +enableGitInfo = true [outputFormats] [outputFormats.Search] @@ -17,10 +18,21 @@ enableRobotsTXT = true mediaType = "application/octet-stream" isPlainText = true notAlternative = true + [outputFormats.LLMsTxt] + baseName = "llms" + mediaType = "text/plain" + isPlainText = true + notAlternative = true + [outputFormats.LLMsFullTxt] + baseName = "llms-full" + mediaType = "text/plain" + isPlainText = true + notAlternative = true [outputs] -home = ["HTML", "Search", "Redirects"] -section = ["HTML"] +home = ["HTML", "Search", "Redirects", "LLMsTxt"] +section = ["HTML", "LLMsTxt", "LLMsFullTxt"] +page = ["HTML"] [markup.goldmark.renderer] unsafe = true diff --git a/layouts/_default/home.llmstxt.txt b/layouts/_default/home.llmstxt.txt new file mode 100644 index 000000000..27af92244 --- /dev/null +++ b/layouts/_default/home.llmstxt.txt @@ -0,0 +1,29 @@ +# {{ .Site.Title }} + +> {{ .Site.Params.description }} + +> Last updated: {{ now.Format "2006-01-02" }} +> Products: {{ len (where (sort hugo.Data.products ".weight") "name" "!=" "") }} + +## Products +{{ range (sort hugo.Data.products ".weight") }} +{{- $key := urlize .name -}} +{{- $values := . -}} +{{- $url := "" -}} +{{- if .versions -}} + {{- $url = (cond (gt (len .versions) 1) (index .versions 1) (index .versions 0)).release -}} +{{- end -}} +{{- $pagePath := cond (ne $url "") (printf "/%s/%s" $key $url) (printf "/%s" $key) -}} +{{- with $.Site.GetPage $pagePath -}} + {{- if not (partial "llms-is-excluded.txt" .) -}} + {{- $llmsURL := "" -}} + {{- with .OutputFormats.Get "LLMsTxt" -}} + {{- $llmsURL = .Permalink -}} + {{- end -}} + {{- if $llmsURL -}} + {{- $title := $values.title | default $values.name }} +- [{{ $title }}]({{ $llmsURL }}): {{ $values.description }} +{{ end -}} +{{- end -}} +{{- end -}} +{{- end }} diff --git a/layouts/_default/section.llmsfulltxt.txt b/layouts/_default/section.llmsfulltxt.txt new file mode 100644 index 000000000..5ef3e57a4 --- /dev/null +++ b/layouts/_default/section.llmsfulltxt.txt @@ -0,0 +1,11 @@ +{{- if not (partial "llms-is-excluded.txt" .) -}} +{{- $segments := split (strings.TrimPrefix "/" (strings.TrimSuffix "/" .RelPermalink)) "/" -}} +{{- if le (len $segments) 3 -}} +# {{ .Title | default .Name | plainify }} +{{ with .Description | default .Params.description }} +> {{ . }} +{{ end }} +{{ partial "llms-decode-entities.txt" .Plain }} +{{ partial "llms-full-content.txt" . }} +{{- end -}} +{{- end -}} diff --git a/layouts/_default/section.llmstxt.txt b/layouts/_default/section.llmstxt.txt new file mode 100644 index 000000000..9cdee9092 --- /dev/null +++ b/layouts/_default/section.llmstxt.txt @@ -0,0 +1,25 @@ +{{- if not (partial "llms-is-excluded.txt" .) -}} +{{- $title := .Title | default .Name | plainify -}} +{{- $pageCount := partial "llms-page-count.txt" . -}} +{{- $segments := split (strings.TrimPrefix "/" (strings.TrimSuffix "/" .RelPermalink)) "/" -}} +# {{ $title }} +{{ with .Description | default .Params.description }} +> {{ . }} +{{ end }} +> Last updated: {{ .Lastmod.Format "2006-01-02" }} +{{- if ge (len $segments) 2 }} +> Version: {{ index $segments 1 }} +{{- end }} +> Pages: {{ $pageCount }} +{{- $llmsFullURL := "" -}} +{{- with .OutputFormats.Get "LLMsFullTxt" -}} + {{- $llmsFullURL = .Permalink -}} +{{- end -}} +{{- with $llmsFullURL }} +## About this section + +- [{{ $title }} — Full Content]({{ . }}) +{{ end }} +## Pages +{{ partial "llms-list-pages.txt" . }} +{{- end -}} diff --git a/layouts/_default/single.llmsfulltxt.txt b/layouts/_default/single.llmsfulltxt.txt new file mode 100644 index 000000000..6bcd45113 --- /dev/null +++ b/layouts/_default/single.llmsfulltxt.txt @@ -0,0 +1,9 @@ +{{- if not (partial "llms-is-excluded.txt" .) -}} +{{- if .Plain }} +# {{ .Title | default .Name | plainify }} +{{ with .Description | default .Params.description }} +> {{ . }} +{{ end }} +{{ partial "llms-decode-entities.txt" .Plain }} +{{- end -}} +{{- end -}} diff --git a/layouts/partials/llms-decode-entities.txt b/layouts/partials/llms-decode-entities.txt new file mode 100644 index 000000000..e40a0d168 --- /dev/null +++ b/layouts/partials/llms-decode-entities.txt @@ -0,0 +1,24 @@ +{{- /* Decodes all HTML entities (named and numeric) from a string. + Hugo's htmlUnescape only handles some named entities, so we also + manually replace numeric and remaining named entities. + IMPORTANT: & must be replaced LAST to avoid double-decoding. */ -}} +{{- $s := . | htmlUnescape -}} +{{- $s = replace $s """ "\"" -}} +{{- $s = replace $s "'" "'" -}} +{{- $s = replace $s "+" "+" -}} +{{- $s = replace $s " " " " -}} +{{- $s = replace $s "<" "<" -}} +{{- $s = replace $s ">" ">" -}} +{{- $s = replace $s """ "\"" -}} +{{- $s = replace $s "<" "<" -}} +{{- $s = replace $s ">" ">" -}} +{{- $s = replace $s "’" "\u2019" -}} +{{- $s = replace $s "‘" "\u2018" -}} +{{- $s = replace $s "”" "\u201D" -}} +{{- $s = replace $s "“" "\u201C" -}} +{{- $s = replace $s "…" "\u2026" -}} +{{- $s = replace $s "—" "\u2014" -}} +{{- $s = replace $s "–" "\u2013" -}} +{{- $s = replace $s "&" "&" -}} +{{- $s = replace $s "&" "&" -}} +{{- return $s -}} diff --git a/layouts/partials/llms-full-content.txt b/layouts/partials/llms-full-content.txt new file mode 100644 index 000000000..12bf58591 --- /dev/null +++ b/layouts/partials/llms-full-content.txt @@ -0,0 +1,21 @@ +{{- /* Recursive partial: outputs full content of all descendant pages. + Context: a page (section or single). + Skips excluded pages and pages without content. + Outputs each page as a markdown section with heading + content. + Uses .Plain to resolve shortcodes and produce clean text. */ -}} +{{- range .Pages -}} + {{- if not (partial "llms-is-excluded.txt" .) -}} + {{- if .Plain -}} + {{- with .Title }} + +--- + +## {{ . | plainify }} +{{ end -}} +{{ partial "llms-decode-entities.txt" .Plain | safeHTML }} +{{ end -}} + {{- if .IsSection -}} + {{- partial "llms-full-content.txt" . -}} + {{- end -}} + {{- end -}} +{{- end -}} diff --git a/layouts/partials/llms-is-excluded.txt b/layouts/partials/llms-is-excluded.txt new file mode 100644 index 000000000..5816be304 --- /dev/null +++ b/layouts/partials/llms-is-excluded.txt @@ -0,0 +1,22 @@ +{{- /* Returns true if a page should be excluded from llms.txt outputs. + Excluded when: + - .Draft is true, OR + - .Params.sitemapexclude is true, OR + - The page does not belong to the most recent product version + (2nd in the versions list from products.yaml, or 1st if only one). */ -}} +{{- $excluded := or .Draft .Params.sitemapexclude -}} +{{- if not $excluded -}} + {{- with index hugo.Data.products .Section -}} + {{- if .versions -}} + {{- $latestVersion := cond (gt (len .versions) 1) (index .versions 1).release (index .versions 0).release -}} + {{- $pathParts := split $.RelPermalink "/" -}} + {{- if gt (len $pathParts) 2 -}} + {{- $pageVersion := index $pathParts 2 -}} + {{- if ne $pageVersion $latestVersion -}} + {{- $excluded = true -}} + {{- end -}} + {{- end -}} + {{- end -}} + {{- end -}} +{{- end -}} +{{- return $excluded -}} diff --git a/layouts/partials/llms-list-pages.txt b/layouts/partials/llms-list-pages.txt new file mode 100644 index 000000000..af3ddefbf --- /dev/null +++ b/layouts/partials/llms-list-pages.txt @@ -0,0 +1,40 @@ +{{- /* Hierarchical partial: list only direct children of the current section. + Context: a page (section). + Skips excluded pages (see llms-is-excluded partial). + Skips pages without main content. + For child sections within depth limit (≤3 URL segments), links to their llms-full.txt. + For child sections beyond depth limit, links to their llms.txt index. + For single pages, lists title and description as plain text. */ -}} +{{- range .Pages -}} + {{- if not (partial "llms-is-excluded.txt" .) -}} + {{- if .Plain -}} + {{- if .IsSection -}} + {{- $segments := split (strings.TrimPrefix "/" (strings.TrimSuffix "/" .RelPermalink)) "/" -}} + {{- if le (len $segments) 3 -}} + {{- $llmsFullURL := "" -}} + {{- with .OutputFormats.Get "LLMsFullTxt" -}} + {{- $llmsFullURL = .Permalink -}} + {{- end -}} + {{- if and .Title $llmsFullURL -}} + {{- $desc := .Description | default .Params.description | default "" -}} +- [{{ .Title | plainify }}]({{ $llmsFullURL }}){{ with $desc }}: {{ . }}{{ end }} +{{ end -}} + {{- else -}} + {{- $llmsTxtURL := "" -}} + {{- with .OutputFormats.Get "LLMsTxt" -}} + {{- $llmsTxtURL = .Permalink -}} + {{- end -}} + {{- if and .Title $llmsTxtURL -}} + {{- $desc := .Description | default .Params.description | default "" -}} +- [{{ .Title | plainify }}]({{ $llmsTxtURL }}){{ with $desc }}: {{ . }}{{ end }} +{{ end -}} + {{- end -}} + {{- else -}} + {{- if .Title -}} + {{- $desc := .Description | default .Params.description | default "" -}} +- {{ .Title | plainify }}{{ with $desc }}: {{ . }}{{ end }} +{{ end -}} + {{- end -}} + {{- end -}} + {{- end -}} +{{- end -}} diff --git a/layouts/partials/llms-page-count.txt b/layouts/partials/llms-page-count.txt new file mode 100644 index 000000000..0ba11da0d --- /dev/null +++ b/layouts/partials/llms-page-count.txt @@ -0,0 +1,14 @@ +{{- /* Returns the count of non-excluded pages with content under a section. + Context: a section page. */ -}} +{{- $count := 0 -}} +{{- range .Pages -}} + {{- if not (partial "llms-is-excluded.txt" .) -}} + {{- if .Plain -}} + {{- $count = add $count 1 -}} + {{- end -}} + {{- if .IsSection -}} + {{- $count = add $count (partial "llms-page-count.txt" .) -}} + {{- end -}} + {{- end -}} +{{- end -}} +{{- return $count -}} diff --git a/layouts/partials/meta.html b/layouts/partials/meta.html new file mode 100644 index 000000000..e0f3ae36e --- /dev/null +++ b/layouts/partials/meta.html @@ -0,0 +1,42 @@ +{{$trimTagsRegex := "(h1|h2|h3|h4|h5|h6|ul|ol|pre|table)"}} +{{$regexPattern := printf "<%[1]s.*?>(.|\n)*?" $trimTagsRegex}} +{{$htmlContent := strings.ReplaceRE $regexPattern "" (.Content | safeHTML)}} +{{$autoDescription := trim (truncate 160 "" (replace ($htmlContent | htmlUnescape | plainify) "\n" " ")) " " | chomp}} +{{$description := replace $autoDescription "&" "and"}} + +{{if $description}} + {{$sentences := split $description ". "}} + {{range $index, $val := $sentences}} + {{$delimiter := ""}} + + {{if lt $index (sub (len $sentences) 2)}} + {{$delimiter = ". "}} + {{end}} + + {{if eq $index 0}} + {{$description = print . $delimiter}} + {{else if lt $index (sub (len $sentences) 1)}} + {{$description = print $description . $delimiter}} + {{end}} + {{end}} +{{end}} + +{{with .Description}} + {{$description = .}} +{{end}} + +{{with .Site.Params.author}}{{end}} + + + +{{$product := index hugo.Data.products .Section | default (dict "shareImage" nil)}} +{{$shareImage := or .Params.shareImage $product.shareImage | default .Site.Params.shareImage}} +{{with $shareImage}} + + +{{end}} + +{{- /* LLMs.txt discoverability */ -}} +{{- with .OutputFormats.Get "LLMsTxt" }} + +{{- end }} diff --git a/layouts/robots.txt b/layouts/robots.txt index 0af428494..b17bee7b0 100644 --- a/layouts/robots.txt +++ b/layouts/robots.txt @@ -8,3 +8,4 @@ User-agent: * Disallow: Sitemap: {{"sitemap.xml" | absURL}} +Llms-txt: {{"llms.txt" | absURL}} From 1f2f4801b80a8f0bfd3d46d530cced0d459ad9ec Mon Sep 17 00:00:00 2001 From: serg Date: Thu, 30 Apr 2026 11:06:30 +0300 Subject: [PATCH 02/18] add redirects for llm agents Signed-off-by: serg --- layouts/_default/home.redirects | 3 +++ 1 file changed, 3 insertions(+) diff --git a/layouts/_default/home.redirects b/layouts/_default/home.redirects index 7c6328f92..043b4d3d7 100644 --- a/layouts/_default/home.redirects +++ b/layouts/_default/home.redirects @@ -7,6 +7,9 @@ https://docs.kubermatic.io/* https://docs.kubermatic.com/:splat 301! # Optional: Redirect default Netlify subdomain to primary domain https://cranky-newton-4e6ed2.netlify.com/* https://docs.kubermatic.com/:splat 301! +# Serve /llms.txt at the well-known discovery path for LLM agent frameworks +/.well-known/llms.txt /llms.txt 200! + /kubermatic/master/cheat_sheets/alerting_runbook/* /kubermatic/main/cheat-sheets/alerting-runbook/:splat 301! /kubermatic/master/tutorials_howtos/oidc_provider_configuration/share-_clusters_via_delegated_oidc_authentication /kubermatic/main/tutorials-howtos/oidc-provider-configuration/share-clusters-via-delegated-oidc-authentication/ 301! /kubermatic/master/installation/start_kkp /kubermatic/main/installation/start-kkp/ 301! From 3f8be7ffece9a374925db9cf676fed0d617836e6 Mon Sep 17 00:00:00 2001 From: serg Date: Thu, 30 Apr 2026 15:16:41 +0300 Subject: [PATCH 03/18] add md files for on-demand crawling, add function for handling request for md version of the page Signed-off-by: serg --- config.toml | 11 +++- layouts/_default/home.markdown.md | 28 ++++++++++ layouts/_default/section.markdown.md | 12 ++++ layouts/_default/single.markdown.md | 12 ++++ netlify/edge-functions/markdown-for-agents.js | 55 +++++++++++++++++++ 5 files changed, 115 insertions(+), 3 deletions(-) create mode 100644 layouts/_default/home.markdown.md create mode 100644 layouts/_default/section.markdown.md create mode 100644 layouts/_default/single.markdown.md create mode 100644 netlify/edge-functions/markdown-for-agents.js diff --git a/config.toml b/config.toml index 1e5107393..258f61702 100644 --- a/config.toml +++ b/config.toml @@ -28,11 +28,16 @@ enableGitInfo = true mediaType = "text/plain" isPlainText = true notAlternative = true + [outputFormats.Markdown] + baseName = "index" + mediaType = "text/markdown" + isPlainText = true + notAlternative = true [outputs] -home = ["HTML", "Search", "Redirects", "LLMsTxt"] -section = ["HTML", "LLMsTxt", "LLMsFullTxt"] -page = ["HTML"] +home = ["HTML", "Search", "Redirects", "LLMsTxt", "Markdown"] +section = ["HTML", "LLMsTxt", "LLMsFullTxt", "Markdown"] +page = ["HTML", "Markdown"] [markup.goldmark.renderer] unsafe = true diff --git a/layouts/_default/home.markdown.md b/layouts/_default/home.markdown.md new file mode 100644 index 000000000..61e2adece --- /dev/null +++ b/layouts/_default/home.markdown.md @@ -0,0 +1,28 @@ +{{- if not .Params.sitemapexclude -}} +--- +title: {{ .Title | plainify }} +url: {{ .Permalink }} +--- + +# {{ .Title | plainify }} +{{ with .Description | default .Params.description }} +> {{ . }} +{{ end }} +## Products +{{ range sort hugo.Data.products "weight" }} +{{- if not .sitemapexclude -}} +{{- $key := .name | urlize -}} +{{- $version := "" -}} +{{- if .versions -}} + {{- $latest := cond (gt (len .versions) 1) (index .versions 1) (index .versions 0) -}} + {{- $version = $latest.release -}} +{{- end -}} +### {{ .title }} + +{{ .description }} +{{ if $version -}} +Documentation: [/{{ $key }}/{{ $version }}/](/{{ $key }}/{{ $version }}/) +{{ end }} +{{ end -}} +{{- end -}} +{{- end -}} diff --git a/layouts/_default/section.markdown.md b/layouts/_default/section.markdown.md new file mode 100644 index 000000000..aca28f466 --- /dev/null +++ b/layouts/_default/section.markdown.md @@ -0,0 +1,12 @@ +{{- if not .Params.sitemapexclude -}} +--- +title: {{ .Title | plainify }} +url: {{ .Permalink }} +--- + +# {{ .Title | plainify }} +{{ with .Description | default .Params.description }} +> {{ . }} +{{ end }} +{{ .RenderShortcodes }} +{{- end -}} diff --git a/layouts/_default/single.markdown.md b/layouts/_default/single.markdown.md new file mode 100644 index 000000000..aca28f466 --- /dev/null +++ b/layouts/_default/single.markdown.md @@ -0,0 +1,12 @@ +{{- if not .Params.sitemapexclude -}} +--- +title: {{ .Title | plainify }} +url: {{ .Permalink }} +--- + +# {{ .Title | plainify }} +{{ with .Description | default .Params.description }} +> {{ . }} +{{ end }} +{{ .RenderShortcodes }} +{{- end -}} diff --git a/netlify/edge-functions/markdown-for-agents.js b/netlify/edge-functions/markdown-for-agents.js new file mode 100644 index 000000000..84e232135 --- /dev/null +++ b/netlify/edge-functions/markdown-for-agents.js @@ -0,0 +1,55 @@ +// Netlify Edge Function: Markdown for Agents +// +// When a request includes "Accept: text/markdown", rewrites to the +// pre-generated index.md that Hugo places alongside each index.html. +// Returns Content-Type: text/markdown and x-markdown-tokens as per +// the Cloudflare Markdown for Agents spec. + +export default async (request, context) => { + const accept = request.headers.get("accept") ?? ""; + if (!accept.includes("text/markdown")) { + return; + } + + const url = new URL(request.url); + let path = url.pathname; + if (!path.endsWith("/")) { + path += "/"; + } + url.pathname = path + "index.md"; + + const response = await context.rewrite(url); + if (!response || !response.ok) { + return; + } + + const body = await response.text(); + if (!body.trim()) { + return; + } + const headers = new Headers(response.headers); + headers.set("content-type", "text/markdown; charset=utf-8"); + headers.set("vary", "Accept"); + // Approximate token count (1 token ≈ 4 chars) + headers.set("x-markdown-tokens", String(Math.ceil(body.length / 4))); + + return new Response(body, { status: 200, headers }); +}; + +export const config = { + path: "/*", + excludedPath: [ + "/_redirects", + "/robots.txt", + "/sitemap.xml", + "/search.json", + "/llms.txt", + "/llms-full.txt", + "/css/*", + "/js/*", + "/fonts/*", + "/webfonts/*", + "/img/*", + "/mermaid/*", + ], +}; From 44fcc33571eed584d828dd899a6ae3dc60276aa4 Mon Sep 17 00:00:00 2001 From: serg Date: Thu, 30 Apr 2026 15:59:20 +0300 Subject: [PATCH 04/18] update issue condition for falling shortcode with non existed url, the CHANGELOG-.md part Signed-off-by: serg --- layouts/shortcodes/render_external_markdown.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/layouts/shortcodes/render_external_markdown.html b/layouts/shortcodes/render_external_markdown.html index ff1914427..ee02e0987 100644 --- a/layouts/shortcodes/render_external_markdown.html +++ b/layouts/shortcodes/render_external_markdown.html @@ -2,5 +2,5 @@ {{- with resources.GetRemote $urlToRender (dict "headers" (dict "Cache-Control" "no-cache")) -}} {{- $.Page.RenderString .Content -}} {{- else -}} - {{- errorf "Unable to get remote resource %q" (.Get 0) -}} + {{- warnf "Unable to get remote resource %q" (.Get 0) -}} {{- end -}} From e11797a24bdb15a7cf1267aba012483bfa19558f Mon Sep 17 00:00:00 2001 From: serg Date: Thu, 30 Apr 2026 16:06:55 +0300 Subject: [PATCH 05/18] add content signals to robots.txt Signed-off-by: serg --- layouts/robots.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/layouts/robots.txt b/layouts/robots.txt index b17bee7b0..517dfcfe7 100644 --- a/layouts/robots.txt +++ b/layouts/robots.txt @@ -7,5 +7,6 @@ User-agent: * Disallow: +Content-Signal: ai-train=yes, search=yes, ai-input=yes Sitemap: {{"sitemap.xml" | absURL}} Llms-txt: {{"llms.txt" | absURL}} From 0990be632c53b9c5b837bad686a3a04da5a4e214 Mon Sep 17 00:00:00 2001 From: serg Date: Thu, 30 Apr 2026 16:15:59 +0300 Subject: [PATCH 06/18] add header for useful resources Signed-off-by: serg --- netlify.toml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/netlify.toml b/netlify.toml index 8f06136b2..0fec45373 100644 --- a/netlify.toml +++ b/netlify.toml @@ -12,3 +12,8 @@ directory = "functions" [functions."deploy-succeeded"] included_files = ["public/search.json"] + +[[headers]] + for = "/" + [headers.values] + Link = '''; rel="llms-txt", ; rel="sitemap", ; rel="search"''' From 784f8936c9fb6dc2cf61f601abfeac7db7af401a Mon Sep 17 00:00:00 2001 From: serg Date: Thu, 30 Apr 2026 16:33:29 +0300 Subject: [PATCH 07/18] remove sitemapexclude for kdp Signed-off-by: serg --- content/developer-platform/_index.en.md | 1 - 1 file changed, 1 deletion(-) diff --git a/content/developer-platform/_index.en.md b/content/developer-platform/_index.en.md index 72cf47939..b61b2d13a 100644 --- a/content/developer-platform/_index.en.md +++ b/content/developer-platform/_index.en.md @@ -1,6 +1,5 @@ +++ title = "Kubermatic Developer Platform" -sitemapexclude = true +++ KDP (Kubermatic Developer Platform) is a new Kubermatic product in development that targets the IDP From b0c64703e127b9def09f18e6b8f590494c82fc47 Mon Sep 17 00:00:00 2001 From: serg Date: Thu, 30 Apr 2026 16:58:27 +0300 Subject: [PATCH 08/18] use plain output instead of rendershortcodes for md files Signed-off-by: serg --- layouts/_default/section.markdown.md | 2 +- layouts/_default/single.markdown.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/layouts/_default/section.markdown.md b/layouts/_default/section.markdown.md index aca28f466..ae7e1c87c 100644 --- a/layouts/_default/section.markdown.md +++ b/layouts/_default/section.markdown.md @@ -8,5 +8,5 @@ url: {{ .Permalink }} {{ with .Description | default .Params.description }} > {{ . }} {{ end }} -{{ .RenderShortcodes }} +{{ partial "llms-decode-entities.txt" .Plain }} {{- end -}} diff --git a/layouts/_default/single.markdown.md b/layouts/_default/single.markdown.md index aca28f466..ae7e1c87c 100644 --- a/layouts/_default/single.markdown.md +++ b/layouts/_default/single.markdown.md @@ -8,5 +8,5 @@ url: {{ .Permalink }} {{ with .Description | default .Params.description }} > {{ . }} {{ end }} -{{ .RenderShortcodes }} +{{ partial "llms-decode-entities.txt" .Plain }} {{- end -}} From e96add0ca7670022506011d6a931806d8c05c07d Mon Sep 17 00:00:00 2001 From: serg Date: Thu, 30 Apr 2026 17:10:12 +0300 Subject: [PATCH 09/18] better organisation for llms partials Signed-off-by: serg --- layouts/_default/home.llmstxt.txt | 2 +- layouts/_default/section.llmsfulltxt.txt | 6 +++--- layouts/_default/section.llmstxt.txt | 6 +++--- layouts/_default/section.markdown.md | 2 +- layouts/_default/single.llmsfulltxt.txt | 4 ++-- layouts/_default/single.markdown.md | 2 +- .../{llms-decode-entities.txt => llms/decode-entities.txt} | 0 .../{llms-full-content.txt => llms/full-content.txt} | 6 +++--- .../partials/{llms-is-excluded.txt => llms/is-excluded.txt} | 0 .../partials/{llms-list-pages.txt => llms/list-pages.txt} | 4 ++-- .../partials/{llms-page-count.txt => llms/page-count.txt} | 4 ++-- 11 files changed, 18 insertions(+), 18 deletions(-) rename layouts/partials/{llms-decode-entities.txt => llms/decode-entities.txt} (100%) rename layouts/partials/{llms-full-content.txt => llms/full-content.txt} (75%) rename layouts/partials/{llms-is-excluded.txt => llms/is-excluded.txt} (100%) rename layouts/partials/{llms-list-pages.txt => llms/list-pages.txt} (93%) rename layouts/partials/{llms-page-count.txt => llms/page-count.txt} (73%) diff --git a/layouts/_default/home.llmstxt.txt b/layouts/_default/home.llmstxt.txt index 27af92244..2f24f2e2d 100644 --- a/layouts/_default/home.llmstxt.txt +++ b/layouts/_default/home.llmstxt.txt @@ -15,7 +15,7 @@ {{- end -}} {{- $pagePath := cond (ne $url "") (printf "/%s/%s" $key $url) (printf "/%s" $key) -}} {{- with $.Site.GetPage $pagePath -}} - {{- if not (partial "llms-is-excluded.txt" .) -}} + {{- if not (partial "llms/is-excluded.txt" .) -}} {{- $llmsURL := "" -}} {{- with .OutputFormats.Get "LLMsTxt" -}} {{- $llmsURL = .Permalink -}} diff --git a/layouts/_default/section.llmsfulltxt.txt b/layouts/_default/section.llmsfulltxt.txt index 5ef3e57a4..59931bbdb 100644 --- a/layouts/_default/section.llmsfulltxt.txt +++ b/layouts/_default/section.llmsfulltxt.txt @@ -1,11 +1,11 @@ -{{- if not (partial "llms-is-excluded.txt" .) -}} +{{- if not (partial "llms/is-excluded.txt" .) -}} {{- $segments := split (strings.TrimPrefix "/" (strings.TrimSuffix "/" .RelPermalink)) "/" -}} {{- if le (len $segments) 3 -}} # {{ .Title | default .Name | plainify }} {{ with .Description | default .Params.description }} > {{ . }} {{ end }} -{{ partial "llms-decode-entities.txt" .Plain }} -{{ partial "llms-full-content.txt" . }} +{{ partial "llms/decode-entities.txt" .Plain }} +{{ partial "llms/full-content.txt" . }} {{- end -}} {{- end -}} diff --git a/layouts/_default/section.llmstxt.txt b/layouts/_default/section.llmstxt.txt index 9cdee9092..910b4d12a 100644 --- a/layouts/_default/section.llmstxt.txt +++ b/layouts/_default/section.llmstxt.txt @@ -1,6 +1,6 @@ -{{- if not (partial "llms-is-excluded.txt" .) -}} +{{- if not (partial "llms/is-excluded.txt" .) -}} {{- $title := .Title | default .Name | plainify -}} -{{- $pageCount := partial "llms-page-count.txt" . -}} +{{- $pageCount := partial "llms/page-count.txt" . -}} {{- $segments := split (strings.TrimPrefix "/" (strings.TrimSuffix "/" .RelPermalink)) "/" -}} # {{ $title }} {{ with .Description | default .Params.description }} @@ -21,5 +21,5 @@ - [{{ $title }} — Full Content]({{ . }}) {{ end }} ## Pages -{{ partial "llms-list-pages.txt" . }} +{{ partial "llms/list-pages.txt" . }} {{- end -}} diff --git a/layouts/_default/section.markdown.md b/layouts/_default/section.markdown.md index ae7e1c87c..377c3fcb1 100644 --- a/layouts/_default/section.markdown.md +++ b/layouts/_default/section.markdown.md @@ -8,5 +8,5 @@ url: {{ .Permalink }} {{ with .Description | default .Params.description }} > {{ . }} {{ end }} -{{ partial "llms-decode-entities.txt" .Plain }} +{{ partial "llms/decode-entities.txt" .Plain }} {{- end -}} diff --git a/layouts/_default/single.llmsfulltxt.txt b/layouts/_default/single.llmsfulltxt.txt index 6bcd45113..a06a239dd 100644 --- a/layouts/_default/single.llmsfulltxt.txt +++ b/layouts/_default/single.llmsfulltxt.txt @@ -1,9 +1,9 @@ -{{- if not (partial "llms-is-excluded.txt" .) -}} +{{- if not (partial "llms/is-excluded.txt" .) -}} {{- if .Plain }} # {{ .Title | default .Name | plainify }} {{ with .Description | default .Params.description }} > {{ . }} {{ end }} -{{ partial "llms-decode-entities.txt" .Plain }} +{{ partial "llms/decode-entities.txt" .Plain }} {{- end -}} {{- end -}} diff --git a/layouts/_default/single.markdown.md b/layouts/_default/single.markdown.md index ae7e1c87c..377c3fcb1 100644 --- a/layouts/_default/single.markdown.md +++ b/layouts/_default/single.markdown.md @@ -8,5 +8,5 @@ url: {{ .Permalink }} {{ with .Description | default .Params.description }} > {{ . }} {{ end }} -{{ partial "llms-decode-entities.txt" .Plain }} +{{ partial "llms/decode-entities.txt" .Plain }} {{- end -}} diff --git a/layouts/partials/llms-decode-entities.txt b/layouts/partials/llms/decode-entities.txt similarity index 100% rename from layouts/partials/llms-decode-entities.txt rename to layouts/partials/llms/decode-entities.txt diff --git a/layouts/partials/llms-full-content.txt b/layouts/partials/llms/full-content.txt similarity index 75% rename from layouts/partials/llms-full-content.txt rename to layouts/partials/llms/full-content.txt index 12bf58591..20c04b7ca 100644 --- a/layouts/partials/llms-full-content.txt +++ b/layouts/partials/llms/full-content.txt @@ -4,7 +4,7 @@ Outputs each page as a markdown section with heading + content. Uses .Plain to resolve shortcodes and produce clean text. */ -}} {{- range .Pages -}} - {{- if not (partial "llms-is-excluded.txt" .) -}} + {{- if not (partial "llms/is-excluded.txt" .) -}} {{- if .Plain -}} {{- with .Title }} @@ -12,10 +12,10 @@ ## {{ . | plainify }} {{ end -}} -{{ partial "llms-decode-entities.txt" .Plain | safeHTML }} +{{ partial "llms/decode-entities.txt" .Plain | safeHTML }} {{ end -}} {{- if .IsSection -}} - {{- partial "llms-full-content.txt" . -}} + {{- partial "llms/full-content.txt" . -}} {{- end -}} {{- end -}} {{- end -}} diff --git a/layouts/partials/llms-is-excluded.txt b/layouts/partials/llms/is-excluded.txt similarity index 100% rename from layouts/partials/llms-is-excluded.txt rename to layouts/partials/llms/is-excluded.txt diff --git a/layouts/partials/llms-list-pages.txt b/layouts/partials/llms/list-pages.txt similarity index 93% rename from layouts/partials/llms-list-pages.txt rename to layouts/partials/llms/list-pages.txt index af3ddefbf..4542a944d 100644 --- a/layouts/partials/llms-list-pages.txt +++ b/layouts/partials/llms/list-pages.txt @@ -1,12 +1,12 @@ {{- /* Hierarchical partial: list only direct children of the current section. Context: a page (section). - Skips excluded pages (see llms-is-excluded partial). + Skips excluded pages (see llms/is-excluded partial). Skips pages without main content. For child sections within depth limit (≤3 URL segments), links to their llms-full.txt. For child sections beyond depth limit, links to their llms.txt index. For single pages, lists title and description as plain text. */ -}} {{- range .Pages -}} - {{- if not (partial "llms-is-excluded.txt" .) -}} + {{- if not (partial "llms/is-excluded.txt" .) -}} {{- if .Plain -}} {{- if .IsSection -}} {{- $segments := split (strings.TrimPrefix "/" (strings.TrimSuffix "/" .RelPermalink)) "/" -}} diff --git a/layouts/partials/llms-page-count.txt b/layouts/partials/llms/page-count.txt similarity index 73% rename from layouts/partials/llms-page-count.txt rename to layouts/partials/llms/page-count.txt index 0ba11da0d..22f903663 100644 --- a/layouts/partials/llms-page-count.txt +++ b/layouts/partials/llms/page-count.txt @@ -2,12 +2,12 @@ Context: a section page. */ -}} {{- $count := 0 -}} {{- range .Pages -}} - {{- if not (partial "llms-is-excluded.txt" .) -}} + {{- if not (partial "llms/is-excluded.txt" .) -}} {{- if .Plain -}} {{- $count = add $count 1 -}} {{- end -}} {{- if .IsSection -}} - {{- $count = add $count (partial "llms-page-count.txt" .) -}} + {{- $count = add $count (partial "llms/page-count.txt" .) -}} {{- end -}} {{- end -}} {{- end -}} From 13dd50da02504511e0a8a55c2214086f325444de Mon Sep 17 00:00:00 2001 From: serg Date: Thu, 30 Apr 2026 18:00:07 +0300 Subject: [PATCH 10/18] disable comments for robots.txt Signed-off-by: serg --- layouts/robots.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/layouts/robots.txt b/layouts/robots.txt index 517dfcfe7..b751ed448 100644 --- a/layouts/robots.txt +++ b/layouts/robots.txt @@ -1,9 +1,10 @@ +{{- /* This is a comment, that is not used in production # We allows Google to crawl website. It should crawl the pages from custom sitemap.xml. # sitemap.xml updates every new release and keep the pages for stable version of product. # The previous versions should be removed from Google index on every new release for product. # These pages are marked with "noindex" rule in the "robots" meta tag on build time and will be deleted from Google after some time. # We dont need to disallow crawling for these pages, because Google should have opportunity to analyze previous pages (with "noindex" tag) and update the state of index for those. - +*/ -}} User-agent: * Disallow: From 2059fab28a70c48f9bebf43e265d4b4e78471b65 Mon Sep 17 00:00:00 2001 From: serg Date: Thu, 30 Apr 2026 18:09:06 +0300 Subject: [PATCH 11/18] prevent generation of md files for old product versions Signed-off-by: serg --- layouts/_default/section.markdown.md | 2 +- layouts/_default/single.markdown.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/layouts/_default/section.markdown.md b/layouts/_default/section.markdown.md index 377c3fcb1..570822aa8 100644 --- a/layouts/_default/section.markdown.md +++ b/layouts/_default/section.markdown.md @@ -1,4 +1,4 @@ -{{- if not .Params.sitemapexclude -}} +{{- if not (partial "llms/is-excluded.txt" .) -}} --- title: {{ .Title | plainify }} url: {{ .Permalink }} diff --git a/layouts/_default/single.markdown.md b/layouts/_default/single.markdown.md index 377c3fcb1..570822aa8 100644 --- a/layouts/_default/single.markdown.md +++ b/layouts/_default/single.markdown.md @@ -1,4 +1,4 @@ -{{- if not .Params.sitemapexclude -}} +{{- if not (partial "llms/is-excluded.txt" .) -}} --- title: {{ .Title | plainify }} url: {{ .Permalink }} From 6bc68bdb9d744f47da430b9264da4244c52063b8 Mon Sep 17 00:00:00 2001 From: serg Date: Thu, 30 Apr 2026 19:58:08 +0300 Subject: [PATCH 12/18] improve relations for llms files Signed-off-by: serg --- layouts/_default/section.llmsfulltxt.txt | 3 --- layouts/_default/section.llmstxt.txt | 2 ++ layouts/partials/llms/list-pages.txt | 27 ++++++++++++------------ 3 files changed, 15 insertions(+), 17 deletions(-) diff --git a/layouts/_default/section.llmsfulltxt.txt b/layouts/_default/section.llmsfulltxt.txt index 59931bbdb..25ba29aa7 100644 --- a/layouts/_default/section.llmsfulltxt.txt +++ b/layouts/_default/section.llmsfulltxt.txt @@ -1,6 +1,4 @@ {{- if not (partial "llms/is-excluded.txt" .) -}} -{{- $segments := split (strings.TrimPrefix "/" (strings.TrimSuffix "/" .RelPermalink)) "/" -}} -{{- if le (len $segments) 3 -}} # {{ .Title | default .Name | plainify }} {{ with .Description | default .Params.description }} > {{ . }} @@ -8,4 +6,3 @@ {{ partial "llms/decode-entities.txt" .Plain }} {{ partial "llms/full-content.txt" . }} {{- end -}} -{{- end -}} diff --git a/layouts/_default/section.llmstxt.txt b/layouts/_default/section.llmstxt.txt index 910b4d12a..51a310b85 100644 --- a/layouts/_default/section.llmstxt.txt +++ b/layouts/_default/section.llmstxt.txt @@ -2,6 +2,7 @@ {{- $title := .Title | default .Name | plainify -}} {{- $pageCount := partial "llms/page-count.txt" . -}} {{- $segments := split (strings.TrimPrefix "/" (strings.TrimSuffix "/" .RelPermalink)) "/" -}} +{{- if gt $pageCount 0 -}} # {{ $title }} {{ with .Description | default .Params.description }} > {{ . }} @@ -23,3 +24,4 @@ ## Pages {{ partial "llms/list-pages.txt" . }} {{- end -}} +{{- end -}} diff --git a/layouts/partials/llms/list-pages.txt b/layouts/partials/llms/list-pages.txt index 4542a944d..e353e0442 100644 --- a/layouts/partials/llms/list-pages.txt +++ b/layouts/partials/llms/list-pages.txt @@ -2,31 +2,30 @@ Context: a page (section). Skips excluded pages (see llms/is-excluded partial). Skips pages without main content. - For child sections within depth limit (≤3 URL segments), links to their llms-full.txt. - For child sections beyond depth limit, links to their llms.txt index. + For child sections with >10 pages, links to their llms.txt (navigate deeper). + For child sections with ≤10 pages, links to their llms-full.txt (content directly). For single pages, lists title and description as plain text. */ -}} {{- range .Pages -}} {{- if not (partial "llms/is-excluded.txt" .) -}} {{- if .Plain -}} {{- if .IsSection -}} - {{- $segments := split (strings.TrimPrefix "/" (strings.TrimSuffix "/" .RelPermalink)) "/" -}} - {{- if le (len $segments) 3 -}} - {{- $llmsFullURL := "" -}} - {{- with .OutputFormats.Get "LLMsFullTxt" -}} - {{- $llmsFullURL = .Permalink -}} - {{- end -}} - {{- if and .Title $llmsFullURL -}} - {{- $desc := .Description | default .Params.description | default "" -}} -- [{{ .Title | plainify }}]({{ $llmsFullURL }}){{ with $desc }}: {{ . }}{{ end }} -{{ end -}} - {{- else -}} + {{- $childCount := partial "llms/page-count.txt" . -}} + {{- $desc := .Description | default .Params.description | default "" -}} + {{- if gt $childCount 10 -}} {{- $llmsTxtURL := "" -}} {{- with .OutputFormats.Get "LLMsTxt" -}} {{- $llmsTxtURL = .Permalink -}} {{- end -}} {{- if and .Title $llmsTxtURL -}} - {{- $desc := .Description | default .Params.description | default "" -}} - [{{ .Title | plainify }}]({{ $llmsTxtURL }}){{ with $desc }}: {{ . }}{{ end }} +{{ end -}} + {{- else -}} + {{- $llmsFullURL := "" -}} + {{- with .OutputFormats.Get "LLMsFullTxt" -}} + {{- $llmsFullURL = .Permalink -}} + {{- end -}} + {{- if and .Title $llmsFullURL -}} +- [{{ .Title | plainify }}]({{ $llmsFullURL }}){{ with $desc }}: {{ . }}{{ end }} {{ end -}} {{- end -}} {{- else -}} From 5151c6444a68b7635b08fa441eedeff5b1299710 Mon Sep 17 00:00:00 2001 From: serg Date: Fri, 1 May 2026 13:31:35 +0300 Subject: [PATCH 13/18] use .RawContent instead .Plain Signed-off-by: serg --- layouts/_default/section.llmsfulltxt.txt | 2 +- layouts/_default/section.markdown.md | 2 +- layouts/_default/single.llmsfulltxt.txt | 4 +- layouts/_default/single.markdown.md | 2 +- layouts/partials/llms/full-content.txt | 4 +- layouts/partials/llms/page-count.txt | 2 +- layouts/partials/llms/render-content.txt | 62 ++++++++++++++++++++++++ 7 files changed, 70 insertions(+), 8 deletions(-) create mode 100644 layouts/partials/llms/render-content.txt diff --git a/layouts/_default/section.llmsfulltxt.txt b/layouts/_default/section.llmsfulltxt.txt index 25ba29aa7..b70daafe7 100644 --- a/layouts/_default/section.llmsfulltxt.txt +++ b/layouts/_default/section.llmsfulltxt.txt @@ -3,6 +3,6 @@ {{ with .Description | default .Params.description }} > {{ . }} {{ end }} -{{ partial "llms/decode-entities.txt" .Plain }} +{{ partial "llms/render-content.txt" . }} {{ partial "llms/full-content.txt" . }} {{- end -}} diff --git a/layouts/_default/section.markdown.md b/layouts/_default/section.markdown.md index 570822aa8..856fceab0 100644 --- a/layouts/_default/section.markdown.md +++ b/layouts/_default/section.markdown.md @@ -8,5 +8,5 @@ url: {{ .Permalink }} {{ with .Description | default .Params.description }} > {{ . }} {{ end }} -{{ partial "llms/decode-entities.txt" .Plain }} +{{ partial "llms/render-content.txt" . }} {{- end -}} diff --git a/layouts/_default/single.llmsfulltxt.txt b/layouts/_default/single.llmsfulltxt.txt index a06a239dd..b92b2cfe1 100644 --- a/layouts/_default/single.llmsfulltxt.txt +++ b/layouts/_default/single.llmsfulltxt.txt @@ -1,9 +1,9 @@ {{- if not (partial "llms/is-excluded.txt" .) -}} -{{- if .Plain }} +{{- if .RawContent }} # {{ .Title | default .Name | plainify }} {{ with .Description | default .Params.description }} > {{ . }} {{ end }} -{{ partial "llms/decode-entities.txt" .Plain }} +{{ partial "llms/render-content.txt" . }} {{- end -}} {{- end -}} diff --git a/layouts/_default/single.markdown.md b/layouts/_default/single.markdown.md index 570822aa8..856fceab0 100644 --- a/layouts/_default/single.markdown.md +++ b/layouts/_default/single.markdown.md @@ -8,5 +8,5 @@ url: {{ .Permalink }} {{ with .Description | default .Params.description }} > {{ . }} {{ end }} -{{ partial "llms/decode-entities.txt" .Plain }} +{{ partial "llms/render-content.txt" . }} {{- end -}} diff --git a/layouts/partials/llms/full-content.txt b/layouts/partials/llms/full-content.txt index 20c04b7ca..0248d983e 100644 --- a/layouts/partials/llms/full-content.txt +++ b/layouts/partials/llms/full-content.txt @@ -5,14 +5,14 @@ Uses .Plain to resolve shortcodes and produce clean text. */ -}} {{- range .Pages -}} {{- if not (partial "llms/is-excluded.txt" .) -}} - {{- if .Plain -}} + {{- if .RawContent -}} {{- with .Title }} --- ## {{ . | plainify }} {{ end -}} -{{ partial "llms/decode-entities.txt" .Plain | safeHTML }} +{{ partial "llms/render-content.txt" . }} {{ end -}} {{- if .IsSection -}} {{- partial "llms/full-content.txt" . -}} diff --git a/layouts/partials/llms/page-count.txt b/layouts/partials/llms/page-count.txt index 22f903663..83e4695e3 100644 --- a/layouts/partials/llms/page-count.txt +++ b/layouts/partials/llms/page-count.txt @@ -3,7 +3,7 @@ {{- $count := 0 -}} {{- range .Pages -}} {{- if not (partial "llms/is-excluded.txt" .) -}} - {{- if .Plain -}} + {{- if .RawContent -}} {{- $count = add $count 1 -}} {{- end -}} {{- if .IsSection -}} diff --git a/layouts/partials/llms/render-content.txt b/layouts/partials/llms/render-content.txt new file mode 100644 index 000000000..466d64dfb --- /dev/null +++ b/layouts/partials/llms/render-content.txt @@ -0,0 +1,62 @@ +{{- /* Hybrid content renderer for LLM/MD output. + For pages that require network fetching (render_external_markdown/code), + falls back to .Plain (HTML-stripped text). + For all other pages, processes .RawContent with regex transformations + to preserve native markdown formatting, especially fenced code blocks. + Context: a single page. + Outputs content directly (no return value). */ -}} + +{{- $raw := .RawContent -}} + +{{- /* Detect pages requiring network-fetched content */ -}} +{{- $needsRender := or (strings.Contains $raw "render_external_markdown") (strings.Contains $raw "render_external_code") -}} +{{- if $needsRender -}} +{{- partial "llms/decode-entities.txt" .Plain | safeHTML -}} +{{- else -}} + +{{- $s := $raw -}} + +{{- /* 1. Strip children shortcode — navigation only, no real content */ -}} +{{- $s = replaceRE `\{\{[%<] *children[^}]*[%>]\}\}` "" $s -}} + +{{- /* 2. Inline readfile shortcodes — output raw file content. + The shortcode is typically placed inside a fenced code block in the source + markdown, so no extra wrapping is added here. */ -}} +{{- range findRE `\{\{< *readfile "[^"]+"[^>]*>\}\}` $s -}} + {{- $match := . -}} + {{- $path := replaceRE `\{\{< *readfile "([^"]+)"[^>]*>\}\}` "$1" $match -}} + {{- with readFile $path -}} + {{- $s = replace $s $match (. | strings.TrimRight "\n") -}} + {{- else -}} + {{- $s = replace $s $match "" -}} + {{- end -}} +{{- end -}} + +{{- /* 3. Convert notice shortcodes to labelled blockquotes. + Types: note, warning, tip, info */ -}} +{{- $s = replaceRE `(?s)\{\{% notice (\w+)[^%]*%\}\}\s*(.*?)\s*\{\{% /notice %\}\}` "\n> **$1:** $2\n" $s -}} + +{{- /* 4. Strip tabs wrapper; convert tab names to level-4 headings */ -}} +{{- $s = replaceRE `\{\{< *tabs [^>]*>\}\}` "" $s -}} +{{- $s = replaceRE `\{\{< */tabs *>\}\}` "" $s -}} +{{- $s = replaceRE `\{\{% tab name="([^"]+)"[^%]*%\}\}` "\n#### $1\n\n" $s -}} +{{- $s = replaceRE `\{\{% /tab *%\}\}` "" $s -}} + +{{- /* 5. Convert details shortcodes to heading + content */ -}} +{{- $s = replaceRE `\{\{< *details summary="([^"]+)"[^>]*>\}\}` "\n#### $1\n\n" $s -}} +{{- $s = replaceRE `\{\{< */details *>\}\}` "" $s -}} + +{{- /* 6. Resolve ref/relref in links: [text]({{< ref "path" >}}) → [text](path) */ -}} +{{- $s = replaceRE `\{\{< *rel?ref "([^"]+)"[^>]*>\}\}` "$1" $s -}} + +{{- /* 7. Replace current_version with actual version string from URL */ -}} +{{- $pathParts := split .RelPermalink "/" -}} +{{- if gt (len $pathParts) 2 -}} + {{- $s = replace $s "{{< current_version >}}" (index $pathParts 2) -}} +{{- end -}} + +{{- /* 8. Strip any remaining unhandled shortcode tags (self-closing and wrappers) */ -}} +{{- $s = replaceRE `\{\{[<%][^}]*[%>]\}\}` "" $s -}} + +{{ $s | safeHTML }} +{{- end -}} From fb96a73889e69f19eefdc0c139e64490c31863f0 Mon Sep 17 00:00:00 2001 From: serg Date: Tue, 5 May 2026 12:43:17 +0300 Subject: [PATCH 14/18] use remote snippets as is for md generation Signed-off-by: serg --- layouts/partials/llms/decode-entities.txt | 24 ----------- layouts/partials/llms/render-content.txt | 50 +++++++++++++---------- 2 files changed, 28 insertions(+), 46 deletions(-) delete mode 100644 layouts/partials/llms/decode-entities.txt diff --git a/layouts/partials/llms/decode-entities.txt b/layouts/partials/llms/decode-entities.txt deleted file mode 100644 index e40a0d168..000000000 --- a/layouts/partials/llms/decode-entities.txt +++ /dev/null @@ -1,24 +0,0 @@ -{{- /* Decodes all HTML entities (named and numeric) from a string. - Hugo's htmlUnescape only handles some named entities, so we also - manually replace numeric and remaining named entities. - IMPORTANT: & must be replaced LAST to avoid double-decoding. */ -}} -{{- $s := . | htmlUnescape -}} -{{- $s = replace $s """ "\"" -}} -{{- $s = replace $s "'" "'" -}} -{{- $s = replace $s "+" "+" -}} -{{- $s = replace $s " " " " -}} -{{- $s = replace $s "<" "<" -}} -{{- $s = replace $s ">" ">" -}} -{{- $s = replace $s """ "\"" -}} -{{- $s = replace $s "<" "<" -}} -{{- $s = replace $s ">" ">" -}} -{{- $s = replace $s "’" "\u2019" -}} -{{- $s = replace $s "‘" "\u2018" -}} -{{- $s = replace $s "”" "\u201D" -}} -{{- $s = replace $s "“" "\u201C" -}} -{{- $s = replace $s "…" "\u2026" -}} -{{- $s = replace $s "—" "\u2014" -}} -{{- $s = replace $s "–" "\u2013" -}} -{{- $s = replace $s "&" "&" -}} -{{- $s = replace $s "&" "&" -}} -{{- return $s -}} diff --git a/layouts/partials/llms/render-content.txt b/layouts/partials/llms/render-content.txt index 466d64dfb..f7fd439b5 100644 --- a/layouts/partials/llms/render-content.txt +++ b/layouts/partials/llms/render-content.txt @@ -1,27 +1,19 @@ {{- /* Hybrid content renderer for LLM/MD output. - For pages that require network fetching (render_external_markdown/code), - falls back to .Plain (HTML-stripped text). - For all other pages, processes .RawContent with regex transformations - to preserve native markdown formatting, especially fenced code blocks. + Processes .RawContent with regex transformations to preserve native + markdown formatting (fenced code blocks, headings, links, etc.). + External content (render_external_markdown/code) is fetched at build + time via resources.GetRemote and inlined directly. Context: a single page. Outputs content directly (no return value). */ -}} -{{- $raw := .RawContent -}} - -{{- /* Detect pages requiring network-fetched content */ -}} -{{- $needsRender := or (strings.Contains $raw "render_external_markdown") (strings.Contains $raw "render_external_code") -}} -{{- if $needsRender -}} -{{- partial "llms/decode-entities.txt" .Plain | safeHTML -}} -{{- else -}} - -{{- $s := $raw -}} +{{- $s := .RawContent -}} {{- /* 1. Strip children shortcode — navigation only, no real content */ -}} {{- $s = replaceRE `\{\{[%<] *children[^}]*[%>]\}\}` "" $s -}} {{- /* 2. Inline readfile shortcodes — output raw file content. - The shortcode is typically placed inside a fenced code block in the source - markdown, so no extra wrapping is added here. */ -}} + The shortcode is typically placed inside a fenced code block in the + source markdown, so no extra wrapping is added here. */ -}} {{- range findRE `\{\{< *readfile "[^"]+"[^>]*>\}\}` $s -}} {{- $match := . -}} {{- $path := replaceRE `\{\{< *readfile "([^"]+)"[^>]*>\}\}` "$1" $match -}} @@ -32,31 +24,45 @@ {{- end -}} {{- end -}} -{{- /* 3. Convert notice shortcodes to labelled blockquotes. +{{- /* 3. Inline render_external_markdown / render_external_code: fetch the remote + resource and embed its content directly. Both shortcodes produce raw + content (markdown or code) that is already wrapped appropriately by + the surrounding source markdown, so no extra wrapping is added. */ -}} +{{- range findRE `\{\{< *render_external(?:_markdown|_code) "[^"]+"[^>]*>\}\}` $s -}} + {{- $match := . -}} + {{- $url := replaceRE `\{\{< *render_external(?:_markdown|_code) "([^"]+)"[^>]*>\}\}` "$1" $match -}} + {{- with resources.GetRemote $url (dict "headers" (dict "Cache-Control" "no-cache")) -}} + {{- $s = replace $s $match (.Content | strings.TrimRight "\n") -}} + {{- else -}} + {{- warnf "llms/render-content: failed to fetch %s" $url -}} + {{- $s = replace $s $match "" -}} + {{- end -}} +{{- end -}} + +{{- /* 4. Convert notice shortcodes to labelled blockquotes. Types: note, warning, tip, info */ -}} {{- $s = replaceRE `(?s)\{\{% notice (\w+)[^%]*%\}\}\s*(.*?)\s*\{\{% /notice %\}\}` "\n> **$1:** $2\n" $s -}} -{{- /* 4. Strip tabs wrapper; convert tab names to level-4 headings */ -}} +{{- /* 5. Strip tabs wrapper; convert tab names to level-4 headings */ -}} {{- $s = replaceRE `\{\{< *tabs [^>]*>\}\}` "" $s -}} {{- $s = replaceRE `\{\{< */tabs *>\}\}` "" $s -}} {{- $s = replaceRE `\{\{% tab name="([^"]+)"[^%]*%\}\}` "\n#### $1\n\n" $s -}} {{- $s = replaceRE `\{\{% /tab *%\}\}` "" $s -}} -{{- /* 5. Convert details shortcodes to heading + content */ -}} +{{- /* 6. Convert details shortcodes to heading + content */ -}} {{- $s = replaceRE `\{\{< *details summary="([^"]+)"[^>]*>\}\}` "\n#### $1\n\n" $s -}} {{- $s = replaceRE `\{\{< */details *>\}\}` "" $s -}} -{{- /* 6. Resolve ref/relref in links: [text]({{< ref "path" >}}) → [text](path) */ -}} +{{- /* 7. Resolve ref/relref in links: [text]({{< ref "path" >}}) → [text](path) */ -}} {{- $s = replaceRE `\{\{< *rel?ref "([^"]+)"[^>]*>\}\}` "$1" $s -}} -{{- /* 7. Replace current_version with actual version string from URL */ -}} +{{- /* 8. Replace current_version with actual version string from URL */ -}} {{- $pathParts := split .RelPermalink "/" -}} {{- if gt (len $pathParts) 2 -}} {{- $s = replace $s "{{< current_version >}}" (index $pathParts 2) -}} {{- end -}} -{{- /* 8. Strip any remaining unhandled shortcode tags (self-closing and wrappers) */ -}} +{{- /* 9. Strip any remaining unhandled shortcode tags (self-closing and wrappers) */ -}} {{- $s = replaceRE `\{\{[<%][^}]*[%>]\}\}` "" $s -}} {{ $s | safeHTML }} -{{- end -}} From 34403525f6ccbdd520a92ae1016ec31295ccdc06 Mon Sep 17 00:00:00 2001 From: serg Date: Tue, 5 May 2026 13:16:22 +0300 Subject: [PATCH 15/18] remove unused llms layout Signed-off-by: serg --- layouts/_default/single.llmsfulltxt.txt | 9 --------- 1 file changed, 9 deletions(-) delete mode 100644 layouts/_default/single.llmsfulltxt.txt diff --git a/layouts/_default/single.llmsfulltxt.txt b/layouts/_default/single.llmsfulltxt.txt deleted file mode 100644 index b92b2cfe1..000000000 --- a/layouts/_default/single.llmsfulltxt.txt +++ /dev/null @@ -1,9 +0,0 @@ -{{- if not (partial "llms/is-excluded.txt" .) -}} -{{- if .RawContent }} -# {{ .Title | default .Name | plainify }} -{{ with .Description | default .Params.description }} -> {{ . }} -{{ end }} -{{ partial "llms/render-content.txt" . }} -{{- end -}} -{{- end -}} From 4d524aeb45d62b53e91ae9da811a2a7bcdcc421a Mon Sep 17 00:00:00 2001 From: serg Date: Tue, 5 May 2026 13:32:03 +0300 Subject: [PATCH 16/18] small improvements Signed-off-by: serg --- layouts/partials/llms/full-content.txt | 2 +- layouts/partials/llms/list-pages.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/layouts/partials/llms/full-content.txt b/layouts/partials/llms/full-content.txt index 0248d983e..733964a93 100644 --- a/layouts/partials/llms/full-content.txt +++ b/layouts/partials/llms/full-content.txt @@ -2,7 +2,7 @@ Context: a page (section or single). Skips excluded pages and pages without content. Outputs each page as a markdown section with heading + content. - Uses .Plain to resolve shortcodes and produce clean text. */ -}} + Uses .RawContent via render-content.txt partial. */ -}} {{- range .Pages -}} {{- if not (partial "llms/is-excluded.txt" .) -}} {{- if .RawContent -}} diff --git a/layouts/partials/llms/list-pages.txt b/layouts/partials/llms/list-pages.txt index e353e0442..8b42117b4 100644 --- a/layouts/partials/llms/list-pages.txt +++ b/layouts/partials/llms/list-pages.txt @@ -7,7 +7,7 @@ For single pages, lists title and description as plain text. */ -}} {{- range .Pages -}} {{- if not (partial "llms/is-excluded.txt" .) -}} - {{- if .Plain -}} + {{- if .RawContent -}} {{- if .IsSection -}} {{- $childCount := partial "llms/page-count.txt" . -}} {{- $desc := .Description | default .Params.description | default "" -}} From 78486dcb2521420cf13db85cc03c82915604ac2c Mon Sep 17 00:00:00 2001 From: serg Date: Tue, 5 May 2026 19:12:11 +0300 Subject: [PATCH 17/18] improve path for relative links Signed-off-by: serg --- layouts/partials/llms/render-content.txt | 53 ++++++++++++++++++++++-- 1 file changed, 49 insertions(+), 4 deletions(-) diff --git a/layouts/partials/llms/render-content.txt b/layouts/partials/llms/render-content.txt index f7fd439b5..104040b89 100644 --- a/layouts/partials/llms/render-content.txt +++ b/layouts/partials/llms/render-content.txt @@ -53,16 +53,61 @@ {{- $s = replaceRE `\{\{< *details summary="([^"]+)"[^>]*>\}\}` "\n#### $1\n\n" $s -}} {{- $s = replaceRE `\{\{< */details *>\}\}` "" $s -}} -{{- /* 7. Resolve ref/relref in links: [text]({{< ref "path" >}}) → [text](path) */ -}} -{{- $s = replaceRE `\{\{< *rel?ref "([^"]+)"[^>]*>\}\}` "$1" $s -}} +{{- /* 7. Resolve ref/relref in links: [text]({{< ref "path" >}}) → [text](path) + Also handles malformed shortcodes with escaped quotes: {{< ref \"path\" >}} */ -}} +{{- $s = replaceRE `\{\{< *(?:rel)?ref \\?"([^"\\]+)\\?"[^>]*>\}\}` "$1" $s -}} -{{- /* 8. Replace current_version with actual version string from URL */ -}} +{{- /* 8. Resolve relative markdown links to absolute URLs. + After step 7, relref-based paths are plain relative paths (e.g. ../../tutorials/kkp). + Plain relative links (e.g. ../../installation/) are also still relative at this point. + AI parsers cannot resolve relative paths without knowing the base URL, so we convert + all [text](../path) and [text](../../path) links to absolute URLs here. + We use manual path resolution (not Hugo's relref) to avoid REF_NOT_FOUND errors for + relative links that point to non-page resources (images, PDFs, etc.). */ -}} +{{- $siteBase := strings.TrimSuffix "/" .Site.BaseURL -}} +{{- $baseParts := slice -}} +{{- range split (strings.TrimSuffix "/" .RelPermalink) "/" -}} + {{- if ne . "" -}} + {{- $baseParts = $baseParts | append . -}} + {{- end -}} +{{- end -}} +{{- range findRE `\[([^\]]*)\]\((\.\.?/[^)]*)\)` $s -}} + {{- $fullMatch := . -}} + {{- $linkText := replaceRE `\[([^\]]*)\]\((\.\.?/[^)]*)\)` "$1" . -}} + {{- $href := replaceRE `\[([^\]]*)\]\((\.\.?/[^)]*)\)` "$2" . -}} + {{- /* Split anchor fragment from path */ -}} + {{- $anchor := replaceRE `^[^#]*` "" $href -}} + {{- $href = replaceRE `#.*$` "" $href -}} + {{- /* Preserve trailing slash */ -}} + {{- $trailingSlash := "" -}} + {{- if strings.HasSuffix $href "/" -}} + {{- $trailingSlash = "/" -}} + {{- end -}} + {{- /* Resolve path segments against the page's directory */ -}} + {{- $parts := slice -}} + {{- range $baseParts -}} + {{- $parts = $parts | append . -}} + {{- end -}} + {{- range split $href "/" -}} + {{- if eq . ".." -}} + {{- if gt (len $parts) 0 -}} + {{- $parts = first (sub (len $parts) 1) $parts -}} + {{- end -}} + {{- else if and (ne . ".") (ne . "") -}} + {{- $parts = $parts | append . -}} + {{- end -}} + {{- end -}} + {{- $absURL := printf "%s/%s%s%s" $siteBase (delimit $parts "/") $trailingSlash $anchor -}} + {{- $s = replace $s $fullMatch (printf "[%s](%s)" $linkText $absURL) -}} +{{- end -}} + +{{- /* 9. Replace current_version with actual version string from URL */ -}} {{- $pathParts := split .RelPermalink "/" -}} {{- if gt (len $pathParts) 2 -}} {{- $s = replace $s "{{< current_version >}}" (index $pathParts 2) -}} {{- end -}} -{{- /* 9. Strip any remaining unhandled shortcode tags (self-closing and wrappers) */ -}} +{{- /* 10. Strip any remaining unhandled shortcode tags (self-closing and wrappers) */ -}} {{- $s = replaceRE `\{\{[<%][^}]*[%>]\}\}` "" $s -}} {{ $s | safeHTML }} From de269e12edb6442cc616c7718d84ede1b73c1f87 Mon Sep 17 00:00:00 2001 From: serg Date: Tue, 5 May 2026 19:36:31 +0300 Subject: [PATCH 18/18] fix case when defining link as [id]: ../path Signed-off-by: serg --- layouts/partials/llms/render-content.txt | 47 +++++++++++++++--------- 1 file changed, 30 insertions(+), 17 deletions(-) diff --git a/layouts/partials/llms/render-content.txt b/layouts/partials/llms/render-content.txt index 104040b89..0a991051e 100644 --- a/layouts/partials/llms/render-content.txt +++ b/layouts/partials/llms/render-content.txt @@ -58,12 +58,14 @@ {{- $s = replaceRE `\{\{< *(?:rel)?ref \\?"([^"\\]+)\\?"[^>]*>\}\}` "$1" $s -}} {{- /* 8. Resolve relative markdown links to absolute URLs. - After step 7, relref-based paths are plain relative paths (e.g. ../../tutorials/kkp). + After step 7, ref/relref shortcodes are plain relative paths (e.g. ../../tutorials/kkp). Plain relative links (e.g. ../../installation/) are also still relative at this point. AI parsers cannot resolve relative paths without knowing the base URL, so we convert - all [text](../path) and [text](../../path) links to absolute URLs here. - We use manual path resolution (not Hugo's relref) to avoid REF_NOT_FOUND errors for - relative links that point to non-page resources (images, PDFs, etc.). */ -}} + them to absolute URLs here using manual path resolution (not Hugo's relref) to avoid + REF_NOT_FOUND errors for links pointing to non-page resources (images, PDFs, etc.). + Two link forms are handled: + a) Inline links: [text](../path) + b) Reference definitions: [id]: ../path */ -}} {{- $siteBase := strings.TrimSuffix "/" .Site.BaseURL -}} {{- $baseParts := slice -}} {{- range split (strings.TrimSuffix "/" .RelPermalink) "/" -}} @@ -71,34 +73,45 @@ {{- $baseParts = $baseParts | append . -}} {{- end -}} {{- end -}} +{{- /* 8a. Inline links: [text](../path) */ -}} {{- range findRE `\[([^\]]*)\]\((\.\.?/[^)]*)\)` $s -}} {{- $fullMatch := . -}} {{- $linkText := replaceRE `\[([^\]]*)\]\((\.\.?/[^)]*)\)` "$1" . -}} {{- $href := replaceRE `\[([^\]]*)\]\((\.\.?/[^)]*)\)` "$2" . -}} - {{- /* Split anchor fragment from path */ -}} {{- $anchor := replaceRE `^[^#]*` "" $href -}} {{- $href = replaceRE `#.*$` "" $href -}} - {{- /* Preserve trailing slash */ -}} {{- $trailingSlash := "" -}} - {{- if strings.HasSuffix $href "/" -}} - {{- $trailingSlash = "/" -}} - {{- end -}} - {{- /* Resolve path segments against the page's directory */ -}} + {{- if strings.HasSuffix $href "/" -}}{{- $trailingSlash = "/" -}}{{- end -}} {{- $parts := slice -}} - {{- range $baseParts -}} - {{- $parts = $parts | append . -}} + {{- range $baseParts -}}{{- $parts = $parts | append . -}}{{- end -}} + {{- range split $href "/" -}} + {{- if eq . ".." -}} + {{- if gt (len $parts) 0 -}}{{- $parts = first (sub (len $parts) 1) $parts -}}{{- end -}} + {{- else if and (ne . ".") (ne . "") -}} + {{- $parts = $parts | append . -}} + {{- end -}} {{- end -}} + {{- $s = replace $s $fullMatch (printf "[%s](%s/%s%s%s)" $linkText $siteBase (delimit $parts "/") $trailingSlash $anchor) -}} +{{- end -}} +{{- /* 8b. Reference definitions: [id]: ../path */ -}} +{{- range findRE `\[[^\]]+\]: *(\.\.?/[^\s]+)` $s -}} + {{- $fullMatch := . -}} + {{- $id := replaceRE `\[([^\]]+)\]: *(\.\.?/[^\s]+)` "$1" . -}} + {{- $href := replaceRE `\[([^\]]+)\]: *(\.\.?/[^\s]+)` "$2" . -}} + {{- $anchor := replaceRE `^[^#]*` "" $href -}} + {{- $href = replaceRE `#.*$` "" $href -}} + {{- $trailingSlash := "" -}} + {{- if strings.HasSuffix $href "/" -}}{{- $trailingSlash = "/" -}}{{- end -}} + {{- $parts := slice -}} + {{- range $baseParts -}}{{- $parts = $parts | append . -}}{{- end -}} {{- range split $href "/" -}} {{- if eq . ".." -}} - {{- if gt (len $parts) 0 -}} - {{- $parts = first (sub (len $parts) 1) $parts -}} - {{- end -}} + {{- if gt (len $parts) 0 -}}{{- $parts = first (sub (len $parts) 1) $parts -}}{{- end -}} {{- else if and (ne . ".") (ne . "") -}} {{- $parts = $parts | append . -}} {{- end -}} {{- end -}} - {{- $absURL := printf "%s/%s%s%s" $siteBase (delimit $parts "/") $trailingSlash $anchor -}} - {{- $s = replace $s $fullMatch (printf "[%s](%s)" $linkText $absURL) -}} + {{- $s = replace $s $fullMatch (printf "[%s]: %s/%s%s%s" $id $siteBase (delimit $parts "/") $trailingSlash $anchor) -}} {{- end -}} {{- /* 9. Replace current_version with actual version string from URL */ -}}