new 'searchable' attribute on page entries - hidden files no longer searchable by default

.changeset/silent-doors-show.md

@@ -0,0 +1,8 @@
+---
+'myst-cli': minor
+'myst-toc': minor
+---
+
+new `searchable` field on TOC file entries
+new `hidden_files_searchable` option on the `site:` config
+as a result hidden inputs are no longer searchable by default

docs/table-of-contents.md

@@ -213,6 +213,43 @@ project:
 
 In particular: hidden pages do not impact numbering; also they can be referred to by other pages in the project.
 
+(searchable-in-toc)=
+
+### Making pages searchable in the Table of Contents
+
+By default:
+- **non-hidden** pages are **searchable** (their content is indexed and can be found via the search bar)
+- **hidden** pages are **not searchable**
+
+There are two levels of settings that let you better control this default behavior:
+- any input in the `toc` section can have a `searchable: true/false` attribute,
+  which will override the default behavior for that specific page or pattern;
+- when a `searchable` attribute is not present, the default behavior is to:
+  - treat a non-hidden page as searchable
+  - and for hidden pages, the value of the global `hidden_files_searchable` - in
+    the project's `site` settings - will determine whether they are searchable
+    or not, default is `false`
+
+```{code} yaml
+:filename: myst.yml
+version: 1
+project:
+  toc:
+    # in the toc and searchable
+    - file: plain-page.md 
+    # not in the toc and not searchable
+    - file: hidden-page.md
+      hidden: true
+    # not in the toc but searchable
+    - file: hidden-but-searchable.md
+      hidden: true
+      searchable: true
+site:
+  # this is the defaut, you can set it to true 
+  # if that's the defaut you want
+  hidden_files_searchable: false
+```
+
 ## In-page table of contents
 
 The {myst:directive}`toc` directive displays a list of titles and links for all headers that follow on the page. This can be done at the `project`, `page`, or `section`, level.

packages/myst-cli/src/process/site.ts

@@ -162,8 +162,31 @@ export async function writeMystXRefJson(session: ISession, states: ReferenceStat
   writeFileToFolder(filename, JSON.stringify(mystXRefs));
 }
 
-export async function writeMystSearchJson(session: ISession, pages: LocalProjectPage[]) {
+/**
+ * Resolve whether a page's content should be included in the search index.
+ *
+ * Precedence: an explicit per-page `searchable` flag wins; otherwise hidden
+ * pages fall back to the site-wide `hidden_files_searchable` default (itself
+ * defaulting to false), and non-hidden pages default to searchable.
+ */
+function pageIsSearchable(page: LocalProjectPage, hiddenFilesSearchable: boolean): boolean {
+  // hidden or not, setting searchable explicitly wins
+  if (page.searchable !== undefined) return page.searchable;
+  // hidden pages fall back to the site-wide default
+  if (page.hidden) return hiddenFilesSearchable;
+  // general case: non-hidden pages are searchable
+  return true;
+}
+
+export async function writeMystSearchJson(
+  session: ISession,
+  pages: LocalProjectPage[],
+  opts?: { hiddenFilesSearchable?: boolean },
+) {
+  const hiddenFilesSearchable = opts?.hiddenFilesSearchable ?? false;
   const records = [...pages]
+    // Drop pages that are not meant to be searchable (see pageIsSearchable)
+    .filter((page) => pageIsSearchable(page, hiddenFilesSearchable))
     // Ensure deterministic ordering
     .sort((left, right) => {
       if (left.file < right.file) {
@@ -740,7 +763,10 @@ export async function processSite(session: ISession, opts?: ProcessSiteOptions):
     await writeObjectsInv(session, states, siteConfig);
     await writeMystXRefJson(session, states);
     // Search does not include parts
-    await writeMystSearchJson(session, allPages);
+    // Hidden pages are excluded from search unless this is explicitly enabled
+    const hiddenFilesSearchable =
+      (siteConfig as any)?.options?.hidden_files_searchable === true;
+    await writeMystSearchJson(session, allPages, { hiddenFilesSearchable });
   }
   return true;
 }

packages/myst-cli/src/project/types.ts

@@ -24,6 +24,9 @@ export type LocalProjectPage = {
   title?: string;
   /** Flag to mark if the page is implied from a TOC pattern or folder structure */
   implicit?: boolean;
+  hidden?: boolean;
+  /** Explicit per-page override for whether the page is included in the search index */
+  searchable?: boolean;
 };
 
 export type ExternalURL = {

packages/myst-toc/src/toc.ts

@@ -23,7 +23,7 @@ import {
   validateChoice,
 } from 'simple-validators';
 
-const COMMON_ENTRY_KEYS = ['title', 'hidden'];
+const COMMON_ENTRY_KEYS = ['title', 'hidden', 'searchable'];
 // const COMMON_ENTRY_KEYS = ['title', 'hidden', 'numbering', 'id', 'class'];
 
 function validateCommonEntry(entry: Record<string, any>, opts: ValidationOptions): CommonEntry {
@@ -36,6 +36,10 @@ function validateCommonEntry(entry: Record<string, any>, opts: ValidationOptions
     output.hidden = validateBoolean(entry.hidden, incrementOptions('hidden', opts));
   }
 
+  if (defined(entry.searchable)) {
+    output.searchable = validateBoolean(entry.searchable, incrementOptions('searchable', opts));
+  }
+
   // if (defined(entry.numbering)) {
   //   output.numbering = validateString(entry.numbering, incrementOptions('numbering', opts));
   // }

packages/myst-toc/src/types.ts

@@ -5,6 +5,7 @@
 export type CommonEntry = {
   title?: string;
   hidden?: boolean;
+  searchable?: boolean;
   // numbering?: string;
   // id?: string;
   // class?: string;