fix(package-deps): materialize the workspace library closure

doc/changes/changed/14373.md

@@ -1,5 +1,5 @@
-- `(deps (package ...))` now exposes only the directly declared packages to
-  the action's environment (`OCAMLPATH`, `PATH`, etc.). Previously, other
-  packages in the workspace could be discoverable via the shared install
-  staging area. Actions that relied on undeclared packages being visible
-  must declare them explicitly. (#14373, @Alizter)
+- `(deps (package ...))` now exposes the declared packages and the workspace
+  libraries in their OCaml library closure to the action's environment.
+  Previously, every package in the workspace could be discoverable via the
+  shared install staging area. Non-library entries and unrelated libraries
+  remain available only from explicitly declared packages. (#14373, @Alizter)

doc/changes/fixed/15511.md

@@ -0,0 +1,7 @@
+- Make `(deps (package ...))` include the workspace library closure in each
+  compilation mode, including PPX runtimes, virtual libraries with or without
+  defaults, serialized subsystem and instrumentation references, redirect
+  chains, and deprecated-package metadata. This preserves installed library
+  semantics without exposing complete owning
+  packages or following the broader package dependency graph. (#15511,
+  @Alizter)

doc/changes/fixed/15994.md

@@ -0,0 +1,2 @@
+- Reject malformed META file templates at their source or generated location,
+  and preserve backslashes in generated values. (#15994, @Alizter)

doc/concepts/dependency-spec.rst

@@ -40,9 +40,16 @@ Dependencies in ``dune`` files can be specified using one of the following:
   from the workspace.
 - ``(package <pkg>)`` builds the files installed by ``<package>`` and adds
   them to the action's environment: bin entries on ``PATH``, libraries on
-  ``OCAMLPATH``, stublibs on ``CAML_LD_LIBRARY_PATH``, and so on. Only the
-  named package is added; transitive package dependencies must be listed
-  separately.
+  ``OCAMLPATH``, stublibs on ``CAML_LD_LIBRARY_PATH``, and so on. For a
+  workspace package, Dune also adds the workspace libraries in the transitive
+  closure of the libraries it installs, in each compilation mode. Only those
+  libraries' artifacts and metadata are added, not the other contents of their
+  owning packages. This makes the same library closure findable before and
+  after installation. A META file template that adds or replaces dependency or
+  artifact metadata cannot supply support metadata for this scoped layout. Dune
+  does not
+  follow dependencies from package metadata; non-library package dependencies
+  must be listed separately.
 - ``(env_var <var>)`` depends on the value of the environment variable ``<var>``.
   If this variable becomes set, becomes unset, or changes value, the target
   will be rebuilt.

doc/dev/install-layouts.md

@@ -1,23 +1,24 @@
-# Install Layouts for Package Sets
+# Install Layouts for Package and Library Sets
 
 ## Overview
 
 `(deps (package ...))` materializes a scoped install layout under
-`_build/install/<context>/.packages/<digest>/` containing only the declared
-package dependencies. This replaces the old alias-based mechanism where `(deps
-(package foo))` depended on the `.foo-files` install alias, which populated the
+`_build/install/<context>/.packages/<digest>/` containing the declared packages
+and the workspace library artifacts needed by their OCaml library closure.
+This replaces the old alias-based mechanism where `(deps (package foo))`
+depended on the `.foo-files` install alias, which populated the
 `_build/install/` staging area shared by all packages.
 
 ## Motivation
 
 The `_build/install/` staging area shared by all packages causes several
 problems:
 
-1. Actions can silently depend on packages they did not declare via the shared
-   environment variables (OCAMLPATH, PATH, etc.). Whether an action succeeds
-   can depend on what other packages happened to be built, making builds
-   non-deterministic. `(strict_package_deps)` validates that dependencies are
-   declared but does not prevent undeclared packages from being visible at
+1. Actions can silently depend on arbitrary packages they did not declare via
+   the shared environment variables (OCAMLPATH, PATH, etc.). Whether an action
+   succeeds can depend on what other packages happened to be built, making
+   builds non-deterministic. `(strict_package_deps)` validates that dependencies
+   are declared but does not prevent unrelated packages from being visible at
    runtime.
 
 2. The shared staging area can cause rule collisions and dependency cycles.
@@ -54,28 +55,111 @@ problems:
   non-lock-dir contexts; with a lock-dir active, external packages can
   only resolve as `Build` (if they are in `dune.lock`) or as not found.
 
-### Immediate deps only
-
-The layout includes only the immediate packages listed in `(deps (package
-...))`. No transitive expansion is performed. Actions should declare what
-they need explicitly.
-
-This is a deliberate design choice:
-
-1. Transitive closure cannot traverse lock-dir packages (they are not
-   workspace packages), so it gives incomplete results in mixed
-   workspace/lock-dir setups. Immediate deps avoid this inconsistency.
-
-2. Workspace package compilation is handled by dune internally via `Lib.DB`,
-   not via OCAMLPATH. The only consumer of OCAMLPATH in the layout is
-   user-written rule actions, where explicit deps are appropriate.
-
-3. Immediate deps keep layout *contents* tractable for the "in-and-out" problem
-   ([#8652]). This is orthogonal to whether per-package layouts exist at all.
-   When a lock-dir package depends on a workspace package, that workspace
-   package's layout can be provided to the lock-dir package's build env without
-   first computing a transitive closure that would have to cross back into other
-   lock-dir packages.
+### Action package dependencies versus package metadata
+
+Two different dependencies are involved here:
+
+- `(deps (package foo))` is an action dependency. It asks Dune to build or
+  locate the installable contents of `foo`, make them available to the action,
+  and track those contents as inputs.
+
+- `(package (name foo) (depends ...))` records package metadata. It describes
+  what must be available when `foo` is built or installed, but it does not say
+  which artifacts an arbitrary action intends to use.
+
+The install layout never computes the transitive closure of the second kind.
+For example, if package `foo` has a package dependency on `bar`, then
+`(deps (package foo))` does not expose `bar`'s executable, data files, or
+unrelated libraries. An action using those must also declare
+`(package bar)`. The only implicit expansion is the narrower library closure
+described below.
+
+### Library closure, not package closure
+
+The layout starts with the packages listed in `(deps (package ...))`. For each
+workspace package, Dune also finds the libraries it installs, computes their
+transitive OCaml and Melange library closures, and adds only the install
+entries belonging to those workspace libraries. It does not add the libraries'
+complete owning packages or traverse dependencies in package metadata.
+
+This distinction preserves two important properties:
+
+1. Installed and workspace packages have the same library semantics. The
+   `META` or `dune-package` entry for an installed library records the other
+   libraries it requires, and OCaml tools recursively resolve those libraries.
+   A scoped workspace layout must therefore make the same libraries findable.
+   Otherwise, metadata in the layout can refer to libraries outside it and a
+   command that works after installation can fail in the workspace.
+
+2. Package closure cannot be defined consistently. Dune knows package
+   dependencies for workspace and lock-directory packages, but `dune-package`
+   does not record them and findlib packages do not provide a reliable package
+   dependency graph. Following package dependencies only when that information
+   happens to be available would make `(package ...)` depend on where a package
+   came from.
+
+Package dependencies are also broader than the requirement imposed by OCaml
+library metadata: they can bring in tools, data, test dependencies, and
+unrelated libraries. Such dependencies must still be listed explicitly when an
+action uses them.
+
+Dune starts from both the libraries and deprecated-library redirects installed
+by each declared package. In each compilation mode, it computes the shared
+compile and link closures of every root library independently rather than
+trying to link all the package's libraries together. Their union retains
+virtual libraries as metadata requirements while also selecting default
+implementations needed at link time. Dune additionally follows PPX runtime
+dependencies and library references serialized by inline-test backends, PPX
+drivers, and instrumentation backends. Every intermediate name in a
+deprecated-library redirect chain is
+retained, including metadata emitted under deprecated package names.
+
+A package may provide independent libraries or competing implementations of a
+virtual library, so its libraries need not form one valid link-time closure.
+Dune requires a public virtual library and its default implementation to belong
+to the same package, but they remain separate libraries and both are selected
+when the default is needed. Resolution errors are reported rather than silently
+producing an incomplete layout.
+
+Library metadata is normally grouped by package. For each owning package in
+the support closure, Dune therefore generates a filtered `META` and
+`dune-package` containing only the selected libraries and relevant deprecated
+library redirects. The layout similarly contains only those libraries'
+interfaces, archives, runtime files, headers, and stubs. Unrelated sibling
+libraries and non-library entries such as binaries, data, and documentation
+are excluded.
+
+At namespace nodes that do not themselves contain a selected library, the
+filtered `META` retains only `directory`, which is the setting inherited by
+findlib subpackages. Other template variables at such a node describe that
+package node rather than its selected descendants and are dropped.
+
+A META file template can replace dependency or artifact rules independently of
+Dune's library graph and selected install entries. Such a template remains
+valid for ordinary installation, but a scoped `(package ...)` layout rejects
+additions or replacements of dependency and artifact-bearing variables in the
+selected package hierarchy. A template may omit generated rules. Rules in
+unselected support subpackages are filtered first, while complete explicitly
+requested packages are checked without filtering. Dune reports an incompatible
+rule instead of emitting dangling metadata or exposing a broader closure than
+the Dune library graph requires.
+
+Package sites and their directories are also excluded. The filtered
+`dune-package` consequently uses an empty sites map instead of advertising
+paths that the support layout does not materialize.
+
+The consuming action depends on every path materialized in the layout. A
+support library therefore adds dependencies on its selected interfaces,
+archives, stubs, headers, runtime files, and filtered metadata—not merely an
+`OCAMLPATH` entry. The symlink or generated-file rule for each layout path then
+tracks the corresponding workspace artifact or metadata input. Changes in a
+transitively required workspace library consequently invalidate the action.
+
+Installed libraries in the closure are already visible through the context
+environment; only workspace library artifacts need adding to the scoped
+layout. Because rule actions are opaque, a support library on `OCAMLPATH` is
+also directly queryable with `ocamlfind`. The distinction is one of dependency
+selection and layout contents, not an access-control boundary.
 
 ### Environment variables
 
@@ -100,12 +184,13 @@ entirely.
 
 The `env` lazy in `Super_context.create` (`src/dune_rules/super_context.ml`) is
 just `Context.installed_env` plus `Site_env.add_packages_env`, with no staging
-paths attached. Actions only see declared dependencies via layout env vars
-consed in `extend_action`. This is what makes the strict-deps property hold for
-workspace build outputs: no workspace-built artifact is reachable via
-dune-managed env vars unless declared. The user's inherited shell env (PATH,
-OCAMLPATH, findlib config) is still visible via `Context.installed_env`, as
-documented in "Environment variables" above.
+paths attached. Actions only see the named packages and the selected artifacts
+of their library closure via layout env vars consed in `extend_action`. This is
+what makes the strict-deps property hold for workspace build outputs: no
+unrelated workspace-built artifact is reachable via dune-managed env vars.
+The user's inherited shell env (PATH, OCAMLPATH, findlib config) is still
+visible via `Context.installed_env`, as documented in "Environment variables"
+above.
 
 ### `dune exec` and the staging area
 
@@ -138,26 +223,22 @@ staging cons above, this means:
   the staging dir is populated. The dune-site / sites cram tests
   consequently run `dune build @install` before `dune exec`.
 - Cram tests that exercise dune-site libraries (`(libraries dune-site
-  dune-site.plugins)`) must declare both `(package dune-site)` and
-  `(package dune-private-libs)` in their cram-level `dune` setup.
-  `dune-site` re-exports `dune-private-libs.dune-section`, and the layout
-  does not auto-expand transitive package deps (see "Immediate deps only"
-  above). The same pattern applies to other re-exporting libraries.
-  `(package stdune)`, for example, requires `(package dyn) (package
-  ordering) (package pp) (package top-closure) (package csexp) (package
-  fs-io)`. This is the explicit-deps tradeoff in its most visible form:
-  the immediate-deps-only design surfaces a library's transitive
-  dependencies at the call site rather than implicitly pulling them in.
+  dune-site.plugins)`) can declare `(package dune-site)` in their cram-level
+  `dune` setup. `dune-site` re-exports `dune-private-libs.dune-section`, so the
+  library closure also adds that library's install artifacts and filtered
+  metadata. It does not add the other contents of `dune-private-libs`. The same
+  rule applies to other re-exporting libraries. Package dependencies that are
+  not represented in the library graph must still be declared explicitly.
 
 The current staging cons is sufficient for the existing test surface and
 matches pre-install-layouts behaviour.
 
 ### Package set structure and `_root` section collisions
 
-The layout merges all packages' install entries into a single directory tree.
-For scoped sections (`lib`, `share`, `doc`, `etc`), each package installs
-under its own subdirectory (`lib/<pkgname>/`), so collisions are impossible
-by construction. The unordered set is the correct data structure.
+The layout merges the explicit packages' install entries and the selected
+support libraries' entries into a single directory tree. For scoped sections
+(`lib`, `share`, `doc`, `etc`), entries install under their owning package's
+subdirectory (`lib/<pkgname>/`), so collisions are impossible by construction.
 
 Collisions can only occur in `_root` sections (`lib_root`, `share_root`,
 `libexec_root`), which install directly to the section root without package
@@ -189,12 +270,15 @@ the core layout mechanism.
 ### Layout key (digest derivation)
 
 The `<digest>` component of `_build/install/<context>/.packages/<digest>/` is
-the hex `Digest.repr` of the sorted package-name list of the set (see
-`Install_layout.Key.encode` in `src/dune_rules/install_layout.ml`). The sort
-makes the digest order-independent. A reverse table maps each digest back to
-its original set so `gen_rules` can decode the layout dir's name when the
-engine asks for rules. Hash collisions are detected at insertion time and raise
-a `Code_error`.
+the hex `Digest.repr` of the sorted explicit-package names, support-library
+identities, and support-redirect identities. Library and redirect identities
+contain both their owning package and library name, so package-private names
+remain unambiguous across workspace scopes. See `Install_layout.Key.encode` in
+`src/dune_rules/install_layout.ml`. The sorting makes the digest
+order-independent. A reverse table maps each digest back to its original
+request so `gen_rules` can decode the layout dir's name when the engine asks
+for rules. Hash collisions are detected at insertion time and raise a
+`Code_error`.
 
 ### `For_rocq_only` escape hatch
 

src/dune_findlib/meta.ml

@@ -52,6 +52,31 @@ let rec filter_variable t ~f =
   }
 ;;
 
+let rec filter_by_package_hierarchy t ~selected =
+  let selected_has_rules =
+    List.exists selected.entries ~f:(function
+      | Rule { var = "directory"; _ } | Comment _ | Package _ -> false
+      | Rule _ -> true)
+  in
+  let selected_packages =
+    List.filter_map selected.entries ~f:(function
+      | Package package -> Some package
+      | Comment _ | Rule _ -> None)
+  in
+  let entries =
+    List.filter_map t.entries ~f:(function
+      | Comment _ -> None
+      | Rule ({ var = "directory"; _ } as rule) -> Some (Rule rule)
+      | Rule _ as rule -> Option.some_if selected_has_rules rule
+      | Package package ->
+        List.find selected_packages ~f:(fun selected ->
+          Option.equal Lib_name.equal selected.name package.name)
+        |> Option.map ~f:(fun selected ->
+          Package (filter_by_package_hierarchy package ~selected)))
+  in
+  { t with entries }
+;;
+
 module Simplified = struct
   module Rules = struct
     type t =
@@ -320,16 +345,16 @@ let pp_predicate p =
      | Neg p -> "-" ^ p)
 ;;
 
+let escape_quoted_string s = String.escape_only '"' (String.escape_only '\\' s)
+
 let pp_print_text s =
   let open Pp.O in
-  Pp.verbatim "\"" ++ Pp.hvbox (Pp.text (String.escape_only '"' s)) ++ Pp.verbatim "\""
+  Pp.verbatim "\"" ++ Pp.hvbox (Pp.text (escape_quoted_string s)) ++ Pp.verbatim "\""
 ;;
 
 let pp_print_string s =
   let open Pp.O in
-  Pp.verbatim "\""
-  ++ Pp.hvbox (Pp.verbatim (String.escape_only '"' s))
-  ++ Pp.verbatim "\""
+  Pp.verbatim "\"" ++ Pp.hvbox (Pp.verbatim (escape_quoted_string s)) ++ Pp.verbatim "\""
 ;;
 
 let pp_quoted_value var =

src/dune_findlib/meta.mli

@@ -29,6 +29,12 @@ and predicate =
 
 val to_dyn : t -> Dyn.t
 val filter_variable : t -> f:(string -> bool) -> t
+
+(** Filter [t] to the package hierarchy present in [selected]. At hierarchy
+    nodes with no selected rules, retain only [directory], which is inherited
+    by findlib subpackages. Comments are discarded. *)
+val filter_by_package_hierarchy : t -> selected:t -> t
+
 val parse_entries : Lexing.lexbuf -> entry list
 
 (** Add version fields to all package in [t] that don't have and have at least