diff --git a/foreign_cc/boost_build.bzl b/foreign_cc/boost_build.bzl index aa5abae81..5742725fa 100644 --- a/foreign_cc/boost_build.bzl +++ b/foreign_cc/boost_build.bzl @@ -28,7 +28,7 @@ def _create_configure_script(configureParameters): return [ "cd $INSTALLDIR", - "##copy_dir_contents_to_dir## $$EXT_BUILD_ROOT$$/{}/. .".format(root), + "##copy_dir_contents_to_dir## $$EXT_BUILD_ROOT$$/{}/. . True".format(root), "chmod -R +w .", "##enable_tracing##", "./bootstrap.sh {}".format(" ".join(ctx.attr.bootstrap_options)), diff --git a/foreign_cc/built_tools/meson_tool_wrapper.py b/foreign_cc/built_tools/meson_tool_wrapper.py index 6e13f0728..0b782482c 100644 --- a/foreign_cc/built_tools/meson_tool_wrapper.py +++ b/foreign_cc/built_tools/meson_tool_wrapper.py @@ -32,13 +32,28 @@ def _find_meson_main(): if not rlocation: raise RuntimeError("REAL_MESON is not set") - path = runfiles.Create().Rlocation(rlocation) + r = runfiles.Create() + path = r.Rlocation(rlocation) if path and os.path.isfile(path): return path - raise RuntimeError( - "Failed to locate meson main from REAL_MESON={!r}".format(rlocation) + # Provide detailed diagnostics for runfiles resolution failures + diag = ["Failed to locate meson main from REAL_MESON={!r}".format(rlocation)] + diag.append(" Rlocation returned: {!r}".format(path)) + diag.append(" RUNFILES_DIR={!r}".format(os.environ.get("RUNFILES_DIR"))) + diag.append( + " RUNFILES_MANIFEST_FILE={!r}".format(os.environ.get("RUNFILES_MANIFEST_FILE")) ) + rf_dir = os.environ.get("RUNFILES_DIR", "") + candidate = os.path.join(rf_dir, rlocation) if rf_dir else None + diag.append( + " candidate path: {!r} exists={}".format( + candidate, os.path.isfile(candidate) if candidate else "N/A" + ) + ) + mf = os.environ.get("RUNFILES_MANIFEST_FILE", "") + diag.append(" manifest exists={}".format(os.path.isfile(mf) if mf else "N/A")) + raise RuntimeError("\n".join(diag)) def main(): diff --git a/foreign_cc/built_tools/private/built_tools_framework.bzl b/foreign_cc/built_tools/private/built_tools_framework.bzl index d17e6248d..3252b1e74 100644 --- a/foreign_cc/built_tools/private/built_tools_framework.bzl +++ b/foreign_cc/built_tools/private/built_tools_framework.bzl @@ -115,7 +115,7 @@ def built_tool_rule_impl(ctx, script_lines, out_dir, mnemonic, additional_tools "##rm_rf## $$BUILD_TMPDIR$$", "##mkdirs## $$INSTALLDIR$$", "##mkdirs## $$BUILD_TMPDIR$$", - "##copy_dir_contents_to_dir## ./{} $$BUILD_TMPDIR$$".format(root), + "##copy_dir_contents_to_dir## ./{} $$BUILD_TMPDIR$$ True".format(root), "cd \"$$BUILD_TMPDIR$$\"", ] diff --git a/foreign_cc/meson.bzl b/foreign_cc/meson.bzl index 7b27129c4..38ce7f7b9 100644 --- a/foreign_cc/meson.bzl +++ b/foreign_cc/meson.bzl @@ -296,6 +296,10 @@ def meson_with_requirements(name, requirements, **kwargs): "REAL_MESON": "$(rlocationpath @meson_src//:meson.py)", }, path = "$(execpath :meson_tool_for_{})".format(name), + staged_path = select({ + "@platforms//os:windows": "bin/meson_tool_for_{}.exe".format(name), + "//conditions:default": "bin/meson_tool_for_{}".format(name), + }), target = ":meson_tool_for_{}".format(name), tools = ["@meson_src//:meson.py"], ) diff --git a/foreign_cc/private/configure_script.bzl b/foreign_cc/private/configure_script.bzl index bbb50569f..e6b324c2c 100644 --- a/foreign_cc/private/configure_script.bzl +++ b/foreign_cc/private/configure_script.bzl @@ -37,7 +37,7 @@ def create_configure_script( root_path = "$$EXT_BUILD_ROOT$$/{}".format(root) configure_path = "{}/{}".format(root_path, configure_command) if configure_in_place: - script.append("##copy_dir_contents_to_dir## $$EXT_BUILD_ROOT$$/{} $$BUILD_TMPDIR$$".format(root)) + script.append("##copy_dir_contents_to_dir## $$EXT_BUILD_ROOT$$/{} $$BUILD_TMPDIR$$ True".format(root)) root_path = "$$BUILD_TMPDIR$$" configure_path = "{}/{}".format(root_path, configure_command) @@ -98,7 +98,7 @@ def create_configure_script( script.extend(make_commands) script.append("##disable_tracing##") - script.append("##copy_dir_contents_to_dir## $$BUILD_TMPDIR$$/$$INSTALL_PREFIX$$ $$INSTALLDIR$$\n") + script.append("##copy_dir_contents_to_dir## $$BUILD_TMPDIR$$/$$INSTALL_PREFIX$$ $$INSTALLDIR$$ True\n") return script diff --git a/foreign_cc/private/framework.bzl b/foreign_cc/private/framework.bzl index 0abad8a7e..212445e82 100644 --- a/foreign_cc/private/framework.bzl +++ b/foreign_cc/private/framework.bzl @@ -858,16 +858,43 @@ def _copy_deps_and_tools(files): if files.tools_files: lines.append("##mkdirs## $$EXT_BUILD_DEPS$$/bin") + lines.append("##mkdirs## $$EXT_BUILD_DEPS$$/tools") for tool in files.tools_files: - tool_prefix = "$EXT_BUILD_ROOT/" - tool = tool[len(tool_prefix):] if tool.startswith(tool_prefix) else tool - tool_runfiles = "{}.runfiles".format(tool) - tool_runfiles_manifest = "{}.runfiles_manifest".format(tool) - tool_exe_runfiles_manifest = "{}.exe.runfiles_manifest".format(tool) - lines.append("##symlink_to_dir## $$EXT_BUILD_ROOT$$/{} $$EXT_BUILD_DEPS$$/bin/ False".format(tool)) - lines.append("##symlink_to_dir## $$EXT_BUILD_ROOT$$/{} $$EXT_BUILD_DEPS$$/bin/ False".format(tool_runfiles)) - lines.append("##symlink_to_dir## $$EXT_BUILD_ROOT$$/{} $$EXT_BUILD_DEPS$$/bin/ False".format(tool_runfiles_manifest)) - lines.append("##symlink_to_dir## $$EXT_BUILD_ROOT$$/{} $$EXT_BUILD_DEPS$$/bin/ False".format(tool_exe_runfiles_manifest)) + tool_path = _file_path(tool).strip() + if not tool_path: + continue + if getattr(tool, "is_directory", False): + lines.append("##copy_dir_contents_to_dir## \"$$EXT_BUILD_ROOT$$/{}\" \"$$EXT_BUILD_DEPS$$/tools/{}\" True".format( + tool_path, + paths.basename(tool_path), + )) + else: + lines.append("##copy_file_to_dir## \"$$EXT_BUILD_ROOT$$/{}\" \"$$EXT_BUILD_DEPS$$/bin\"".format(tool_path)) + + for tool in files.tools_runfiles: + staged_runfiles = "{}.runfiles".format(tool.staged_path) + source_runfiles = "$$EXT_BUILD_ROOT$$/{}.runfiles".format(tool.invoke_path) + + # Bulk-copy the entire runfiles tree instead of copying files + # individually. Individual cp commands have high fork+exec overhead + # (~15s for ~2500 files on Linux, far worse on Windows). + lines.append("##copy_dir_contents_to_dir## \"{}\" \"{}\" False".format( + source_runfiles, + staged_runfiles, + )) + + # Tell the Python runfiles library where to find the staged tree. + # This must happen here (not in env_prelude) so that $EXT_BUILD_DEPS + # has already been aliased to the short path on Windows — using the + # long exec-root path would exceed MAX_PATH. + # + # Do NOT set RUNFILES_MANIFEST_FILE: the manifest contains absolute + # paths from the original build machine that are not valid in the + # staged tree. _repo_mapping is inside the .runfiles tree and is + # already covered by the bulk copy. + lines.append("export RUNFILES_DIR=\"{staged_runfiles}\"".format( + staged_runfiles = staged_runfiles, + )) for ext_dir in files.ext_build_dirs: lines.append("##symlink_to_dir## $$EXT_BUILD_ROOT$$/{} $$EXT_BUILD_DEPS$$ True".format(_file_path(ext_dir))) @@ -1081,9 +1108,10 @@ InputFiles = provider( ), libs = "Library files built by Bazel. Will be copied into $EXT_BUILD_DEPS/lib.", tools_files = ( - "Files and directories with tools needed for configuration/building " + - "to be copied into the bin folder, which is added to the PATH" + "Tool runtime files and directories that should be staged under " + + "$EXT_BUILD_DEPS for foreign builds." ), + tools_runfiles = "Structured runfiles trees that should be staged next to staged tools.", ext_build_dirs = ( "Directories with libraries, built by framework function. " + "This directories should be copied into $EXT_BUILD_DEPS/lib-name as is, with all contents." @@ -1133,14 +1161,23 @@ def _define_inputs(attrs): # but filter out repeating directories ext_build_dirs = uniq_list_keep_order(ext_build_dirs) - tools = [] tools_files = [] + tools_runfiles = [] input_files = [] for tool in attrs.tools_data: - tools.append(tool.path) - if tool.target: - for file_list in tool.target.files.to_list(): - tools_files += _list(file_list) + if tool.target and getattr(tool, "stage_runtime", False): + tools_files += tool.runtime_files.to_list() + runfiles_files = tool.runfiles_files.to_list() + runfiles_manifest = tool.runfiles_manifest + repo_mapping_manifest = tool.repo_mapping_manifest + if runfiles_files or runfiles_manifest or repo_mapping_manifest: + tools_runfiles.append(struct( + files = runfiles_files, + invoke_path = tool.invoke_path, + runfiles_manifest = runfiles_manifest, + repo_mapping_manifest = repo_mapping_manifest, + staged_path = tool.path, + )) # TODO: Remove, `additional_tools` is deprecated. for tool in attrs.additional_tools: @@ -1159,13 +1196,17 @@ def _define_inputs(attrs): headers = bazel_headers, include_dirs = bazel_system_includes, libs = bazel_libs, - tools_files = tools, + tools_files = tools_files, + tools_runfiles = tools_runfiles, deps_compilation_info = cc_info_merged.compilation_context, deps_linking_info = cc_info_merged.linking_context, ext_build_dirs = ext_build_dirs, declared_inputs = filter_containing_dirs_from_inputs(attrs.lib_source.files.to_list()) + bazel_libs + tools_files + + [runfile for tool in tools_runfiles for runfile in tool.files] + + [tool.runfiles_manifest for tool in tools_runfiles if tool.runfiles_manifest] + + [tool.repo_mapping_manifest for tool in tools_runfiles if tool.repo_mapping_manifest] + input_files + cc_info_merged.compilation_context.headers.to_list() + _collect_libs(cc_info_merged.linking_context) + ext_build_dirs, diff --git a/foreign_cc/private/framework/toolchains/commands.bzl b/foreign_cc/private/framework/toolchains/commands.bzl index e4f107318..764c000a7 100644 --- a/foreign_cc/private/framework/toolchains/commands.bzl +++ b/foreign_cc/private/framework/toolchains/commands.bzl @@ -73,9 +73,28 @@ PLATFORM_COMMANDS = { doc = "Source directory, immediate children of which are copied", ), _argument_info(name = "target", data_type = type(""), doc = "Target directory"), + _argument_info( + name = "flatten_timestamps", + data_type = type(""), + doc = "If True, flatten all file timestamps to the source directory mtime (prevents autotools reruns)", + ), ], doc = "Copies contents of the directory to target directory", ), + "copy_file": _command_info( + arguments = [ + _argument_info(name = "source", data_type = type(""), doc = "Source file"), + _argument_info(name = "target", data_type = type(""), doc = "Target file"), + ], + doc = "Copies a file to an exact target path", + ), + "copy_file_to_dir": _command_info( + arguments = [ + _argument_info(name = "source", data_type = type(""), doc = "Source file"), + _argument_info(name = "target", data_type = type(""), doc = "Target directory"), + ], + doc = "Copies a file into the target directory", + ), "define_absolute_paths": _command_info( arguments = [ _argument_info(name = "dir_", data_type = type(""), doc = "Directory where to replace"), diff --git a/foreign_cc/private/framework/toolchains/freebsd_commands.bzl b/foreign_cc/private/framework/toolchains/freebsd_commands.bzl index 0aaff5588..c9ed403cb 100644 --- a/foreign_cc/private/framework/toolchains/freebsd_commands.bzl +++ b/foreign_cc/private/framework/toolchains/freebsd_commands.bzl @@ -11,6 +11,18 @@ https://docs.bazel.build/versions/main/install-compile-source.html#bootstrap-uni load(":commands.bzl", "FunctionAndCallInfo") +def _strip_outer_quotes(val): + """Remove one layer of surrounding double-quotes if present. + + The ##command## arg1 arg2 directive parser (split_arguments) keeps quoted + tokens together but includes the quotes in the value. Platform command + functions already add their own quotes, so we strip the outer layer to + avoid double-quoting. + """ + if len(val) >= 2 and val[0] == '"' and val[-1] == '"': + return val[1:-1] + return val + def shebang(): return "#!/usr/bin/env bash" @@ -48,7 +60,7 @@ def disable_tracing(): return "set +x" def mkdirs(path): - return "mkdir -p \"{path}\"".format(path = path) + return "mkdir -p \"{path}\"".format(path = _strip_outer_quotes(path)) def rm_rf(path): return "rm -rf \"{path}\"".format(path = path) @@ -94,10 +106,25 @@ fi """, ) -def copy_dir_contents_to_dir(source, target): +def copy_dir_contents_to_dir(source, target, flatten_timestamps): + """Copy directory contents to target, optionally flattening timestamps. + + Args: + source: Source directory whose contents are copied. + target: Target directory. + flatten_timestamps: If "True", set all file timestamps to the source + directory mtime (prevents autotools regeneration). + + Returns: + str: Shell command string. + """ + source = _strip_outer_quotes(source) + target = _strip_outer_quotes(target) + # Beause FreeBSD `cp` doesn't have `--no-target-directory`, we have to # do something more complex for this environment. - return """\ + if flatten_timestamps == "True": + return """\ if [[ -d "{source}" ]]; then cp -L -R "{source}"/. "{target}" else @@ -105,6 +132,37 @@ else fi find "{target}" -type f -exec touch -r "{source}" "{{}}" \\; """.format( + source = source, + target = target, + ) + + # Without flatten_timestamps we skip -L (dereference symlinks) and + # tolerate "File exists" errors. Runfiles trees contain repo-mapping + # symlinks (apparent → canonical) that create duplicate destination + # paths; cp errors on the second copy but the file is already present. + return """\ +if [[ -d "{source}" ]]; then + cp -R "{source}"/. "{target}" 2>&1 | grep -v "File exists" >&2 || true +else + cp -R "{source}" "{target}" 2>&1 | grep -v "File exists" >&2 || true +fi +""".format( + source = source, + target = target, + ) + +def copy_file_to_dir(source, target): + source = _strip_outer_quotes(source) + target = _strip_outer_quotes(target) + return """cp -L "{source}" "{target}/" && touch -r "{source}" "{target}/$(basename "{source}")" """.format( + source = source, + target = target, + ) + +def copy_file(source, target): + source = _strip_outer_quotes(source) + target = _strip_outer_quotes(target) + return """cp -L "{source}" "{target}" && touch -r "{source}" "{target}" """.format( source = source, target = target, ) @@ -295,6 +353,8 @@ commands = struct( children_to_path = children_to_path, cleanup_function = cleanup_function, copy_dir_contents_to_dir = copy_dir_contents_to_dir, + copy_file_to_dir = copy_file_to_dir, + copy_file = copy_file, define_absolute_paths = define_absolute_paths, define_function = define_function, define_sandbox_paths = define_sandbox_paths, diff --git a/foreign_cc/private/framework/toolchains/linux_commands.bzl b/foreign_cc/private/framework/toolchains/linux_commands.bzl index e7ac137b2..9283a3085 100644 --- a/foreign_cc/private/framework/toolchains/linux_commands.bzl +++ b/foreign_cc/private/framework/toolchains/linux_commands.bzl @@ -2,6 +2,18 @@ load(":commands.bzl", "FunctionAndCallInfo") +def _strip_outer_quotes(val): + """Remove one layer of surrounding double-quotes if present. + + The ##command## arg1 arg2 directive parser (split_arguments) keeps quoted + tokens together but includes the quotes in the value. Platform command + functions already add their own quotes, so we strip the outer layer to + avoid double-quoting. + """ + if len(val) >= 2 and val[0] == '"' and val[-1] == '"': + return val[1:-1] + return val + def shebang(): return "#!/usr/bin/env bash" @@ -39,7 +51,7 @@ def disable_tracing(): return "set +x" def mkdirs(path): - return "mkdir -p \"{path}\"".format(path = path) + return "mkdir -p \"{path}\"".format(path = _strip_outer_quotes(path)) def rm_rf(path): return "rm -rf \"{path}\"".format(path = path) @@ -85,8 +97,47 @@ fi """, ) -def copy_dir_contents_to_dir(source, target): - return """cp -L -r --no-target-directory "{source}" "{target}" && find "{target}" -type f -exec touch -r "{source}" "{{}}" \\;""".format( +def copy_dir_contents_to_dir(source, target, flatten_timestamps): + """Copy directory contents to target, optionally flattening timestamps. + + Args: + source: Source directory whose contents are copied. + target: Target directory. + flatten_timestamps: If "True", set all file timestamps to the source + directory mtime (prevents autotools regeneration). + + Returns: + str: Shell command string. + """ + source = _strip_outer_quotes(source) + target = _strip_outer_quotes(target) + if flatten_timestamps == "True": + return """cp -L -r --no-target-directory "{source}" "{target}" && find "{target}" -type f -exec touch -r "{source}" "{{}}" \\;""".format( + source = source, + target = target, + ) + + # Without flatten_timestamps we skip -L (dereference symlinks) and + # tolerate "File exists" errors. Runfiles trees contain repo-mapping + # symlinks (apparent → canonical) that create duplicate destination + # paths; cp errors on the second copy but the file is already present. + return """cp -r --no-target-directory "{source}" "{target}" 2>&1 | grep -v "File exists" >&2 || true""".format( + source = source, + target = target, + ) + +def copy_file_to_dir(source, target): + source = _strip_outer_quotes(source) + target = _strip_outer_quotes(target) + return """cp -L "{source}" "{target}/" && touch -r "{source}" "{target}/$(basename "{source}")" """.format( + source = source, + target = target, + ) + +def copy_file(source, target): + source = _strip_outer_quotes(source) + target = _strip_outer_quotes(target) + return """cp -L "{source}" "{target}" && touch -r "{source}" "{target}" """.format( source = source, target = target, ) @@ -277,6 +328,8 @@ commands = struct( children_to_path = children_to_path, cleanup_function = cleanup_function, copy_dir_contents_to_dir = copy_dir_contents_to_dir, + copy_file_to_dir = copy_file_to_dir, + copy_file = copy_file, define_absolute_paths = define_absolute_paths, define_function = define_function, define_sandbox_paths = define_sandbox_paths, diff --git a/foreign_cc/private/framework/toolchains/macos_commands.bzl b/foreign_cc/private/framework/toolchains/macos_commands.bzl index 622fdc225..6e9a97421 100644 --- a/foreign_cc/private/framework/toolchains/macos_commands.bzl +++ b/foreign_cc/private/framework/toolchains/macos_commands.bzl @@ -2,6 +2,18 @@ load(":commands.bzl", "FunctionAndCallInfo") +def _strip_outer_quotes(val): + """Remove one layer of surrounding double-quotes if present. + + The ##command## arg1 arg2 directive parser (split_arguments) keeps quoted + tokens together but includes the quotes in the value. Platform command + functions already add their own quotes, so we strip the outer layer to + avoid double-quoting. + """ + if len(val) >= 2 and val[0] == '"' and val[-1] == '"': + return val[1:-1] + return val + def shebang(): return "#!/usr/bin/env bash" @@ -39,7 +51,7 @@ def disable_tracing(): return "set +x" def mkdirs(path): - return "mkdir -p \"{path}\"".format(path = path) + return "mkdir -p \"{path}\"".format(path = _strip_outer_quotes(path)) def rm_rf(path): return "rm -rf \"{path}\"".format(path = path) @@ -85,10 +97,25 @@ fi """, ) -def copy_dir_contents_to_dir(source, target): +def copy_dir_contents_to_dir(source, target, flatten_timestamps): + """Copy directory contents to target, optionally flattening timestamps. + + Args: + source: Source directory whose contents are copied. + target: Target directory. + flatten_timestamps: If "True", set all file timestamps to the source + directory mtime (prevents autotools regeneration). + + Returns: + str: Shell command string. + """ + source = _strip_outer_quotes(source) + target = _strip_outer_quotes(target) + # Beause macos `cp` doesn't have `--no-target-directory`, we have to # do something more complext for this environment. - return """\ + if flatten_timestamps == "True": + return """\ if [[ -d "{source}" ]]; then cp -L -R "{source}"/. "{target}" else @@ -96,6 +123,37 @@ else fi find "{target}" -type f -exec touch -r "{source}" "{{}}" \\; """.format( + source = source, + target = target, + ) + + # Without flatten_timestamps we skip -L (dereference symlinks) and + # tolerate "File exists" errors. Runfiles trees contain repo-mapping + # symlinks (apparent → canonical) that create duplicate destination + # paths; cp errors on the second copy but the file is already present. + return """\ +if [[ -d "{source}" ]]; then + cp -R "{source}"/. "{target}" 2>&1 | grep -v "File exists" >&2 || true +else + cp -R "{source}" "{target}" 2>&1 | grep -v "File exists" >&2 || true +fi +""".format( + source = source, + target = target, + ) + +def copy_file_to_dir(source, target): + source = _strip_outer_quotes(source) + target = _strip_outer_quotes(target) + return """cp -L "{source}" "{target}/" && touch -r "{source}" "{target}/$(basename "{source}")" """.format( + source = source, + target = target, + ) + +def copy_file(source, target): + source = _strip_outer_quotes(source) + target = _strip_outer_quotes(target) + return """cp -L "{source}" "{target}" && touch -r "{source}" "{target}" """.format( source = source, target = target, ) @@ -290,6 +348,8 @@ commands = struct( children_to_path = children_to_path, cleanup_function = cleanup_function, copy_dir_contents_to_dir = copy_dir_contents_to_dir, + copy_file_to_dir = copy_file_to_dir, + copy_file = copy_file, define_absolute_paths = define_absolute_paths, define_function = define_function, define_sandbox_paths = define_sandbox_paths, diff --git a/foreign_cc/private/framework/toolchains/windows_commands.bzl b/foreign_cc/private/framework/toolchains/windows_commands.bzl index 950509290..4ae2cc7c5 100644 --- a/foreign_cc/private/framework/toolchains/windows_commands.bzl +++ b/foreign_cc/private/framework/toolchains/windows_commands.bzl @@ -2,6 +2,18 @@ load(":commands.bzl", "FunctionAndCallInfo") +def _strip_outer_quotes(val): + """Remove one layer of surrounding double-quotes if present. + + The ##command## arg1 arg2 directive parser (split_arguments) keeps quoted + tokens together but includes the quotes in the value. Platform command + functions already add their own quotes, so we strip the outer layer to + avoid double-quoting. + """ + if len(val) >= 2 and val[0] == '"' and val[-1] == '"': + return val[1:-1] + return val + def shebang(): return "#!/usr/bin/env bash" @@ -57,7 +69,7 @@ def disable_tracing(): return "set +x" def mkdirs(path): - return "mkdir -p \"{path}\"".format(path = path) + return "mkdir -p \"{path}\"".format(path = _strip_outer_quotes(path)) def rm_rf(path): return "rm -rf \"{path}\"".format(path = path) @@ -107,8 +119,47 @@ fi """, ) -def copy_dir_contents_to_dir(source, target): - return """cp -L -r --no-target-directory "{source}" "{target}" && $REAL_FIND "{target}" -type f -exec touch -r "{source}" "{{}}" \\;""".format( +def copy_dir_contents_to_dir(source, target, flatten_timestamps): + """Copy directory contents to target, optionally flattening timestamps. + + Args: + source: Source directory whose contents are copied. + target: Target directory. + flatten_timestamps: If "True", set all file timestamps to the source + directory mtime (prevents autotools regeneration). + + Returns: + str: Shell command string. + """ + source = _strip_outer_quotes(source) + target = _strip_outer_quotes(target) + if flatten_timestamps == "True": + return """cp -L -r --no-target-directory "{source}" "{target}" && $REAL_FIND "{target}" -type f -exec touch -r "{source}" "{{}}" \\;""".format( + source = source, + target = target, + ) + + # Without flatten_timestamps we skip -L (dereference symlinks) and + # tolerate "File exists" errors. Runfiles trees contain repo-mapping + # symlinks (apparent → canonical) that create duplicate destination + # paths; cp errors on the second copy but the file is already present. + return """cp -r --no-target-directory "{source}" "{target}" 2>&1 | grep -v "File exists" >&2 || true""".format( + source = source, + target = target, + ) + +def copy_file_to_dir(source, target): + source = _strip_outer_quotes(source) + target = _strip_outer_quotes(target) + return """cp -L "{source}" "{target}/" && touch -r "{source}" "{target}/$(basename "{source}")" """.format( + source = source, + target = target, + ) + +def copy_file(source, target): + source = _strip_outer_quotes(source) + target = _strip_outer_quotes(target) + return """cp -L "{source}" "{target}" && touch -r "{source}" "{target}" """.format( source = source, target = target, ) @@ -323,6 +374,8 @@ commands = struct( children_to_path = children_to_path, cleanup_function = cleanup_function, copy_dir_contents_to_dir = copy_dir_contents_to_dir, + copy_file_to_dir = copy_file_to_dir, + copy_file = copy_file, define_absolute_paths = define_absolute_paths, define_function = define_function, define_sandbox_paths = define_sandbox_paths, diff --git a/foreign_cc/private/run_shell_file_utils.bzl b/foreign_cc/private/run_shell_file_utils.bzl index 89b5e3367..99e844e2a 100644 --- a/foreign_cc/private/run_shell_file_utils.bzl +++ b/foreign_cc/private/run_shell_file_utils.bzl @@ -47,7 +47,7 @@ def copy_directory(actions, orig_path, copy_path): file = dir_copy, script = "\n".join([ "##mkdirs## $$EXT_BUILD_ROOT$$/" + dir_copy.path, - "##copy_dir_contents_to_dir## {} $$EXT_BUILD_ROOT$$/{}".format( + "##copy_dir_contents_to_dir## {} $$EXT_BUILD_ROOT$$/{} True".format( orig_path, dir_copy.path, ), diff --git a/toolchains/native_tools/native_tools_toolchain.bzl b/toolchains/native_tools/native_tools_toolchain.bzl index cc443f8fd..c3aa922ff 100644 --- a/toolchains/native_tools/native_tools_toolchain.bzl +++ b/toolchains/native_tools/native_tools_toolchain.bzl @@ -5,12 +5,18 @@ ToolInfo = provider( doc = "Information about the native tool", fields = { "env": "Environment variables to set when using this tool e.g. M4", + "invoke_path": "Path the foreign build should invoke for this tool.", "path": ( "Absolute path to the tool in case the tool is preinstalled on the machine. " + "Relative path to the tool in case the tool is built as part of a build; the path should be relative " + "to the bazel-genfiles, i.e. it should start with the name of the top directory of the built tree " + "artifact. (Please see the example `//examples:built_cmake_toolchain`)" ), + "repo_mapping_manifest": "Optional repo mapping manifest for staged tool runfiles.", + "runfiles_files": "Runfiles that should be staged next to this tool when used by foreign rules.", + "runfiles_manifest": "Optional runfiles manifest for staged tool launchers.", + "runtime_files": "Files that should be staged with this tool when used by foreign rules.", + "staged_path": "Optional path for invoking the tool from inside EXT_BUILD_DEPS.", "target": ( "If the tool is preinstalled, must be None. " + "If the tool is built as part of the build, the corresponding build target, which should produce " + @@ -42,18 +48,43 @@ def _native_tool_toolchain_impl(ctx): fail("Either path or target (and path) should be defined for the tool.") path = None env = {} + runfiles_manifest = None + repo_mapping_manifest = None + runfiles_files = depset() + staged_path = None + runtime_files = depset() if ctx.attr.target: path = _resolve_tool_path(ctx, ctx.attr.path, ctx.attr.target, ctx.attr.tools) + staged_path = ctx.attr.staged_path and _resolve_tool_path(ctx, ctx.attr.staged_path, ctx.attr.target, ctx.attr.tools) for k, v in ctx.attr.env.items(): env[k] = _resolve_tool_path(ctx, v, ctx.attr.target, ctx.attr.tools) + runtime_inputs = [ctx.attr.target] + runtime_inputs.extend(ctx.attr.tools) + runtime_files = depset(transitive = [ + target[DefaultInfo].files + for target in runtime_inputs + ]) + runfiles_files = depset(transitive = [ + target[DefaultInfo].default_runfiles.files + for target in runtime_inputs + ]) + runfiles_manifest = ctx.attr.target[DefaultInfo].files_to_run.runfiles_manifest + repo_mapping_manifest = ctx.attr.target[DefaultInfo].files_to_run.repo_mapping_manifest + else: path = ctx.expand_location(ctx.attr.path) env = {k: ctx.expand_location(v) for (k, v) in ctx.attr.env.items()} return platform_common.ToolchainInfo(data = ToolInfo( env = env, + invoke_path = path, path = path, + runfiles_manifest = runfiles_manifest, + repo_mapping_manifest = repo_mapping_manifest, + runfiles_files = runfiles_files, + runtime_files = runtime_files, + staged_path = staged_path, target = ctx.attr.target, )) @@ -78,6 +109,13 @@ native_tool_toolchain = rule( "of the built tree artifact. (Please see the example `//examples:built_cmake_toolchain`)" ), ), + "staged_path": attr.string( + mandatory = False, + doc = ( + "Optional path to invoke after the tool has been staged into EXT_BUILD_DEPS. " + + "Use this for tools whose runtime closure must run from the staged tree." + ), + ), "target": attr.label( mandatory = False, cfg = "exec", diff --git a/toolchains/native_tools/tool_access.bzl b/toolchains/native_tools/tool_access.bzl index da3bc677e..0ce19c534 100644 --- a/toolchains/native_tools/tool_access.bzl +++ b/toolchains/native_tools/tool_access.bzl @@ -44,31 +44,35 @@ def get_pkgconfig_data(ctx): def _access_and_expect_label_copied(toolchain_type_, ctx): tool_data = access_tool(toolchain_type_, ctx) if tool_data.target: - # This could be made more efficient by changing the - # toolchain to provide the executable as a target - cmd_file = tool_data tool_env = dict(tool_data.env) + resolved_tool_path = "$$EXT_BUILD_ROOT$$/{}".format(tool_data.invoke_path) + if tool_data.staged_path: + resolved_tool_path = "$$EXT_BUILD_DEPS$$/{}".format(tool_data.staged_path) - for f in tool_data.target.files.to_list(): - if f.path.endswith("/" + tool_data.path): - cmd_file = f - break - - # Environment vars for tools such as MAKE and CMAKE needs to be absolute - # as they are used from build_tmpdir and not bazel's exec/sandbox root for k, v in tool_env.items(): - if v.endswith(tool_data.path): - tool_env[k] = "$EXT_BUILD_ROOT/{}".format(cmd_file.path) + if v.endswith(tool_data.invoke_path): + tool_env[k] = resolved_tool_path return struct( target = tool_data.target, env = tool_env, - # as the tool will be copied into tools directory - path = "$EXT_BUILD_ROOT/{}".format(cmd_file.path), + invoke_path = tool_data.invoke_path, + path = resolved_tool_path, + runfiles_manifest = tool_data.runfiles_manifest, + repo_mapping_manifest = tool_data.repo_mapping_manifest, + runfiles_files = tool_data.runfiles_files, + runtime_files = tool_data.runtime_files, + stage_runtime = bool(tool_data.staged_path), ) else: return struct( target = None, env = tool_data.env, + invoke_path = tool_data.path, path = tool_data.path, + runfiles_manifest = None, + repo_mapping_manifest = None, + runfiles_files = depset(), + runtime_files = depset(), + stage_runtime = False, ) diff --git a/toolchains/prebuilt_toolchains.bzl b/toolchains/prebuilt_toolchains.bzl index ae15b7056..206ee2a52 100644 --- a/toolchains/prebuilt_toolchains.bzl +++ b/toolchains/prebuilt_toolchains.bzl @@ -36,7 +36,7 @@ filegroup( native_tool_toolchain( name = "cmake_tool", - path = "bin/{bin}", + path = "$(execpath :cmake_bin)", target = ":cmake_data", env = {env}, tools = [":cmake_bin"], diff --git a/toolchains/prebuilt_toolchains.py b/toolchains/prebuilt_toolchains.py index 9ef467308..f079f5556 100755 --- a/toolchains/prebuilt_toolchains.py +++ b/toolchains/prebuilt_toolchains.py @@ -193,7 +193,7 @@ def repo_definition(name, url, sha256, prefix, template, **template_substitution native_tool_toolchain( name = "cmake_tool", - path = "bin/{{bin}}", + path = "$(execpath :cmake_bin)", target = ":cmake_data", env = {{env}}, tools = [":cmake_bin"], diff --git a/toolchains/private/BUILD.bazel b/toolchains/private/BUILD.bazel index a1ce9c84a..13113780d 100644 --- a/toolchains/private/BUILD.bazel +++ b/toolchains/private/BUILD.bazel @@ -70,6 +70,12 @@ native_tool_toolchain( "@platforms//os:windows": "$(execpath :cmake_tool)/bin/cmake.exe", "//conditions:default": "$(execpath :cmake_tool)/bin/cmake", }), + # cmake_tool is a staged install tree, not a standalone executable. Run it + # from the staged tree so CMAKE_ROOT and companion tools resolve correctly. + staged_path = select({ + "@platforms//os:windows": "tools/cmake_tool_default/bin/cmake.exe", + "//conditions:default": "tools/cmake_tool_default/bin/cmake", + }), target = ":cmake_tool", ) @@ -152,6 +158,10 @@ native_tool_toolchain( "REAL_MESON": "$(rlocationpath @meson_src//:meson.py)", }, path = "$(execpath :meson_tool)", + staged_path = select({ + "@platforms//os:windows": "bin/meson_tool.exe", + "//conditions:default": "bin/meson_tool", + }), target = ":meson_tool", tools = ["@meson_src//:meson.py"], )