Support both local installation and hermetic build for Python.

BUILD

@@ -2,6 +2,7 @@ exports_files([
     "._dummy_.py",
     "pywrapper.sh",
     "coverage_report.sh",
+    "external/python3.9.BUILD"
 ])
 
 sh_library(

bazel_python.bzl

@@ -1,25 +1,129 @@
 load("@bazel_tools//tools/python:toolchain.bzl", "py_runtime_pair")
+load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive", "http_file")
 
-def bazel_python(venv_name = "bazel_python_venv"):
+""" Using a local pre-installed Python. """
+
+_LOCAL_PYTHON_REPO_BUILD="""
+cc_library(
+    name = "python",
+    srcs = glob(["lib/*.a"]),
+    hdrs = glob([
+        "include/**/*.h",
+        "include/**/**/*.h",
+    ]),
+    includes = ["include/python3.9"],
+    visibility = ["//visibility:public"],
+)
+
+exports_files(glob(["bin/**"]))
+"""
+
+def _setup_local_python_repository_impl(repository_ctx):
+    BAZEL_PYTHON_DIR = repository_ctx.attr.dir \
+                    or repository_ctx.os.environ.get("BAZEL_PYTHON_DIR", None)
+    if BAZEL_PYTHON_DIR == None:
+        fail("Environment variable `BAZEL_PYTHON_DIR` is missing.")
+
+    path = "{}/{}".format(BAZEL_PYTHON_DIR, repository_ctx.attr.python_version)
+    repository_ctx.symlink(path, "")
+    repository_ctx.file("BUILD",
+        content = _LOCAL_PYTHON_REPO_BUILD,
+        executable = False,
+        legacy_utf8 = True
+    )
+    return BAZEL_PYTHON_DIR
+
+setup_local_python_repository = repository_rule(
+    implementation = _setup_local_python_repository_impl,
+    attrs = {
+        "python_version": attr.string(),
+        "dir": attr.string()
+    },
+    local = True,
+)
+
+def bazel_local_python(
+    python_version = "3.9.7",
+    name = "python",
+    venv_name = "bazel_python_venv"
+):
+    """Workspace rule setting up bazel_python for a repository from a
+       local Python installation.
+
+    Arguments
+    =========
+    @python_version should be a Python version number in `major.minor.patch`.
+    @name is the repository name.
+    @venv_name should match the 'name' argument given to the
+        bazel_python_interpreter call in the BUILD file.
+    """
+    native.register_toolchains("//:" + venv_name + "_toolchain")
+    setup_local_python_repository(
+        name=name,
+        python_version=python_version
+    )
+
+
+""" Build a hermetic Python. """
+
+_known_python_archives_sha256 = {
+    3: {
+        9: {
+            7: "a838d3f9360d157040142b715db34f0218e535333696a5569dc6f854604eb9d1"
+        }
+    }
+}
+
+def bazel_hermetic_python(
+    python_version = "3.9.7",
+    name = "python",
+    venv_name = "bazel_python_venv"
+):
     """Workspace rule setting up bazel_python for a repository.
 
     Arguments
     =========
+    @python_version should be a Python version number in `major.minor.patch`.
+    @name is the repository name.
     @venv_name should match the 'name' argument given to the
         bazel_python_interpreter call in the BUILD file.
     """
     native.register_toolchains("//:" + venv_name + "_toolchain")
 
+    print("""
+    =========================================================================================
+    Warning: build a hermetic Python is experimental.
+
+    Know issues:
+    1. "Value for scheme.headers does not match" due to the `bazel-out/host/bin` prefix.
+    2. "[33/43] test_pprint -- test_pickle failed (6 errors)" during PGO.
+    3. (Optional) `libsqlite3-dev` is required for Jupyter Notebook.
+
+    TODOs:
+    1. Build with tcmalloc.
+    =========================================================================================
+    """)
+    py_major, py_minor, py_patch = [int(num) for num in python_version.split(".")]
+    http_archive(
+        name = name,
+        build_file = "@bazel_python//:external/python{major}.{minor}.BUILD".format(
+            major=py_major, minor=py_minor),
+        sha256 = _known_python_archives_sha256[py_major][py_minor][py_patch],
+        strip_prefix = "Python-{}".format(python_version),
+        urls = ["https://www.python.org/ftp/python/{0}/Python-{0}.tgz".format(
+            python_version)],
+    )
+
+
 def bazel_python_interpreter(
-        python_version,
         name = "bazel_python_venv",
         requirements_file = None,
         **kwargs):
     """BUILD rule setting up a bazel_python interpreter (venv).
 
     Arguments
     =========
-    @python_version should be the Python version string to use (e.g. 3.7.4 is
+    @python_version (deprecated) should be the Python version string to use (e.g. 3.7.4 is
         the standard for DARG projects). You must run the setup_python.sh
         script with this version number.
     @name is your preferred Bazel name for referencing this. The default should
@@ -30,7 +134,6 @@ def bazel_python_interpreter(
     """
     bazel_python_venv(
         name = name,
-        python_version = python_version,
         requirements_file = requirements_file,
         **kwargs
     )
@@ -73,66 +176,64 @@ def _bazel_python_venv_impl(ctx):
 
     Also installs requirements specified by @ctx.attr.requirements_file.
     """
-    python_version = ctx.attr.python_version
-    use_system = False
-    only_warn = ctx.var.get("BAZEL_PYTHON_ONLY_WARN", "false").lower() == "true"
-    if "BAZEL_PYTHON_DIR" not in ctx.var:
-        if only_warn:
-            print("A bazel-python installation was not found. Falling back to the system python. For reproducibility, please run setup_python.sh for " + python_version)
-            use_system = True
-        else:
-            fail("You must run setup_python.sh for " + python_version)
-    if use_system:
-        python_dir = ""
-    else:
-        python_parent_dir = ctx.var.get("BAZEL_PYTHON_DIR")
-        python_dir = python_parent_dir + "/" + python_version
-
-    # TODO: Fail if python_dir does not exist.
+    tool_inputs, tool_input_mfs = ctx.resolve_tools(tools = [ctx.attr.python])
     venv_dir = ctx.actions.declare_directory("bazel_python_venv_installed")
     inputs = []
-    if use_system:
-        command = ""
-    else:
-        command = """
-            export PATH={py_dir}/bin:$PATH
-            export PATH={py_dir}/include:$PATH
-            export PATH={py_dir}/lib:$PATH
-            export PATH={py_dir}/share:$PATH
-            export PYTHON_PATH={py_dir}:{py_dir}/bin:{py_dir}/include:{py_dir}/lib:{py_dir}/share
-        """
-    command += """
-        python3 -m venv {out_dir} || exit 1
-        source {out_dir}/bin/activate || exit 1
+
+    """ Setup venv. """
+    command = """
+        $(readlink -f {py_executable}) -m venv {venv_dir} || exit 1
+        source {venv_dir}/bin/activate || exit 1
+        export PATH=$PWD/{venv_dir}/bin:$PWD/{venv_dir}/include:$PWD/{venv_dir}/lib:$PWD/{venv_dir}/share:$PATH
+        export PYTHON_PATH=$PWD/{venv_dir}:$PWD/{venv_dir}/bin:$PWD/{venv_dir}/include:$PWD/{venv_dir}/lib:$PWD/{venv_dir}/share
     """
+
+    """ Install requirements.txt. """
     if ctx.attr.requirements_file:
-        command += "pip3 install -r " + ctx.file.requirements_file.path + " || exit 1"
+        command += "python3 -m pip install -r " + ctx.file.requirements_file.path + " || exit 1"
         inputs.append(ctx.file.requirements_file)
-    for src in ctx.attr.run_after_pip_srcs:
+
+    """ Include resources. """
+    for src in ctx.attr.data:
         inputs.extend(src.files.to_list())
+
+    """ Append post-pip scripts. """
     command += ctx.attr.run_after_pip
+
     command += """
-        REPLACEME=$PWD/'{out_dir}'
+        REPLACEME=$PWD/'{venv_dir}'
         REPLACEWITH='$PWD/bazel_python_venv_installed'
         # This prevents sed from trying to modify the directory. We may want to
         # do a more targeted sed in the future.
-        rm -rf {out_dir}/bin/__pycache__ || exit 1
-        sed -i'' -e s:$REPLACEME:$REPLACEWITH:g {out_dir}/bin/* || exit 1
+        rm -rf {venv_dir}/bin/__pycache__ || exit 1
+        sed -i'' -e s:$REPLACEME:$REPLACEWITH:g {venv_dir}/bin/* || exit 1
     """
+    command = command.format(
+        py_executable = ctx.executable.python.path,
+        venv_dir = venv_dir.path
+    )
+    # print(command)
     ctx.actions.run_shell(
-        command = command.format(py_dir = python_dir, out_dir = venv_dir.path),
         inputs = inputs,
         outputs = [venv_dir],
+        tools = tool_inputs,
+        command = command
     )
+
     return [DefaultInfo(files = depset([venv_dir]))]
 
 bazel_python_venv = rule(
     implementation = _bazel_python_venv_impl,
     attrs = {
-        "python_version": attr.string(),
         "requirements_file": attr.label(allow_single_file = True),
         "run_after_pip": attr.string(),
-        "run_after_pip_srcs": attr.label_list(allow_files = True),
+        "data": attr.label_list(allow_files = True),
+        "python": attr.label(
+            executable = True,
+            allow_single_file = True,
+            cfg = 'exec',
+            default = Label("@python//:bin/python3"),
+        ),
     },
 )