[WIP] Added --cache-dir flag and test.

SCons/CacheDir.py

@@ -44,12 +44,15 @@
     b"# SCons cache directory - see https://bford.info/cachedir/\n"
 )
 
+# Defaults for the cache subsystem's globals. Most of these are filled in by
+# SCons.Script.Main._build_targets, once the command line has been parsed.
 cache_enabled = True
 cache_debug = False
 cache_force = False
 cache_show = False
 cache_readonly = False
 cache_tmp_uuid = uuid.uuid4().hex
+cli_cache_dir = ""
 
 def CacheRetrieveFunc(target, source, env) -> int:
     t = target[0]
@@ -210,9 +213,11 @@ def _mkdir_atomic(self, path: str) -> bool:
             return False
 
         try:
+            parent_dir = os.path.dirname(directory)
+            os.makedirs(parent_dir, exist_ok=True)
             # TODO: Python 3.7. See comment below.
-            # tempdir = tempfile.TemporaryDirectory(dir=os.path.dirname(directory))
-            tempdir = tempfile.mkdtemp(dir=os.path.dirname(directory))
+            # tempdir = tempfile.TemporaryDirectory(dir=os.path.dirname(parent_dir))
+            tempdir = tempfile.mkdtemp(dir=os.path.dirname(parent_dir))
         except OSError as e:
             msg = "Failed to create cache directory " + path
             raise SCons.Errors.SConsEnvironmentError(msg) from e
@@ -273,18 +278,33 @@ def CacheDebug(self, fmt, target, cachefile) -> None:
             if cache_debug == '-':
                 self.debugFP = sys.stdout
             elif cache_debug:
+                # TODO: this seems fragile. There can be only one debug output
+                #   (terminal, file or none) per run, so it should not
+                #   be reopened. Testing multiple caches showed a problem
+                #   where reopening with 'w' mode meant some of the output
+                #   was lost, so for the moment switched to append mode.
+                # . Keeping better track of the output file, or switching to
+                #   using the logging module should help. The "persistence"
+                #   of using append mode breaks test/CacheDir/debug.py
                 def debug_cleanup(debugFP) -> None:
                     debugFP.close()
 
-                self.debugFP = open(cache_debug, 'w')
+                self.debugFP = open(cache_debug, 'a')
                 atexit.register(debug_cleanup, self.debugFP)
             else:
                 self.debugFP = None
             self.current_cache_debug = cache_debug
         if self.debugFP:
+            # TODO: consider emitting more than the base filename to help
+            #   distinguish retrievals across variantdirs (target.relpath?).
+            #   Separately, showing more of the cache entry path would be
+            #   useful for testing, though possibly not otherwise. How else
+            #   can you tell which target went to which cache if there are >1?
             self.debugFP.write(fmt % (target, os.path.split(cachefile)[1]))
-            self.debugFP.write("requests: %d, hits: %d, misses: %d, hit rate: %.2f%%\n" %
-                               (self.requests, self.hits, self.misses, self.hit_ratio))
+            self.debugFP.write(
+                "requests: %d, hits: %d, misses: %d, hit rate: %.2f%%\n" %
+                (self.requests, self.hits, self.misses, self.hit_ratio)
+            )
 
     @classmethod
     def copy_from_cache(cls, env, src, dst) -> str:
@@ -336,7 +356,7 @@ def cachepath(self, node) -> tuple:
         Given a Node, obtain the configured cache directory and
         the path to the cached file, which is generated from the
         node's build signature. If caching is not enabled for the
-        None, return a tuple of None.
+        node, return a tuple of ``None``.
         """
         if not self.is_enabled():
             return None, None
@@ -349,11 +369,11 @@ def cachepath(self, node) -> tuple:
     def retrieve(self, node) -> bool:
         """Retrieve a node from cache.
 
-        Returns True if a successful retrieval resulted.
+        Returns ``True`` if a successful retrieval resulted.
 
         This method is called from multiple threads in a parallel build,
         so only do thread safe stuff here. Do thread unsafe stuff in
-        built().
+        :meth:`built`.
 
         Note that there's a special trick here with the execute flag
         (one that's not normally done for other actions).  Basically

SCons/Environment.py

@@ -1362,6 +1362,12 @@ def __init__(
         self._init_special()
         self.added_methods = []
 
+        # If user specifies a --cache-dir on the command line, then
+        # use that for all created Environments, user can alter this
+        # by specifying CacheDir() per environment.
+        if SCons.CacheDir.cli_cache_dir:
+            self.CacheDir(SCons.CacheDir.cli_cache_dir)
+
         # We don't use AddMethod, or define these as methods in this
         # class, because we *don't* want these functions to be bound
         # methods.  They need to operate independently so that the

SCons/Script/Main.py

@@ -1042,6 +1042,9 @@ def _main(parser):
     SCons.Node.implicit_deps_changed = options.implicit_deps_changed
     SCons.Node.implicit_deps_unchanged = options.implicit_deps_unchanged
 
+    if options.cache_dir:
+        SCons.CacheDir.cli_cache_dir = options.cache_dir
+
     if options.no_exec:
         SCons.SConf.dryrun = 1
         SCons.Action.execute_actions = None

SCons/Script/SConsOptions.py

@@ -799,6 +799,13 @@ def opt_ignore(option, opt, value, parser) -> None:
                   help="Print CacheDir debug info to FILE",
                   metavar="FILE")
 
+    op.add_option('--cache-dir',
+                  nargs=1,
+                  dest='cache_dir',
+                  metavar='CACHEDIR',
+                  help='Enable the derived‑file cache and set its directory to CACHEDIR',
+                  default="")
+
     op.add_option('--cache-disable', '--no-cache',
                   dest='cache_disable', default=False,
                   action="store_true",

SCons/Script/SConscript.py

@@ -669,41 +669,42 @@ def get_DefaultEnvironmentProxy():
     return _DefaultEnvironmentProxy
 
 class DefaultEnvironmentCall:
-    """A class that implements "global function" calls of
-    Environment methods by fetching the specified method from the
-    DefaultEnvironment's class.  Note that this uses an intermediate
-    proxy class instead of calling the DefaultEnvironment method
-    directly so that the proxy can override the subst() method and
+    """Create a "global function" from an Environment method.
+
+    Fetches the *method_name* from the Environment instance created to hold
+    the Default Environment.  Uses an intermediate proxy class instead of
+    calling the :meth:`~SCons.Defaults.DefaultEnvironment` function directly,
+    so that the proxy can override the ``subst()`` method and
     thereby prevent expansion of construction variables (since from
     the user's point of view this was called as a global function,
-    with no associated construction environment)."""
-    def __init__(self, method_name, subst: int=0) -> None:
+    with no associated construction environment).
+    """
+
+    def __init__(self, method_name, subst: bool = False) -> None:
         self.method_name = method_name
         if subst:
             self.factory = SCons.Defaults.DefaultEnvironment
         else:
             self.factory = get_DefaultEnvironmentProxy
+
     def __call__(self, *args, **kw):
         env = self.factory()
         method = getattr(env, self.method_name)
         return method(*args, **kw)
 
-
-def BuildDefaultGlobals():
-    """
-    Create a dictionary containing all the default globals for
-    SConstruct and SConscript files.
-    """
-
+def BuildDefaultGlobals() -> dict:
+    """Create a dict containing all the default globals for SConscript files."""
     global GlobalDict
     if GlobalDict is None:
-        GlobalDict = {}
-
         import SCons.Script
+
+        GlobalDict = {}
         d = SCons.Script.__dict__
-        def not_a_module(m, d=d, mtype=type(SCons.Script)) -> bool:
-             return not isinstance(d[m], mtype)
+
+        def not_a_module(m, d=d) -> bool:
+            return not isinstance(d[m], types.ModuleType)
+
         for m in filter(not_a_module, dir(SCons.Script)):
-             GlobalDict[m] = d[m]
+            GlobalDict[m] = d[m]
 
     return GlobalDict.copy()

doc/man/scons.xml

@@ -722,6 +722,25 @@ derived-file cache specified by &f-link-CacheDir;.</para>
   </listitem>
   </varlistentry>
 
+  <varlistentry id="opt-cache-dir">
+    <term>-
+      <option>-cache-dir=
+        <replaceable>cachedir</replaceable>
+      </option>
+    </term>
+    <listitem>
+      <para>Enable derived-file caching globally, using
+        <replaceable>cachedir</replaceable>
+        as the cache directory.
+        An individual &consenv; may still specify a different
+        cache directory by calling &f-link-env-CacheDir;.
+      </para>
+      <para>
+        <emphasis>Added in version NEXT_RELEASE.</emphasis>
+      </para>
+    </listitem>
+  </varlistentry>
+
   <varlistentry id="opt-cache-disable">
   <term>
     <option>--cache-disable</option>,
@@ -9084,7 +9103,7 @@ However, the following variables are imported by
 </para>
 
 <variablelist>
-  <varlistentry>
+  <varlistentry id="v-SCONS_LIB_DIR">
     <term><envar>SCONS_LIB_DIR</envar></term>
     <listitem>
 <para>Specifies the directory that contains the &scons;
@@ -9110,7 +9129,7 @@ so the command line can be used to override
     </listitem>
   </varlistentry>
 
-  <varlistentry>
+  <varlistentry id="v-SCONS_CACHE_MSVC_CONFIG">
     <term><envar>SCONS_CACHE_MSVC_CONFIG</envar></term>
     <listitem>
 <para>(Windows only). If set, save the shell environment variables

doc/user/caching.xml

@@ -72,23 +72,34 @@ CacheDir('/usr/local/build_cache')
        </file>
     </scons_example>
 
+    <para>
+    A cache directory can also be specified on the command line
+    (<literal>--cache-dir=CACHEDIR</literal>) - useful for the case
+    where you have a permanent &f-link-CacheDir; call
+    but want to override it temporarily for a given build.
+    You can also create a cache directory specific to a &consenv;
+    by using the <literal>env.CacheDir()</literal> form.
+    </para>
+
     <para>
 
     The cache directory you specify must
     have read and write access for all developers
-    who will be accessing the cached files
-    (if <option>--cache-readonly</option> is used,
-    only read access is required).
+    who will be accessing the cached files.
+    If <option>--cache-readonly</option> is used,
+    only read access is required.
     It should also be in some central location
     that all builds will be able to access.
     In environments where developers are using separate systems
     (like individual workstations) for builds,
     this directory would typically be
     on a shared or NFS-mounted file system.
     While &SCons; will create the specified cache directory as needed,
-    in this multiuser scenario it is usually best
-    to create it ahead of time, so the access rights
-    can be set up correctly.
+    the underlying &Python; library function used for this will
+    create it accessbile only for the creating user ID,
+    so for a shared cache, it is usually best
+    to create it ahead of time, and manually set up
+    the access rights as needed.
 
     </para>