diff --git a/SCons/CacheDir.py b/SCons/CacheDir.py index b91ddc633e..8dac91a699 100644 --- a/SCons/CacheDir.py +++ b/SCons/CacheDir.py @@ -44,12 +44,15 @@ b"# SCons cache directory - see https://bford.info/cachedir/\n" ) +# Defaults for the cache subsystem's globals. Most of these are filled in by +# SCons.Script.Main._build_targets, once the command line has been parsed. cache_enabled = True cache_debug = False cache_force = False cache_show = False cache_readonly = False cache_tmp_uuid = uuid.uuid4().hex +cli_cache_dir = "" def CacheRetrieveFunc(target, source, env) -> int: t = target[0] @@ -210,9 +213,11 @@ def _mkdir_atomic(self, path: str) -> bool: return False try: + parent_dir = os.path.dirname(directory) + os.makedirs(parent_dir, exist_ok=True) # TODO: Python 3.7. See comment below. - # tempdir = tempfile.TemporaryDirectory(dir=os.path.dirname(directory)) - tempdir = tempfile.mkdtemp(dir=os.path.dirname(directory)) + # tempdir = tempfile.TemporaryDirectory(dir=os.path.dirname(parent_dir)) + tempdir = tempfile.mkdtemp(dir=os.path.dirname(parent_dir)) except OSError as e: msg = "Failed to create cache directory " + path raise SCons.Errors.SConsEnvironmentError(msg) from e @@ -273,18 +278,33 @@ def CacheDebug(self, fmt, target, cachefile) -> None: if cache_debug == '-': self.debugFP = sys.stdout elif cache_debug: + # TODO: this seems fragile. There can be only one debug output + # (terminal, file or none) per run, so it should not + # be reopened. Testing multiple caches showed a problem + # where reopening with 'w' mode meant some of the output + # was lost, so for the moment switched to append mode. + # . Keeping better track of the output file, or switching to + # using the logging module should help. The "persistence" + # of using append mode breaks test/CacheDir/debug.py def debug_cleanup(debugFP) -> None: debugFP.close() - self.debugFP = open(cache_debug, 'w') + self.debugFP = open(cache_debug, 'a') atexit.register(debug_cleanup, self.debugFP) else: self.debugFP = None self.current_cache_debug = cache_debug if self.debugFP: + # TODO: consider emitting more than the base filename to help + # distinguish retrievals across variantdirs (target.relpath?). + # Separately, showing more of the cache entry path would be + # useful for testing, though possibly not otherwise. How else + # can you tell which target went to which cache if there are >1? self.debugFP.write(fmt % (target, os.path.split(cachefile)[1])) - self.debugFP.write("requests: %d, hits: %d, misses: %d, hit rate: %.2f%%\n" % - (self.requests, self.hits, self.misses, self.hit_ratio)) + self.debugFP.write( + "requests: %d, hits: %d, misses: %d, hit rate: %.2f%%\n" % + (self.requests, self.hits, self.misses, self.hit_ratio) + ) @classmethod def copy_from_cache(cls, env, src, dst) -> str: @@ -336,7 +356,7 @@ def cachepath(self, node) -> tuple: Given a Node, obtain the configured cache directory and the path to the cached file, which is generated from the node's build signature. If caching is not enabled for the - None, return a tuple of None. + node, return a tuple of ``None``. """ if not self.is_enabled(): return None, None @@ -349,11 +369,11 @@ def cachepath(self, node) -> tuple: def retrieve(self, node) -> bool: """Retrieve a node from cache. - Returns True if a successful retrieval resulted. + Returns ``True`` if a successful retrieval resulted. This method is called from multiple threads in a parallel build, so only do thread safe stuff here. Do thread unsafe stuff in - built(). + :meth:`built`. Note that there's a special trick here with the execute flag (one that's not normally done for other actions). Basically diff --git a/SCons/Environment.py b/SCons/Environment.py index 024c643dbd..a27c4784fb 100644 --- a/SCons/Environment.py +++ b/SCons/Environment.py @@ -1362,6 +1362,12 @@ def __init__( self._init_special() self.added_methods = [] + # If user specifies a --cache-dir on the command line, then + # use that for all created Environments, user can alter this + # by specifying CacheDir() per environment. + if SCons.CacheDir.cli_cache_dir: + self.CacheDir(SCons.CacheDir.cli_cache_dir) + # We don't use AddMethod, or define these as methods in this # class, because we *don't* want these functions to be bound # methods. They need to operate independently so that the diff --git a/SCons/Script/Main.py b/SCons/Script/Main.py index 5ee0c8b499..7bbec18c58 100644 --- a/SCons/Script/Main.py +++ b/SCons/Script/Main.py @@ -1042,6 +1042,9 @@ def _main(parser): SCons.Node.implicit_deps_changed = options.implicit_deps_changed SCons.Node.implicit_deps_unchanged = options.implicit_deps_unchanged + if options.cache_dir: + SCons.CacheDir.cli_cache_dir = options.cache_dir + if options.no_exec: SCons.SConf.dryrun = 1 SCons.Action.execute_actions = None diff --git a/SCons/Script/SConsOptions.py b/SCons/Script/SConsOptions.py index 52d7fca52e..5f3cd39d9b 100644 --- a/SCons/Script/SConsOptions.py +++ b/SCons/Script/SConsOptions.py @@ -799,6 +799,13 @@ def opt_ignore(option, opt, value, parser) -> None: help="Print CacheDir debug info to FILE", metavar="FILE") + op.add_option('--cache-dir', + nargs=1, + dest='cache_dir', + metavar='CACHEDIR', + help='Enable the derived‑file cache and set its directory to CACHEDIR', + default="") + op.add_option('--cache-disable', '--no-cache', dest='cache_disable', default=False, action="store_true", diff --git a/SCons/Script/SConscript.py b/SCons/Script/SConscript.py index e62563fc6a..3793705bc0 100644 --- a/SCons/Script/SConscript.py +++ b/SCons/Script/SConscript.py @@ -669,41 +669,42 @@ def get_DefaultEnvironmentProxy(): return _DefaultEnvironmentProxy class DefaultEnvironmentCall: - """A class that implements "global function" calls of - Environment methods by fetching the specified method from the - DefaultEnvironment's class. Note that this uses an intermediate - proxy class instead of calling the DefaultEnvironment method - directly so that the proxy can override the subst() method and + """Create a "global function" from an Environment method. + + Fetches the *method_name* from the Environment instance created to hold + the Default Environment. Uses an intermediate proxy class instead of + calling the :meth:`~SCons.Defaults.DefaultEnvironment` function directly, + so that the proxy can override the ``subst()`` method and thereby prevent expansion of construction variables (since from the user's point of view this was called as a global function, - with no associated construction environment).""" - def __init__(self, method_name, subst: int=0) -> None: + with no associated construction environment). + """ + + def __init__(self, method_name, subst: bool = False) -> None: self.method_name = method_name if subst: self.factory = SCons.Defaults.DefaultEnvironment else: self.factory = get_DefaultEnvironmentProxy + def __call__(self, *args, **kw): env = self.factory() method = getattr(env, self.method_name) return method(*args, **kw) - -def BuildDefaultGlobals(): - """ - Create a dictionary containing all the default globals for - SConstruct and SConscript files. - """ - +def BuildDefaultGlobals() -> dict: + """Create a dict containing all the default globals for SConscript files.""" global GlobalDict if GlobalDict is None: - GlobalDict = {} - import SCons.Script + + GlobalDict = {} d = SCons.Script.__dict__ - def not_a_module(m, d=d, mtype=type(SCons.Script)) -> bool: - return not isinstance(d[m], mtype) + + def not_a_module(m, d=d) -> bool: + return not isinstance(d[m], types.ModuleType) + for m in filter(not_a_module, dir(SCons.Script)): - GlobalDict[m] = d[m] + GlobalDict[m] = d[m] return GlobalDict.copy() diff --git a/doc/man/scons.xml b/doc/man/scons.xml index fa18060b8e..accf4a1789 100644 --- a/doc/man/scons.xml +++ b/doc/man/scons.xml @@ -722,6 +722,25 @@ derived-file cache specified by &f-link-CacheDir;. + + - + + + + Enable derived-file caching globally, using + cachedir + as the cache directory. + An individual &consenv; may still specify a different + cache directory by calling &f-link-env-CacheDir;. + + + Added in version NEXT_RELEASE. + + + + , @@ -9084,7 +9103,7 @@ However, the following variables are imported by - + SCONS_LIB_DIR Specifies the directory that contains the &scons; @@ -9110,7 +9129,7 @@ so the command line can be used to override - + SCONS_CACHE_MSVC_CONFIG (Windows only). If set, save the shell environment variables diff --git a/doc/user/caching.xml b/doc/user/caching.xml index 044422686b..af7e00dfe2 100644 --- a/doc/user/caching.xml +++ b/doc/user/caching.xml @@ -72,13 +72,22 @@ CacheDir('/usr/local/build_cache') + + A cache directory can also be specified on the command line + (--cache-dir=CACHEDIR) - useful for the case + where you have a permanent &f-link-CacheDir; call + but want to override it temporarily for a given build. + You can also create a cache directory specific to a &consenv; + by using the env.CacheDir() form. + + The cache directory you specify must have read and write access for all developers - who will be accessing the cached files - (if is used, - only read access is required). + who will be accessing the cached files. + If is used, + only read access is required. It should also be in some central location that all builds will be able to access. In environments where developers are using separate systems @@ -86,9 +95,11 @@ CacheDir('/usr/local/build_cache') this directory would typically be on a shared or NFS-mounted file system. While &SCons; will create the specified cache directory as needed, - in this multiuser scenario it is usually best - to create it ahead of time, so the access rights - can be set up correctly. + the underlying &Python; library function used for this will + create it accessbile only for the creating user ID, + so for a shared cache, it is usually best + to create it ahead of time, and manually set up + the access rights as needed. diff --git a/test/CacheDir/CacheDir_cli.py b/test/CacheDir/CacheDir_cli.py new file mode 100644 index 0000000000..7849f0cbc9 --- /dev/null +++ b/test/CacheDir/CacheDir_cli.py @@ -0,0 +1,157 @@ +#!/usr/bin/env python +# +# MIT License +# +# Copyright The SCons Foundation +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY +# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE +# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +""" +Test retrieving derived files from a CacheDir. +""" + +import os + +import TestSCons + +test = TestSCons.TestSCons() + +cache = test.workpath('cache') + +src_aaa_out = test.workpath('src', 'aaa.out') +src_bbb_out = test.workpath('src', 'bbb.out') +src_ccc_out = test.workpath('src', 'ccc.out') +src_cat_out = test.workpath('src', 'cat.out') +src_all = test.workpath('src', 'all') + +test.subdir('src') + +test.write(['src', 'SConstruct'], """\ +DefaultEnvironment(tools=[]) +SConscript('SConscript') +""" % locals()) + +test.write(['src', 'SConscript'], """\ +def cat(env, source, target): + target = str(target[0]) + with open('cat.out', 'a') as f: + f.write(target + "\\n") + with open(target, "w") as f: + for src in source: + with open(src, "r") as f2: + f.write(f2.read()) +env = Environment(tools=[], BUILDERS={'Cat':Builder(action=cat)}) +env.Cat('aaa.out', 'aaa.in') +env.Cat('bbb.out', 'bbb.in') +env.Cat('ccc.out', 'ccc.in') +env.Cat('all', ['aaa.out', 'bbb.out', 'ccc.out']) +""") + +test.write(['src', 'aaa.in'], "aaa.in\n") +test.write(['src', 'bbb.in'], "bbb.in\n") +test.write(['src', 'ccc.in'], "ccc.in\n") + +# Verify that building with -n and an empty cache reports that proper +# build operations would be taken, but that nothing is actually built +# and that the cache is still empty. +test.run(chdir='src', arguments=f'--cache-dir={cache} -n .', stdout=test.wrap_stdout("""\ +cat(["aaa.out"], ["aaa.in"]) +cat(["bbb.out"], ["bbb.in"]) +cat(["ccc.out"], ["ccc.in"]) +cat(["all"], ["aaa.out", "bbb.out", "ccc.out"]) +""")) + +test.must_not_exist(src_aaa_out) +test.must_not_exist(src_bbb_out) +test.must_not_exist(src_ccc_out) +test.must_not_exist(src_all) +# Even if you do -n, the cache will be configured. +expect = ['CACHEDIR.TAG', 'config'] +found = sorted(os.listdir(cache)) +test.fail_test( + expect != found, + message=f"expected cachedir contents {expect}, found {found}", +) +# Verify that a normal build works correctly, and clean up. +# This should populate the cache with our derived files. +test.run(chdir='src', arguments=f'--cache-dir={cache} .') +test.must_match(['src', 'all'], "aaa.in\nbbb.in\nccc.in\n", mode='r') +test.must_match(['src', 'cat.out'], "aaa.out\nbbb.out\nccc.out\nall\n", mode='r') +test.up_to_date(chdir='src', arguments='.') +test.run(chdir='src', arguments='-c .') +test.unlink(['src', 'cat.out']) + +# Verify that we now retrieve the derived files from cache, +# not rebuild them. Then clean up. +test.run(chdir='src', arguments=f'--cache-dir={cache} .', stdout=test.wrap_stdout("""\ +Retrieved `aaa.out' from cache +Retrieved `bbb.out' from cache +Retrieved `ccc.out' from cache +Retrieved `all' from cache +""")) +test.must_not_exist(src_cat_out) +test.up_to_date(chdir='src', arguments='.') +test.run(chdir='src', arguments='-c .') + +# Verify that rebuilding with -n reports that everything was retrieved +# from the cache, but that nothing really was. +test.run(chdir='src', arguments=f'--cache-dir={cache} -n .', stdout=test.wrap_stdout("""\ +Retrieved `aaa.out' from cache +Retrieved `bbb.out' from cache +Retrieved `ccc.out' from cache +Retrieved `all' from cache +""")) +test.must_not_exist(src_aaa_out) +test.must_not_exist(src_bbb_out) +test.must_not_exist(src_ccc_out) +test.must_not_exist(src_all) + +# Verify that rebuilding with -s retrieves everything from the cache +# even though it doesn't report anything. +test.run(chdir='src', arguments=f'--cache-dir={cache} -s .', stdout="") +test.must_match(['src', 'all'], "aaa.in\nbbb.in\nccc.in\n", mode='r') +test.must_not_exist(src_cat_out) +test.up_to_date(chdir='src', arguments='.') + +test.run(chdir='src', arguments='-c .') +# Verify that updating one input file builds its derived file and +# dependency but that the other files are retrieved from cache. +test.write(['src', 'bbb.in'], "bbb.in 2\n") + +test.run(chdir='src', arguments=f'--cache-dir={cache} .', stdout=test.wrap_stdout("""\ +Retrieved `aaa.out' from cache +cat(["bbb.out"], ["bbb.in"]) +Retrieved `ccc.out' from cache +cat(["all"], ["aaa.out", "bbb.out", "ccc.out"]) +""")) + +test.must_match(['src', 'all'], "aaa.in\nbbb.in 2\nccc.in\n", mode='r') +test.must_match(['src', 'cat.out'], "bbb.out\nall\n", mode='r') + +test.up_to_date(chdir='src', arguments='.') + + +test.pass_test() + +# Local Variables: +# tab-width:4 +# indent-tabs-mode:nil +# End: +# vim: set expandtab tabstop=4 shiftwidth=4: