From cd930697c11709867a787fe7347b018d132a6bb2 Mon Sep 17 00:00:00 2001 From: oreo639 Date: Mon, 22 Jan 2024 02:41:52 -0800 Subject: [PATCH] jack: fix build for python3.12 --- srcpkgs/jack/patches/update-waf.patch | 3477 +++++++++++++++++++++++++ 1 file changed, 3477 insertions(+) create mode 100644 srcpkgs/jack/patches/update-waf.patch diff --git a/srcpkgs/jack/patches/update-waf.patch b/srcpkgs/jack/patches/update-waf.patch new file mode 100644 index 0000000000000..aef5047131d61 --- /dev/null +++ b/srcpkgs/jack/patches/update-waf.patch @@ -0,0 +1,3477 @@ +From 250420381b1a6974798939ad7104ab1a4b9a9994 Mon Sep 17 00:00:00 2001 +From: Nils Philippsen +Date: Tue, 29 Aug 2023 12:41:46 +0200 +Subject: [PATCH] Update to waf 2.0.26 + +This makes waf compatible with Python 3.12 again. + +Also, apply modifications needed for MacOS and add as a patch file (see +commits 0f2e3b2 and dc6c995). + +Signed-off-by: Nils Philippsen +--- + waf | 7 +- + waflib-macos-mods.patch | 18 + + waflib/Build.py | 47 +- + waflib/ConfigSet.py | 2 +- + waflib/Configure.py | 46 +- + waflib/Context.py | 24 +- + waflib/Logs.py | 9 +- + waflib/Node.py | 3 +- + waflib/Options.py | 31 +- + waflib/Runner.py | 27 +- + waflib/Scripting.py | 27 +- + waflib/Task.py | 48 ++- + waflib/TaskGen.py | 16 +- + waflib/Tools/c_aliases.py | 10 +- + waflib/Tools/c_config.py | 37 +- + waflib/Tools/c_preproc.py | 6 +- + waflib/Tools/c_tests.py | 18 +- + waflib/Tools/ccroot.py | 20 +- + waflib/Tools/compiler_c.py | 25 +- + waflib/Tools/compiler_cxx.py | 25 +- + waflib/Tools/irixcc.py | 14 +- + waflib/Tools/msvc.py | 45 +- + waflib/Tools/waf_unit_test.py | 14 +- + waflib/Utils.py | 60 ++- + waflib/ansiterm.py | 2 +- + waflib/extras/clang_cross.py | 92 ++++ + waflib/extras/clang_cross_common.py | 113 +++++ + waflib/extras/clangxx_cross.py | 106 +++++ + waflib/extras/classic_runner.py | 68 +++ + waflib/extras/color_msvc.py | 59 +++ + waflib/extras/fc_fujitsu.py | 52 +++ + waflib/extras/fc_nfort.py | 52 +++ + waflib/extras/genpybind.py | 194 +++++++++ + waflib/extras/haxe.py | 154 +++++++ + waflib/extras/msvc_pdb.py | 46 ++ + waflib/extras/sphinx.py | 120 ++++++ + waflib/extras/wafcache.py | 648 ++++++++++++++++++++++++++++ + waflib/extras/xcode6.py | 18 +- + waflib/fixpy2.py | 2 +- + waflib/processor.py | 4 + + 40 files changed, 2114 insertions(+), 195 deletions(-) + create mode 100644 waflib-macos-mods.patch + create mode 100644 waflib/extras/clang_cross.py + create mode 100644 waflib/extras/clang_cross_common.py + create mode 100644 waflib/extras/clangxx_cross.py + create mode 100644 waflib/extras/classic_runner.py + create mode 100644 waflib/extras/color_msvc.py + create mode 100644 waflib/extras/fc_fujitsu.py + create mode 100644 waflib/extras/fc_nfort.py + create mode 100644 waflib/extras/genpybind.py + create mode 100644 waflib/extras/haxe.py + create mode 100644 waflib/extras/msvc_pdb.py + create mode 100644 waflib/extras/sphinx.py + create mode 100644 waflib/extras/wafcache.py + +diff --git a/waf b/waf +index 845fba5e9..38b2c9106 100755 +--- a/waf ++++ b/waf +@@ -1,4 +1,4 @@ +-#!/usr/bin/python3 ++#!/usr/bin/env python + # encoding: latin-1 + # Thomas Nagy, 2005-2018 + # +@@ -32,7 +32,7 @@ POSSIBILITY OF SUCH DAMAGE. + + import os, sys, inspect + +-VERSION="2.0.12" ++VERSION="2.0.26" + REVISION="x" + GIT="x" + INSTALL="x" +@@ -142,6 +142,9 @@ def find_lib(): + if name.endswith('waf-light'): + w = test(base) + if w: return w ++ for dir in sys.path: ++ if test(dir): ++ return dir + err('waf-light requires waflib -> export WAFDIR=/folder') + + dirname = '%s-%s-%s' % (WAF, VERSION, REVISION) +diff --git a/waflib-macos-mods.patch b/waflib-macos-mods.patch +new file mode 100644 +index 000000000..9e2c8a3de +--- /dev/null ++++ b/waflib-macos-mods.patch +@@ -0,0 +1,18 @@ ++diff --git a/waflib/Tools/ccroot.py b/waflib/Tools/ccroot.py ++index cfef8bf5..484846f5 100644 ++--- a/waflib/Tools/ccroot.py +++++ b/waflib/Tools/ccroot.py ++@@ -575,12 +575,10 @@ def apply_vnum(self): ++ ++ cnum = getattr(self, 'cnum', str(nums[0])) ++ cnums = cnum.split('.') ++- if len(cnums)>len(nums) or nums[0:len(cnums)] != cnums: ++- raise Errors.WafError('invalid compatibility version %s' % cnum) ++ ++ libname = node.name ++ if libname.endswith('.dylib'): ++- name3 = libname.replace('.dylib', '.%s.dylib' % self.vnum) +++ name3 = libname.replace('.dylib', '.%s.dylib' % cnums[0]) ++ name2 = libname.replace('.dylib', '.%s.dylib' % cnum) ++ else: ++ name3 = libname + '.' + self.vnum +diff --git a/waflib/Build.py b/waflib/Build.py +index c9661df15..b49dd8302 100644 +--- a/waflib/Build.py ++++ b/waflib/Build.py +@@ -104,7 +104,7 @@ def __init__(self, **kw): + """Amount of jobs to run in parallel""" + + self.targets = Options.options.targets +- """List of targets to build (default: \*)""" ++ """List of targets to build (default: \\*)""" + + self.keep = Options.options.keep + """Whether the build should continue past errors""" +@@ -753,10 +753,12 @@ def tgpost(tg): + else: + ln = self.launch_node() + if ln.is_child_of(self.bldnode): +- Logs.warn('Building from the build directory, forcing --targets=*') ++ if Logs.verbose > 1: ++ Logs.warn('Building from the build directory, forcing --targets=*') + ln = self.srcnode + elif not ln.is_child_of(self.srcnode): +- Logs.warn('CWD %s is not under %s, forcing --targets=* (run distclean?)', ln.abspath(), self.srcnode.abspath()) ++ if Logs.verbose > 1: ++ Logs.warn('CWD %s is not under %s, forcing --targets=* (run distclean?)', ln.abspath(), self.srcnode.abspath()) + ln = self.srcnode + + def is_post(tg, ln): +@@ -1054,7 +1056,7 @@ def post_run(self): + def get_install_path(self, destdir=True): + """ + Returns the destination path where files will be installed, pre-pending `destdir`. +- ++ + Relative paths will be interpreted relative to `PREFIX` if no `destdir` is given. + + :rtype: string +@@ -1062,11 +1064,11 @@ def get_install_path(self, destdir=True): + if isinstance(self.install_to, Node.Node): + dest = self.install_to.abspath() + else: +- dest = Utils.subst_vars(self.install_to, self.env) ++ dest = os.path.normpath(Utils.subst_vars(self.install_to, self.env)) + if not os.path.isabs(dest): +- dest = os.path.join(self.env.PREFIX, dest) ++ dest = os.path.join(self.env.PREFIX, dest) + if destdir and Options.options.destdir: +- dest = os.path.join(Options.options.destdir, os.path.splitdrive(dest)[1].lstrip(os.sep)) ++ dest = Options.options.destdir.rstrip(os.sep) + os.sep + os.path.splitdrive(dest)[1].lstrip(os.sep) + return dest + + def copy_fun(self, src, tgt): +@@ -1160,11 +1162,19 @@ def do_install(self, src, tgt, lbl, **kw): + # same size and identical timestamps -> make no copy + if st1.st_mtime + 2 >= st2.st_mtime and st1.st_size == st2.st_size: + if not self.generator.bld.progress_bar: +- Logs.info('- install %s (from %s)', tgt, lbl) ++ ++ c1 = Logs.colors.NORMAL ++ c2 = Logs.colors.BLUE ++ ++ Logs.info('%s- install %s%s%s (from %s)', c1, c2, tgt, c1, lbl) + return False + + if not self.generator.bld.progress_bar: +- Logs.info('+ install %s (from %s)', tgt, lbl) ++ ++ c1 = Logs.colors.NORMAL ++ c2 = Logs.colors.BLUE ++ ++ Logs.info('%s+ install %s%s%s (from %s)', c1, c2, tgt, c1, lbl) + + # Give best attempt at making destination overwritable, + # like the 'install' utility used by 'make install' does. +@@ -1221,14 +1231,18 @@ def do_link(self, src, tgt, **kw): + """ + if os.path.islink(tgt) and os.readlink(tgt) == src: + if not self.generator.bld.progress_bar: +- Logs.info('- symlink %s (to %s)', tgt, src) ++ c1 = Logs.colors.NORMAL ++ c2 = Logs.colors.BLUE ++ Logs.info('%s- symlink %s%s%s (to %s)', c1, c2, tgt, c1, src) + else: + try: + os.remove(tgt) + except OSError: + pass + if not self.generator.bld.progress_bar: +- Logs.info('+ symlink %s (to %s)', tgt, src) ++ c1 = Logs.colors.NORMAL ++ c2 = Logs.colors.BLUE ++ Logs.info('%s+ symlink %s%s%s (to %s)', c1, c2, tgt, c1, src) + os.symlink(src, tgt) + self.fix_perms(tgt) + +@@ -1237,7 +1251,9 @@ def do_uninstall(self, src, tgt, lbl, **kw): + See :py:meth:`waflib.Build.inst.do_install` + """ + if not self.generator.bld.progress_bar: +- Logs.info('- remove %s', tgt) ++ c1 = Logs.colors.NORMAL ++ c2 = Logs.colors.BLUE ++ Logs.info('%s- remove %s%s%s', c1, c2, tgt, c1) + + #self.uninstall.append(tgt) + try: +@@ -1257,7 +1273,9 @@ def do_unlink(self, src, tgt, **kw): + """ + try: + if not self.generator.bld.progress_bar: +- Logs.info('- remove %s', tgt) ++ c1 = Logs.colors.NORMAL ++ c2 = Logs.colors.BLUE ++ Logs.info('%s- remove %s%s%s', c1, c2, tgt, c1) + os.remove(tgt) + except OSError: + pass +@@ -1318,7 +1336,8 @@ def build(bld): + lst = [] + for env in self.all_envs.values(): + lst.extend(self.root.find_or_declare(f) for f in env[CFG_FILES]) +- for n in self.bldnode.ant_glob('**/*', excl='.lock* *conf_check_*/** config.log c4che/*', quiet=True): ++ excluded_dirs = '.lock* *conf_check_*/** config.log %s/*' % CACHE_DIR ++ for n in self.bldnode.ant_glob('**/*', excl=excluded_dirs, quiet=True): + if n in lst: + continue + n.delete() +diff --git a/waflib/ConfigSet.py b/waflib/ConfigSet.py +index 84736c9c8..901fba6c0 100644 +--- a/waflib/ConfigSet.py ++++ b/waflib/ConfigSet.py +@@ -11,7 +11,7 @@ + + import copy, re, os + from waflib import Logs, Utils +-re_imp = re.compile('^(#)*?([^#=]*?)\ =\ (.*?)$', re.M) ++re_imp = re.compile(r'^(#)*?([^#=]*?)\ =\ (.*?)$', re.M) + + class ConfigSet(object): + """ +diff --git a/waflib/Configure.py b/waflib/Configure.py +index d0a4793a8..f6fdc4e94 100644 +--- a/waflib/Configure.py ++++ b/waflib/Configure.py +@@ -125,7 +125,7 @@ def init_dirs(self): + self.bldnode.mkdir() + + if not os.path.isdir(self.bldnode.abspath()): +- conf.fatal('Could not create the build directory %s' % self.bldnode.abspath()) ++ self.fatal('Could not create the build directory %s' % self.bldnode.abspath()) + + def execute(self): + """ +@@ -180,6 +180,7 @@ def execute(self): + env.hash = self.hash + env.files = self.files + env.environ = dict(self.environ) ++ env.launch_dir = Context.launch_dir + + if not (self.env.NO_LOCK_IN_RUN or env.environ.get('NO_LOCK_IN_RUN') or getattr(Options.options, 'no_lock_in_run')): + env.store(os.path.join(Context.run_dir, Options.lockfile)) +@@ -438,7 +439,7 @@ def find_program(self, filename, **kw): + + var = kw.get('var', '') + if not var: +- var = re.sub(r'[-.]', '_', filename[0].upper()) ++ var = re.sub(r'\W', '_', filename[0].upper()) + + path_list = kw.get('path_list', '') + if path_list: +@@ -507,23 +508,27 @@ def find_binary(self, filenames, exts, paths): + @conf + def run_build(self, *k, **kw): + """ +- Create a temporary build context to execute a build. A reference to that build +- context is kept on self.test_bld for debugging purposes, and you should not rely +- on it too much (read the note on the cache below). +- The parameters given in the arguments to this function are passed as arguments for +- a single task generator created in the build. Only three parameters are obligatory: ++ Create a temporary build context to execute a build. A temporary reference to that build ++ context is kept on self.test_bld for debugging purposes. ++ The arguments to this function are passed to a single task generator for that build. ++ Only three parameters are mandatory: + + :param features: features to pass to a task generator created in the build + :type features: list of string + :param compile_filename: file to create for the compilation (default: *test.c*) + :type compile_filename: string +- :param code: code to write in the filename to compile ++ :param code: input file contents + :type code: string + +- Though this function returns *0* by default, the build may set an attribute named *retval* on the ++ Though this function returns *0* by default, the build may bind attribute named *retval* on the + build context object to return a particular value. See :py:func:`waflib.Tools.c_config.test_exec_fun` for example. + +- This function also provides a limited cache. To use it, provide the following option:: ++ The temporary builds creates a temporary folder; the name of that folder is calculated ++ by hashing input arguments to this function, with the exception of :py:class:`waflib.ConfigSet.ConfigSet` ++ objects which are used for both reading and writing values. ++ ++ This function also features a cache which is disabled by default; that cache relies ++ on the hash value calculated as indicated above:: + + def options(opt): + opt.add_option('--confcache', dest='confcache', default=0, +@@ -534,10 +539,24 @@ def options(opt): + $ waf configure --confcache + + """ +- lst = [str(v) for (p, v) in kw.items() if p != 'env'] +- h = Utils.h_list(lst) ++ buf = [] ++ for key in sorted(kw.keys()): ++ v = kw[key] ++ if isinstance(v, ConfigSet.ConfigSet): ++ # values are being written to, so they are excluded from contributing to the hash ++ continue ++ elif hasattr(v, '__call__'): ++ buf.append(Utils.h_fun(v)) ++ else: ++ buf.append(str(v)) ++ h = Utils.h_list(buf) + dir = self.bldnode.abspath() + os.sep + (not Utils.is_win32 and '.' or '') + 'conf_check_' + Utils.to_hex(h) + ++ cachemode = kw.get('confcache', getattr(Options.options, 'confcache', None)) ++ ++ if not cachemode and os.path.exists(dir): ++ shutil.rmtree(dir) ++ + try: + os.makedirs(dir) + except OSError: +@@ -548,7 +567,6 @@ def options(opt): + except OSError: + self.fatal('cannot use the configuration test folder %r' % dir) + +- cachemode = getattr(Options.options, 'confcache', None) + if cachemode == 1: + try: + proj = ConfigSet.ConfigSet(os.path.join(dir, 'cache_run_build')) +@@ -588,7 +606,7 @@ def options(opt): + else: + ret = getattr(bld, 'retval', 0) + finally: +- if cachemode == 1: ++ if cachemode: + # cache the results each time + proj = ConfigSet.ConfigSet() + proj['cache_run_build'] = ret +diff --git a/waflib/Context.py b/waflib/Context.py +index 761b521f5..369664819 100644 +--- a/waflib/Context.py ++++ b/waflib/Context.py +@@ -6,20 +6,30 @@ + Classes and functions enabling the command system + """ + +-import os, re, imp, sys ++import os, re, sys + from waflib import Utils, Errors, Logs + import waflib.Node + ++if sys.hexversion > 0x3040000: ++ import types ++ class imp(object): ++ new_module = lambda x: types.ModuleType(x) ++else: ++ import imp ++ + # the following 3 constants are updated on each new release (do not touch) +-HEXVERSION=0x2000c00 ++HEXVERSION=0x2001a00 + """Constant updated on new releases""" + +-WAFVERSION="2.0.12" ++WAFVERSION="2.0.26" + """Constant updated on new releases""" + +-WAFREVISION="54841218840ffa34fddf834680a5a17db69caa12" ++WAFREVISION="0fb985ce1932c6f3e7533f435e4ee209d673776e" + """Git revision when the waf version is updated""" + ++WAFNAME="waf" ++"""Application name displayed on --help""" ++ + ABI = 20 + """Version of the build data cache file format (used in :py:const:`waflib.Context.DBFILE`)""" + +@@ -134,7 +144,7 @@ def foo(ctx): + :type fun: string + + .. inheritance-diagram:: waflib.Context.Context waflib.Build.BuildContext waflib.Build.InstallContext waflib.Build.UninstallContext waflib.Build.StepContext waflib.Build.ListContext waflib.Configure.ConfigurationContext waflib.Scripting.Dist waflib.Scripting.DistCheck waflib.Build.CleanContext +- ++ :top-classes: waflib.Context.Context + """ + + errors = Errors +@@ -613,7 +623,7 @@ def load_special_tools(self, var, ban=[]): + is typically called once for a programming language group, see for + example :py:mod:`waflib.Tools.compiler_c` + +- :param var: glob expression, for example 'cxx\_\*.py' ++ :param var: glob expression, for example 'cxx\\_\\*.py' + :type var: string + :param ban: list of exact file names to exclude + :type ban: list of string +@@ -678,7 +688,7 @@ def load_module(path, encoding=None): + + def load_tool(tool, tooldir=None, ctx=None, with_sys_path=True): + """ +- Importx a Waf tool as a python module, and stores it in the dict :py:const:`waflib.Context.Context.tools` ++ Imports a Waf tool as a python module, and stores it in the dict :py:const:`waflib.Context.Context.tools` + + :type tool: string + :param tool: Name of the tool +diff --git a/waflib/Logs.py b/waflib/Logs.py +index 2a475169b..298411db5 100644 +--- a/waflib/Logs.py ++++ b/waflib/Logs.py +@@ -237,7 +237,10 @@ def format(self, rec): + if rec.levelno >= logging.INFO: + # the goal of this is to format without the leading "Logs, hour" prefix + if rec.args: +- return msg % rec.args ++ try: ++ return msg % rec.args ++ except UnicodeDecodeError: ++ return msg.encode('utf-8') % rec.args + return msg + + rec.msg = msg +@@ -276,9 +279,9 @@ def error(*k, **kw): + + def warn(*k, **kw): + """ +- Wraps logging.warn ++ Wraps logging.warning + """ +- log.warn(*k, **kw) ++ log.warning(*k, **kw) + + def info(*k, **kw): + """ +diff --git a/waflib/Node.py b/waflib/Node.py +index 4ac1ea8a0..2ad184669 100644 +--- a/waflib/Node.py ++++ b/waflib/Node.py +@@ -73,7 +73,7 @@ def ant_matcher(s, ignorecase): + if k == '**': + accu.append(k) + else: +- k = k.replace('.', '[.]').replace('*','.*').replace('?', '.').replace('+', '\\+') ++ k = k.replace('.', '[.]').replace('*', '.*').replace('?', '.').replace('+', '\\+') + k = '^%s$' % k + try: + exp = re.compile(k, flags=reflags) +@@ -595,7 +595,6 @@ def ant_iter(self, accept=None, maxdepth=25, pats=[], dir=False, src=True, remov + :rtype: iterator + """ + dircont = self.listdir() +- dircont.sort() + + try: + lst = set(self.children.keys()) +diff --git a/waflib/Options.py b/waflib/Options.py +index ad802d4b9..d4104917c 100644 +--- a/waflib/Options.py ++++ b/waflib/Options.py +@@ -44,7 +44,7 @@ class opt_parser(optparse.OptionParser): + """ + def __init__(self, ctx, allow_unknown=False): + optparse.OptionParser.__init__(self, conflict_handler='resolve', add_help_option=False, +- version='waf %s (%s)' % (Context.WAFVERSION, Context.WAFREVISION)) ++ version='%s %s (%s)' % (Context.WAFNAME, Context.WAFVERSION, Context.WAFREVISION)) + self.formatter.width = Logs.get_term_cols() + self.ctx = ctx + self.allow_unknown = allow_unknown +@@ -62,6 +62,21 @@ def _process_args(self, largs, rargs, values): + else: + self.error(str(e)) + ++ def _process_long_opt(self, rargs, values): ++ # --custom-option=-ftxyz is interpreted as -f -t... see #2280 ++ if self.allow_unknown: ++ back = [] + rargs ++ try: ++ optparse.OptionParser._process_long_opt(self, rargs, values) ++ except optparse.BadOptionError: ++ while rargs: ++ rargs.pop() ++ rargs.extend(back) ++ rargs.pop(0) ++ raise ++ else: ++ optparse.OptionParser._process_long_opt(self, rargs, values) ++ + def print_usage(self, file=None): + return self.print_help(file) + +@@ -96,11 +111,11 @@ def get_usage(self): + lst.sort() + ret = '\n'.join(lst) + +- return '''waf [commands] [options] ++ return '''%s [commands] [options] + +-Main commands (example: ./waf build -j4) ++Main commands (example: ./%s build -j4) + %s +-''' % ret ++''' % (Context.WAFNAME, Context.WAFNAME, ret) + + + class OptionsContext(Context.Context): +@@ -141,9 +156,9 @@ def __init__(self, **kw): + gr.add_option('-o', '--out', action='store', default='', help='build dir for the project', dest='out') + gr.add_option('-t', '--top', action='store', default='', help='src dir for the project', dest='top') + +- gr.add_option('--no-lock-in-run', action='store_true', default='', help=optparse.SUPPRESS_HELP, dest='no_lock_in_run') +- gr.add_option('--no-lock-in-out', action='store_true', default='', help=optparse.SUPPRESS_HELP, dest='no_lock_in_out') +- gr.add_option('--no-lock-in-top', action='store_true', default='', help=optparse.SUPPRESS_HELP, dest='no_lock_in_top') ++ gr.add_option('--no-lock-in-run', action='store_true', default=os.environ.get('NO_LOCK_IN_RUN', ''), help=optparse.SUPPRESS_HELP, dest='no_lock_in_run') ++ gr.add_option('--no-lock-in-out', action='store_true', default=os.environ.get('NO_LOCK_IN_OUT', ''), help=optparse.SUPPRESS_HELP, dest='no_lock_in_out') ++ gr.add_option('--no-lock-in-top', action='store_true', default=os.environ.get('NO_LOCK_IN_TOP', ''), help=optparse.SUPPRESS_HELP, dest='no_lock_in_top') + + default_prefix = getattr(Context.g_module, 'default_prefix', os.environ.get('PREFIX')) + if not default_prefix: +@@ -282,6 +297,8 @@ def parse_cmd_args(self, _args=None, cwd=None, allow_unknown=False): + elif arg != 'options': + commands.append(arg) + ++ if options.jobs < 1: ++ options.jobs = 1 + for name in 'top out destdir prefix bindir libdir'.split(): + # those paths are usually expanded from Context.launch_dir + if getattr(options, name, None): +diff --git a/waflib/Runner.py b/waflib/Runner.py +index 261084d27..350c86a22 100644 +--- a/waflib/Runner.py ++++ b/waflib/Runner.py +@@ -37,6 +37,8 @@ def __len__(self): + return len(self.lst) + def __iter__(self): + return iter(self.lst) ++ def __str__(self): ++ return 'PriorityTasks: [%s]' % '\n '.join(str(x) for x in self.lst) + def clear(self): + self.lst = [] + def append(self, task): +@@ -69,7 +71,7 @@ def __init__(self, spawner, task): + """Task to execute""" + self.spawner = spawner + """Coordinator object""" +- self.setDaemon(1) ++ self.daemon = True + self.start() + def run(self): + """ +@@ -96,7 +98,7 @@ def __init__(self, master): + """:py:class:`waflib.Runner.Parallel` producer instance""" + self.sem = Utils.threading.Semaphore(master.numjobs) + """Bounded semaphore that prevents spawning more than *n* concurrent consumers""" +- self.setDaemon(1) ++ self.daemon = True + self.start() + def run(self): + """ +@@ -181,10 +183,12 @@ def __init__(self, bld, j=2): + The reverse dependency graph of dependencies obtained from Task.run_after + """ + +- self.spawner = Spawner(self) ++ self.spawner = None + """ + Coordinating daemon thread that spawns thread consumers + """ ++ if self.numjobs > 1: ++ self.spawner = Spawner(self) + + def get_next_task(self): + """ +@@ -254,6 +258,8 @@ def refill_task_list(self): + self.outstanding.append(x) + break + else: ++ if self.stop or self.error: ++ break + raise Errors.WafError('Broken revdeps detected on %r' % self.incomplete) + else: + tasks = next(self.biter) +@@ -331,11 +337,16 @@ def try_unfreeze(x): + + if hasattr(tsk, 'semaphore'): + sem = tsk.semaphore +- sem.release(tsk) +- while sem.waiting and not sem.is_locked(): +- # take a frozen task, make it ready to run +- x = sem.waiting.pop() +- self._add_task(x) ++ try: ++ sem.release(tsk) ++ except KeyError: ++ # TODO ++ pass ++ else: ++ while sem.waiting and not sem.is_locked(): ++ # take a frozen task, make it ready to run ++ x = sem.waiting.pop() ++ self._add_task(x) + + def get_out(self): + """ +diff --git a/waflib/Scripting.py b/waflib/Scripting.py +index 749d4f2e6..a80cb3678 100644 +--- a/waflib/Scripting.py ++++ b/waflib/Scripting.py +@@ -216,7 +216,10 @@ def parse_options(): + ctx = Context.create_context('options') + ctx.execute() + if not Options.commands: +- Options.commands.append(default_cmd) ++ if isinstance(default_cmd, list): ++ Options.commands.extend(default_cmd) ++ else: ++ Options.commands.append(default_cmd) + if Options.options.whelp: + ctx.parser.print_help() + sys.exit(0) +@@ -280,7 +283,7 @@ def distclean_dir(dirname): + pass + + try: +- shutil.rmtree('c4che') ++ shutil.rmtree(Build.CACHE_DIR) + except OSError: + pass + +@@ -303,7 +306,7 @@ def remove_and_log(k, fun): + + # remove a build folder, if any + cur = '.' +- if ctx.options.no_lock_in_top: ++ if os.environ.get('NO_LOCK_IN_TOP') or ctx.options.no_lock_in_top: + cur = ctx.options.out + + try: +@@ -329,7 +332,12 @@ def remove_and_log(k, fun): + else: + remove_and_log(env.out_dir, shutil.rmtree) + +- for k in (env.out_dir, env.top_dir, env.run_dir): ++ env_dirs = [env.out_dir] ++ if not (os.environ.get('NO_LOCK_IN_TOP') or ctx.options.no_lock_in_top): ++ env_dirs.append(env.top_dir) ++ if not (os.environ.get('NO_LOCK_IN_RUN') or ctx.options.no_lock_in_run): ++ env_dirs.append(env.run_dir) ++ for k in env_dirs: + p = os.path.join(k, Options.lockfile) + remove_and_log(p, os.remove) + +@@ -380,7 +388,11 @@ def archive(self): + + for x in files: + archive_name = self.get_base_name() + '/' + x.path_from(self.base_path) +- zip.write(x.abspath(), archive_name, zipfile.ZIP_DEFLATED) ++ if os.environ.get('SOURCE_DATE_EPOCH'): ++ # TODO: parse that timestamp ++ zip.writestr(zipfile.ZipInfo(archive_name), x.read(), zipfile.ZIP_DEFLATED) ++ else: ++ zip.write(x.abspath(), archive_name, zipfile.ZIP_DEFLATED) + zip.close() + else: + self.fatal('Valid algo types are tar.bz2, tar.gz, tar.xz or zip') +@@ -417,6 +429,8 @@ def add_tar_file(self, x, tar): + tinfo.gid = 0 + tinfo.uname = 'root' + tinfo.gname = 'root' ++ if os.environ.get('SOURCE_DATE_EPOCH'): ++ tinfo.mtime = int(os.environ.get('SOURCE_DATE_EPOCH')) + + if os.path.isfile(p): + with open(p, 'rb') as f: +@@ -598,12 +612,15 @@ def execute(self): + cmd = env.config_cmd or 'configure' + if Configure.autoconfig == 'clobber': + tmp = Options.options.__dict__ ++ launch_dir_tmp = Context.launch_dir + if env.options: + Options.options.__dict__ = env.options ++ Context.launch_dir = env.launch_dir + try: + run_command(cmd) + finally: + Options.options.__dict__ = tmp ++ Context.launch_dir = launch_dir_tmp + else: + run_command(cmd) + run_command(self.cmd) +diff --git a/waflib/Task.py b/waflib/Task.py +index 6aebc6074..cb49a7394 100644 +--- a/waflib/Task.py ++++ b/waflib/Task.py +@@ -163,10 +163,10 @@ class Task(evil): + """File extensions that objects of this task class may create""" + + before = [] +- """List of task class names to execute before instances of this class""" ++ """The instances of this class are executed before the instances of classes whose names are in this list""" + + after = [] +- """List of task class names to execute after instances of this class""" ++ """The instances of this class are executed after the instances of classes whose names are in this list""" + + hcode = Utils.SIG_NIL + """String representing an additional hash for the class representation""" +@@ -306,25 +306,31 @@ def exec_command(self, cmd, **kw): + if hasattr(self, 'stderr'): + kw['stderr'] = self.stderr + +- # workaround for command line length limit: +- # http://support.microsoft.com/kb/830473 +- if not isinstance(cmd, str) and (len(repr(cmd)) >= 8192 if Utils.is_win32 else len(cmd) > 200000): +- cmd, args = self.split_argfile(cmd) +- try: +- (fd, tmp) = tempfile.mkstemp() +- os.write(fd, '\r\n'.join(args).encode()) +- os.close(fd) +- if Logs.verbose: +- Logs.debug('argfile: @%r -> %r', tmp, args) +- return self.generator.bld.exec_command(cmd + ['@' + tmp], **kw) +- finally: ++ if not isinstance(cmd, str): ++ if Utils.is_win32: ++ # win32 compares the resulting length http://support.microsoft.com/kb/830473 ++ too_long = sum([len(arg) for arg in cmd]) + len(cmd) > 8192 ++ else: ++ # non-win32 counts the amount of arguments (200k) ++ too_long = len(cmd) > 200000 ++ ++ if too_long and getattr(self, 'allow_argsfile', True): ++ # Shunt arguments to a temporary file if the command is too long. ++ cmd, args = self.split_argfile(cmd) + try: +- os.remove(tmp) +- except OSError: +- # anti-virus and indexers can keep files open -_- +- pass +- else: +- return self.generator.bld.exec_command(cmd, **kw) ++ (fd, tmp) = tempfile.mkstemp() ++ os.write(fd, '\r\n'.join(args).encode()) ++ os.close(fd) ++ if Logs.verbose: ++ Logs.debug('argfile: @%r -> %r', tmp, args) ++ return self.generator.bld.exec_command(cmd + ['@' + tmp], **kw) ++ finally: ++ try: ++ os.remove(tmp) ++ except OSError: ++ # anti-virus and indexers can keep files open -_- ++ pass ++ return self.generator.bld.exec_command(cmd, **kw) + + def process(self): + """ +@@ -1044,7 +1050,7 @@ def funex(c): + exec(c, dc) + return dc['f'] + +-re_cond = re.compile('(?P\w+)|(?P\|)|(?P&)') ++re_cond = re.compile(r'(?P\w+)|(?P\|)|(?P&)') + re_novar = re.compile(r'^(SRC|TGT)\W+.*?$') + reg_act = re.compile(r'(?P\\)|(?P\$\$)|(?P\$\{(?P\w+)(?P.*?)\})', re.M) + def compile_fun_shell(line): +diff --git a/waflib/TaskGen.py b/waflib/TaskGen.py +index a74e6431d..32468f03d 100644 +--- a/waflib/TaskGen.py ++++ b/waflib/TaskGen.py +@@ -74,7 +74,7 @@ def __init__(self, *k, **kw): + else: + self.bld = kw['bld'] + self.env = self.bld.env.derive() +- self.path = self.bld.path # emulate chdir when reading scripts ++ self.path = kw.get('path', self.bld.path) # by default, emulate chdir when reading scripts + + # Provide a unique index per folder + # This is part of a measure to prevent output file name collisions +@@ -400,7 +400,7 @@ def feature(*k): + Decorator that registers a task generator method that will be executed when the + object attribute ``feature`` contains the corresponding key(s):: + +- from waflib.Task import feature ++ from waflib.TaskGen import feature + @feature('myfeature') + def myfunction(self): + print('that is my feature!') +@@ -631,12 +631,8 @@ def chmod_fun(tsk): + cls.scan = self.scan + elif has_deps: + def scan(self): +- nodes = [] +- for x in self.generator.to_list(getattr(self.generator, 'deps', None)): +- node = self.generator.path.find_resource(x) +- if not node: +- self.generator.bld.fatal('Could not find %r (was it declared?)' % x) +- nodes.append(node) ++ deps = getattr(self.generator, 'deps', None) ++ nodes = self.generator.to_nodes(deps) + return [nodes, []] + cls.scan = scan + +@@ -727,7 +723,7 @@ def sequence_order(self): + self.bld.prev = self + + +-re_m4 = re.compile('@(\w+)@', re.M) ++re_m4 = re.compile(r'@(\w+)@', re.M) + + class subst_pc(Task.Task): + """ +@@ -905,7 +901,7 @@ def build(bld): + # paranoid safety measure for the general case foo.in->foo.h with ambiguous dependencies + for xt in HEADER_EXTS: + if b.name.endswith(xt): +- tsk.ext_in = tsk.ext_in + ['.h'] ++ tsk.ext_out = tsk.ext_out + ['.h'] + break + + inst_to = getattr(self, 'install_path', None) +diff --git a/waflib/Tools/c_aliases.py b/waflib/Tools/c_aliases.py +index c9d53692e..928cfe29c 100644 +--- a/waflib/Tools/c_aliases.py ++++ b/waflib/Tools/c_aliases.py +@@ -38,7 +38,7 @@ def sniff_features(**kw): + :return: the list of features for a task generator processing the source files + :rtype: list of string + """ +- exts = get_extensions(kw['source']) ++ exts = get_extensions(kw.get('source', [])) + typ = kw['typ'] + feats = [] + +@@ -47,10 +47,12 @@ def sniff_features(**kw): + if x in exts: + feats.append('cxx') + break +- + if 'c' in exts or 'vala' in exts or 'gs' in exts: + feats.append('c') + ++ if 's' in exts or 'S' in exts: ++ feats.append('asm') ++ + for x in 'f f90 F F90 for FOR'.split(): + if x in exts: + feats.append('fc') +@@ -66,11 +68,11 @@ def sniff_features(**kw): + if typ in ('program', 'shlib', 'stlib'): + will_link = False + for x in feats: +- if x in ('cxx', 'd', 'fc', 'c'): ++ if x in ('cxx', 'd', 'fc', 'c', 'asm'): + feats.append(x + typ) + will_link = True + if not will_link and not kw.get('features', []): +- raise Errors.WafError('Cannot link from %r, try passing eg: features="c cprogram"?' % kw) ++ raise Errors.WafError('Unable to determine how to link %r, try adding eg: features="c cshlib"?' % kw) + return feats + + def set_features(kw, typ): +diff --git a/waflib/Tools/c_config.py b/waflib/Tools/c_config.py +index d2b3c0d8f..f5ab19bf6 100644 +--- a/waflib/Tools/c_config.py ++++ b/waflib/Tools/c_config.py +@@ -68,6 +68,8 @@ + '__s390__' : 's390', + '__sh__' : 'sh', + '__xtensa__' : 'xtensa', ++'__e2k__' : 'e2k', ++'__riscv' : 'riscv', + } + + @conf +@@ -86,6 +88,10 @@ def configure(conf): + :type uselib_store: string + :param env: config set or conf.env by default + :type env: :py:class:`waflib.ConfigSet.ConfigSet` ++ :param force_static: force usage of static libraries ++ :type force_static: bool default False ++ :param posix: usage of POSIX mode for shlex lexical analiysis library ++ :type posix: bool default True + """ + + assert(isinstance(line, str)) +@@ -103,6 +109,8 @@ def configure(conf): + lex.commenters = '' + lst = list(lex) + ++ so_re = re.compile(r"\.so(?:\.[0-9]+)*$") ++ + # append_unique is not always possible + # for example, apple flags may require both -arch i386 and -arch ppc + uselib = uselib_store +@@ -144,7 +152,7 @@ def appu(var, val): + elif x.startswith('-std='): + prefix = 'CXXFLAGS' if '++' in x else 'CFLAGS' + app(prefix, x) +- elif x.startswith('+') or x in ('-pthread', '-fPIC', '-fpic', '-fPIE', '-fpie'): ++ elif x.startswith('+') or x in ('-pthread', '-fPIC', '-fpic', '-fPIE', '-fpie', '-flto', '-fno-lto'): + app('CFLAGS', x) + app('CXXFLAGS', x) + app('LINKFLAGS', x) +@@ -180,7 +188,7 @@ def appu(var, val): + app('CFLAGS', tmp) + app('CXXFLAGS', tmp) + app('LINKFLAGS', tmp) +- elif x.endswith(('.a', '.so', '.dylib', '.lib')): ++ elif x.endswith(('.a', '.dylib', '.lib')) or so_re.search(x): + appu('LINKFLAGS', x) # not cool, #762 + else: + self.to_log('Unhandled flag %r' % x) +@@ -246,13 +254,15 @@ def exec_cfg(self, kw): + * if modversion is given, then return the module version + * else, execute the *-config* program with the *args* and *variables* given, and set the flags on the *conf.env.FLAGS_name* variable + ++ :param path: the **-config program to use** ++ :type path: list of string + :param atleast_pkgconfig_version: minimum pkg-config version to use (disable other tests) + :type atleast_pkgconfig_version: string + :param package: package name, for example *gtk+-2.0* + :type package: string +- :param uselib_store: if the test is successful, define HAVE\_*name*. It is also used to define *conf.env.FLAGS_name* variables. ++ :param uselib_store: if the test is successful, define HAVE\\_*name*. It is also used to define *conf.env.FLAGS_name* variables. + :type uselib_store: string +- :param modversion: if provided, return the version of the given module and define *name*\_VERSION ++ :param modversion: if provided, return the version of the given module and define *name*\\_VERSION + :type modversion: string + :param args: arguments to give to *package* when retrieving flags + :type args: list of string +@@ -260,6 +270,12 @@ def exec_cfg(self, kw): + :type variables: list of string + :param define_variable: additional variables to define (also in conf.env.PKG_CONFIG_DEFINES) + :type define_variable: dict(string: string) ++ :param pkg_config_path: paths where pkg-config should search for .pc config files (overrides env.PKG_CONFIG_PATH if exists) ++ :type pkg_config_path: string, list of directories separated by colon ++ :param force_static: force usage of static libraries ++ :type force_static: bool default False ++ :param posix: usage of POSIX mode for shlex lexical analiysis library ++ :type posix: bool default True + """ + + path = Utils.to_list(kw['path']) +@@ -334,6 +350,7 @@ def check_cfg(self, *k, **kw): + """ + Checks for configuration flags using a **-config**-like program (pkg-config, sdl-config, etc). + This wraps internal calls to :py:func:`waflib.Tools.c_config.validate_cfg` and :py:func:`waflib.Tools.c_config.exec_cfg` ++ so check exec_cfg parameters descriptions for more details on kw passed + + A few examples:: + +@@ -659,20 +676,21 @@ class test_exec(Task.Task): + """ + color = 'PINK' + def run(self): ++ cmd = [self.inputs[0].abspath()] + getattr(self.generator, 'test_args', []) + if getattr(self.generator, 'rpath', None): + if getattr(self.generator, 'define_ret', False): +- self.generator.bld.retval = self.generator.bld.cmd_and_log([self.inputs[0].abspath()]) ++ self.generator.bld.retval = self.generator.bld.cmd_and_log(cmd) + else: +- self.generator.bld.retval = self.generator.bld.exec_command([self.inputs[0].abspath()]) ++ self.generator.bld.retval = self.generator.bld.exec_command(cmd) + else: + env = self.env.env or {} + env.update(dict(os.environ)) + for var in ('LD_LIBRARY_PATH', 'DYLD_LIBRARY_PATH', 'PATH'): + env[var] = self.inputs[0].parent.abspath() + os.path.pathsep + env.get(var, '') + if getattr(self.generator, 'define_ret', False): +- self.generator.bld.retval = self.generator.bld.cmd_and_log([self.inputs[0].abspath()], env=env) ++ self.generator.bld.retval = self.generator.bld.cmd_and_log(cmd, env=env) + else: +- self.generator.bld.retval = self.generator.bld.exec_command([self.inputs[0].abspath()], env=env) ++ self.generator.bld.retval = self.generator.bld.exec_command(cmd, env=env) + + @feature('test_exec') + @after_method('apply_link') +@@ -1266,10 +1284,11 @@ def to_log(self, *k, **kw): + tasks = [] + + id_to_task = {} +- for dct in k: ++ for counter, dct in enumerate(k): + x = Task.classes['cfgtask'](bld=bld, env=None) + tasks.append(x) + x.args = dct ++ x.args['multicheck_counter'] = counter + x.bld = bld + x.conf = self + x.args = dct +diff --git a/waflib/Tools/c_preproc.py b/waflib/Tools/c_preproc.py +index 7e04b4a7c..68e5f5aea 100644 +--- a/waflib/Tools/c_preproc.py ++++ b/waflib/Tools/c_preproc.py +@@ -75,13 +75,13 @@ class PreprocError(Errors.WafError): + re.IGNORECASE | re.MULTILINE) + """Match #include lines""" + +-re_mac = re.compile("^[a-zA-Z_]\w*") ++re_mac = re.compile(r"^[a-zA-Z_]\w*") + """Match macro definitions""" + + re_fun = re.compile('^[a-zA-Z_][a-zA-Z0-9_]*[(]') + """Match macro functions""" + +-re_pragma_once = re.compile('^\s*once\s*', re.IGNORECASE) ++re_pragma_once = re.compile(r'^\s*once\s*', re.IGNORECASE) + """Match #pragma once statements""" + + re_nl = re.compile('\\\\\r*\n', re.MULTILINE) +@@ -660,7 +660,7 @@ def extract_macro(txt): + # empty define, assign an empty token + return (v, [[], [('T','')]]) + +-re_include = re.compile('^\s*(<(?:.*)>|"(?:.*)")') ++re_include = re.compile(r'^\s*(<(?:.*)>|"(?:.*)")') + def extract_include(txt, defs): + """ + Process a line in the form:: +diff --git a/waflib/Tools/c_tests.py b/waflib/Tools/c_tests.py +index f858df576..bdd186c6b 100644 +--- a/waflib/Tools/c_tests.py ++++ b/waflib/Tools/c_tests.py +@@ -180,9 +180,15 @@ def check_large_file(self, **kw): + ######################################################################################## + + ENDIAN_FRAGMENT = ''' ++#ifdef _MSC_VER ++#define testshlib_EXPORT __declspec(dllexport) ++#else ++#define testshlib_EXPORT ++#endif ++ + short int ascii_mm[] = { 0x4249, 0x4765, 0x6E44, 0x6961, 0x6E53, 0x7953, 0 }; + short int ascii_ii[] = { 0x694C, 0x5454, 0x656C, 0x6E45, 0x6944, 0x6E61, 0 }; +-int use_ascii (int i) { ++int testshlib_EXPORT use_ascii (int i) { + return ascii_mm[i] + ascii_ii[i]; + } + short int ebcdic_ii[] = { 0x89D3, 0xE3E3, 0x8593, 0x95C5, 0x89C4, 0x9581, 0 }; +@@ -208,12 +214,12 @@ def run(self): + return -1 + + @feature('grep_for_endianness') +-@after_method('process_source') ++@after_method('apply_link') + def grep_for_endianness_fun(self): + """ + Used by the endianness configuration test + """ +- self.create_task('grep_for_endianness', self.compiled_tasks[0].outputs[0]) ++ self.create_task('grep_for_endianness', self.link_task.outputs[0]) + + @conf + def check_endianness(self): +@@ -223,7 +229,9 @@ def check_endianness(self): + tmp = [] + def check_msg(self): + return tmp[0] +- self.check(fragment=ENDIAN_FRAGMENT, features='c grep_for_endianness', +- msg='Checking for endianness', define='ENDIANNESS', tmp=tmp, okmsg=check_msg) ++ ++ self.check(fragment=ENDIAN_FRAGMENT, features='c cshlib grep_for_endianness', ++ msg='Checking for endianness', define='ENDIANNESS', tmp=tmp, ++ okmsg=check_msg, confcache=None) + return tmp[0] + +diff --git a/waflib/Tools/ccroot.py b/waflib/Tools/ccroot.py +index 484846f5f..533992903 100644 +--- a/waflib/Tools/ccroot.py ++++ b/waflib/Tools/ccroot.py +@@ -111,7 +111,7 @@ def apply_incpaths(self): + tg = bld(features='includes', includes='.') + + The folders only need to be relative to the current directory, the equivalent build directory is +- added automatically (for headers created in the build directory). This enable using a build directory ++ added automatically (for headers created in the build directory). This enables using a build directory + or not (``top == out``). + + This method will add a list of nodes read by :py:func:`waflib.Tools.ccroot.to_incnodes` in ``tg.env.INCPATHS``, +@@ -128,6 +128,7 @@ class link_task(Task.Task): + Base class for all link tasks. A task generator is supposed to have at most one link task bound in the attribute *link_task*. See :py:func:`waflib.Tools.ccroot.apply_link`. + + .. inheritance-diagram:: waflib.Tools.ccroot.stlink_task waflib.Tools.c.cprogram waflib.Tools.c.cshlib waflib.Tools.cxx.cxxstlib waflib.Tools.cxx.cxxprogram waflib.Tools.cxx.cxxshlib waflib.Tools.d.dprogram waflib.Tools.d.dshlib waflib.Tools.d.dstlib waflib.Tools.ccroot.fake_shlib waflib.Tools.ccroot.fake_stlib waflib.Tools.asm.asmprogram waflib.Tools.asm.asmshlib waflib.Tools.asm.asmstlib ++ :top-classes: waflib.Tools.ccroot.link_task + """ + color = 'YELLOW' + +@@ -238,6 +239,17 @@ def wrap(self): + setattr(cls, 'run', wrap) + rm_tgt(stlink_task) + ++@feature('skip_stlib_link_deps') ++@before_method('process_use') ++def apply_skip_stlib_link_deps(self): ++ """ ++ This enables an optimization in the :py:func:wafilb.Tools.ccroot.processes_use: method that skips dependency and ++ link flag optimizations for targets that generate static libraries (via the :py:class:Tools.ccroot.stlink_task task). ++ The actual behavior is implemented in :py:func:wafilb.Tools.ccroot.processes_use: method so this feature only tells waf ++ to enable the new behavior. ++ """ ++ self.env.SKIP_STLIB_LINK_DEPS = True ++ + @feature('c', 'cxx', 'd', 'fc', 'asm') + @after_method('process_source') + def apply_link(self): +@@ -386,7 +398,11 @@ def build(bld): + y = self.bld.get_tgen_by_name(x) + var = y.tmp_use_var + if var and link_task: +- if var == 'LIB' or y.tmp_use_stlib or x in names: ++ if self.env.SKIP_STLIB_LINK_DEPS and isinstance(link_task, stlink_task): ++ # If the skip_stlib_link_deps feature is enabled then we should ++ # avoid adding lib deps to the stlink_task instance. ++ pass ++ elif var == 'LIB' or y.tmp_use_stlib or x in names: + self.env.append_value(var, [y.target[y.target.rfind(os.sep) + 1:]]) + self.link_task.dep_nodes.extend(y.link_task.outputs) + tmp_path = y.link_task.outputs[0].parent.path_from(self.get_cwd()) +diff --git a/waflib/Tools/compiler_c.py b/waflib/Tools/compiler_c.py +index 2dba3f827..e033ce6c5 100644 +--- a/waflib/Tools/compiler_c.py ++++ b/waflib/Tools/compiler_c.py +@@ -36,18 +36,19 @@ def build(bld): + from waflib.Logs import debug + + c_compiler = { +-'win32': ['msvc', 'gcc', 'clang'], +-'cygwin': ['gcc'], +-'darwin': ['clang', 'gcc'], +-'aix': ['xlc', 'gcc', 'clang'], +-'linux': ['gcc', 'clang', 'icc'], +-'sunos': ['suncc', 'gcc'], +-'irix': ['gcc', 'irixcc'], +-'hpux': ['gcc'], +-'osf1V': ['gcc'], +-'gnu': ['gcc', 'clang'], +-'java': ['gcc', 'msvc', 'clang', 'icc'], +-'default':['clang', 'gcc'], ++'win32': ['msvc', 'gcc', 'clang'], ++'cygwin': ['gcc', 'clang'], ++'darwin': ['clang', 'gcc'], ++'aix': ['xlc', 'gcc', 'clang'], ++'linux': ['gcc', 'clang', 'icc'], ++'sunos': ['suncc', 'gcc'], ++'irix': ['gcc', 'irixcc'], ++'hpux': ['gcc'], ++'osf1V': ['gcc'], ++'gnu': ['gcc', 'clang'], ++'java': ['gcc', 'msvc', 'clang', 'icc'], ++'gnukfreebsd': ['gcc', 'clang'], ++'default': ['clang', 'gcc'], + } + """ + Dict mapping platform names to Waf tools finding specific C compilers:: +diff --git a/waflib/Tools/compiler_cxx.py b/waflib/Tools/compiler_cxx.py +index 1af65a226..42658c584 100644 +--- a/waflib/Tools/compiler_cxx.py ++++ b/waflib/Tools/compiler_cxx.py +@@ -37,18 +37,19 @@ def build(bld): + from waflib.Logs import debug + + cxx_compiler = { +-'win32': ['msvc', 'g++', 'clang++'], +-'cygwin': ['g++'], +-'darwin': ['clang++', 'g++'], +-'aix': ['xlc++', 'g++', 'clang++'], +-'linux': ['g++', 'clang++', 'icpc'], +-'sunos': ['sunc++', 'g++'], +-'irix': ['g++'], +-'hpux': ['g++'], +-'osf1V': ['g++'], +-'gnu': ['g++', 'clang++'], +-'java': ['g++', 'msvc', 'clang++', 'icpc'], +-'default': ['clang++', 'g++'] ++'win32': ['msvc', 'g++', 'clang++'], ++'cygwin': ['g++', 'clang++'], ++'darwin': ['clang++', 'g++'], ++'aix': ['xlc++', 'g++', 'clang++'], ++'linux': ['g++', 'clang++', 'icpc'], ++'sunos': ['sunc++', 'g++'], ++'irix': ['g++'], ++'hpux': ['g++'], ++'osf1V': ['g++'], ++'gnu': ['g++', 'clang++'], ++'java': ['g++', 'msvc', 'clang++', 'icpc'], ++'gnukfreebsd': ['g++', 'clang++'], ++'default': ['clang++', 'g++'] + } + """ + Dict mapping the platform names to Waf tools finding specific C++ compilers:: +diff --git a/waflib/Tools/irixcc.py b/waflib/Tools/irixcc.py +index c3ae1ac91..0335c13cb 100644 +--- a/waflib/Tools/irixcc.py ++++ b/waflib/Tools/irixcc.py +@@ -13,22 +13,11 @@ + @conf + def find_irixcc(conf): + v = conf.env +- cc = None +- if v.CC: +- cc = v.CC +- elif 'CC' in conf.environ: +- cc = conf.environ['CC'] +- if not cc: +- cc = conf.find_program('cc', var='CC') +- if not cc: +- conf.fatal('irixcc was not found') +- ++ cc = conf.find_program('cc', var='CC') + try: + conf.cmd_and_log(cc + ['-version']) + except Errors.WafError: + conf.fatal('%r -version could not be executed' % cc) +- +- v.CC = cc + v.CC_NAME = 'irix' + + @conf +@@ -57,7 +46,6 @@ def irixcc_common_flags(conf): + + def configure(conf): + conf.find_irixcc() +- conf.find_cpp() + conf.find_ar() + conf.irixcc_common_flags() + conf.cc_load_tools() +diff --git a/waflib/Tools/msvc.py b/waflib/Tools/msvc.py +index 17b347d45..d60f67026 100644 +--- a/waflib/Tools/msvc.py ++++ b/waflib/Tools/msvc.py +@@ -99,10 +99,31 @@ def build(bld): + """List of icl platforms""" + + def options(opt): +- opt.add_option('--msvc_version', type='string', help = 'msvc version, eg: "msvc 10.0,msvc 9.0"', default='') ++ default_ver = '' ++ vsver = os.getenv('VSCMD_VER') ++ if vsver: ++ m = re.match(r'(^\d+\.\d+).*', vsver) ++ if m: ++ default_ver = 'msvc %s' % m.group(1) ++ opt.add_option('--msvc_version', type='string', help = 'msvc version, eg: "msvc 10.0,msvc 9.0"', default=default_ver) + opt.add_option('--msvc_targets', type='string', help = 'msvc targets, eg: "x64,arm"', default='') + opt.add_option('--no-msvc-lazy', action='store_false', help = 'lazily check msvc target environments', default=True, dest='msvc_lazy') + ++class MSVCVersion(object): ++ def __init__(self, ver): ++ m = re.search(r'^(.*)\s+(\d+[.]\d+)', ver) ++ if m: ++ self.name = m.group(1) ++ self.number = float(m.group(2)) ++ else: ++ self.name = ver ++ self.number = 0. ++ ++ def __lt__(self, other): ++ if self.number == other.number: ++ return self.name < other.name ++ return self.number < other.number ++ + @conf + def setup_msvc(conf, versiondict): + """ +@@ -119,7 +140,7 @@ def setup_msvc(conf, versiondict): + platforms=Utils.to_list(conf.env.MSVC_TARGETS) or [i for i,j in all_msvc_platforms+all_icl_platforms+all_wince_platforms] + desired_versions = getattr(Options.options, 'msvc_version', '').split(',') + if desired_versions == ['']: +- desired_versions = conf.env.MSVC_VERSIONS or list(reversed(sorted(versiondict.keys()))) ++ desired_versions = conf.env.MSVC_VERSIONS or list(sorted(versiondict.keys(), key=MSVCVersion, reverse=True)) + + # Override lazy detection by evaluating after the fact. + lazy_detect = getattr(Options.options, 'msvc_lazy', True) +@@ -187,7 +208,7 @@ def get_msvc_version(conf, compiler, version, target, vcvars): + echo INCLUDE=%%INCLUDE%% + echo LIB=%%LIB%%;%%LIBPATH%% + """ % (vcvars,target)) +- sout = conf.cmd_and_log(['cmd.exe', '/E:on', '/V:on', '/C', batfile.abspath()]) ++ sout = conf.cmd_and_log(['cmd.exe', '/E:on', '/V:on', '/C', batfile.abspath()], stdin=getattr(Utils.subprocess, 'DEVNULL', None)) + lines = sout.splitlines() + + if not lines[0]: +@@ -281,7 +302,7 @@ def gather_wince_supported_platforms(): + + def gather_msvc_detected_versions(): + #Detected MSVC versions! +- version_pattern = re.compile('^(\d\d?\.\d\d?)(Exp)?$') ++ version_pattern = re.compile(r'^(\d\d?\.\d\d?)(Exp)?$') + detected_versions = [] + for vcver,vcvar in (('VCExpress','Exp'), ('VisualStudio','')): + prefix = 'SOFTWARE\\Wow6432node\\Microsoft\\' + vcver +@@ -367,7 +388,7 @@ def gather_wsdk_versions(conf, versions): + :param versions: list to modify + :type versions: list + """ +- version_pattern = re.compile('^v..?.?\...?.?') ++ version_pattern = re.compile(r'^v..?.?\...?.?') + try: + all_versions = Utils.winreg.OpenKey(Utils.winreg.HKEY_LOCAL_MACHINE, 'SOFTWARE\\Wow6432node\\Microsoft\\Microsoft SDKs\\Windows') + except OSError: +@@ -525,7 +546,7 @@ def gather_icl_versions(conf, versions): + :param versions: list to modify + :type versions: list + """ +- version_pattern = re.compile('^...?.?\....?.?') ++ version_pattern = re.compile(r'^...?.?\....?.?') + try: + all_versions = Utils.winreg.OpenKey(Utils.winreg.HKEY_LOCAL_MACHINE, 'SOFTWARE\\Wow6432node\\Intel\\Compilers\\C++') + except OSError: +@@ -579,7 +600,7 @@ def gather_intel_composer_versions(conf, versions): + :param versions: list to modify + :type versions: list + """ +- version_pattern = re.compile('^...?.?\...?.?.?') ++ version_pattern = re.compile(r'^...?.?\...?.?.?') + try: + all_versions = Utils.winreg.OpenKey(Utils.winreg.HKEY_LOCAL_MACHINE, 'SOFTWARE\\Wow6432node\\Intel\\Suites') + except OSError: +@@ -683,7 +704,7 @@ def find_lt_names_msvc(self, libname, is_static=False): + if not is_static and ltdict.get('library_names', ''): + dllnames=ltdict['library_names'].split() + dll=dllnames[0].lower() +- dll=re.sub('\.dll$', '', dll) ++ dll=re.sub(r'\.dll$', '', dll) + return (lt_libdir, dll, False) + elif ltdict.get('old_library', ''): + olib=ltdict['old_library'] +@@ -700,7 +721,7 @@ def find_lt_names_msvc(self, libname, is_static=False): + @conf + def libname_msvc(self, libname, is_static=False): + lib = libname.lower() +- lib = re.sub('\.lib$','',lib) ++ lib = re.sub(r'\.lib$','',lib) + + if lib in g_msvc_systemlibs: + return lib +@@ -747,11 +768,11 @@ def libname_msvc(self, libname, is_static=False): + for libn in libnames: + if os.path.exists(os.path.join(path, libn)): + Logs.debug('msvc: lib found: %s', os.path.join(path,libn)) +- return re.sub('\.lib$', '',libn) ++ return re.sub(r'\.lib$', '',libn) + + #if no lib can be found, just return the libname as msvc expects it + self.fatal('The library %r could not be found' % libname) +- return re.sub('\.lib$', '', libname) ++ return re.sub(r'\.lib$', '', libname) + + @conf + def check_lib_msvc(self, libname, is_static=False, uselib_store=None): +@@ -969,7 +990,7 @@ def build(bld): + if not is_static: + for f in self.env.LINKFLAGS: + d = f.lower() +- if d[1:] == 'debug': ++ if d[1:] in ('debug', 'debug:full', 'debug:fastlink'): + pdbnode = self.link_task.outputs[0].change_ext('.pdb') + self.link_task.outputs.append(pdbnode) + +diff --git a/waflib/Tools/waf_unit_test.py b/waflib/Tools/waf_unit_test.py +index 74d6c0561..8cff89bde 100644 +--- a/waflib/Tools/waf_unit_test.py ++++ b/waflib/Tools/waf_unit_test.py +@@ -97,6 +97,7 @@ def make_interpreted_test(self): + if isinstance(v, str): + v = v.split(os.pathsep) + self.ut_env[k] = os.pathsep.join(p + v) ++ self.env.append_value('UT_DEPS', ['%r%r' % (key, self.ut_env[key]) for key in self.ut_env]) + + @feature('test') + @after_method('apply_link', 'process_use') +@@ -108,7 +109,8 @@ def make_test(self): + tsk = self.create_task('utest', self.link_task.outputs) + if getattr(self, 'ut_str', None): + self.ut_run, lst = Task.compile_fun(self.ut_str, shell=getattr(self, 'ut_shell', False)) +- tsk.vars = lst + tsk.vars ++ tsk.vars = tsk.vars + lst ++ self.env.append_value('UT_DEPS', self.ut_str) + + self.handle_ut_cwd('ut_cwd') + +@@ -139,6 +141,10 @@ def add_path(var): + if not hasattr(self, 'ut_cmd'): + self.ut_cmd = getattr(Options.options, 'testcmd', False) + ++ self.env.append_value('UT_DEPS', str(self.ut_cmd)) ++ self.env.append_value('UT_DEPS', self.ut_paths) ++ self.env.append_value('UT_DEPS', ['%r%r' % (key, self.ut_env[key]) for key in self.ut_env]) ++ + @taskgen_method + def add_test_results(self, tup): + """Override and return tup[1] to interrupt the build immediately if a test does not run""" +@@ -159,7 +165,7 @@ class utest(Task.Task): + """ + color = 'PINK' + after = ['vnum', 'inst'] +- vars = [] ++ vars = ['UT_DEPS'] + + def runnable_status(self): + """ +@@ -200,7 +206,7 @@ def run(self): + self.ut_exec = getattr(self.generator, 'ut_exec', [self.inputs[0].abspath()]) + ut_cmd = getattr(self.generator, 'ut_cmd', False) + if ut_cmd: +- self.ut_exec = shlex.split(ut_cmd % ' '.join(self.ut_exec)) ++ self.ut_exec = shlex.split(ut_cmd % Utils.shell_escape(self.ut_exec)) + + return self.exec_command(self.ut_exec) + +@@ -214,7 +220,7 @@ def exec_command(self, cmd, **kw): + 'cmd': cmd + } + script_file = self.inputs[0].abspath() + '_run.py' +- Utils.writef(script_file, script_code) ++ Utils.writef(script_file, script_code, encoding='utf-8') + os.chmod(script_file, Utils.O755) + if Logs.verbose > 1: + Logs.info('Test debug file written as %r' % script_file) +diff --git a/waflib/Utils.py b/waflib/Utils.py +index a0cc2a09d..ea0f7a9db 100644 +--- a/waflib/Utils.py ++++ b/waflib/Utils.py +@@ -11,7 +11,7 @@ + + from __future__ import with_statement + +-import atexit, os, sys, errno, inspect, re, datetime, platform, base64, signal, functools, time ++import atexit, os, sys, errno, inspect, re, datetime, platform, base64, signal, functools, time, shlex + + try: + import cPickle +@@ -49,10 +49,16 @@ class TimeoutExpired(Exception): + from hashlib import md5 + except ImportError: + try: +- from md5 import md5 ++ from hashlib import sha1 as md5 + except ImportError: +- # never fail to enable fixes from another module ++ # never fail to enable potential fixes from another module + pass ++else: ++ try: ++ md5().digest() ++ except ValueError: ++ # Fips? #2213 ++ from hashlib import sha1 as md5 + + try: + import threading +@@ -202,7 +208,7 @@ def __next__(self): + + next = __next__ + +-is_win32 = os.sep == '\\' or sys.platform == 'win32' # msys2 ++is_win32 = os.sep == '\\' or sys.platform == 'win32' or os.name == 'nt' # msys2 + """ + Whether this system is a Windows series + """ +@@ -446,6 +452,8 @@ def console_encoding(): + pass + else: + if codepage: ++ if 65001 == codepage and sys.version_info < (3, 3): ++ return 'utf-8' + return 'cp%d' % codepage + return sys.stdout.encoding or ('cp1252' if is_win32 else 'latin-1') + +@@ -484,7 +492,9 @@ def split_path_msys(path): + if sys.platform == 'cygwin': + split_path = split_path_cygwin + elif is_win32: +- if os.environ.get('MSYSTEM'): ++ # Consider this an MSYSTEM environment if $MSYSTEM is set and python ++ # reports is executable from a unix like path on a windows host. ++ if os.environ.get('MSYSTEM') and sys.executable.startswith('/'): + split_path = split_path_msys + else: + split_path = split_path_win32 +@@ -569,10 +579,13 @@ def quote_define_name(s): + fu = fu.upper() + return fu + +-re_sh = re.compile('\\s|\'|"') +-""" +-Regexp used for shell_escape below +-""" ++# shlex.quote didn't exist until python 3.3. Prior to that it was a non-documented ++# function in pipes. ++try: ++ shell_quote = shlex.quote ++except AttributeError: ++ import pipes ++ shell_quote = pipes.quote + + def shell_escape(cmd): + """ +@@ -581,7 +594,7 @@ def shell_escape(cmd): + """ + if isinstance(cmd, str): + return cmd +- return ' '.join(repr(x) if re_sh.search(x) else x for x in cmd) ++ return ' '.join(shell_quote(x) for x in cmd) + + def h_list(lst): + """ +@@ -596,6 +609,12 @@ def h_list(lst): + """ + return md5(repr(lst).encode()).digest() + ++if sys.hexversion < 0x3000000: ++ def h_list_python2(lst): ++ return md5(repr(lst)).digest() ++ h_list_python2.__doc__ = h_list.__doc__ ++ h_list = h_list_python2 ++ + def h_fun(fun): + """ + Hash functions +@@ -615,7 +634,7 @@ def h_fun(fun): + # + # The sorting result outcome will be consistent because: + # 1. tuples are compared in order of their elements +- # 2. optional argument names are unique ++ # 2. optional argument namess are unique + code.extend(sorted(fun.keywords.items())) + code.append(h_fun(fun.func)) + fun.code = h_list(code) +@@ -730,7 +749,7 @@ def unversioned_sys_platform(): + if s == 'cli' and os.name == 'nt': + # ironpython is only on windows as far as we know + return 'win32' +- return re.split('\d+$', s)[0] ++ return re.split(r'\d+$', s)[0] + + def nada(*k, **kw): + """ +@@ -851,6 +870,19 @@ def lib64(): + return '64' + return '' + ++def loose_version(ver_str): ++ # private for the time being! ++ # see #2402 ++ lst = re.split(r'([.]|\\d+|[a-zA-Z])', ver_str) ++ ver = [] ++ for i, val in enumerate(lst): ++ try: ++ ver.append(int(val)) ++ except ValueError: ++ if val != '.': ++ ver.append(val) ++ return ver ++ + def sane_path(p): + # private function for the time being! + return os.path.abspath(os.path.expanduser(p)) +@@ -871,13 +903,13 @@ def get_process(): + except IndexError: + filepath = os.path.dirname(os.path.abspath(__file__)) + os.sep + 'processor.py' + cmd = [sys.executable, '-c', readf(filepath)] +- return subprocess.Popen(cmd, stdout=subprocess.PIPE, stdin=subprocess.PIPE, bufsize=0) ++ return subprocess.Popen(cmd, stdout=subprocess.PIPE, stdin=subprocess.PIPE, bufsize=0, close_fds=not is_win32) + + def run_prefork_process(cmd, kwargs, cargs): + """ + Delegates process execution to a pre-forked process instance. + """ +- if not 'env' in kwargs: ++ if not kwargs.get('env'): + kwargs['env'] = dict(os.environ) + try: + obj = base64.b64encode(cPickle.dumps([cmd, kwargs, cargs])) +diff --git a/waflib/ansiterm.py b/waflib/ansiterm.py +index 0d20c6374..027f0ad68 100644 +--- a/waflib/ansiterm.py ++++ b/waflib/ansiterm.py +@@ -264,7 +264,7 @@ def hide_cursor(self,param): + 'u': pop_cursor, + } + # Match either the escape sequence or text not containing escape sequence +- ansi_tokens = re.compile('(?:\x1b\[([0-9?;]*)([a-zA-Z])|([^\x1b]+))') ++ ansi_tokens = re.compile(r'(?:\x1b\[([0-9?;]*)([a-zA-Z])|([^\x1b]+))') + def write(self, text): + try: + wlock.acquire() +diff --git a/waflib/extras/clang_cross.py b/waflib/extras/clang_cross.py +new file mode 100644 +index 000000000..1b51e2886 +--- /dev/null ++++ b/waflib/extras/clang_cross.py +@@ -0,0 +1,92 @@ ++#!/usr/bin/env python ++# encoding: utf-8 ++# Krzysztof Kosiński 2014 ++# DragoonX6 2018 ++ ++""" ++Detect the Clang C compiler ++This version is an attempt at supporting the -target and -sysroot flag of Clang. ++""" ++ ++from waflib.Tools import ccroot, ar, gcc ++from waflib.Configure import conf ++import waflib.Context ++import waflib.extras.clang_cross_common ++ ++def options(opt): ++ """ ++ Target triplet for clang:: ++ $ waf configure --clang-target-triple=x86_64-pc-linux-gnu ++ """ ++ cc_compiler_opts = opt.add_option_group('Configuration options') ++ cc_compiler_opts.add_option('--clang-target-triple', default=None, ++ help='Target triple for clang', ++ dest='clang_target_triple') ++ cc_compiler_opts.add_option('--clang-sysroot', default=None, ++ help='Sysroot for clang', ++ dest='clang_sysroot') ++ ++@conf ++def find_clang(conf): ++ """ ++ Finds the program clang and executes it to ensure it really is clang ++ """ ++ ++ import os ++ ++ cc = conf.find_program('clang', var='CC') ++ ++ if conf.options.clang_target_triple != None: ++ conf.env.append_value('CC', ['-target', conf.options.clang_target_triple]) ++ ++ if conf.options.clang_sysroot != None: ++ sysroot = str() ++ ++ if os.path.isabs(conf.options.clang_sysroot): ++ sysroot = conf.options.clang_sysroot ++ else: ++ sysroot = os.path.normpath(os.path.join(os.getcwd(), conf.options.clang_sysroot)) ++ ++ conf.env.append_value('CC', ['--sysroot', sysroot]) ++ ++ conf.get_cc_version(cc, clang=True) ++ conf.env.CC_NAME = 'clang' ++ ++@conf ++def clang_modifier_x86_64_w64_mingw32(conf): ++ conf.gcc_modifier_win32() ++ ++@conf ++def clang_modifier_i386_w64_mingw32(conf): ++ conf.gcc_modifier_win32() ++ ++@conf ++def clang_modifier_x86_64_windows_msvc(conf): ++ conf.clang_modifier_msvc() ++ ++ # Allow the user to override any flags if they so desire. ++ clang_modifier_user_func = getattr(conf, 'clang_modifier_x86_64_windows_msvc_user', None) ++ if clang_modifier_user_func: ++ clang_modifier_user_func() ++ ++@conf ++def clang_modifier_i386_windows_msvc(conf): ++ conf.clang_modifier_msvc() ++ ++ # Allow the user to override any flags if they so desire. ++ clang_modifier_user_func = getattr(conf, 'clang_modifier_i386_windows_msvc_user', None) ++ if clang_modifier_user_func: ++ clang_modifier_user_func() ++ ++def configure(conf): ++ conf.find_clang() ++ conf.find_program(['llvm-ar', 'ar'], var='AR') ++ conf.find_ar() ++ conf.gcc_common_flags() ++ # Allow the user to provide flags for the target platform. ++ conf.gcc_modifier_platform() ++ # And allow more fine grained control based on the compiler's triplet. ++ conf.clang_modifier_target_triple() ++ conf.cc_load_tools() ++ conf.cc_add_flags() ++ conf.link_add_flags() +diff --git a/waflib/extras/clang_cross_common.py b/waflib/extras/clang_cross_common.py +new file mode 100644 +index 000000000..b76a07006 +--- /dev/null ++++ b/waflib/extras/clang_cross_common.py +@@ -0,0 +1,113 @@ ++#!/usr/bin/env python ++# encoding: utf-8 ++# DragoonX6 2018 ++ ++""" ++Common routines for cross_clang.py and cross_clangxx.py ++""" ++ ++from waflib.Configure import conf ++import waflib.Context ++ ++def normalize_target_triple(target_triple): ++ target_triple = target_triple[:-1] ++ normalized_triple = target_triple.replace('--', '-unknown-') ++ ++ if normalized_triple.startswith('-'): ++ normalized_triple = 'unknown' + normalized_triple ++ ++ if normalized_triple.endswith('-'): ++ normalized_triple += 'unknown' ++ ++ # Normalize MinGW builds to *arch*-w64-mingw32 ++ if normalized_triple.endswith('windows-gnu'): ++ normalized_triple = normalized_triple[:normalized_triple.index('-')] + '-w64-mingw32' ++ ++ # Strip the vendor when doing msvc builds, since it's unused anyway. ++ if normalized_triple.endswith('windows-msvc'): ++ normalized_triple = normalized_triple[:normalized_triple.index('-')] + '-windows-msvc' ++ ++ return normalized_triple.replace('-', '_') ++ ++@conf ++def clang_modifier_msvc(conf): ++ import os ++ ++ """ ++ Really basic setup to use clang in msvc mode. ++ We actually don't really want to do a lot, even though clang is msvc compatible ++ in this mode, that doesn't mean we're actually using msvc. ++ It's probably the best to leave it to the user, we can assume msvc mode if the user ++ uses the clang-cl frontend, but this module only concerns itself with the gcc-like frontend. ++ """ ++ v = conf.env ++ v.cprogram_PATTERN = '%s.exe' ++ ++ v.cshlib_PATTERN = '%s.dll' ++ v.implib_PATTERN = '%s.lib' ++ v.IMPLIB_ST = '-Wl,-IMPLIB:%s' ++ v.SHLIB_MARKER = [] ++ ++ v.CFLAGS_cshlib = [] ++ v.LINKFLAGS_cshlib = ['-Wl,-DLL'] ++ v.cstlib_PATTERN = '%s.lib' ++ v.STLIB_MARKER = [] ++ ++ del(v.AR) ++ conf.find_program(['llvm-lib', 'lib'], var='AR') ++ v.ARFLAGS = ['-nologo'] ++ v.AR_TGT_F = ['-out:'] ++ ++ # Default to the linker supplied with llvm instead of link.exe or ld ++ v.LINK_CC = v.CC + ['-fuse-ld=lld', '-nostdlib'] ++ v.CCLNK_TGT_F = ['-o'] ++ v.def_PATTERN = '-Wl,-def:%s' ++ ++ v.LINKFLAGS = [] ++ ++ v.LIB_ST = '-l%s' ++ v.LIBPATH_ST = '-Wl,-LIBPATH:%s' ++ v.STLIB_ST = '-l%s' ++ v.STLIBPATH_ST = '-Wl,-LIBPATH:%s' ++ ++ CFLAGS_CRT_COMMON = [ ++ '-Xclang', '--dependent-lib=oldnames', ++ '-Xclang', '-fno-rtti-data', ++ '-D_MT' ++ ] ++ ++ v.CFLAGS_CRT_MULTITHREADED = CFLAGS_CRT_COMMON + [ ++ '-Xclang', '-flto-visibility-public-std', ++ '-Xclang', '--dependent-lib=libcmt', ++ ] ++ v.CXXFLAGS_CRT_MULTITHREADED = v.CFLAGS_CRT_MULTITHREADED ++ ++ v.CFLAGS_CRT_MULTITHREADED_DBG = CFLAGS_CRT_COMMON + [ ++ '-D_DEBUG', ++ '-Xclang', '-flto-visibility-public-std', ++ '-Xclang', '--dependent-lib=libcmtd', ++ ] ++ v.CXXFLAGS_CRT_MULTITHREADED_DBG = v.CFLAGS_CRT_MULTITHREADED_DBG ++ ++ v.CFLAGS_CRT_MULTITHREADED_DLL = CFLAGS_CRT_COMMON + [ ++ '-D_DLL', ++ '-Xclang', '--dependent-lib=msvcrt' ++ ] ++ v.CXXFLAGS_CRT_MULTITHREADED_DLL = v.CFLAGS_CRT_MULTITHREADED_DLL ++ ++ v.CFLAGS_CRT_MULTITHREADED_DLL_DBG = CFLAGS_CRT_COMMON + [ ++ '-D_DLL', ++ '-D_DEBUG', ++ '-Xclang', '--dependent-lib=msvcrtd', ++ ] ++ v.CXXFLAGS_CRT_MULTITHREADED_DLL_DBG = v.CFLAGS_CRT_MULTITHREADED_DLL_DBG ++ ++@conf ++def clang_modifier_target_triple(conf, cpp=False): ++ compiler = conf.env.CXX if cpp else conf.env.CC ++ output = conf.cmd_and_log(compiler + ['-dumpmachine'], output=waflib.Context.STDOUT) ++ ++ modifier = ('clangxx' if cpp else 'clang') + '_modifier_' ++ clang_modifier_func = getattr(conf, modifier + normalize_target_triple(output), None) ++ if clang_modifier_func: ++ clang_modifier_func() +diff --git a/waflib/extras/clangxx_cross.py b/waflib/extras/clangxx_cross.py +new file mode 100644 +index 000000000..0ad38ad46 +--- /dev/null ++++ b/waflib/extras/clangxx_cross.py +@@ -0,0 +1,106 @@ ++#!/usr/bin/env python ++# encoding: utf-8 ++# Thomas Nagy 2009-2018 (ita) ++# DragoonX6 2018 ++ ++""" ++Detect the Clang++ C++ compiler ++This version is an attempt at supporting the -target and -sysroot flag of Clang++. ++""" ++ ++from waflib.Tools import ccroot, ar, gxx ++from waflib.Configure import conf ++import waflib.extras.clang_cross_common ++ ++def options(opt): ++ """ ++ Target triplet for clang++:: ++ $ waf configure --clangxx-target-triple=x86_64-pc-linux-gnu ++ """ ++ cxx_compiler_opts = opt.add_option_group('Configuration options') ++ cxx_compiler_opts.add_option('--clangxx-target-triple', default=None, ++ help='Target triple for clang++', ++ dest='clangxx_target_triple') ++ cxx_compiler_opts.add_option('--clangxx-sysroot', default=None, ++ help='Sysroot for clang++', ++ dest='clangxx_sysroot') ++ ++@conf ++def find_clangxx(conf): ++ """ ++ Finds the program clang++, and executes it to ensure it really is clang++ ++ """ ++ ++ import os ++ ++ cxx = conf.find_program('clang++', var='CXX') ++ ++ if conf.options.clangxx_target_triple != None: ++ conf.env.append_value('CXX', ['-target', conf.options.clangxx_target_triple]) ++ ++ if conf.options.clangxx_sysroot != None: ++ sysroot = str() ++ ++ if os.path.isabs(conf.options.clangxx_sysroot): ++ sysroot = conf.options.clangxx_sysroot ++ else: ++ sysroot = os.path.normpath(os.path.join(os.getcwd(), conf.options.clangxx_sysroot)) ++ ++ conf.env.append_value('CXX', ['--sysroot', sysroot]) ++ ++ conf.get_cc_version(cxx, clang=True) ++ conf.env.CXX_NAME = 'clang' ++ ++@conf ++def clangxx_modifier_x86_64_w64_mingw32(conf): ++ conf.gcc_modifier_win32() ++ ++@conf ++def clangxx_modifier_i386_w64_mingw32(conf): ++ conf.gcc_modifier_win32() ++ ++@conf ++def clangxx_modifier_msvc(conf): ++ v = conf.env ++ v.cxxprogram_PATTERN = v.cprogram_PATTERN ++ v.cxxshlib_PATTERN = v.cshlib_PATTERN ++ ++ v.CXXFLAGS_cxxshlib = [] ++ v.LINKFLAGS_cxxshlib = v.LINKFLAGS_cshlib ++ v.cxxstlib_PATTERN = v.cstlib_PATTERN ++ ++ v.LINK_CXX = v.CXX + ['-fuse-ld=lld', '-nostdlib'] ++ v.CXXLNK_TGT_F = v.CCLNK_TGT_F ++ ++@conf ++def clangxx_modifier_x86_64_windows_msvc(conf): ++ conf.clang_modifier_msvc() ++ conf.clangxx_modifier_msvc() ++ ++ # Allow the user to override any flags if they so desire. ++ clang_modifier_user_func = getattr(conf, 'clangxx_modifier_x86_64_windows_msvc_user', None) ++ if clang_modifier_user_func: ++ clang_modifier_user_func() ++ ++@conf ++def clangxx_modifier_i386_windows_msvc(conf): ++ conf.clang_modifier_msvc() ++ conf.clangxx_modifier_msvc() ++ ++ # Allow the user to override any flags if they so desire. ++ clang_modifier_user_func = getattr(conf, 'clangxx_modifier_i386_windows_msvc_user', None) ++ if clang_modifier_user_func: ++ clang_modifier_user_func() ++ ++def configure(conf): ++ conf.find_clangxx() ++ conf.find_program(['llvm-ar', 'ar'], var='AR') ++ conf.find_ar() ++ conf.gxx_common_flags() ++ # Allow the user to provide flags for the target platform. ++ conf.gxx_modifier_platform() ++ # And allow more fine grained control based on the compiler's triplet. ++ conf.clang_modifier_target_triple(cpp=True) ++ conf.cxx_load_tools() ++ conf.cxx_add_flags() ++ conf.link_add_flags() +diff --git a/waflib/extras/classic_runner.py b/waflib/extras/classic_runner.py +new file mode 100644 +index 000000000..b08c794e8 +--- /dev/null ++++ b/waflib/extras/classic_runner.py +@@ -0,0 +1,68 @@ ++#!/usr/bin/env python ++# encoding: utf-8 ++# Thomas Nagy, 2021 (ita) ++ ++from waflib import Utils, Runner ++ ++""" ++Re-enable the classic threading system from waf 1.x ++ ++def configure(conf): ++ conf.load('classic_runner') ++""" ++ ++class TaskConsumer(Utils.threading.Thread): ++ """ ++ Task consumers belong to a pool of workers ++ ++ They wait for tasks in the queue and then use ``task.process(...)`` ++ """ ++ def __init__(self, spawner): ++ Utils.threading.Thread.__init__(self) ++ """ ++ Obtain :py:class:`waflib.Task.TaskBase` instances from this queue. ++ """ ++ self.spawner = spawner ++ self.daemon = True ++ self.start() ++ ++ def run(self): ++ """ ++ Loop over the tasks to execute ++ """ ++ try: ++ self.loop() ++ except Exception: ++ pass ++ ++ def loop(self): ++ """ ++ Obtain tasks from :py:attr:`waflib.Runner.TaskConsumer.ready` and call ++ :py:meth:`waflib.Task.TaskBase.process`. If the object is a function, execute it. ++ """ ++ master = self.spawner.master ++ while 1: ++ if not master.stop: ++ try: ++ tsk = master.ready.get() ++ if tsk: ++ tsk.log_display(tsk.generator.bld) ++ master.process_task(tsk) ++ else: ++ break ++ finally: ++ master.out.put(tsk) ++ ++class Spawner(object): ++ """ ++ Daemon thread that consumes tasks from :py:class:`waflib.Runner.Parallel` producer and ++ spawns a consuming thread :py:class:`waflib.Runner.Consumer` for each ++ :py:class:`waflib.Task.Task` instance. ++ """ ++ def __init__(self, master): ++ self.master = master ++ """:py:class:`waflib.Runner.Parallel` producer instance""" ++ ++ self.pool = [TaskConsumer(self) for i in range(master.numjobs)] ++ ++Runner.Spawner = Spawner +diff --git a/waflib/extras/color_msvc.py b/waflib/extras/color_msvc.py +new file mode 100644 +index 000000000..60bacb7b2 +--- /dev/null ++++ b/waflib/extras/color_msvc.py +@@ -0,0 +1,59 @@ ++#!/usr/bin/env python ++# encoding: utf-8 ++ ++# Replaces the default formatter by one which understands MSVC output and colorizes it. ++# Modified from color_gcc.py ++ ++__author__ = __maintainer__ = "Alibek Omarov " ++__copyright__ = "Alibek Omarov, 2019" ++ ++import sys ++from waflib import Logs ++ ++class ColorMSVCFormatter(Logs.formatter): ++ def __init__(self, colors): ++ self.colors = colors ++ Logs.formatter.__init__(self) ++ ++ def parseMessage(self, line, color): ++ # Split messaage from 'disk:filepath: type: message' ++ arr = line.split(':', 3) ++ if len(arr) < 4: ++ return line ++ ++ colored = self.colors.BOLD + arr[0] + ':' + arr[1] + ':' + self.colors.NORMAL ++ colored += color + arr[2] + ':' + self.colors.NORMAL ++ colored += arr[3] ++ return colored ++ ++ def format(self, rec): ++ frame = sys._getframe() ++ while frame: ++ func = frame.f_code.co_name ++ if func == 'exec_command': ++ cmd = frame.f_locals.get('cmd') ++ if isinstance(cmd, list): ++ # Fix file case, it may be CL.EXE or cl.exe ++ argv0 = cmd[0].lower() ++ if 'cl.exe' in argv0: ++ lines = [] ++ # This will not work with "localized" versions ++ # of MSVC ++ for line in rec.msg.splitlines(): ++ if ': warning ' in line: ++ lines.append(self.parseMessage(line, self.colors.YELLOW)) ++ elif ': error ' in line: ++ lines.append(self.parseMessage(line, self.colors.RED)) ++ elif ': fatal error ' in line: ++ lines.append(self.parseMessage(line, self.colors.RED + self.colors.BOLD)) ++ elif ': note: ' in line: ++ lines.append(self.parseMessage(line, self.colors.CYAN)) ++ else: ++ lines.append(line) ++ rec.msg = "\n".join(lines) ++ frame = frame.f_back ++ return Logs.formatter.format(self, rec) ++ ++def options(opt): ++ Logs.log.handlers[0].setFormatter(ColorMSVCFormatter(Logs.colors)) ++ +diff --git a/waflib/extras/fc_fujitsu.py b/waflib/extras/fc_fujitsu.py +new file mode 100644 +index 000000000..cae676c20 +--- /dev/null ++++ b/waflib/extras/fc_fujitsu.py +@@ -0,0 +1,52 @@ ++#! /usr/bin/env python ++# encoding: utf-8 ++# Detection of the Fujitsu Fortran compiler for ARM64FX ++ ++import re ++from waflib.Tools import fc,fc_config,fc_scan ++from waflib.Configure import conf ++from waflib.Tools.compiler_fc import fc_compiler ++fc_compiler['linux'].append('fc_fujitsu') ++ ++@conf ++def find_fujitsu(conf): ++ fc=conf.find_program(['frtpx'],var='FC') ++ conf.get_fujitsu_version(fc) ++ conf.env.FC_NAME='FUJITSU' ++ conf.env.FC_MOD_CAPITALIZATION='lower' ++ ++@conf ++def fujitsu_flags(conf): ++ v=conf.env ++ v['_FCMODOUTFLAGS']=[] ++ v['FCFLAGS_DEBUG']=[] ++ v['FCFLAGS_fcshlib']=[] ++ v['LINKFLAGS_fcshlib']=[] ++ v['FCSTLIB_MARKER']='' ++ v['FCSHLIB_MARKER']='' ++ ++@conf ++def get_fujitsu_version(conf,fc): ++ version_re=re.compile(r"frtpx\s*\(FRT\)\s*(?P\d+)\.(?P\d+)\.",re.I).search ++ cmd=fc+['--version'] ++ out,err=fc_config.getoutput(conf,cmd,stdin=False) ++ if out: ++ match=version_re(out) ++ else: ++ match=version_re(err) ++ if not match: ++ return(False) ++ conf.fatal('Could not determine the Fujitsu FRT Fortran compiler version.') ++ else: ++ k=match.groupdict() ++ conf.env['FC_VERSION']=(k['major'],k['minor']) ++ ++def configure(conf): ++ conf.find_fujitsu() ++ conf.find_program('ar',var='AR') ++ conf.add_os_flags('ARFLAGS') ++ if not conf.env.ARFLAGS: ++ conf.env.ARFLAGS=['rcs'] ++ conf.fc_flags() ++ conf.fc_add_flags() ++ conf.fujitsu_flags() +diff --git a/waflib/extras/fc_nfort.py b/waflib/extras/fc_nfort.py +new file mode 100644 +index 000000000..c25886b8e +--- /dev/null ++++ b/waflib/extras/fc_nfort.py +@@ -0,0 +1,52 @@ ++#! /usr/bin/env python ++# encoding: utf-8 ++# Detection of the NEC Fortran compiler for Aurora Tsubasa ++ ++import re ++from waflib.Tools import fc,fc_config,fc_scan ++from waflib.Configure import conf ++from waflib.Tools.compiler_fc import fc_compiler ++fc_compiler['linux'].append('fc_nfort') ++ ++@conf ++def find_nfort(conf): ++ fc=conf.find_program(['nfort'],var='FC') ++ conf.get_nfort_version(fc) ++ conf.env.FC_NAME='NFORT' ++ conf.env.FC_MOD_CAPITALIZATION='lower' ++ ++@conf ++def nfort_flags(conf): ++ v=conf.env ++ v['_FCMODOUTFLAGS']=[] ++ v['FCFLAGS_DEBUG']=[] ++ v['FCFLAGS_fcshlib']=[] ++ v['LINKFLAGS_fcshlib']=[] ++ v['FCSTLIB_MARKER']='' ++ v['FCSHLIB_MARKER']='' ++ ++@conf ++def get_nfort_version(conf,fc): ++ version_re=re.compile(r"nfort\s*\(NFORT\)\s*(?P\d+)\.(?P\d+)\.",re.I).search ++ cmd=fc+['--version'] ++ out,err=fc_config.getoutput(conf,cmd,stdin=False) ++ if out: ++ match=version_re(out) ++ else: ++ match=version_re(err) ++ if not match: ++ return(False) ++ conf.fatal('Could not determine the NEC NFORT Fortran compiler version.') ++ else: ++ k=match.groupdict() ++ conf.env['FC_VERSION']=(k['major'],k['minor']) ++ ++def configure(conf): ++ conf.find_nfort() ++ conf.find_program('nar',var='AR') ++ conf.add_os_flags('ARFLAGS') ++ if not conf.env.ARFLAGS: ++ conf.env.ARFLAGS=['rcs'] ++ conf.fc_flags() ++ conf.fc_add_flags() ++ conf.nfort_flags() +diff --git a/waflib/extras/genpybind.py b/waflib/extras/genpybind.py +new file mode 100644 +index 000000000..ac206ee8a +--- /dev/null ++++ b/waflib/extras/genpybind.py +@@ -0,0 +1,194 @@ ++import os ++import pipes ++import subprocess ++import sys ++ ++from waflib import Logs, Task, Context ++from waflib.Tools.c_preproc import scan as scan_impl ++# ^-- Note: waflib.extras.gccdeps.scan does not work for us, ++# due to its current implementation: ++# The -MD flag is injected into the {C,CXX}FLAGS environment variable and ++# dependencies are read out in a separate step after compiling by reading ++# the .d file saved alongside the object file. ++# As the genpybind task refers to a header file that is never compiled itself, ++# gccdeps will not be able to extract the list of dependencies. ++ ++from waflib.TaskGen import feature, before_method ++ ++ ++def join_args(args): ++ return " ".join(pipes.quote(arg) for arg in args) ++ ++ ++def configure(cfg): ++ cfg.load("compiler_cxx") ++ cfg.load("python") ++ cfg.check_python_version(minver=(2, 7)) ++ if not cfg.env.LLVM_CONFIG: ++ cfg.find_program("llvm-config", var="LLVM_CONFIG") ++ if not cfg.env.GENPYBIND: ++ cfg.find_program("genpybind", var="GENPYBIND") ++ ++ # find clang reasource dir for builtin headers ++ cfg.env.GENPYBIND_RESOURCE_DIR = os.path.join( ++ cfg.cmd_and_log(cfg.env.LLVM_CONFIG + ["--libdir"]).strip(), ++ "clang", ++ cfg.cmd_and_log(cfg.env.LLVM_CONFIG + ["--version"]).strip()) ++ if os.path.exists(cfg.env.GENPYBIND_RESOURCE_DIR): ++ cfg.msg("Checking clang resource dir", cfg.env.GENPYBIND_RESOURCE_DIR) ++ else: ++ cfg.fatal("Clang resource dir not found") ++ ++ ++@feature("genpybind") ++@before_method("process_source") ++def generate_genpybind_source(self): ++ """ ++ Run genpybind on the headers provided in `source` and compile/link the ++ generated code instead. This works by generating the code on the fly and ++ swapping the source node before `process_source` is run. ++ """ ++ # name of module defaults to name of target ++ module = getattr(self, "module", self.target) ++ ++ # create temporary source file in build directory to hold generated code ++ out = "genpybind-%s.%d.cpp" % (module, self.idx) ++ out = self.path.get_bld().find_or_declare(out) ++ ++ task = self.create_task("genpybind", self.to_nodes(self.source), out) ++ # used to detect whether CFLAGS or CXXFLAGS should be passed to genpybind ++ task.features = self.features ++ task.module = module ++ # can be used to select definitions to include in the current module ++ # (when header files are shared by more than one module) ++ task.genpybind_tags = self.to_list(getattr(self, "genpybind_tags", [])) ++ # additional include directories ++ task.includes = self.to_list(getattr(self, "includes", [])) ++ task.genpybind = self.env.GENPYBIND ++ ++ # Tell waf to compile/link the generated code instead of the headers ++ # originally passed-in via the `source` parameter. (see `process_source`) ++ self.source = [out] ++ ++ ++class genpybind(Task.Task): # pylint: disable=invalid-name ++ """ ++ Runs genpybind on headers provided as input to this task. ++ Generated code will be written to the first (and only) output node. ++ """ ++ quiet = True ++ color = "PINK" ++ scan = scan_impl ++ ++ @staticmethod ++ def keyword(): ++ return "Analyzing" ++ ++ def run(self): ++ if not self.inputs: ++ return ++ ++ args = self.find_genpybind() + self._arguments( ++ resource_dir=self.env.GENPYBIND_RESOURCE_DIR) ++ ++ output = self.run_genpybind(args) ++ ++ # For debugging / log output ++ pasteable_command = join_args(args) ++ ++ # write generated code to file in build directory ++ # (will be compiled during process_source stage) ++ (output_node,) = self.outputs ++ output_node.write("// {}\n{}\n".format( ++ pasteable_command.replace("\n", "\n// "), output)) ++ ++ def find_genpybind(self): ++ return self.genpybind ++ ++ def run_genpybind(self, args): ++ bld = self.generator.bld ++ ++ kwargs = dict(cwd=bld.variant_dir) ++ if hasattr(bld, "log_command"): ++ bld.log_command(args, kwargs) ++ else: ++ Logs.debug("runner: {!r}".format(args)) ++ proc = subprocess.Popen( ++ args, stdout=subprocess.PIPE, stderr=subprocess.PIPE, **kwargs) ++ stdout, stderr = proc.communicate() ++ ++ if not isinstance(stdout, str): ++ stdout = stdout.decode(sys.stdout.encoding, errors="replace") ++ if not isinstance(stderr, str): ++ stderr = stderr.decode(sys.stderr.encoding, errors="replace") ++ ++ if proc.returncode != 0: ++ bld.fatal( ++ "genpybind returned {code} during the following call:" ++ "\n{command}\n\n{stdout}\n\n{stderr}".format( ++ code=proc.returncode, ++ command=join_args(args), ++ stdout=stdout, ++ stderr=stderr, ++ )) ++ ++ if stderr.strip(): ++ Logs.debug("non-fatal warnings during genpybind run:\n{}".format(stderr)) ++ ++ return stdout ++ ++ def _include_paths(self): ++ return self.generator.to_incnodes(self.includes + self.env.INCLUDES) ++ ++ def _inputs_as_relative_includes(self): ++ include_paths = self._include_paths() ++ relative_includes = [] ++ for node in self.inputs: ++ for inc in include_paths: ++ if node.is_child_of(inc): ++ relative_includes.append(node.path_from(inc)) ++ break ++ else: ++ self.generator.bld.fatal("could not resolve {}".format(node)) ++ return relative_includes ++ ++ def _arguments(self, genpybind_parse=None, resource_dir=None): ++ args = [] ++ relative_includes = self._inputs_as_relative_includes() ++ is_cxx = "cxx" in self.features ++ ++ # options for genpybind ++ args.extend(["--genpybind-module", self.module]) ++ if self.genpybind_tags: ++ args.extend(["--genpybind-tag"] + self.genpybind_tags) ++ if relative_includes: ++ args.extend(["--genpybind-include"] + relative_includes) ++ if genpybind_parse: ++ args.extend(["--genpybind-parse", genpybind_parse]) ++ ++ args.append("--") ++ ++ # headers to be processed by genpybind ++ args.extend(node.abspath() for node in self.inputs) ++ ++ args.append("--") ++ ++ # options for clang/genpybind-parse ++ args.append("-D__GENPYBIND__") ++ args.append("-xc++" if is_cxx else "-xc") ++ has_std_argument = False ++ for flag in self.env["CXXFLAGS" if is_cxx else "CFLAGS"]: ++ flag = flag.replace("-std=gnu", "-std=c") ++ if flag.startswith("-std=c"): ++ has_std_argument = True ++ args.append(flag) ++ if not has_std_argument: ++ args.append("-std=c++14") ++ args.extend("-I{}".format(n.abspath()) for n in self._include_paths()) ++ args.extend("-D{}".format(p) for p in self.env.DEFINES) ++ ++ # point to clang resource dir, if specified ++ if resource_dir: ++ args.append("-resource-dir={}".format(resource_dir)) ++ ++ return args +diff --git a/waflib/extras/haxe.py b/waflib/extras/haxe.py +new file mode 100644 +index 000000000..4ff374579 +--- /dev/null ++++ b/waflib/extras/haxe.py +@@ -0,0 +1,154 @@ ++import re ++ ++from waflib import Utils, Task, Errors, Logs ++from waflib.Configure import conf ++from waflib.TaskGen import extension, taskgen_method ++ ++HAXE_COMPILERS = { ++ 'JS': {'tgt': '--js', 'ext_out': ['.js']}, ++ 'LUA': {'tgt': '--lua', 'ext_out': ['.lua']}, ++ 'SWF': {'tgt': '--swf', 'ext_out': ['.swf']}, ++ 'NEKO': {'tgt': '--neko', 'ext_out': ['.n']}, ++ 'PHP': {'tgt': '--php', 'ext_out': ['.php']}, ++ 'CPP': {'tgt': '--cpp', 'ext_out': ['.h', '.cpp']}, ++ 'CPPIA': {'tgt': '--cppia', 'ext_out': ['.cppia']}, ++ 'CS': {'tgt': '--cs', 'ext_out': ['.cs']}, ++ 'JAVA': {'tgt': '--java', 'ext_out': ['.java']}, ++ 'JVM': {'tgt': '--jvm', 'ext_out': ['.jar']}, ++ 'PYTHON': {'tgt': '--python', 'ext_out': ['.py']}, ++ 'HL': {'tgt': '--hl', 'ext_out': ['.hl']}, ++ 'HLC': {'tgt': '--hl', 'ext_out': ['.h', '.c']}, ++} ++ ++@conf ++def check_haxe_pkg(self, **kw): ++ self.find_program('haxelib') ++ libs = kw.get('libs') ++ if not libs or not (type(libs) == str or (type(libs) == list and all(isinstance(s, str) for s in libs))): ++ self.fatal('Specify correct libs value in ensure call') ++ return ++ fetch = kw.get('fetch') ++ if not fetch is None and not type(fetch) == bool: ++ self.fatal('Specify correct fetch value in ensure call') ++ ++ libs = [libs] if type(libs) == str else libs ++ halt = False ++ for lib in libs: ++ try: ++ self.start_msg('Checking for library %s' % lib) ++ output = self.cmd_and_log(self.env.HAXELIB + ['list', lib]) ++ except Errors.WafError: ++ self.end_msg(False) ++ self.fatal('Can\'t run haxelib list, ensuring halted') ++ return ++ ++ if lib in output: ++ self.end_msg(lib in output) ++ else: ++ if not fetch: ++ self.end_msg(False) ++ halt = True ++ continue ++ try: ++ status = self.exec_command(self.env.HAXELIB + ['install', lib]) ++ if status: ++ self.end_msg(False) ++ self.fatal('Can\'t get %s with haxelib, ensuring halted' % lib) ++ return ++ else: ++ self.end_msg('downloaded', color='YELLOW') ++ except Errors.WafError: ++ self.end_msg(False) ++ self.fatal('Can\'t run haxelib install, ensuring halted') ++ return ++ postfix = kw.get('uselib_store') or lib.upper() ++ self.env.append_unique('LIB_' + postfix, lib) ++ ++ if halt: ++ self.fatal('Can\'t find libraries in haxelib list, ensuring halted') ++ return ++ ++class haxe(Task.Task): ++ vars = ['HAXE_VERSION', 'HAXE_FLAGS'] ++ ext_in = ['.hx'] ++ ++ def run(self): ++ cmd = self.env.HAXE + self.env.HAXE_FLAGS_DEFAULT + self.env.HAXE_FLAGS ++ return self.exec_command(cmd) ++ ++for COMP in HAXE_COMPILERS: ++ # create runners for each compile target ++ type("haxe_" + COMP, (haxe,), {'ext_out': HAXE_COMPILERS[COMP]['ext_out']}) ++ ++@taskgen_method ++def init_haxe(self): ++ errmsg = '%s not found, specify correct value' ++ try: ++ compiler = HAXE_COMPILERS[self.compiler] ++ comp_tgt = compiler['tgt'] ++ comp_mod = '/main.c' if self.compiler == 'HLC' else '' ++ except (AttributeError, KeyError): ++ self.bld.fatal(errmsg % 'COMPILER' + ': ' + ', '.join(HAXE_COMPILERS.keys())) ++ return ++ ++ self.env.append_value( ++ 'HAXE_FLAGS', ++ [comp_tgt, self.path.get_bld().make_node(self.target + comp_mod).abspath()]) ++ if hasattr(self, 'use'): ++ if not (type(self.use) == str or type(self.use) == list): ++ self.bld.fatal(errmsg % 'USE') ++ return ++ self.use = [self.use] if type(self.use) == str else self.use ++ ++ for dep in self.use: ++ if self.env['LIB_' + dep]: ++ for lib in self.env['LIB_' + dep]: ++ self.env.append_value('HAXE_FLAGS', ['-lib', lib]) ++ ++ if hasattr(self, 'res'): ++ if not type(self.res) == str: ++ self.bld.fatal(errmsg % 'RES') ++ return ++ self.env.append_value('HAXE_FLAGS', ['-D', 'resourcesPath=%s' % self.res]) ++ ++@extension('.hx') ++def haxe_hook(self, node): ++ if len(self.source) > 1: ++ self.bld.fatal('Use separate task generators for multiple files') ++ return ++ ++ src = node ++ tgt = self.path.get_bld().find_or_declare(self.target) ++ ++ self.init_haxe() ++ self.create_task('haxe_' + self.compiler, src, tgt) ++ ++@conf ++def check_haxe(self, mini=None, maxi=None): ++ self.start_msg('Checking for haxe version') ++ try: ++ curr = re.search( ++ r'(\d+.?)+', ++ self.cmd_and_log(self.env.HAXE + ['-version'])).group() ++ except Errors.WafError: ++ self.end_msg(False) ++ self.fatal('Can\'t get haxe version') ++ return ++ ++ if mini and Utils.num2ver(curr) < Utils.num2ver(mini): ++ self.end_msg('wrong', color='RED') ++ self.fatal('%s is too old, need >= %s' % (curr, mini)) ++ return ++ if maxi and Utils.num2ver(curr) > Utils.num2ver(maxi): ++ self.end_msg('wrong', color='RED') ++ self.fatal('%s is too new, need <= %s' % (curr, maxi)) ++ return ++ self.end_msg(curr, color='GREEN') ++ self.env.HAXE_VERSION = curr ++ ++def configure(self): ++ self.env.append_value( ++ 'HAXE_FLAGS_DEFAULT', ++ ['-D', 'no-compilation', '-cp', self.path.abspath()]) ++ Logs.warn('Default flags: %s' % ' '.join(self.env.HAXE_FLAGS_DEFAULT)) ++ self.find_program('haxe') +diff --git a/waflib/extras/msvc_pdb.py b/waflib/extras/msvc_pdb.py +new file mode 100644 +index 000000000..077656b4f +--- /dev/null ++++ b/waflib/extras/msvc_pdb.py +@@ -0,0 +1,46 @@ ++#!/usr/bin/env python ++# encoding: utf-8 ++# Rafaël Kooi 2019 ++ ++from waflib import TaskGen ++ ++@TaskGen.feature('c', 'cxx', 'fc') ++@TaskGen.after_method('propagate_uselib_vars') ++def add_pdb_per_object(self): ++ """For msvc/fortran, specify a unique compile pdb per object, to work ++ around LNK4099. Flags are updated with a unique /Fd flag based on the ++ task output name. This is separate from the link pdb. ++ """ ++ if not hasattr(self, 'compiled_tasks'): ++ return ++ ++ link_task = getattr(self, 'link_task', None) ++ ++ for task in self.compiled_tasks: ++ if task.inputs and task.inputs[0].name.lower().endswith('.rc'): ++ continue ++ ++ add_pdb = False ++ for flagname in ('CFLAGS', 'CXXFLAGS', 'FCFLAGS'): ++ # several languages may be used at once ++ for flag in task.env[flagname]: ++ if flag[1:].lower() == 'zi': ++ add_pdb = True ++ break ++ ++ if add_pdb: ++ node = task.outputs[0].change_ext('.pdb') ++ pdb_flag = '/Fd:' + node.abspath() ++ ++ for flagname in ('CFLAGS', 'CXXFLAGS', 'FCFLAGS'): ++ buf = [pdb_flag] ++ for flag in task.env[flagname]: ++ if flag[1:3] == 'Fd' or flag[1:].lower() == 'fs' or flag[1:].lower() == 'mp': ++ continue ++ buf.append(flag) ++ task.env[flagname] = buf ++ ++ if link_task and not node in link_task.dep_nodes: ++ link_task.dep_nodes.append(node) ++ if not node in task.outputs: ++ task.outputs.append(node) +diff --git a/waflib/extras/sphinx.py b/waflib/extras/sphinx.py +new file mode 100644 +index 000000000..08f3cfd8a +--- /dev/null ++++ b/waflib/extras/sphinx.py +@@ -0,0 +1,120 @@ ++"""Support for Sphinx documentation ++ ++This is a wrapper for sphinx-build program. Please note that sphinx-build supports only ++one output format at a time, but the tool can create multiple tasks to handle more. ++The output formats can be passed via the sphinx_output_format, which is an array of ++strings. For backwards compatibility if only one output is needed, it can be passed ++as a single string. ++The default output format is html. ++ ++Specific formats can be installed in different directories by specifying the ++install_path_ attribute. If not defined, the standard install_path ++will be used instead. ++ ++Example wscript: ++ ++def configure(cnf): ++ conf.load('sphinx') ++ ++def build(bld): ++ bld( ++ features='sphinx', ++ sphinx_source='sources', # path to source directory ++ sphinx_options='-a -v', # sphinx-build program additional options ++ sphinx_output_format=['html', 'man'], # output format of sphinx documentation ++ install_path_man='${DOCDIR}/man' # put man pages in a specific directory ++ ) ++ ++""" ++ ++from waflib.Node import Node ++from waflib import Utils ++from waflib import Task ++from waflib.TaskGen import feature, after_method ++ ++ ++def configure(cnf): ++ """Check if sphinx-build program is available and loads gnu_dirs tool.""" ++ cnf.find_program('sphinx-build', var='SPHINX_BUILD', mandatory=False) ++ cnf.load('gnu_dirs') ++ ++ ++@feature('sphinx') ++def build_sphinx(self): ++ """Builds sphinx sources. ++ """ ++ if not self.env.SPHINX_BUILD: ++ self.bld.fatal('Program SPHINX_BUILD not defined.') ++ if not getattr(self, 'sphinx_source', None): ++ self.bld.fatal('Attribute sphinx_source not defined.') ++ if not isinstance(self.sphinx_source, Node): ++ self.sphinx_source = self.path.find_node(self.sphinx_source) ++ if not self.sphinx_source: ++ self.bld.fatal('Can\'t find sphinx_source: %r' % self.sphinx_source) ++ ++ # In the taskgen we have the complete list of formats ++ Utils.def_attrs(self, sphinx_output_format='html') ++ self.sphinx_output_format = Utils.to_list(self.sphinx_output_format) ++ ++ self.env.SPHINX_OPTIONS = getattr(self, 'sphinx_options', []) ++ ++ for source_file in self.sphinx_source.ant_glob('**/*'): ++ self.bld.add_manual_dependency(self.sphinx_source, source_file) ++ ++ for cfmt in self.sphinx_output_format: ++ sphinx_build_task = self.create_task('SphinxBuildingTask') ++ sphinx_build_task.set_inputs(self.sphinx_source) ++ # In task we keep the specific format this task is generating ++ sphinx_build_task.env.SPHINX_OUTPUT_FORMAT = cfmt ++ ++ # the sphinx-build results are in directory ++ sphinx_build_task.sphinx_output_directory = self.path.get_bld().make_node(cfmt) ++ sphinx_build_task.set_outputs(sphinx_build_task.sphinx_output_directory) ++ sphinx_build_task.sphinx_output_directory.mkdir() ++ ++ Utils.def_attrs(sphinx_build_task, install_path=getattr(self, 'install_path_' + cfmt, getattr(self, 'install_path', get_install_path(sphinx_build_task)))) ++ ++ ++def get_install_path(object): ++ if object.env.SPHINX_OUTPUT_FORMAT == 'man': ++ return object.env.MANDIR ++ elif object.env.SPHINX_OUTPUT_FORMAT == 'info': ++ return object.env.INFODIR ++ else: ++ return object.env.DOCDIR ++ ++ ++class SphinxBuildingTask(Task.Task): ++ color = 'BOLD' ++ run_str = '${SPHINX_BUILD} -M ${SPHINX_OUTPUT_FORMAT} ${SRC} ${TGT} -d ${TGT[0].bld_dir()}/doctrees-${SPHINX_OUTPUT_FORMAT} ${SPHINX_OPTIONS}' ++ ++ def keyword(self): ++ return 'Compiling (%s)' % self.env.SPHINX_OUTPUT_FORMAT ++ ++ def runnable_status(self): ++ ++ for x in self.run_after: ++ if not x.hasrun: ++ return Task.ASK_LATER ++ ++ self.signature() ++ ret = Task.Task.runnable_status(self) ++ if ret == Task.SKIP_ME: ++ # in case the files were removed ++ self.add_install() ++ return ret ++ ++ ++ def post_run(self): ++ self.add_install() ++ return Task.Task.post_run(self) ++ ++ ++ def add_install(self): ++ nodes = self.sphinx_output_directory.ant_glob('**/*', quiet=True) ++ self.outputs += nodes ++ self.generator.add_install_files(install_to=self.install_path, ++ install_from=nodes, ++ postpone=False, ++ cwd=self.sphinx_output_directory.make_node(self.env.SPHINX_OUTPUT_FORMAT), ++ relative_trick=True) +diff --git a/waflib/extras/wafcache.py b/waflib/extras/wafcache.py +new file mode 100644 +index 000000000..30ac3ef51 +--- /dev/null ++++ b/waflib/extras/wafcache.py +@@ -0,0 +1,648 @@ ++#! /usr/bin/env python ++# encoding: utf-8 ++# Thomas Nagy, 2019 (ita) ++ ++""" ++Filesystem-based cache system to share and re-use build artifacts ++ ++Cache access operations (copy to and from) are delegated to ++independent pre-forked worker subprocesses. ++ ++The following environment variables may be set: ++* WAFCACHE: several possibilities: ++ - File cache: ++ absolute path of the waf cache (~/.cache/wafcache_user, ++ where `user` represents the currently logged-in user) ++ - URL to a cache server, for example: ++ export WAFCACHE=http://localhost:8080/files/ ++ in that case, GET/POST requests are made to urls of the form ++ http://localhost:8080/files/000000000/0 (cache management is delegated to the server) ++ - GCS, S3 or MINIO bucket ++ gs://my-bucket/ (uses gsutil command line tool or WAFCACHE_CMD) ++ s3://my-bucket/ (uses aws command line tool or WAFCACHE_CMD) ++ minio://my-bucket/ (uses mc command line tool or WAFCACHE_CMD) ++* WAFCACHE_CMD: bucket upload/download command, for example: ++ WAFCACHE_CMD="gsutil cp %{SRC} %{TGT}" ++ Note that the WAFCACHE bucket value is used for the source or destination ++ depending on the operation (upload or download). For example, with: ++ WAFCACHE="gs://mybucket/" ++ the following commands may be run: ++ gsutil cp build/myprogram gs://mybucket/aa/aaaaa/1 ++ gsutil cp gs://mybucket/bb/bbbbb/2 build/somefile ++* WAFCACHE_NO_PUSH: if set, disables pushing to the cache ++* WAFCACHE_VERBOSITY: if set, displays more detailed cache operations ++* WAFCACHE_STATS: if set, displays cache usage statistics on exit ++ ++File cache specific options: ++ Files are copied using hard links by default; if the cache is located ++ onto another partition, the system switches to file copies instead. ++* WAFCACHE_TRIM_MAX_FOLDER: maximum amount of tasks to cache (1M) ++* WAFCACHE_EVICT_MAX_BYTES: maximum amount of cache size in bytes (10GB) ++* WAFCACHE_EVICT_INTERVAL_MINUTES: minimum time interval to try ++ and trim the cache (3 minutes) ++ ++Upload specific options: ++* WAFCACHE_ASYNC_WORKERS: define a number of workers to upload results asynchronously ++ this may improve build performance with many/long file uploads ++ the default is unset (synchronous uploads) ++* WAFCACHE_ASYNC_NOWAIT: do not wait for uploads to complete (default: False) ++ this requires asynchonous uploads to have an effect ++ ++Usage:: ++ ++ def build(bld): ++ bld.load('wafcache') ++ ... ++ ++To troubleshoot:: ++ ++ waf clean build --zone=wafcache ++""" ++ ++import atexit, base64, errno, fcntl, getpass, os, re, shutil, sys, time, threading, traceback, urllib3, shlex ++try: ++ import subprocess32 as subprocess ++except ImportError: ++ import subprocess ++ ++base_cache = os.path.expanduser('~/.cache/') ++if not os.path.isdir(base_cache): ++ base_cache = '/tmp/' ++default_wafcache_dir = os.path.join(base_cache, 'wafcache_' + getpass.getuser()) ++ ++CACHE_DIR = os.environ.get('WAFCACHE', default_wafcache_dir) ++WAFCACHE_CMD = os.environ.get('WAFCACHE_CMD') ++TRIM_MAX_FOLDERS = int(os.environ.get('WAFCACHE_TRIM_MAX_FOLDER', 1000000)) ++EVICT_INTERVAL_MINUTES = int(os.environ.get('WAFCACHE_EVICT_INTERVAL_MINUTES', 3)) ++EVICT_MAX_BYTES = int(os.environ.get('WAFCACHE_EVICT_MAX_BYTES', 10**10)) ++WAFCACHE_NO_PUSH = 1 if os.environ.get('WAFCACHE_NO_PUSH') else 0 ++WAFCACHE_VERBOSITY = 1 if os.environ.get('WAFCACHE_VERBOSITY') else 0 ++WAFCACHE_STATS = 1 if os.environ.get('WAFCACHE_STATS') else 0 ++WAFCACHE_ASYNC_WORKERS = os.environ.get('WAFCACHE_ASYNC_WORKERS') ++WAFCACHE_ASYNC_NOWAIT = os.environ.get('WAFCACHE_ASYNC_NOWAIT') ++OK = "ok" ++ ++re_waf_cmd = re.compile('(?P%{SRC})|(?P%{TGT})') ++ ++try: ++ import cPickle ++except ImportError: ++ import pickle as cPickle ++ ++if __name__ != '__main__': ++ from waflib import Task, Logs, Utils, Build ++ ++def can_retrieve_cache(self): ++ """ ++ New method for waf Task classes ++ """ ++ if not self.outputs: ++ return False ++ ++ self.cached = False ++ ++ sig = self.signature() ++ ssig = Utils.to_hex(self.uid() + sig) ++ ++ if WAFCACHE_STATS: ++ self.generator.bld.cache_reqs += 1 ++ ++ files_to = [node.abspath() for node in self.outputs] ++ proc = get_process() ++ err = cache_command(proc, ssig, [], files_to) ++ process_pool.append(proc) ++ if err.startswith(OK): ++ if WAFCACHE_VERBOSITY: ++ Logs.pprint('CYAN', ' Fetched %r from cache' % files_to) ++ else: ++ Logs.debug('wafcache: fetched %r from cache', files_to) ++ if WAFCACHE_STATS: ++ self.generator.bld.cache_hits += 1 ++ else: ++ if WAFCACHE_VERBOSITY: ++ Logs.pprint('YELLOW', ' No cache entry %s' % files_to) ++ else: ++ Logs.debug('wafcache: No cache entry %s: %s', files_to, err) ++ return False ++ ++ self.cached = True ++ return True ++ ++def put_files_cache(self): ++ """ ++ New method for waf Task classes ++ """ ++ if WAFCACHE_NO_PUSH or getattr(self, 'cached', None) or not self.outputs: ++ return ++ ++ files_from = [] ++ for node in self.outputs: ++ path = node.abspath() ++ if not os.path.isfile(path): ++ return ++ files_from.append(path) ++ ++ bld = self.generator.bld ++ old_sig = self.signature() ++ ++ for node in self.inputs: ++ try: ++ del node.ctx.cache_sig[node] ++ except KeyError: ++ pass ++ ++ delattr(self, 'cache_sig') ++ sig = self.signature() ++ ++ def _async_put_files_cache(bld, ssig, files_from): ++ proc = get_process() ++ if WAFCACHE_ASYNC_WORKERS: ++ with bld.wafcache_lock: ++ if bld.wafcache_stop: ++ process_pool.append(proc) ++ return ++ bld.wafcache_procs.add(proc) ++ ++ err = cache_command(proc, ssig, files_from, []) ++ process_pool.append(proc) ++ if err.startswith(OK): ++ if WAFCACHE_VERBOSITY: ++ Logs.pprint('CYAN', ' Successfully uploaded %s to cache' % files_from) ++ else: ++ Logs.debug('wafcache: Successfully uploaded %r to cache', files_from) ++ if WAFCACHE_STATS: ++ bld.cache_puts += 1 ++ else: ++ if WAFCACHE_VERBOSITY: ++ Logs.pprint('RED', ' Error caching step results %s: %s' % (files_from, err)) ++ else: ++ Logs.debug('wafcache: Error caching results %s: %s', files_from, err) ++ ++ if old_sig == sig: ++ ssig = Utils.to_hex(self.uid() + sig) ++ if WAFCACHE_ASYNC_WORKERS: ++ fut = bld.wafcache_executor.submit(_async_put_files_cache, bld, ssig, files_from) ++ bld.wafcache_uploads.append(fut) ++ else: ++ _async_put_files_cache(bld, ssig, files_from) ++ else: ++ Logs.debug('wafcache: skipped %r upload due to late input modifications %r', self.outputs, self.inputs) ++ ++ bld.task_sigs[self.uid()] = self.cache_sig ++ ++def hash_env_vars(self, env, vars_lst): ++ """ ++ Reimplement BuildContext.hash_env_vars so that the resulting hash does not depend on local paths ++ """ ++ if not env.table: ++ env = env.parent ++ if not env: ++ return Utils.SIG_NIL ++ ++ idx = str(id(env)) + str(vars_lst) ++ try: ++ cache = self.cache_env ++ except AttributeError: ++ cache = self.cache_env = {} ++ else: ++ try: ++ return self.cache_env[idx] ++ except KeyError: ++ pass ++ ++ v = str([env[a] for a in vars_lst]) ++ v = v.replace(self.srcnode.abspath().__repr__()[:-1], '') ++ m = Utils.md5() ++ m.update(v.encode()) ++ ret = m.digest() ++ ++ Logs.debug('envhash: %r %r', ret, v) ++ ++ cache[idx] = ret ++ ++ return ret ++ ++def uid(self): ++ """ ++ Reimplement Task.uid() so that the signature does not depend on local paths ++ """ ++ try: ++ return self.uid_ ++ except AttributeError: ++ m = Utils.md5() ++ src = self.generator.bld.srcnode ++ up = m.update ++ up(self.__class__.__name__.encode()) ++ for x in self.inputs + self.outputs: ++ up(x.path_from(src).encode()) ++ self.uid_ = m.digest() ++ return self.uid_ ++ ++ ++def make_cached(cls): ++ """ ++ Enable the waf cache for a given task class ++ """ ++ if getattr(cls, 'nocache', None) or getattr(cls, 'has_cache', False): ++ return ++ ++ full_name = "%s.%s" % (cls.__module__, cls.__name__) ++ if full_name in ('waflib.Tools.ccroot.vnum', 'waflib.Build.inst'): ++ return ++ ++ m1 = getattr(cls, 'run', None) ++ def run(self): ++ if getattr(self, 'nocache', False): ++ return m1(self) ++ if self.can_retrieve_cache(): ++ return 0 ++ return m1(self) ++ cls.run = run ++ ++ m2 = getattr(cls, 'post_run', None) ++ def post_run(self): ++ if getattr(self, 'nocache', False): ++ return m2(self) ++ ret = m2(self) ++ self.put_files_cache() ++ return ret ++ cls.post_run = post_run ++ cls.has_cache = True ++ ++process_pool = [] ++def get_process(): ++ """ ++ Returns a worker process that can process waf cache commands ++ The worker process is assumed to be returned to the process pool when unused ++ """ ++ try: ++ return process_pool.pop() ++ except IndexError: ++ filepath = os.path.dirname(os.path.abspath(__file__)) + os.sep + 'wafcache.py' ++ cmd = [sys.executable, '-c', Utils.readf(filepath)] ++ return subprocess.Popen(cmd, stdout=subprocess.PIPE, stdin=subprocess.PIPE, bufsize=0) ++ ++def atexit_pool(): ++ for proc in process_pool: ++ proc.kill() ++atexit.register(atexit_pool) ++ ++def build(bld): ++ """ ++ Called during the build process to enable file caching ++ """ ++ ++ if WAFCACHE_ASYNC_WORKERS: ++ try: ++ num_workers = int(WAFCACHE_ASYNC_WORKERS) ++ except ValueError: ++ Logs.warn('Invalid WAFCACHE_ASYNC_WORKERS specified: %r' % WAFCACHE_ASYNC_WORKERS) ++ else: ++ from concurrent.futures import ThreadPoolExecutor ++ bld.wafcache_executor = ThreadPoolExecutor(max_workers=num_workers) ++ bld.wafcache_uploads = [] ++ bld.wafcache_procs = set([]) ++ bld.wafcache_stop = False ++ bld.wafcache_lock = threading.Lock() ++ ++ def finalize_upload_async(bld): ++ if WAFCACHE_ASYNC_NOWAIT: ++ with bld.wafcache_lock: ++ bld.wafcache_stop = True ++ ++ for fut in reversed(bld.wafcache_uploads): ++ fut.cancel() ++ ++ for proc in bld.wafcache_procs: ++ proc.kill() ++ ++ bld.wafcache_procs.clear() ++ else: ++ Logs.pprint('CYAN', '... waiting for wafcache uploads to complete (%s uploads)' % len(bld.wafcache_uploads)) ++ bld.wafcache_executor.shutdown(wait=True) ++ bld.add_post_fun(finalize_upload_async) ++ ++ if WAFCACHE_STATS: ++ # Init counter for statistics and hook to print results at the end ++ bld.cache_reqs = bld.cache_hits = bld.cache_puts = 0 ++ ++ def printstats(bld): ++ hit_ratio = 0 ++ if bld.cache_reqs > 0: ++ hit_ratio = (bld.cache_hits / bld.cache_reqs) * 100 ++ Logs.pprint('CYAN', ' wafcache stats: %s requests, %s hits (ratio: %.2f%%), %s writes' % ++ (bld.cache_reqs, bld.cache_hits, hit_ratio, bld.cache_puts) ) ++ bld.add_post_fun(printstats) ++ ++ if process_pool: ++ # already called once ++ return ++ ++ # pre-allocation ++ processes = [get_process() for x in range(bld.jobs)] ++ process_pool.extend(processes) ++ ++ Task.Task.can_retrieve_cache = can_retrieve_cache ++ Task.Task.put_files_cache = put_files_cache ++ Task.Task.uid = uid ++ Build.BuildContext.hash_env_vars = hash_env_vars ++ for x in reversed(list(Task.classes.values())): ++ make_cached(x) ++ ++def cache_command(proc, sig, files_from, files_to): ++ """ ++ Create a command for cache worker processes, returns a pickled ++ base64-encoded tuple containing the task signature, a list of files to ++ cache and a list of files files to get from cache (one of the lists ++ is assumed to be empty) ++ """ ++ obj = base64.b64encode(cPickle.dumps([sig, files_from, files_to])) ++ proc.stdin.write(obj) ++ proc.stdin.write('\n'.encode()) ++ proc.stdin.flush() ++ obj = proc.stdout.readline() ++ if not obj: ++ raise OSError('Preforked sub-process %r died' % proc.pid) ++ return cPickle.loads(base64.b64decode(obj)) ++ ++try: ++ copyfun = os.link ++except NameError: ++ copyfun = shutil.copy2 ++ ++def atomic_copy(orig, dest): ++ """ ++ Copy files to the cache, the operation is atomic for a given file ++ """ ++ global copyfun ++ tmp = dest + '.tmp' ++ up = os.path.dirname(dest) ++ try: ++ os.makedirs(up) ++ except OSError: ++ pass ++ ++ try: ++ copyfun(orig, tmp) ++ except OSError as e: ++ if e.errno == errno.EXDEV: ++ copyfun = shutil.copy2 ++ copyfun(orig, tmp) ++ else: ++ raise ++ os.rename(tmp, dest) ++ ++def lru_trim(): ++ """ ++ the cache folders take the form: ++ `CACHE_DIR/0b/0b180f82246d726ece37c8ccd0fb1cde2650d7bfcf122ec1f169079a3bfc0ab9` ++ they are listed in order of last access, and then removed ++ until the amount of folders is within TRIM_MAX_FOLDERS and the total space ++ taken by files is less than EVICT_MAX_BYTES ++ """ ++ lst = [] ++ for up in os.listdir(CACHE_DIR): ++ if len(up) == 2: ++ sub = os.path.join(CACHE_DIR, up) ++ for hval in os.listdir(sub): ++ path = os.path.join(sub, hval) ++ ++ size = 0 ++ for fname in os.listdir(path): ++ try: ++ size += os.lstat(os.path.join(path, fname)).st_size ++ except OSError: ++ pass ++ lst.append((os.stat(path).st_mtime, size, path)) ++ ++ lst.sort(key=lambda x: x[0]) ++ lst.reverse() ++ ++ tot = sum(x[1] for x in lst) ++ while tot > EVICT_MAX_BYTES or len(lst) > TRIM_MAX_FOLDERS: ++ _, tmp_size, path = lst.pop() ++ tot -= tmp_size ++ ++ tmp = path + '.remove' ++ try: ++ shutil.rmtree(tmp) ++ except OSError: ++ pass ++ try: ++ os.rename(path, tmp) ++ except OSError: ++ sys.stderr.write('Could not rename %r to %r\n' % (path, tmp)) ++ else: ++ try: ++ shutil.rmtree(tmp) ++ except OSError: ++ sys.stderr.write('Could not remove %r\n' % tmp) ++ sys.stderr.write("Cache trimmed: %r bytes in %r folders left\n" % (tot, len(lst))) ++ ++ ++def lru_evict(): ++ """ ++ Reduce the cache size ++ """ ++ lockfile = os.path.join(CACHE_DIR, 'all.lock') ++ try: ++ st = os.stat(lockfile) ++ except EnvironmentError as e: ++ if e.errno == errno.ENOENT: ++ with open(lockfile, 'w') as f: ++ f.write('') ++ return ++ else: ++ raise ++ ++ if st.st_mtime < time.time() - EVICT_INTERVAL_MINUTES * 60: ++ # check every EVICT_INTERVAL_MINUTES minutes if the cache is too big ++ # OCLOEXEC is unnecessary because no processes are spawned ++ fd = os.open(lockfile, os.O_RDWR | os.O_CREAT, 0o755) ++ try: ++ try: ++ fcntl.flock(fd, fcntl.LOCK_EX | fcntl.LOCK_NB) ++ except EnvironmentError: ++ if WAFCACHE_VERBOSITY: ++ sys.stderr.write('wafcache: another cleaning process is running\n') ++ else: ++ # now dow the actual cleanup ++ lru_trim() ++ os.utime(lockfile, None) ++ finally: ++ os.close(fd) ++ ++class netcache(object): ++ def __init__(self): ++ self.http = urllib3.PoolManager() ++ ++ def url_of(self, sig, i): ++ return "%s/%s/%s" % (CACHE_DIR, sig, i) ++ ++ def upload(self, file_path, sig, i): ++ url = self.url_of(sig, i) ++ with open(file_path, 'rb') as f: ++ file_data = f.read() ++ r = self.http.request('POST', url, timeout=60, ++ fields={ 'file': ('%s/%s' % (sig, i), file_data), }) ++ if r.status >= 400: ++ raise OSError("Invalid status %r %r" % (url, r.status)) ++ ++ def download(self, file_path, sig, i): ++ url = self.url_of(sig, i) ++ with self.http.request('GET', url, preload_content=False, timeout=60) as inf: ++ if inf.status >= 400: ++ raise OSError("Invalid status %r %r" % (url, inf.status)) ++ with open(file_path, 'wb') as out: ++ shutil.copyfileobj(inf, out) ++ ++ def copy_to_cache(self, sig, files_from, files_to): ++ try: ++ for i, x in enumerate(files_from): ++ if not os.path.islink(x): ++ self.upload(x, sig, i) ++ except Exception: ++ return traceback.format_exc() ++ return OK ++ ++ def copy_from_cache(self, sig, files_from, files_to): ++ try: ++ for i, x in enumerate(files_to): ++ self.download(x, sig, i) ++ except Exception: ++ return traceback.format_exc() ++ return OK ++ ++class fcache(object): ++ def __init__(self): ++ if not os.path.exists(CACHE_DIR): ++ try: ++ os.makedirs(CACHE_DIR) ++ except OSError: ++ pass ++ if not os.path.exists(CACHE_DIR): ++ raise ValueError('Could not initialize the cache directory') ++ ++ def copy_to_cache(self, sig, files_from, files_to): ++ """ ++ Copy files to the cache, existing files are overwritten, ++ and the copy is atomic only for a given file, not for all files ++ that belong to a given task object ++ """ ++ try: ++ for i, x in enumerate(files_from): ++ dest = os.path.join(CACHE_DIR, sig[:2], sig, str(i)) ++ atomic_copy(x, dest) ++ except Exception: ++ return traceback.format_exc() ++ else: ++ # attempt trimming if caching was successful: ++ # we may have things to trim! ++ try: ++ lru_evict() ++ except Exception: ++ return traceback.format_exc() ++ return OK ++ ++ def copy_from_cache(self, sig, files_from, files_to): ++ """ ++ Copy files from the cache ++ """ ++ try: ++ for i, x in enumerate(files_to): ++ orig = os.path.join(CACHE_DIR, sig[:2], sig, str(i)) ++ atomic_copy(orig, x) ++ ++ # success! update the cache time ++ os.utime(os.path.join(CACHE_DIR, sig[:2], sig), None) ++ except Exception: ++ return traceback.format_exc() ++ return OK ++ ++class bucket_cache(object): ++ def bucket_copy(self, source, target): ++ if WAFCACHE_CMD: ++ def replacer(match): ++ if match.group('src'): ++ return source ++ elif match.group('tgt'): ++ return target ++ cmd = [re_waf_cmd.sub(replacer, x) for x in shlex.split(WAFCACHE_CMD)] ++ elif CACHE_DIR.startswith('s3://'): ++ cmd = ['aws', 's3', 'cp', source, target] ++ elif CACHE_DIR.startswith('gs://'): ++ cmd = ['gsutil', 'cp', source, target] ++ else: ++ cmd = ['mc', 'cp', source, target] ++ ++ proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) ++ out, err = proc.communicate() ++ if proc.returncode: ++ raise OSError('Error copy %r to %r using: %r (exit %r):\n out:%s\n err:%s' % ( ++ source, target, cmd, proc.returncode, out.decode(errors='replace'), err.decode(errors='replace'))) ++ ++ def copy_to_cache(self, sig, files_from, files_to): ++ try: ++ for i, x in enumerate(files_from): ++ dest = os.path.join(CACHE_DIR, sig[:2], sig, str(i)) ++ self.bucket_copy(x, dest) ++ except Exception: ++ return traceback.format_exc() ++ return OK ++ ++ def copy_from_cache(self, sig, files_from, files_to): ++ try: ++ for i, x in enumerate(files_to): ++ orig = os.path.join(CACHE_DIR, sig[:2], sig, str(i)) ++ self.bucket_copy(orig, x) ++ except EnvironmentError: ++ return traceback.format_exc() ++ return OK ++ ++def loop(service): ++ """ ++ This function is run when this file is run as a standalone python script, ++ it assumes a parent process that will communicate the commands to it ++ as pickled-encoded tuples (one line per command) ++ ++ The commands are to copy files to the cache or copy files from the ++ cache to a target destination ++ """ ++ # one operation is performed at a single time by a single process ++ # therefore stdin never has more than one line ++ txt = sys.stdin.readline().strip() ++ if not txt: ++ # parent process probably ended ++ sys.exit(1) ++ ret = OK ++ ++ [sig, files_from, files_to] = cPickle.loads(base64.b64decode(txt)) ++ if files_from: ++ # TODO return early when pushing files upstream ++ ret = service.copy_to_cache(sig, files_from, files_to) ++ elif files_to: ++ # the build process waits for workers to (possibly) obtain files from the cache ++ ret = service.copy_from_cache(sig, files_from, files_to) ++ else: ++ ret = "Invalid command" ++ ++ obj = base64.b64encode(cPickle.dumps(ret)) ++ sys.stdout.write(obj.decode()) ++ sys.stdout.write('\n') ++ sys.stdout.flush() ++ ++if __name__ == '__main__': ++ if CACHE_DIR.startswith('s3://') or CACHE_DIR.startswith('gs://') or CACHE_DIR.startswith('minio://'): ++ if CACHE_DIR.startswith('minio://'): ++ CACHE_DIR = CACHE_DIR[8:] # minio doesn't need the protocol part, uses config aliases ++ service = bucket_cache() ++ elif CACHE_DIR.startswith('http'): ++ service = netcache() ++ else: ++ service = fcache() ++ while 1: ++ try: ++ loop(service) ++ except KeyboardInterrupt: ++ break ++ +diff --git a/waflib/extras/xcode6.py b/waflib/extras/xcode6.py +index 91bbff181..c5b309120 100644 +--- a/waflib/extras/xcode6.py ++++ b/waflib/extras/xcode6.py +@@ -99,7 +99,7 @@ def delete_invalid_values(dct): + ... + } + 'Release': { +- 'ARCHS' x86_64' ++ 'ARCHS': x86_64' + ... + } + } +@@ -163,12 +163,12 @@ def tostring(self, value): + result = result + "\t\t}" + return result + elif isinstance(value, str): +- return "\"%s\"" % value ++ return '"%s"' % value.replace('"', '\\\\\\"') + elif isinstance(value, list): + result = "(\n" + for i in value: +- result = result + "\t\t\t%s,\n" % self.tostring(i) +- result = result + "\t\t)" ++ result = result + "\t\t\t\t%s,\n" % self.tostring(i) ++ result = result + "\t\t\t)" + return result + elif isinstance(value, XCodeNode): + return value._id +@@ -565,13 +565,13 @@ def process_xcode(self): + # Override target specific build settings + bldsettings = { + 'HEADER_SEARCH_PATHS': ['$(inherited)'] + self.env['INCPATHS'], +- 'LIBRARY_SEARCH_PATHS': ['$(inherited)'] + Utils.to_list(self.env.LIBPATH) + Utils.to_list(self.env.STLIBPATH) + Utils.to_list(self.env.LIBDIR) , ++ 'LIBRARY_SEARCH_PATHS': ['$(inherited)'] + Utils.to_list(self.env.LIBPATH) + Utils.to_list(self.env.STLIBPATH) + Utils.to_list(self.env.LIBDIR), + 'FRAMEWORK_SEARCH_PATHS': ['$(inherited)'] + Utils.to_list(self.env.FRAMEWORKPATH), +- 'OTHER_LDFLAGS': libs + ' ' + frameworks, +- 'OTHER_LIBTOOLFLAGS': bld.env['LINKFLAGS'], ++ 'OTHER_LDFLAGS': libs + ' ' + frameworks + ' ' + ' '.join(bld.env['LINKFLAGS']), + 'OTHER_CPLUSPLUSFLAGS': Utils.to_list(self.env['CXXFLAGS']), + 'OTHER_CFLAGS': Utils.to_list(self.env['CFLAGS']), +- 'INSTALL_PATH': [] ++ 'INSTALL_PATH': [], ++ 'GCC_PREPROCESSOR_DEFINITIONS': self.env['DEFINES'] + } + + # Install path +@@ -591,7 +591,7 @@ def process_xcode(self): + + # The keys represents different build configuration, e.g. Debug, Release and so on.. + # Insert our generated build settings to all configuration names +- keys = set(settings.keys() + bld.env.PROJ_CONFIGURATION.keys()) ++ keys = set(settings.keys()) | set(bld.env.PROJ_CONFIGURATION.keys()) + for k in keys: + if k in settings: + settings[k].update(bldsettings) +diff --git a/waflib/fixpy2.py b/waflib/fixpy2.py +index 24176e066..c99bff4b9 100644 +--- a/waflib/fixpy2.py ++++ b/waflib/fixpy2.py +@@ -56,7 +56,7 @@ def r1(code): + @subst('Runner.py') + def r4(code): + "generator syntax" +- return code.replace('next(self.biter)', 'self.biter.next()') ++ return code.replace('next(self.biter)', 'self.biter.next()').replace('self.daemon = True', 'self.setDaemon(1)') + + @subst('Context.py') + def r5(code): +diff --git a/waflib/processor.py b/waflib/processor.py +index 2eecf3bd9..eff2e69ad 100755 +--- a/waflib/processor.py ++++ b/waflib/processor.py +@@ -27,6 +27,10 @@ def run(): + [cmd, kwargs, cargs] = cPickle.loads(base64.b64decode(txt)) + cargs = cargs or {} + ++ if not 'close_fds' in kwargs: ++ # workers have no fds ++ kwargs['close_fds'] = False ++ + ret = 1 + out, err, ex, trace = (None, None, None, None) + try: