#!/usr/bin/env python # encoding: utf-8 # Thomas Nagy, 2005-2008 (ita) """ Running tasks in parallel is a simple problem, but in practice it is more complicated: * dependencies discovered during the build (dynamic task creation) * dependencies discovered after files are compiled * the amount of tasks and dependencies (graph size) can be huge This is why the dependency management is split on three different levels: 1. groups of tasks that run all after another group of tasks 2. groups of tasks that can be run in parallel 3. tasks that can run in parallel, but with possible unknown ad-hoc dependencies The point #1 represents a strict sequential order between groups of tasks, for example a compiler is produced and used to compile the rest, whereas #2 and #3 represent partial order constraints where #2 applies to the kind of task and #3 applies to the task instances. #1 is held by the task manager: ordered list of TaskGroups (see bld.add_group) #2 is held by the task groups and the task types: precedence after/before (topological sort), and the constraints extracted from file extensions #3 is held by the tasks individually (attribute run_after), and the scheduler (Runner.py) use Task::runnable_status to reorder the tasks -- To try, use something like this in your code: import Constants, Task Task.algotype = Constants.MAXPARALLEL -- There are two concepts with the tasks (individual units of change): * dependency (if 1 is recompiled, recompile 2) * order (run 2 after 1) example 1: if t1 depends on t2 and t2 depends on t3 it is not necessary to make t1 depend on t3 (dependency is transitive) example 2: if t1 depends on a node produced by t2, it is not immediately obvious that t1 must run after t2 (order is not obvious) The role of the Task Manager is to give the tasks in order (groups of task that may be run in parallel one after the other) """ import os, shutil, sys, re, random, datetime, tempfile, shlex from Utils import md5 import Build, Runner, Utils, Node, Logs, Options from Logs import debug, warn, error from Constants import * algotype = NORMAL #algotype = JOBCONTROL #algotype = MAXPARALLEL COMPILE_TEMPLATE_SHELL = ''' def f(task): env = task.env wd = getattr(task, 'cwd', None) p = env.get_flat cmd = \'\'\' %s \'\'\' % s return task.exec_command(cmd, cwd=wd) ''' COMPILE_TEMPLATE_NOSHELL = ''' def f(task): env = task.env wd = getattr(task, 'cwd', None) def to_list(xx): if isinstance(xx, str): return [xx] return xx lst = [] %s lst = [x for x in lst if x] return task.exec_command(lst, cwd=wd) ''' """ Enable different kind of dependency algorithms: 1 make groups: first compile all cpps and then compile all links (NORMAL) 2 parallelize all (each link task run after its dependencies) (MAXPARALLEL) 3 like 1 but provide additional constraints for the parallelization (MAXJOBS) In theory 1. will be faster than 2 for waf, but might be slower for builds The scheme 2 will not allow for running tasks one by one so it can cause disk thrashing on huge builds """ file_deps = Utils.nada """ Additional dependency pre-check may be added by replacing the function file_deps. e.g. extract_outputs, extract_deps below. """ class TaskManager(object): """The manager is attached to the build object, it holds a list of TaskGroup""" def __init__(self): self.groups = [] self.tasks_done = [] self.current_group = 0 self.groups_names = {} def group_name(self, g): """name for the group g (utility)""" if not isinstance(g, TaskGroup): g = self.groups[g] for x in self.groups_names: if id(self.groups_names[x]) == id(g): return x return '' def group_idx(self, tg): """group the task generator tg is in""" se = id(tg) for i in range(len(self.groups)): g = self.groups[i] for t in g.tasks_gen: if id(t) == se: return i return None def get_next_set(self): """return the next set of tasks to execute the first parameter is the maximum amount of parallelization that may occur""" ret = None while not ret and self.current_group < len(self.groups): ret = self.groups[self.current_group].get_next_set() if ret: return ret else: self.groups[self.current_group].process_install() self.current_group += 1 return (None, None) def add_group(self, name=None, set=True): #if self.groups and not self.groups[0].tasks: # error('add_group: an empty group is already present') g = TaskGroup() if name and name in self.groups_names: error('add_group: name %s already present' % name) self.groups_names[name] = g self.groups.append(g) if set: self.current_group = len(self.groups) - 1 def set_group(self, idx): if isinstance(idx, str): g = self.groups_names[idx] for x in xrange(len(self.groups)): if id(g) == id(self.groups[x]): self.current_group = x else: self.current_group = idx def add_task_gen(self, tgen): if not self.groups: self.add_group() self.groups[self.current_group].tasks_gen.append(tgen) def add_task(self, task): if not self.groups: self.add_group() self.groups[self.current_group].tasks.append(task) def total(self): total = 0 if not self.groups: return 0 for group in self.groups: total += len(group.tasks) return total def add_finished(self, tsk): self.tasks_done.append(tsk) bld = tsk.generator.bld if bld.is_install: f = None if 'install' in tsk.__dict__: f = tsk.__dict__['install'] # install=0 to prevent installation if f: f(tsk) else: tsk.install() class TaskGroup(object): "the compilation of one group does not begin until the previous group has finished (in the manager)" def __init__(self): self.tasks = [] # this list will be consumed self.tasks_gen = [] self.cstr_groups = Utils.DefaultDict(list) # tasks having equivalent constraints self.cstr_order = Utils.DefaultDict(set) # partial order between the cstr groups self.temp_tasks = [] # tasks put on hold self.ready = 0 self.post_funs = [] def reset(self): "clears the state of the object (put back the tasks into self.tasks)" for x in self.cstr_groups: self.tasks += self.cstr_groups[x] self.tasks = self.temp_tasks + self.tasks self.temp_tasks = [] self.cstr_groups = Utils.DefaultDict(list) self.cstr_order = Utils.DefaultDict(set) self.ready = 0 def process_install(self): for (f, k, kw) in self.post_funs: f(*k, **kw) def prepare(self): "prepare the scheduling" self.ready = 1 file_deps(self.tasks) self.make_cstr_groups() self.extract_constraints() def get_next_set(self): "next list of tasks to execute using max job settings, returns (maxjobs, task_list)" global algotype if algotype == NORMAL: tasks = self.tasks_in_parallel() maxj = MAXJOBS elif algotype == JOBCONTROL: (maxj, tasks) = self.tasks_by_max_jobs() elif algotype == MAXPARALLEL: tasks = self.tasks_with_inner_constraints() maxj = MAXJOBS else: raise Utils.WafError("unknown algorithm type %s" % (algotype)) if not tasks: return () return (maxj, tasks) def make_cstr_groups(self): "unite the tasks that have similar constraints" self.cstr_groups = Utils.DefaultDict(list) for x in self.tasks: h = x.hash_constraints() self.cstr_groups[h].append(x) def set_order(self, a, b): self.cstr_order[a].add(b) def compare_exts(self, t1, t2): "extension production" x = "ext_in" y = "ext_out" in_ = t1.attr(x, ()) out_ = t2.attr(y, ()) for k in in_: if k in out_: return -1 in_ = t2.attr(x, ()) out_ = t1.attr(y, ()) for k in in_: if k in out_: return 1 return 0 def compare_partial(self, t1, t2): "partial relations after/before" m = "after" n = "before" name = t2.__class__.__name__ if name in Utils.to_list(t1.attr(m, ())): return -1 elif name in Utils.to_list(t1.attr(n, ())): return 1 name = t1.__class__.__name__ if name in Utils.to_list(t2.attr(m, ())): return 1 elif name in Utils.to_list(t2.attr(n, ())): return -1 return 0 def extract_constraints(self): "extract the parallelization constraints from the tasks with different constraints" keys = self.cstr_groups.keys() max = len(keys) # hopefully the length of this list is short for i in xrange(max): t1 = self.cstr_groups[keys[i]][0] for j in xrange(i + 1, max): t2 = self.cstr_groups[keys[j]][0] # add the constraints based on the comparisons val = (self.compare_exts(t1, t2) or self.compare_partial(t1, t2) ) if val > 0: self.set_order(keys[i], keys[j]) elif val < 0: self.set_order(keys[j], keys[i]) def tasks_in_parallel(self): "(NORMAL) next list of tasks that may be executed in parallel" if not self.ready: self.prepare() keys = self.cstr_groups.keys() unconnected = [] remainder = [] for u in keys: for k in self.cstr_order.values(): if u in k: remainder.append(u) break else: unconnected.append(u) toreturn = [] for y in unconnected: toreturn.extend(self.cstr_groups[y]) # remove stuff only after for y in unconnected: try: self.cstr_order.__delitem__(y) except KeyError: pass self.cstr_groups.__delitem__(y) if not toreturn and remainder: raise Utils.WafError("circular order constraint detected %r" % remainder) return toreturn def tasks_by_max_jobs(self): "(JOBCONTROL) returns the tasks that can run in parallel with the max amount of jobs" if not self.ready: self.prepare() if not self.temp_tasks: self.temp_tasks = self.tasks_in_parallel() if not self.temp_tasks: return (None, None) maxjobs = MAXJOBS ret = [] remaining = [] for t in self.temp_tasks: m = getattr(t, "maxjobs", getattr(self.__class__, "maxjobs", MAXJOBS)) if m > maxjobs: remaining.append(t) elif m < maxjobs: remaining += ret ret = [t] maxjobs = m else: ret.append(t) self.temp_tasks = remaining return (maxjobs, ret) def tasks_with_inner_constraints(self): """(MAXPARALLEL) returns all tasks in this group, but add the constraints on each task instance as an optimization, it might be desirable to discard the tasks which do not have to run""" if not self.ready: self.prepare() if getattr(self, "done", None): return None for p in self.cstr_order: for v in self.cstr_order[p]: for m in self.cstr_groups[p]: for n in self.cstr_groups[v]: n.set_run_after(m) self.cstr_order = Utils.DefaultDict(set) self.cstr_groups = Utils.DefaultDict(list) self.done = 1 return self.tasks[:] # make a copy class store_task_type(type): "store the task types that have a name ending in _task into a map (remember the existing task types)" def __init__(cls, name, bases, dict): super(store_task_type, cls).__init__(name, bases, dict) name = cls.__name__ if name.endswith('_task'): name = name.replace('_task', '') if name != 'TaskBase': TaskBase.classes[name] = cls class TaskBase(object): """Base class for all Waf tasks The most important methods are (by usual order of call): 1 runnable_status: ask the task if it should be run, skipped, or if we have to ask later 2 __str__: string to display to the user 3 run: execute the task 4 post_run: after the task is run, update the cache about the task This class should be seen as an interface, it provides the very minimum necessary for the scheduler so it does not do much. For illustration purposes, TaskBase instances try to execute self.fun (if provided) """ __metaclass__ = store_task_type color = "GREEN" maxjobs = MAXJOBS classes = {} stat = None def __init__(self, *k, **kw): self.hasrun = NOT_RUN try: self.generator = kw['generator'] except KeyError: self.generator = self self.bld = Build.bld if kw.get('normal', 1): self.generator.bld.task_manager.add_task(self) def __repr__(self): "used for debugging" return '\n\t{task: %s %s}' % (self.__class__.__name__, str(getattr(self, "fun", ""))) def __str__(self): "string to display to the user" if hasattr(self, 'fun'): return 'executing: %s\n' % self.fun.__name__ return self.__class__.__name__ + '\n' def exec_command(self, *k, **kw): "use this for executing commands from tasks" # TODO in waf 1.6, eliminate bld.exec_command, and move the cwd processing to here if self.env['env']: kw['env'] = self.env['env'] return self.generator.bld.exec_command(*k, **kw) def runnable_status(self): "RUN_ME SKIP_ME or ASK_LATER" return RUN_ME def can_retrieve_cache(self): return False def call_run(self): if self.can_retrieve_cache(): return 0 return self.run() def run(self): "called if the task must run" if hasattr(self, 'fun'): return self.fun(self) return 0 def post_run(self): "update the dependency tree (node stats)" pass def display(self): "print either the description (using __str__) or the progress bar or the ide output" col1 = Logs.colors(self.color) col2 = Logs.colors.NORMAL if Options.options.progress_bar == 1: return self.generator.bld.progress_line(self.position[0], self.position[1], col1, col2) if Options.options.progress_bar == 2: ela = Utils.get_elapsed_time(self.generator.bld.ini) try: ins = ','.join([n.name for n in self.inputs]) except AttributeError: ins = '' try: outs = ','.join([n.name for n in self.outputs]) except AttributeError: outs = '' return '|Total %s|Current %s|Inputs %s|Outputs %s|Time %s|\n' % (self.position[1], self.position[0], ins, outs, ela) total = self.position[1] n = len(str(total)) fs = '[%%%dd/%%%dd] %%s%%s%%s' % (n, n) return fs % (self.position[0], self.position[1], col1, str(self), col2) def attr(self, att, default=None): "retrieve an attribute from the instance or from the class (microoptimization here)" ret = getattr(self, att, self) if ret is self: return getattr(self.__class__, att, default) return ret def hash_constraints(self): "identify a task type for all the constraints relevant for the scheduler: precedence, file production" a = self.attr sum = hash((self.__class__.__name__, str(a('before', '')), str(a('after', '')), str(a('ext_in', '')), str(a('ext_out', '')), self.__class__.maxjobs)) return sum def format_error(self): "error message to display to the user (when a build fails)" if getattr(self, "err_msg", None): return self.err_msg elif self.hasrun == CRASHED: try: return " -> task failed (err #%d): %r" % (self.err_code, self) except AttributeError: return " -> task failed: %r" % self elif self.hasrun == MISSING: return " -> missing files: %r" % self else: return '' def install(self): """ installation is performed by looking at the task attributes: * install_path: installation path like "${PREFIX}/bin" * filename: install the first node in the outputs as a file with a particular name, be certain to give os.sep * chmod: permissions """ bld = self.generator.bld d = self.attr('install') if self.attr('install_path'): lst = [a.relpath_gen(bld.srcnode) for a in self.outputs] perm = self.attr('chmod', O644) if self.attr('src'): # if src is given, install the sources too lst += [a.relpath_gen(bld.srcnode) for a in self.inputs] if self.attr('filename'): dir = self.install_path.rstrip(os.sep) + os.sep + self.attr('filename') bld.install_as(dir, lst[0], self.env, perm) else: bld.install_files(self.install_path, lst, self.env, perm) class Task(TaskBase): """The parent class is quite limited, in this version: * file system interaction: input and output nodes * persistence: do not re-execute tasks that have already run * caching: same files can be saved and retrieved from a cache directory * dependencies: implicit, like .c files depending on .h files explicit, like the input nodes or the dep_nodes environment variables, like the CXXFLAGS in self.env """ vars = [] def __init__(self, env, **kw): TaskBase.__init__(self, **kw) self.env = env # inputs and outputs are nodes # use setters when possible self.inputs = [] self.outputs = [] self.dep_nodes = [] self.run_after = [] # Additionally, you may define the following #self.dep_vars = 'PREFIX DATADIR' def __str__(self): "string to display to the user" env = self.env src_str = ' '.join([a.nice_path(env) for a in self.inputs]) tgt_str = ' '.join([a.nice_path(env) for a in self.outputs]) if self.outputs: sep = ' -> ' else: sep = '' return '%s: %s%s%s\n' % (self.__class__.__name__.replace('_task', ''), src_str, sep, tgt_str) def __repr__(self): return "".join(['\n\t{task: ', self.__class__.__name__, " ", ",".join([x.name for x in self.inputs]), " -> ", ",".join([x.name for x in self.outputs]), '}']) def unique_id(self): "get a unique id: hash the node paths, the variant, the class, the function" try: return self.uid except AttributeError: "this is not a real hot zone, but we want to avoid surprizes here" m = md5() up = m.update up(self.__class__.__name__) up(self.env.variant()) p = None for x in self.inputs + self.outputs: if p != x.parent.id: p = x.parent.id up(x.parent.abspath()) up(x.name) self.uid = m.digest() return self.uid def set_inputs(self, inp): if isinstance(inp, list): self.inputs += inp else: self.inputs.append(inp) def set_outputs(self, out): if isinstance(out, list): self.outputs += out else: self.outputs.append(out) def set_run_after(self, task): "set (scheduler) order on another task" # TODO: handle list or object assert isinstance(task, TaskBase) self.run_after.append(task) def add_file_dependency(self, filename): "TODO user-provided file dependencies" node = self.generator.bld.path.find_resource(filename) self.dep_nodes.append(node) def signature(self): # compute the result one time, and suppose the scan_signature will give the good result try: return self.cache_sig[0] except AttributeError: pass self.m = md5() # explicit deps exp_sig = self.sig_explicit_deps() # env vars var_sig = self.sig_vars() # implicit deps imp_sig = SIG_NIL if self.scan: try: imp_sig = self.sig_implicit_deps() except ValueError: return self.signature() # we now have the signature (first element) and the details (for debugging) ret = self.m.digest() self.cache_sig = (ret, exp_sig, imp_sig, var_sig) return ret def runnable_status(self): "SKIP_ME RUN_ME or ASK_LATER" #return 0 # benchmarking if self.inputs and (not self.outputs): if not getattr(self.__class__, 'quiet', None): warn("invalid task (no inputs OR outputs): override in a Task subclass or set the attribute 'quiet' %r" % self) for t in self.run_after: if not t.hasrun: return ASK_LATER env = self.env bld = self.generator.bld # first compute the signature new_sig = self.signature() # compare the signature to a signature computed previously key = self.unique_id() try: prev_sig = bld.task_sigs[key][0] except KeyError: debug("task: task %r must run as it was never run before or the task code changed", self) return RUN_ME # compare the signatures of the outputs for node in self.outputs: variant = node.variant(env) try: if bld.node_sigs[variant][node.id] != new_sig: return RUN_ME except KeyError: debug("task: task %r must run as the output nodes do not exist", self) return RUN_ME # debug if asked to if Logs.verbose: self.debug_why(bld.task_sigs[key]) if new_sig != prev_sig: return RUN_ME return SKIP_ME def post_run(self): "called after a successful task run" bld = self.generator.bld env = self.env sig = self.signature() ssig = sig.encode('hex') variant = env.variant() for node in self.outputs: # check if the node exists .. try: os.stat(node.abspath(env)) except OSError: self.hasrun = MISSING self.err_msg = '-> missing file: %r' % node.abspath(env) raise Utils.WafError # important, store the signature for the next run bld.node_sigs[variant][node.id] = sig bld.task_sigs[self.unique_id()] = self.cache_sig # file caching, if possible # try to avoid data corruption as much as possible if not Options.cache_global or Options.options.nocache or not self.outputs: return None if getattr(self, 'cached', None): return None dname = os.path.join(Options.cache_global, ssig) tmpdir = tempfile.mkdtemp(prefix=Options.cache_global + os.sep + 'waf') try: shutil.rmtree(dname) except: pass try: i = 0 for node in self.outputs: variant = node.variant(env) dest = os.path.join(tmpdir, str(i) + node.name) shutil.copy2(node.abspath(env), dest) i += 1 except (OSError, IOError): try: shutil.rmtree(tmpdir) except: pass else: try: os.rename(tmpdir, dname) except OSError: try: shutil.rmtree(tmpdir) except: pass else: try: os.chmod(dname, O755) except: pass def can_retrieve_cache(self): """ Retrieve build nodes from the cache update the file timestamps to help cleaning the least used entries from the cache additionally, set an attribute 'cached' to avoid re-creating the same cache files suppose there are files in cache/dir1/file1 and cache/dir2/file2 first, read the timestamp of dir1 then try to copy the files then look at the timestamp again, if it has changed, the data may have been corrupt (cache update by another process) should an exception occur, ignore the data """ if not Options.cache_global or Options.options.nocache or not self.outputs: return None env = self.env sig = self.signature() ssig = sig.encode('hex') # first try to access the cache folder for the task dname = os.path.join(Options.cache_global, ssig) try: t1 = os.stat(dname).st_mtime except OSError: return None i = 0 for node in self.outputs: variant = node.variant(env) orig = os.path.join(dname, str(i) + node.name) try: shutil.copy2(orig, node.abspath(env)) # mark the cache file as used recently (modified) os.utime(orig, None) except (OSError, IOError): debug('task: failed retrieving file') return None i += 1 # is it the same folder? try: t2 = os.stat(dname).st_mtime except OSError: return None if t1 != t2: return None for node in self.outputs: self.generator.bld.node_sigs[variant][node.id] = sig if Options.options.progress_bar < 1: self.generator.bld.printout('restoring from cache %r\n' % node.bldpath(env)) self.cached = True return 1 def debug_why(self, old_sigs): "explains why a task is run" new_sigs = self.cache_sig def v(x): return x.encode('hex') debug("Task %r", self) msgs = ['Task must run', '* Source file or manual dependency', '* Implicit dependency', '* Environment variable'] tmp = 'task: -> %s: %s %s' for x in xrange(len(msgs)): if (new_sigs[x] != old_sigs[x]): debug(tmp, msgs[x], v(old_sigs[x]), v(new_sigs[x])) def sig_explicit_deps(self): bld = self.generator.bld up = self.m.update # the inputs for x in self.inputs + getattr(self, 'dep_nodes', []): if not x.parent.id in bld.cache_scanned_folders: bld.rescan(x.parent) variant = x.variant(self.env) try: up(bld.node_sigs[variant][x.id]) except KeyError: raise Utils.WafError('Missing node signature for %r (required by %r)' % (x, self)) # manual dependencies, they can slow down the builds if bld.deps_man: additional_deps = bld.deps_man for x in self.inputs + self.outputs: try: d = additional_deps[x.id] except KeyError: continue for v in d: if isinstance(v, Node.Node): bld.rescan(v.parent) variant = v.variant(self.env) try: v = bld.node_sigs[variant][v.id] except KeyError: raise Utils.WafError('Missing node signature for %r (required by %r)' % (v, self)) elif hasattr(v, '__call__'): v = v() # dependency is a function, call it up(v) for x in self.dep_nodes: v = bld.node_sigs[x.variant(self.env)][x.id] up(v) return self.m.digest() def sig_vars(self): bld = self.generator.bld env = self.env # dependencies on the environment vars act_sig = bld.hash_env_vars(env, self.__class__.vars) self.m.update(act_sig) # additional variable dependencies, if provided dep_vars = getattr(self, 'dep_vars', None) if dep_vars: self.m.update(bld.hash_env_vars(env, dep_vars)) return self.m.digest() #def scan(self, node): # """this method returns a tuple containing: # * a list of nodes corresponding to real files # * a list of names for files not found in path_lst # the input parameters may have more parameters that the ones used below # """ # return ((), ()) scan = None # compute the signature, recompute it if there is no match in the cache def sig_implicit_deps(self): "the signature obtained may not be the one if the files have changed, we do it in two steps" bld = self.generator.bld # get the task signatures from previous runs key = self.unique_id() prev_sigs = bld.task_sigs.get(key, ()) if prev_sigs: try: # for issue #379 if prev_sigs[2] == self.compute_sig_implicit_deps(): return prev_sigs[2] except (KeyError, OSError): pass del bld.task_sigs[key] raise ValueError('rescan') # no previous run or the signature of the dependencies has changed, rescan the dependencies (nodes, names) = self.scan() if Logs.verbose: debug('deps: scanner for %s returned %s %s', str(self), str(nodes), str(names)) # store the dependencies in the cache bld.node_deps[key] = nodes bld.raw_deps[key] = names # recompute the signature and return it try: sig = self.compute_sig_implicit_deps() except KeyError: try: nodes = [] for k in bld.node_deps.get(self.unique_id(), []): if k.id & 3 == 2: # Node.FILE: if not k.id in bld.node_sigs[0]: nodes.append(k) else: if not k.id in bld.node_sigs[self.env.variant()]: nodes.append(k) except: nodes = '?' raise Utils.WafError('Missing node signature for %r (for implicit dependencies %r)' % (nodes, self)) return sig def compute_sig_implicit_deps(self): """it is intended for .cpp and inferred .h files there is a single list (no tree traversal) this is the hot spot so ... do not touch""" upd = self.m.update bld = self.generator.bld tstamp = bld.node_sigs env = self.env for k in bld.node_deps.get(self.unique_id(), []): # unlikely but necessary if it happens if not k.parent.id in bld.cache_scanned_folders: # if the parent folder is removed, an OSError may be thrown bld.rescan(k.parent) # if the parent folder is removed, a KeyError will be thrown if k.id & 3 == 2: # Node.FILE: upd(tstamp[0][k.id]) else: upd(tstamp[env.variant()][k.id]) return self.m.digest() def funex(c): dc = {} exec(c, dc) return dc['f'] reg_act = re.compile(r"(?P<backslash>\\)|(?P<dollar>\$\$)|(?P<subst>\$\{(?P<var>\w+)(?P<code>.*?)\})", re.M) def compile_fun_shell(name, line): """Compiles a string (once) into a function, eg: simple_task_type('c++', '${CXX} -o ${TGT[0]} ${SRC} -I ${SRC[0].parent.bldpath()}') The env variables (CXX, ..) on the task must not hold dicts (order) The reserved keywords TGT and SRC represent the task input and output nodes quick test: bld(source='wscript', rule='echo "foo\\${SRC[0].name}\\bar"') """ extr = [] def repl(match): g = match.group if g('dollar'): return "$" elif g('backslash'): return '\\\\' elif g('subst'): extr.append((g('var'), g('code'))); return "%s" return None line = reg_act.sub(repl, line) or line parm = [] dvars = [] app = parm.append for (var, meth) in extr: if var == 'SRC': if meth: app('task.inputs%s' % meth) else: app('" ".join([a.srcpath(env) for a in task.inputs])') elif var == 'TGT': if meth: app('task.outputs%s' % meth) else: app('" ".join([a.bldpath(env) for a in task.outputs])') else: if not var in dvars: dvars.append(var) app("p('%s')" % var) if parm: parm = "%% (%s) " % (',\n\t\t'.join(parm)) else: parm = '' c = COMPILE_TEMPLATE_SHELL % (line, parm) debug('action: %s', c) return (funex(c), dvars) def compile_fun_noshell(name, line): extr = [] def repl(match): g = match.group if g('dollar'): return "$" elif g('subst'): extr.append((g('var'), g('code'))); return "<<|@|>>" return None line2 = reg_act.sub(repl, line) params = line2.split('<<|@|>>') buf = [] dvars = [] app = buf.append for x in xrange(len(extr)): params[x] = params[x].strip() if params[x]: app("lst.extend(%r)" % params[x].split()) (var, meth) = extr[x] if var == 'SRC': if meth: app('lst.append(task.inputs%s)' % meth) else: app("lst.extend([a.srcpath(env) for a in task.inputs])") elif var == 'TGT': if meth: app('lst.append(task.outputs%s)' % meth) else: app("lst.extend([a.bldpath(env) for a in task.outputs])") else: app('lst.extend(to_list(env[%r]))' % var) if not var in dvars: dvars.append(var) if params[-1]: app("lst.extend(%r)" % shlex.split(params[-1])) fun = COMPILE_TEMPLATE_NOSHELL % "\n\t".join(buf) debug('action: %s', fun) return (funex(fun), dvars) def compile_fun(name, line, shell=None): "commands can be launched by the shell or not" if line.find('<') > 0 or line.find('>') > 0 or line.find('&&') > 0: shell = True #else: # shell = False if shell is None: if sys.platform == 'win32': shell = False else: shell = True if shell: return compile_fun_shell(name, line) else: return compile_fun_noshell(name, line) def simple_task_type(name, line, color='GREEN', vars=[], ext_in=[], ext_out=[], before=[], after=[], shell=None): """return a new Task subclass with the function run compiled from the line given""" (fun, dvars) = compile_fun(name, line, shell) fun.code = line return task_type_from_func(name, fun, vars or dvars, color, ext_in, ext_out, before, after) def task_type_from_func(name, func, vars=[], color='GREEN', ext_in=[], ext_out=[], before=[], after=[]): """return a new Task subclass with the function run compiled from the line given""" params = { 'run': func, 'vars': vars, 'color': color, 'name': name, 'ext_in': Utils.to_list(ext_in), 'ext_out': Utils.to_list(ext_out), 'before': Utils.to_list(before), 'after': Utils.to_list(after), } cls = type(Task)(name, (Task,), params) TaskBase.classes[name] = cls return cls def always_run(cls): """Set all task instances of this class to be executed whenever a build is started The task signature is calculated, but the result of the comparation between task signatures is bypassed """ old = cls.runnable_status def always(self): ret = old(self) if ret == SKIP_ME: return RUN_ME return ret cls.runnable_status = always def update_outputs(cls): """When a command is always run, it is possible that the output only change sometimes. By default the build node have as a hash the signature of the task which may not change. With this, the output nodes (produced) are hashed, and the hashes are set to the build nodes This may avoid unnecessary recompilations, but it uses more resources (hashing the output files) so it is not used by default """ old_post_run = cls.post_run def post_run(self): old_post_run(self) bld = self.generator.bld for output in self.outputs: bld.node_sigs[self.env.variant()][output.id] = Utils.h_file(output.abspath(self.env)) bld.task_sigs[output.id] = self.unique_id() cls.post_run = post_run old_runnable_status = cls.runnable_status def runnable_status(self): status = old_runnable_status(self) if status != RUN_ME: return status uid = self.unique_id() try: bld = self.outputs[0].__class__.bld new_sig = self.signature() prev_sig = bld.task_sigs[uid][0] if prev_sig == new_sig: for x in self.outputs: if not x.id in bld.node_sigs[self.env.variant()]: return RUN_ME if bld.task_sigs[x.id] != uid: # ensure the outputs are associated with *this* task return RUN_ME return SKIP_ME except KeyError: pass except IndexError: pass return RUN_ME cls.runnable_status = runnable_status def extract_outputs(tasks): """file_deps: Infer additional dependencies from task input and output nodes """ v = {} for x in tasks: try: (ins, outs) = v[x.env.variant()] except KeyError: ins = {} outs = {} v[x.env.variant()] = (ins, outs) for a in getattr(x, 'inputs', []): try: ins[a.id].append(x) except KeyError: ins[a.id] = [x] for a in getattr(x, 'outputs', []): try: outs[a.id].append(x) except KeyError: outs[a.id] = [x] for (ins, outs) in v.values(): links = set(ins.iterkeys()).intersection(outs.iterkeys()) for k in links: for a in ins[k]: for b in outs[k]: a.set_run_after(b) def extract_deps(tasks): """file_deps: Infer additional dependencies from task input and output nodes and from implicit dependencies returned by the scanners - that will only work if all tasks are created this is aimed at people who have pathological builds and who do not care enough to implement the build dependencies properly with two loops over the list of tasks, do not expect this to be really fast """ # first reuse the function above extract_outputs(tasks) # map the output nodes to the tasks producing them out_to_task = {} for x in tasks: v = x.env.variant() try: lst = x.outputs except AttributeError: pass else: for node in lst: out_to_task[(v, node.id)] = x # map the dependencies found to the tasks compiled dep_to_task = {} for x in tasks: try: x.signature() except: # this is on purpose pass v = x.env.variant() key = x.unique_id() for k in x.generator.bld.node_deps.get(x.unique_id(), []): try: dep_to_task[(v, k.id)].append(x) except KeyError: dep_to_task[(v, k.id)] = [x] # now get the intersection deps = set(dep_to_task.keys()).intersection(set(out_to_task.keys())) # and add the dependencies from task to task for idx in deps: for k in dep_to_task[idx]: k.set_run_after(out_to_task[idx]) # cleanup, remove the signatures for x in tasks: try: delattr(x, 'cache_sig') except AttributeError: pass