diff options
Diffstat (limited to 'buildtools')
| -rw-r--r-- | buildtools/wafadmin/Tools/preproc.py | 7 |
1 files changed, 5 insertions, 2 deletions
diff --git a/buildtools/wafadmin/Tools/preproc.py b/buildtools/wafadmin/Tools/preproc.py index 5055456054..6c49326ec3 100644 --- a/buildtools/wafadmin/Tools/preproc.py +++ b/buildtools/wafadmin/Tools/preproc.py @@ -555,8 +555,7 @@ def parse_char(txt): except KeyError: raise PreprocError("could not parse char literal '%s'" % txt) @Utils.run_once -def tokenize(s): - """convert a string into a list of tokens (shlex.split does not apply to c/c++/d)""" +def tokenize_private(s): ret = [] for match in re_clexer.finditer(s): m = match.group @@ -591,6 +590,10 @@ def tokenize(s): break return ret +def tokenize(s): + """convert a string into a list of tokens (shlex.split does not apply to c/c++/d)""" + return tokenize_private(s)[:] + @Utils.run_once def define_name(line): return re_mac.match(line).group(0) |