summaryrefslogtreecommitdiff
path: root/buildtools/wafadmin/Tools
diff options
context:
space:
mode:
Diffstat (limited to 'buildtools/wafadmin/Tools')
-rw-r--r--buildtools/wafadmin/Tools/preproc.py7
1 files changed, 5 insertions, 2 deletions
diff --git a/buildtools/wafadmin/Tools/preproc.py b/buildtools/wafadmin/Tools/preproc.py
index 5055456054..6c49326ec3 100644
--- a/buildtools/wafadmin/Tools/preproc.py
+++ b/buildtools/wafadmin/Tools/preproc.py
@@ -555,8 +555,7 @@ def parse_char(txt):
except KeyError: raise PreprocError("could not parse char literal '%s'" % txt)
@Utils.run_once
-def tokenize(s):
- """convert a string into a list of tokens (shlex.split does not apply to c/c++/d)"""
+def tokenize_private(s):
ret = []
for match in re_clexer.finditer(s):
m = match.group
@@ -591,6 +590,10 @@ def tokenize(s):
break
return ret
+def tokenize(s):
+ """convert a string into a list of tokens (shlex.split does not apply to c/c++/d)"""
+ return tokenize_private(s)[:]
+
@Utils.run_once
def define_name(line):
return re_mac.match(line).group(0)