waf-abi: auto-generate per-symbol versions from ABI files

This changes our version-script generation to use the ABI files that
are saved in git with each version number change of our public
libraries.

We use these ABI files to generate a linker version script that gives
the exact version number that each symbol was introduced. This
provides us with automatic fine grained symbol versioning.

Pair-Programmed-With: Jelmer Vernooij <jelmer@samba.org>
Pair-Programmed-With: Andrew Bartlett <abartlet@samba.org>

7 files changed, 92 insertions, 23 deletions

diff --git a/buildtools/wafsamba/samba_abi.py b/buildtools/wafsamba/samba_abi.py
index cdce58763c..4059fe3378 100644
--- a/buildtools/wafsamba/samba_abi.py
+++ b/buildtools/wafsamba/samba_abi.py
@@ -1,6 +1,6 @@
 # functions for handling ABI checking of libraries
 
-import Options, Utils, os, Logs, samba_utils, sys, Task, fnmatch, re
+import Options, Utils, os, Logs, samba_utils, sys, Task, fnmatch, re, Build
 from TaskGen import feature, before, after
 
 # these type maps cope with platform specific names for common types
@@ -117,7 +117,7 @@ if '--abi-check' in sys.argv:
 def abi_check(self):
     '''check that ABI matches saved signatures'''
     env = self.bld.env
-    if not env.ABI_CHECK or self.abi_file is None:
+    if not env.ABI_CHECK or self.abi_directory is None:
         return
 
     # if the platform doesn't support -fvisibility=hidden then the ABI
@@ -128,7 +128,80 @@ def abi_check(self):
     topsrc = self.bld.srcnode.abspath()
     abi_gen = os.path.join(topsrc, 'buildtools/scripts/abi_gen.sh')
 
+    abi_file = "%s/%s-%s.sigs" % (self.abi_directory, self.name, self.vnum)
+
     tsk = self.create_task('abi_check', self.link_task.outputs[0])
-    tsk.ABI_FILE = self.abi_file
+    tsk.ABI_FILE = abi_file
     tsk.ABI_MATCH = self.abi_match
     tsk.ABI_GEN = abi_gen
+
+
+
+def abi_process_file(fname, version, symmap):
+    '''process one ABI file, adding new symbols to the symmap'''
+    f = open(fname, mode='r')
+    for line in f:
+        symname = line.split(":")[0]
+        if not symname in symmap:
+            symmap[symname] = version
+    f.close()
+
+def abi_write_vscript(vscript, libname, vnum, symmap):
+    '''write a vscript file for a library in --version-script format'''
+
+    libname = libname.replace("-", "_").replace("+","_").upper()
+
+    invmap = {}
+    for s in symmap:
+        invmap.setdefault(symmap[s], []).append(s)
+
+    f = open(vscript, mode='w')
+    last_key = ""
+    for k in sorted(invmap):
+        symver = "%s_%s" % (libname, k)
+        if symver == version:
+            break
+        f.write("%s {\n\tglobal: \n" % symver)
+        for s in invmap[k]:
+            f.write("\t\t%s;\n" % s);
+        f.write("}%s;\n\n" % last_key)
+        last_key = " %s" % symver
+    f.write("%s { global: *;};\n" % version)
+    f.close()
+
+
+def abi_build_vscript(task):
+    '''generate a vscript file for our public libraries'''
+
+    tgt = task.outputs[0].bldpath(task.env)
+
+    symmap = {}
+
+    for f in task.inputs:
+        fname = f.abspath(task.env)
+        basename = os.path.basename(fname)
+        version = basename[len(task.env.LIBNAME)+1:-len(".sigs")]
+        abi_process_file(fname, version, symmap)
+    abi_write_vscript(tgt, task.env.LIBNAME, task.env.VNUM, symmap)
+
+
+def ABI_VSCRIPT(bld, libname, abi_directory, vnum, vscript):
+    '''generate a vscript file for our public libraries'''
+    if abi_directory:
+        source = bld.path.ant_glob('%s/%s-[0-9]*.sigs' % (abi_directory, libname))
+        source = sorted(source.split())
+    else:
+        source = ''
+
+    libname = libname.replace("-", "_").replace("+","_").upper()
+    version = version.replace("-", "_").replace("+","_").upper()
+
+    t = bld.SAMBA_GENERATOR(vscript,
+                            rule=abi_build_vscript,
+                            source=source,
+                            group='vscripts',
+                            target=vscript)
+    t.env.VNUM = vnum
+    t.env.LIBNAME = libname
+    t.vars = [vnum, vscript]
+Build.BuildContext.ABI_VSCRIPT = ABI_VSCRIPT
diff --git a/buildtools/wafsamba/samba_utils.py b/buildtools/wafsamba/samba_utils.py
index 5811cb3240..563009e093 100644
--- a/buildtools/wafsamba/samba_utils.py
+++ b/buildtools/wafsamba/samba_utils.py
@@ -339,6 +339,9 @@ def EXPAND_VARIABLES(ctx, varstr, vars=None):
             ret.append(EXPAND_VARIABLES(ctx, s, vars=vars))
         return ret
 
+    if not isinstance(varstr, str):
+        return varstr
+
     import Environment
     env = Environment.Environment()
     ret = varstr
diff --git a/buildtools/wafsamba/wafsamba.py b/buildtools/wafsamba/wafsamba.py
index e7a23fb169..aea9d2bf30 100644
--- a/buildtools/wafsamba/wafsamba.py
+++ b/buildtools/wafsamba/wafsamba.py
@@ -118,7 +118,7 @@ def SAMBA_LIBRARY(bld, libname, source,
                   target_type='LIBRARY',
                   bundled_extension=True,
                   link_name=None,
-                  abi_file=None,
+                  abi_directory=None,
                   abi_match=None,
                   hide_symbols=False,
                   manpages=None,
@@ -200,14 +200,6 @@ def SAMBA_LIBRARY(bld, libname, source,
     else:
         version = "%s_%s" % (Utils.g_module.APPNAME, Utils.g_module.VERSION.split(".")[0])
 
-    if bld.env.HAVE_LD_VERSION_SCRIPT:
-        vscript = "%s.vscript" % libname
-        bld.SAMBA_GENERATOR(vscript,
-                            rule="echo %s \{ global: \*\; \}\; > ${TGT}" % version.replace("-","_").replace("+","_").upper(),
-                            group='vscripts',
-                            target=vscript)
-        ldflags.append("-Wl,--version-script=%s/%s" % (bld.path.abspath(bld.env), vscript))
-
     features = 'cc cshlib symlink_lib install_lib'
     if target_type == 'PYTHON':
         features += ' pyext'
@@ -215,11 +207,13 @@ def SAMBA_LIBRARY(bld, libname, source,
         # this is quite strange. we should add pyext feature for pyext
         # but that breaks the build. This may be a bug in the waf python tool
         features += ' pyembed'
-    if abi_file:
-        features += ' abi_check'
 
-    if abi_file:
-        abi_file = os.path.join(bld.curdir, abi_file)
+    if abi_directory:
+        features += ' abi_check'
+        if bld.env.HAVE_LD_VERSION_SCRIPT:
+            vscript = "%s.vscript" % libname
+            bld.ABI_VSCRIPT(libname, abi_directory, vnum, vscript)
+            ldflags.append("-Wl,--version-script=%s/%s" % (bld.path.abspath(bld.env), vscript))
 
     bld.SET_BUILD_GROUP(group)
     t = bld(
@@ -238,7 +232,7 @@ def SAMBA_LIBRARY(bld, libname, source,
         name            = libname,
         samba_realname  = realname,
         samba_install   = install,
-        abi_file        = abi_file,
+        abi_directory   = abi_directory,
         abi_match       = abi_match,
         private_library = private_library,
         grouping_library=grouping_library
diff --git a/lib/talloc/wscript b/lib/talloc/wscript
index e83caa5f4e..3d359c1281 100644
--- a/lib/talloc/wscript
+++ b/lib/talloc/wscript
@@ -96,7 +96,7 @@ def build(bld):
         bld.SAMBA_LIBRARY('talloc',
                           'talloc.c',
                           deps='replace',
-                          abi_file='ABI/talloc-%s.sigs' % VERSION,
+                          abi_directory='ABI',
                           abi_match='talloc* _talloc*',
                           hide_symbols=True,
                           vnum=vnum,
@@ -108,7 +108,7 @@ def build(bld):
         bld.SAMBA_LIBRARY('pytalloc-util',
             source='pytalloc_util.c',
             public_deps='talloc',
-            abi_file='ABI/pytalloc-util-%s.sigs' % VERSION,
+            abi_directory='ABI',
             abi_match='py* Py*',
             pyext=True,
             vnum=vnum,
diff --git a/lib/tdb/wscript b/lib/tdb/wscript
index febd76b043..5fc64aa9e6 100644
--- a/lib/tdb/wscript
+++ b/lib/tdb/wscript
@@ -77,7 +77,7 @@ def build(bld):
                           COMMON_SRC,
                           deps='replace',
                           includes='include',
-                          abi_file='ABI/tdb-%s.sigs' % VERSION,
+                          abi_directory='ABI',
                           abi_match='tdb_*',
                           hide_symbols=True,
                           vnum=vnum,
diff --git a/lib/tevent/wscript b/lib/tevent/wscript
index 8f8820f5a1..81cc594fc2 100644
--- a/lib/tevent/wscript
+++ b/lib/tevent/wscript
@@ -81,7 +81,7 @@ def build(bld):
                           SRC,
                           deps='replace talloc',
                           enabled= not bld.CONFIG_SET('USING_SYSTEM_TEVENT'),
-                          abi_file='ABI/tevent-%s.sigs' % VERSION,
+                          abi_directory='ABI',
                           abi_match='tevent_* _tevent_*',
                           vnum=vnum,
                           private_library=private_library)
diff --git a/source4/lib/ldb/wscript b/source4/lib/ldb/wscript
index 8a4e83b69d..b785fefa10 100644
--- a/source4/lib/ldb/wscript
+++ b/source4/lib/ldb/wscript
@@ -121,7 +121,6 @@ def build(bld):
             # the current modules, not the installed ones
             modules_dir = os.path.join(os.getcwd(), 'bin/modules/ldb')
 
-        abi_file = 'ABI/ldb-%s.sigs' % VERSION
         abi_match = '!ldb_*module_ops !ldb_*backend_ops ldb_*'
 
         bld.SAMBA_LIBRARY('ldb',
@@ -134,7 +133,7 @@ def build(bld):
                           vnum=vnum,
                           private_library=private_library,
                           manpages='man/ldb.3',
-                          abi_file = abi_file,
+                          abi_directory = 'ABI',
                           abi_match = abi_match)