lib: import ccan modules for tdb2

Imported from git://git.ozlabs.org/~ccan/ccan init-1161-g661d41f

Signed-off-by: Rusty Russell <rusty@rustcorp.com.au>

9 files changed, 1713 insertions, 0 deletions

diff --git a/lib/ccan/htable/LICENSE b/lib/ccan/htable/LICENSE
new file mode 100644
index 0000000000..d511905c16
--- /dev/null
+++ b/lib/ccan/htable/LICENSE
@@ -0,0 +1,339 @@
+		    GNU GENERAL PUBLIC LICENSE
+		       Version 2, June 1991
+
+ Copyright (C) 1989, 1991 Free Software Foundation, Inc.,
+ 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+ Everyone is permitted to copy and distribute verbatim copies
+ of this license document, but changing it is not allowed.
+
+			    Preamble
+
+  The licenses for most software are designed to take away your
+freedom to share and change it.  By contrast, the GNU General Public
+License is intended to guarantee your freedom to share and change free
+software--to make sure the software is free for all its users.  This
+General Public License applies to most of the Free Software
+Foundation's software and to any other program whose authors commit to
+using it.  (Some other Free Software Foundation software is covered by
+the GNU Lesser General Public License instead.)  You can apply it to
+your programs, too.
+
+  When we speak of free software, we are referring to freedom, not
+price.  Our General Public Licenses are designed to make sure that you
+have the freedom to distribute copies of free software (and charge for
+this service if you wish), that you receive source code or can get it
+if you want it, that you can change the software or use pieces of it
+in new free programs; and that you know you can do these things.
+
+  To protect your rights, we need to make restrictions that forbid
+anyone to deny you these rights or to ask you to surrender the rights.
+These restrictions translate to certain responsibilities for you if you
+distribute copies of the software, or if you modify it.
+
+  For example, if you distribute copies of such a program, whether
+gratis or for a fee, you must give the recipients all the rights that
+you have.  You must make sure that they, too, receive or can get the
+source code.  And you must show them these terms so they know their
+rights.
+
+  We protect your rights with two steps: (1) copyright the software, and
+(2) offer you this license which gives you legal permission to copy,
+distribute and/or modify the software.
+
+  Also, for each author's protection and ours, we want to make certain
+that everyone understands that there is no warranty for this free
+software.  If the software is modified by someone else and passed on, we
+want its recipients to know that what they have is not the original, so
+that any problems introduced by others will not reflect on the original
+authors' reputations.
+
+  Finally, any free program is threatened constantly by software
+patents.  We wish to avoid the danger that redistributors of a free
+program will individually obtain patent licenses, in effect making the
+program proprietary.  To prevent this, we have made it clear that any
+patent must be licensed for everyone's free use or not licensed at all.
+
+  The precise terms and conditions for copying, distribution and
+modification follow.
+
+		    GNU GENERAL PUBLIC LICENSE
+   TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION
+
+  0. This License applies to any program or other work which contains
+a notice placed by the copyright holder saying it may be distributed
+under the terms of this General Public License.  The "Program", below,
+refers to any such program or work, and a "work based on the Program"
+means either the Program or any derivative work under copyright law:
+that is to say, a work containing the Program or a portion of it,
+either verbatim or with modifications and/or translated into another
+language.  (Hereinafter, translation is included without limitation in
+the term "modification".)  Each licensee is addressed as "you".
+
+Activities other than copying, distribution and modification are not
+covered by this License; they are outside its scope.  The act of
+running the Program is not restricted, and the output from the Program
+is covered only if its contents constitute a work based on the
+Program (independent of having been made by running the Program).
+Whether that is true depends on what the Program does.
+
+  1. You may copy and distribute verbatim copies of the Program's
+source code as you receive it, in any medium, provided that you
+conspicuously and appropriately publish on each copy an appropriate
+copyright notice and disclaimer of warranty; keep intact all the
+notices that refer to this License and to the absence of any warranty;
+and give any other recipients of the Program a copy of this License
+along with the Program.
+
+You may charge a fee for the physical act of transferring a copy, and
+you may at your option offer warranty protection in exchange for a fee.
+
+  2. You may modify your copy or copies of the Program or any portion
+of it, thus forming a work based on the Program, and copy and
+distribute such modifications or work under the terms of Section 1
+above, provided that you also meet all of these conditions:
+
+    a) You must cause the modified files to carry prominent notices
+    stating that you changed the files and the date of any change.
+
+    b) You must cause any work that you distribute or publish, that in
+    whole or in part contains or is derived from the Program or any
+    part thereof, to be licensed as a whole at no charge to all third
+    parties under the terms of this License.
+
+    c) If the modified program normally reads commands interactively
+    when run, you must cause it, when started running for such
+    interactive use in the most ordinary way, to print or display an
+    announcement including an appropriate copyright notice and a
+    notice that there is no warranty (or else, saying that you provide
+    a warranty) and that users may redistribute the program under
+    these conditions, and telling the user how to view a copy of this
+    License.  (Exception: if the Program itself is interactive but
+    does not normally print such an announcement, your work based on
+    the Program is not required to print an announcement.)
+
+These requirements apply to the modified work as a whole.  If
+identifiable sections of that work are not derived from the Program,
+and can be reasonably considered independent and separate works in
+themselves, then this License, and its terms, do not apply to those
+sections when you distribute them as separate works.  But when you
+distribute the same sections as part of a whole which is a work based
+on the Program, the distribution of the whole must be on the terms of
+this License, whose permissions for other licensees extend to the
+entire whole, and thus to each and every part regardless of who wrote it.
+
+Thus, it is not the intent of this section to claim rights or contest
+your rights to work written entirely by you; rather, the intent is to
+exercise the right to control the distribution of derivative or
+collective works based on the Program.
+
+In addition, mere aggregation of another work not based on the Program
+with the Program (or with a work based on the Program) on a volume of
+a storage or distribution medium does not bring the other work under
+the scope of this License.
+
+  3. You may copy and distribute the Program (or a work based on it,
+under Section 2) in object code or executable form under the terms of
+Sections 1 and 2 above provided that you also do one of the following:
+
+    a) Accompany it with the complete corresponding machine-readable
+    source code, which must be distributed under the terms of Sections
+    1 and 2 above on a medium customarily used for software interchange; or,
+
+    b) Accompany it with a written offer, valid for at least three
+    years, to give any third party, for a charge no more than your
+    cost of physically performing source distribution, a complete
+    machine-readable copy of the corresponding source code, to be
+    distributed under the terms of Sections 1 and 2 above on a medium
+    customarily used for software interchange; or,
+
+    c) Accompany it with the information you received as to the offer
+    to distribute corresponding source code.  (This alternative is
+    allowed only for noncommercial distribution and only if you
+    received the program in object code or executable form with such
+    an offer, in accord with Subsection b above.)
+
+The source code for a work means the preferred form of the work for
+making modifications to it.  For an executable work, complete source
+code means all the source code for all modules it contains, plus any
+associated interface definition files, plus the scripts used to
+control compilation and installation of the executable.  However, as a
+special exception, the source code distributed need not include
+anything that is normally distributed (in either source or binary
+form) with the major components (compiler, kernel, and so on) of the
+operating system on which the executable runs, unless that component
+itself accompanies the executable.
+
+If distribution of executable or object code is made by offering
+access to copy from a designated place, then offering equivalent
+access to copy the source code from the same place counts as
+distribution of the source code, even though third parties are not
+compelled to copy the source along with the object code.
+
+  4. You may not copy, modify, sublicense, or distribute the Program
+except as expressly provided under this License.  Any attempt
+otherwise to copy, modify, sublicense or distribute the Program is
+void, and will automatically terminate your rights under this License.
+However, parties who have received copies, or rights, from you under
+this License will not have their licenses terminated so long as such
+parties remain in full compliance.
+
+  5. You are not required to accept this License, since you have not
+signed it.  However, nothing else grants you permission to modify or
+distribute the Program or its derivative works.  These actions are
+prohibited by law if you do not accept this License.  Therefore, by
+modifying or distributing the Program (or any work based on the
+Program), you indicate your acceptance of this License to do so, and
+all its terms and conditions for copying, distributing or modifying
+the Program or works based on it.
+
+  6. Each time you redistribute the Program (or any work based on the
+Program), the recipient automatically receives a license from the
+original licensor to copy, distribute or modify the Program subject to
+these terms and conditions.  You may not impose any further
+restrictions on the recipients' exercise of the rights granted herein.
+You are not responsible for enforcing compliance by third parties to
+this License.
+
+  7. If, as a consequence of a court judgment or allegation of patent
+infringement or for any other reason (not limited to patent issues),
+conditions are imposed on you (whether by court order, agreement or
+otherwise) that contradict the conditions of this License, they do not
+excuse you from the conditions of this License.  If you cannot
+distribute so as to satisfy simultaneously your obligations under this
+License and any other pertinent obligations, then as a consequence you
+may not distribute the Program at all.  For example, if a patent
+license would not permit royalty-free redistribution of the Program by
+all those who receive copies directly or indirectly through you, then
+the only way you could satisfy both it and this License would be to
+refrain entirely from distribution of the Program.
+
+If any portion of this section is held invalid or unenforceable under
+any particular circumstance, the balance of the section is intended to
+apply and the section as a whole is intended to apply in other
+circumstances.
+
+It is not the purpose of this section to induce you to infringe any
+patents or other property right claims or to contest validity of any
+such claims; this section has the sole purpose of protecting the
+integrity of the free software distribution system, which is
+implemented by public license practices.  Many people have made
+generous contributions to the wide range of software distributed
+through that system in reliance on consistent application of that
+system; it is up to the author/donor to decide if he or she is willing
+to distribute software through any other system and a licensee cannot
+impose that choice.
+
+This section is intended to make thoroughly clear what is believed to
+be a consequence of the rest of this License.
+
+  8. If the distribution and/or use of the Program is restricted in
+certain countries either by patents or by copyrighted interfaces, the
+original copyright holder who places the Program under this License
+may add an explicit geographical distribution limitation excluding
+those countries, so that distribution is permitted only in or among
+countries not thus excluded.  In such case, this License incorporates
+the limitation as if written in the body of this License.
+
+  9. The Free Software Foundation may publish revised and/or new versions
+of the General Public License from time to time.  Such new versions will
+be similar in spirit to the present version, but may differ in detail to
+address new problems or concerns.
+
+Each version is given a distinguishing version number.  If the Program
+specifies a version number of this License which applies to it and "any
+later version", you have the option of following the terms and conditions
+either of that version or of any later version published by the Free
+Software Foundation.  If the Program does not specify a version number of
+this License, you may choose any version ever published by the Free Software
+Foundation.
+
+  10. If you wish to incorporate parts of the Program into other free
+programs whose distribution conditions are different, write to the author
+to ask for permission.  For software which is copyrighted by the Free
+Software Foundation, write to the Free Software Foundation; we sometimes
+make exceptions for this.  Our decision will be guided by the two goals
+of preserving the free status of all derivatives of our free software and
+of promoting the sharing and reuse of software generally.
+
+			    NO WARRANTY
+
+  11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY
+FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW.  EXCEPT WHEN
+OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES
+PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED
+OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE.  THE ENTIRE RISK AS
+TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU.  SHOULD THE
+PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING,
+REPAIR OR CORRECTION.
+
+  12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING
+WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR
+REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES,
+INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING
+OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED
+TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY
+YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER
+PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE
+POSSIBILITY OF SUCH DAMAGES.
+
+		     END OF TERMS AND CONDITIONS
+
+	    How to Apply These Terms to Your New Programs
+
+  If you develop a new program, and you want it to be of the greatest
+possible use to the public, the best way to achieve this is to make it
+free software which everyone can redistribute and change under these terms.
+
+  To do so, attach the following notices to the program.  It is safest
+to attach them to the start of each source file to most effectively
+convey the exclusion of warranty; and each file should have at least
+the "copyright" line and a pointer to where the full notice is found.
+
+    <one line to give the program's name and a brief idea of what it does.>
+    Copyright (C) <year>  <name of author>
+
+    This program is free software; you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation; either version 2 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License along
+    with this program; if not, write to the Free Software Foundation, Inc.,
+    51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+Also add information on how to contact you by electronic and paper mail.
+
+If the program is interactive, make it output a short notice like this
+when it starts in an interactive mode:
+
+    Gnomovision version 69, Copyright (C) year name of author
+    Gnomovision comes with ABSOLUTELY NO WARRANTY; for details type `show w'.
+    This is free software, and you are welcome to redistribute it
+    under certain conditions; type `show c' for details.
+
+The hypothetical commands `show w' and `show c' should show the appropriate
+parts of the General Public License.  Of course, the commands you use may
+be called something other than `show w' and `show c'; they could even be
+mouse-clicks or menu items--whatever suits your program.
+
+You should also get your employer (if you work as a programmer) or your
+school, if any, to sign a "copyright disclaimer" for the program, if
+necessary.  Here is a sample; alter the names:
+
+  Yoyodyne, Inc., hereby disclaims all copyright interest in the program
+  `Gnomovision' (which makes passes at compilers) written by James Hacker.
+
+  <signature of Ty Coon>, 1 April 1989
+  Ty Coon, President of Vice
+
+This General Public License does not permit incorporating your program into
+proprietary programs.  If your program is a subroutine library, you may
+consider it more useful to permit linking proprietary applications with the
+library.  If this is what you want to do, use the GNU Lesser General
+Public License instead of this License.
diff --git a/lib/ccan/htable/_info b/lib/ccan/htable/_info
new file mode 100644
index 0000000000..8dabe46a50
--- /dev/null
+++ b/lib/ccan/htable/_info
@@ -0,0 +1,115 @@
+#include <string.h>
+#include <stdio.h>
+
+/**
+ * htable - hash table routines
+ *
+ * A hash table is an efficient structure for looking up keys.  This version
+ * grows with usage and allows efficient deletion.
+ *
+ * Example:
+ *	#include <ccan/htable/htable.h>
+ *	#include <ccan/hash/hash.h>
+ *	#include <stdio.h>
+ *	#include <err.h>
+ *	#include <string.h>
+ *
+ *	struct name_to_digit {
+ *		const char *name;
+ *		unsigned int val;
+ *	};
+ *
+ *	static struct name_to_digit map[] = {
+ *		{ "zero", 0},
+ *		{ "one", 1 },
+ *		{ "two", 2 },
+ *		{ "three", 3 },
+ *		{ "four", 4 },
+ *		{ "five", 5 },
+ *		{ "six", 6 },
+ *		{ "seven", 7 },
+ *		{ "eight", 8 },
+ *		{ "nine", 9 }
+ *	};
+ *
+ *	// Wrapper for rehash function pointer.
+ *	static size_t rehash(const void *e, void *unused)
+ *	{
+ *		return hash_string(((struct name_to_digit *)e)->name);
+ *	}
+ *
+ *	// Comparison function.
+ *	static bool streq(const void *e, void *string)
+ *	{
+ *		return strcmp(((struct name_to_digit *)e)->name, string) == 0;
+ *	}
+ *
+ *	// We let them add their own aliases, eg. --alias=v=5
+ *	static void add_alias(struct htable *ht, const char *alias)
+ *	{
+ *		char *eq;
+ *		struct name_to_digit *n;
+ *
+ *		n = malloc(sizeof(*n));
+ *		n->name = strdup(alias);
+ *
+ *		eq = strchr(n->name, '=');
+ *		if (!eq || ((n->val = atoi(eq+1)) == 0 && !strcmp(eq+1, "0")))
+ *			errx(1, "Usage: --alias=<name>=<value>");
+ *		*eq = '\0';
+ *		htable_add(ht, hash_string(n->name), n);
+ *	}
+ *
+ *	int main(int argc, char *argv[])
+ *	{
+ *		struct htable *ht;
+ *		unsigned int i;
+ *		unsigned long val;
+ *
+ *		if (argc < 2)
+ *			errx(1, "Usage: %s [--alias=<name>=<val>]... <str>...",
+ *			     argv[0]);
+ *
+ *		// Create and populate hash table.
+ *		ht = htable_new(rehash, NULL);
+ *		for (i = 0; i < sizeof(map)/sizeof(map[0]); i++)
+ *			htable_add(ht, hash_string(map[i].name), &map[i]);
+ *
+ *		// Add any aliases to the hash table.
+ *		for (i = 1; i < argc; i++) {
+ *			if (!strncmp(argv[i], "--alias=", strlen("--alias=")))
+ *				add_alias(ht, argv[i] + strlen("--alias="));
+ *			else
+ *				break;
+ *		}
+ *
+ *		// Find the other args in the hash table.
+ *		for (val = 0; i < argc; i++) {
+ *			struct name_to_digit *n;
+ *			n = htable_get(ht, hash_string(argv[i]),
+ *				       streq, argv[i]);
+ *			if (!n)
+ *				errx(1, "Invalid digit name %s", argv[i]);
+ *			// Append it to the value we are building up.
+ *			val *= 10;
+ *			val += n->val;
+ *		}
+ *		printf("%lu\n", val);
+ *		return 0;
+ *	}
+ *
+ * License: GPLv2 (or later)
+ * Author: Rusty Russell <rusty@rustcorp.com.au>
+ */
+int main(int argc, char *argv[])
+{
+	if (argc != 2)
+		return 1;
+
+	if (strcmp(argv[1], "depends") == 0) {
+		printf("ccan/compiler\n");
+		return 0;
+	}
+
+	return 1;
+}
diff --git a/lib/ccan/htable/htable.c b/lib/ccan/htable/htable.c
new file mode 100644
index 0000000000..a15c54d795
--- /dev/null
+++ b/lib/ccan/htable/htable.c
@@ -0,0 +1,290 @@
+#include <ccan/htable/htable.h>
+#include <ccan/compiler/compiler.h>
+#include <stdint.h>
+#include <stdlib.h>
+#include <limits.h>
+#include <stdbool.h>
+#include <assert.h>
+
+/* This means a struct htable takes at least 512 bytes / 1k (32/64 bits). */
+#define HTABLE_BASE_BITS 7
+
+/* We use 0x1 as deleted marker. */
+#define HTABLE_DELETED (0x1)
+
+struct htable {
+	size_t (*rehash)(const void *elem, void *priv);
+	void *priv;
+	unsigned int bits;
+	size_t elems, deleted, max, max_with_deleted;
+	/* These are the bits which are the same in all pointers. */
+	uintptr_t common_mask, common_bits;
+	uintptr_t perfect_bit;
+	uintptr_t *table;
+};
+
+/* We clear out the bits which are always the same, and put metadata there. */
+static inline uintptr_t get_extra_ptr_bits(const struct htable *ht,
+					   uintptr_t e)
+{
+	return e & ht->common_mask;
+}
+
+static inline void *get_raw_ptr(const struct htable *ht, uintptr_t e)
+{
+	return (void *)((e & ~ht->common_mask) | ht->common_bits);
+}
+
+static inline uintptr_t make_hval(const struct htable *ht,
+				  const void *p, uintptr_t bits)
+{
+	return ((uintptr_t)p & ~ht->common_mask) | bits;
+}
+
+static inline bool entry_is_valid(uintptr_t e)
+{
+	return e > HTABLE_DELETED;
+}
+
+static inline uintptr_t get_hash_ptr_bits(const struct htable *ht,
+					  size_t hash)
+{
+	/* Shuffling the extra bits (as specified in mask) down the
+	 * end is quite expensive.  But the lower bits are redundant, so
+	 * we fold the value first. */
+	return (hash ^ (hash >> ht->bits))
+		& ht->common_mask & ~ht->perfect_bit;
+}
+
+struct htable *htable_new(size_t (*rehash)(const void *elem, void *priv),
+			  void *priv)
+{
+	struct htable *ht = malloc(sizeof(struct htable));
+	if (ht) {
+		ht->bits = HTABLE_BASE_BITS;
+		ht->rehash = rehash;
+		ht->priv = priv;
+		ht->elems = 0;
+		ht->deleted = 0;
+		ht->max = ((size_t)1 << ht->bits) * 3 / 4;
+		ht->max_with_deleted = ((size_t)1 << ht->bits) * 9 / 10;
+		/* This guarantees we enter update_common first add. */
+		ht->common_mask = -1;
+		ht->common_bits = 0;
+		ht->perfect_bit = 0;
+		ht->table = calloc(1 << ht->bits, sizeof(uintptr_t));
+		if (!ht->table) {
+			free(ht);
+			ht = NULL;
+		}
+	}
+	return ht;
+}
+
+void htable_free(const struct htable *ht)
+{
+	free((void *)ht->table);
+	free((void *)ht);
+}
+
+static size_t hash_bucket(const struct htable *ht, size_t h)
+{
+	return h & ((1 << ht->bits)-1);
+}
+
+static void *htable_val(const struct htable *ht,
+			struct htable_iter *i, size_t hash, uintptr_t perfect)
+{
+	uintptr_t h2 = get_hash_ptr_bits(ht, hash) | perfect;
+
+	while (ht->table[i->off]) {
+		if (ht->table[i->off] != HTABLE_DELETED) {
+			if (get_extra_ptr_bits(ht, ht->table[i->off]) == h2)
+				return get_raw_ptr(ht, ht->table[i->off]);
+		}
+		i->off = (i->off + 1) & ((1 << ht->bits)-1);
+		h2 &= ~perfect;
+	}
+	return NULL;
+}
+
+void *htable_firstval(const struct htable *ht,
+		      struct htable_iter *i, size_t hash)
+{
+	i->off = hash_bucket(ht, hash);
+	return htable_val(ht, i, hash, ht->perfect_bit);
+}
+
+void *htable_nextval(const struct htable *ht,
+		     struct htable_iter *i, size_t hash)
+{
+	i->off = (i->off + 1) & ((1 << ht->bits)-1);
+	return htable_val(ht, i, hash, 0);
+}
+
+void *htable_first(const struct htable *ht, struct htable_iter *i)
+{
+	for (i->off = 0; i->off < (size_t)1 << ht->bits; i->off++) {
+		if (entry_is_valid(ht->table[i->off]))
+			return get_raw_ptr(ht, ht->table[i->off]);
+	}
+	return NULL;
+}
+
+void *htable_next(const struct htable *ht, struct htable_iter *i)
+{
+	for (i->off++; i->off < (size_t)1 << ht->bits; i->off++) {
+		if (entry_is_valid(ht->table[i->off]))
+			return get_raw_ptr(ht, ht->table[i->off]);
+	}
+	return NULL;
+}
+
+/* This does not expand the hash table, that's up to caller. */
+static void ht_add(struct htable *ht, const void *new, size_t h)
+{
+	size_t i;
+	uintptr_t perfect = ht->perfect_bit;
+
+	i = hash_bucket(ht, h);
+
+	while (entry_is_valid(ht->table[i])) {
+		perfect = 0;
+		i = (i + 1) & ((1 << ht->bits)-1);
+	}
+	ht->table[i] = make_hval(ht, new, get_hash_ptr_bits(ht, h)|perfect);
+}
+
+static COLD bool double_table(struct htable *ht)
+{
+	unsigned int i;
+	size_t oldnum = (size_t)1 << ht->bits;
+	uintptr_t *oldtable, e;
+
+	oldtable = ht->table;
+	ht->table = calloc(1 << (ht->bits+1), sizeof(size_t));
+	if (!ht->table) {
+		ht->table = oldtable;
+		return false;
+	}
+	ht->bits++;
+	ht->max *= 2;
+	ht->max_with_deleted *= 2;
+
+	/* If we lost our "perfect bit", get it back now. */
+	if (!ht->perfect_bit && ht->common_mask) {
+		for (i = 0; i < sizeof(ht->common_mask) * CHAR_BIT; i++) {
+			if (ht->common_mask & ((size_t)1 << i)) {
+				ht->perfect_bit = (size_t)1 << i;
+				break;
+			}
+		}
+	}
+
+	for (i = 0; i < oldnum; i++) {
+		if (entry_is_valid(e = oldtable[i])) {
+			void *p = get_raw_ptr(ht, e);
+			ht_add(ht, p, ht->rehash(p, ht->priv));
+		}
+	}
+	ht->deleted = 0;
+	free(oldtable);
+	return true;
+}
+
+static COLD void rehash_table(struct htable *ht)
+{
+	size_t start, i;
+	uintptr_t e;
+
+	/* Beware wrap cases: we need to start from first empty bucket. */
+	for (start = 0; ht->table[start]; start++);
+
+	for (i = 0; i < (size_t)1 << ht->bits; i++) {
+		size_t h = (i + start) & ((1 << ht->bits)-1);
+		e = ht->table[h];
+		if (!e)
+			continue;
+		if (e == HTABLE_DELETED)
+			ht->table[h] = 0;
+		else if (!(e & ht->perfect_bit)) {
+			void *p = get_raw_ptr(ht, e);
+			ht->table[h] = 0;
+			ht_add(ht, p, ht->rehash(p, ht->priv));
+		}
+	}
+	ht->deleted = 0;
+}
+
+/* We stole some bits, now we need to put them back... */
+static COLD void update_common(struct htable *ht, const void *p)
+{
+	unsigned int i;
+	uintptr_t maskdiff, bitsdiff;
+
+	if (ht->elems == 0) {
+		ht->common_mask = -1;
+		ht->common_bits = (uintptr_t)p;
+		ht->perfect_bit = 1;
+		return;
+	}
+
+	/* Find bits which are unequal to old common set. */
+	maskdiff = ht->common_bits ^ ((uintptr_t)p & ht->common_mask);
+
+	/* These are the bits which go there in existing entries. */
+	bitsdiff = ht->common_bits & maskdiff;
+
+	for (i = 0; i < (size_t)1 << ht->bits; i++) {
+		if (!entry_is_valid(ht->table[i]))
+			continue;
+		/* Clear the bits no longer in the mask, set them as
+		 * expected. */
+		ht->table[i] &= ~maskdiff;
+		ht->table[i] |= bitsdiff;
+	}
+
+	/* Take away those bits from our mask, bits and perfect bit. */
+	ht->common_mask &= ~maskdiff;
+	ht->common_bits &= ~maskdiff;
+	ht->perfect_bit &= ~maskdiff;
+}
+
+bool htable_add(struct htable *ht, size_t hash, const void *p)
+{
+	if (ht->elems+1 > ht->max && !double_table(ht))
+		return false;
+	if (ht->elems+1 + ht->deleted > ht->max_with_deleted)
+		rehash_table(ht);
+	assert(p);
+	if (((uintptr_t)p & ht->common_mask) != ht->common_bits)
+		update_common(ht, p);
+
+	ht_add(ht, p, hash);
+	ht->elems++;
+	return true;
+}
+
+bool htable_del(struct htable *ht, size_t h, const void *p)
+{
+	struct htable_iter i;
+	void *c;
+
+	for (c = htable_firstval(ht,&i,h); c; c = htable_nextval(ht,&i,h)) {
+		if (c == p) {
+			htable_delval(ht, &i);
+			return true;
+		}
+	}
+	return false;
+}
+
+void htable_delval(struct htable *ht, struct htable_iter *i)
+{
+	assert(i->off < (size_t)1 << ht->bits);
+	assert(entry_is_valid(ht->table[i->off]));
+
+	ht->elems--;
+	ht->table[i->off] = HTABLE_DELETED;
+	ht->deleted++;
+}
diff --git a/lib/ccan/htable/htable.h b/lib/ccan/htable/htable.h
new file mode 100644
index 0000000000..b68442972c
--- /dev/null
+++ b/lib/ccan/htable/htable.h
@@ -0,0 +1,138 @@
+#ifndef CCAN_HTABLE_H
+#define CCAN_HTABLE_H
+#include "config.h"
+#include <stdbool.h>
+#include <stdlib.h>
+
+struct htable;
+
+/**
+ * htable_new - allocate a hash tree.
+ * @rehash: hash function to use for rehashing.
+ * @priv: private argument to @rehash function.
+ */
+struct htable *htable_new(size_t (*hash)(const void *elem, void *priv),
+			  void *priv);
+
+/**
+ * htable_free - dellocate a hash tree.
+ *
+ * This doesn't do anything to any pointers left in it.
+ */
+void htable_free(const struct htable *);
+
+/**
+ * htable_rehash - use a hashtree's rehash function
+ * @elem: the argument to rehash()
+ *
+ */
+size_t htable_rehash(const void *elem);
+
+/**
+ * htable_add - add a pointer into a hash tree.
+ * @ht: the htable
+ * @hash: the hash value of the object
+ * @p: the non-NULL pointer
+ *
+ * Also note that this can only fail due to allocation failure.  Otherwise, it
+ * returns true.
+ */
+bool htable_add(struct htable *ht, size_t hash, const void *p);
+
+/**
+ * htable_del - remove a pointer from a hash tree
+ * @ht: the htable
+ * @hash: the hash value of the object
+ * @p: the pointer
+ *
+ * Returns true if the pointer was found (and deleted).
+ */
+bool htable_del(struct htable *ht, size_t hash, const void *p);
+
+/**
+ * struct htable_iter - iterator or htable_first or htable_firstval etc.
+ *
+ * This refers to a location inside the hashtable.
+ */
+struct htable_iter {
+	size_t off;
+};
+
+/**
+ * htable_firstval - find a candidate for a given hash value
+ * @htable: the hashtable
+ * @i: the struct htable_iter to initialize
+ * @hash: the hash value
+ *
+ * You'll need to check the value is what you want; returns NULL if none.
+ * See Also:
+ *	htable_delval()
+ */
+void *htable_firstval(const struct htable *htable,
+		      struct htable_iter *i, size_t hash);
+
+/**
+ * htable_nextval - find another candidate for a given hash value
+ * @htable: the hashtable
+ * @i: the struct htable_iter to initialize
+ * @hash: the hash value
+ *
+ * You'll need to check the value is what you want; returns NULL if no more.
+ */
+void *htable_nextval(const struct htable *htable,
+		     struct htable_iter *i, size_t hash);
+
+/**
+ * htable_get - find an entry in the hash table
+ * @ht: the hashtable
+ * @h: the hash value of the entry
+ * @cmp: the comparison function
+ * @ptr: the pointer to hand to the comparison function.
+ *
+ * Convenient inline wrapper for htable_firstval/htable_nextval loop.
+ */
+static inline void *htable_get(const struct htable *ht,
+			       size_t h,
+			       bool (*cmp)(const void *candidate, void *ptr),
+			       const void *ptr)
+{
+	struct htable_iter i;
+	void *c;
+
+	for (c = htable_firstval(ht,&i,h); c; c = htable_nextval(ht,&i,h)) {
+		if (cmp(c, (void *)ptr))
+			return c;
+	}
+	return NULL;
+}
+
+/**
+ * htable_first - find an entry in the hash table
+ * @ht: the hashtable
+ * @i: the struct htable_iter to initialize
+ *
+ * Get an entry in the hashtable; NULL if empty.
+ */
+void *htable_first(const struct htable *htable, struct htable_iter *i);
+
+/**
+ * htable_next - find another entry in the hash table
+ * @ht: the hashtable
+ * @i: the struct htable_iter to use
+ *
+ * Get another entry in the hashtable; NULL if all done.
+ * This is usually used after htable_first or prior non-NULL htable_next.
+ */
+void *htable_next(const struct htable *htable, struct htable_iter *i);
+
+/**
+ * htable_delval - remove an iterated pointer from a hash tree
+ * @ht: the htable
+ * @i: the htable_iter
+ *
+ * Usually used to delete a hash entry after it has been found with
+ * htable_firstval etc.
+ */
+void htable_delval(struct htable *ht, struct htable_iter *i);
+
+#endif /* CCAN_HTABLE_H */
diff --git a/lib/ccan/htable/htable_type.h b/lib/ccan/htable/htable_type.h
new file mode 100644
index 0000000000..0d9e3fbb2d
--- /dev/null
+++ b/lib/ccan/htable/htable_type.h
@@ -0,0 +1,97 @@
+#ifndef CCAN_HTABLE_TYPE_H
+#define CCAN_HTABLE_TYPE_H
+#include <ccan/htable/htable.h>
+#include "config.h"
+
+/**
+ * HTABLE_DEFINE_TYPE - create a set of htable ops for a type
+ * @type: a type whose pointers will be values in the hash.
+ * @keyof: a function/macro to extract a key from a @type element.
+ * @hashfn: a hash function for a @key
+ * @cmpfn: a comparison function for two keyof()s.
+ * @name: a name for all the functions to define (of form htable_<name>_*)
+ *
+ * NULL values may not be placed into the hash table.
+ *
+ * The following wrapper functions are defined; each one is a
+ * simplified version of the htable.h equivalent:
+ *
+ *	// Creating and freeing.
+ *	struct htable_@name *htable_@name_new(void);
+ *	void htable_@name_free(const struct htable_@name *ht);
+ *
+ *	// Add, delete and find.
+ *	bool htable_@name_add(struct htable_@name *ht, const type *e);
+ *	bool htable_@name_del(struct htable_@name *ht, const type *e);
+ *	bool htable_@name_delkey(struct htable_@name *ht, const ktype *k);
+ *	type *htable_@name_get(const struct htable_@name *ht, const ktype *k);
+ *
+ *	// Iteration.
+ *	struct htable_@name_iter;
+ *	type *htable_@name_first(const struct htable_@name *ht,
+ *				 struct htable_@name_iter *i);
+ *	type *htable_@name_next(const struct htable_@name *ht,
+ *				struct htable_@name_iter *i);
+ */
+#define HTABLE_DEFINE_TYPE(type, keyof, hashfn, cmpfn, name)		\
+struct htable_##name;							\
+struct htable_##name##_iter { struct htable_iter i; };			\
+static inline size_t htable_##name##_hash(const void *elem, void *priv)	\
+{									\
+	return hashfn(keyof((const type *)elem));			\
+}									\
+static inline struct htable_##name *htable_##name##_new(void)		\
+{									\
+	return (struct htable_##name *)htable_new(htable_##name##_hash,	\
+						  NULL);		\
+}									\
+static inline void htable_##name##_free(const struct htable_##name *ht)	\
+{									\
+	htable_free((const struct htable *)ht);				\
+}									\
+static inline bool htable_##name##_add(struct htable_##name *ht,	\
+				       const type *elem)		\
+{									\
+	return htable_add((struct htable *)ht, hashfn(keyof(elem)), elem); \
+}									\
+static inline bool htable_##name##_del(const struct htable_##name *ht,	\
+				       const type *elem)		\
+{									\
+	return htable_del((struct htable *)ht, hashfn(keyof(elem)), elem); \
+}									\
+static inline type *htable_##name##_get(const struct htable_##name *ht,	\
+					const HTABLE_KTYPE(keyof) k)	\
+{									\
+	/* Typecheck for cmpfn */					\
+	(void)sizeof(cmpfn((const type *)NULL,				\
+			   keyof((const type *)NULL)));			\
+	return (type *)htable_get((const struct htable *)ht,		\
+				  hashfn(k),				\
+				  (bool (*)(const void *, void *))(cmpfn), \
+				  k);					\
+}									\
+static inline bool htable_##name##_delkey(struct htable_##name *ht,	\
+					  const HTABLE_KTYPE(keyof) k) \
+{									\
+	type *elem = htable_##name##_get(ht, k);			\
+	if (elem)							\
+		return htable_##name##_del(ht, elem);			\
+	return false;							\
+}									\
+static inline type *htable_##name##_first(const struct htable_##name *ht, \
+					  struct htable_##name##_iter *iter) \
+{									\
+	return htable_first((const struct htable *)ht, &iter->i);	\
+}									\
+static inline type *htable_##name##_next(const struct htable_##name *ht, \
+					 struct htable_##name##_iter *iter) \
+{									\
+	return htable_next((const struct htable *)ht, &iter->i);	\
+}
+
+#if HAVE_TYPEOF
+#define HTABLE_KTYPE(keyof) typeof(keyof(NULL))
+#else
+#define HTABLE_KTYPE(keyof) void *
+#endif
+#endif /* CCAN_HTABLE_TYPE_H */
diff --git a/lib/ccan/htable/test/run-type.c b/lib/ccan/htable/test/run-type.c
new file mode 100644
index 0000000000..02dac29e10
--- /dev/null
+++ b/lib/ccan/htable/test/run-type.c
@@ -0,0 +1,176 @@
+#include <ccan/htable/htable_type.h>
+#include <ccan/htable/htable.c>
+#include <ccan/tap/tap.h>
+#include <stdbool.h>
+#include <string.h>
+
+#define NUM_VALS (1 << HTABLE_BASE_BITS)
+
+struct obj {
+	/* Makes sure we don't try to treat and obj as a key or vice versa */
+	unsigned char unused;
+	unsigned int key;
+};
+
+static const unsigned int *objkey(const struct obj *obj)
+{
+	return &obj->key;
+}
+
+/* We use the number divided by two as the hash (for lots of
+   collisions), plus set all the higher bits so we can detect if they
+   don't get masked out. */
+static size_t objhash(const unsigned int *key)
+{
+	size_t h = *key / 2;
+	h |= -1UL << HTABLE_BASE_BITS;
+	return h;
+}
+
+static bool cmp(const struct obj *obj, const unsigned int *key)
+{
+	return obj->key == *key;
+}
+
+HTABLE_DEFINE_TYPE(struct obj, objkey, objhash, cmp, obj);
+
+static void add_vals(struct htable_obj *ht,
+		     struct obj val[], unsigned int num)
+{
+	unsigned int i;
+
+	for (i = 0; i < num; i++) {
+		if (htable_obj_get(ht, &i)) {
+			fail("%u already in hash", i);
+			return;
+		}
+		htable_obj_add(ht, &val[i]);
+		if (htable_obj_get(ht, &i) != &val[i]) {
+			fail("%u not added to hash", i);
+			return;
+		}
+	}
+	pass("Added %u numbers to hash", i);
+}
+
+static void find_vals(const struct htable_obj *ht,
+		      const struct obj val[], unsigned int num)
+{
+	unsigned int i;
+
+	for (i = 0; i < num; i++) {
+		if (htable_obj_get(ht, &i) != &val[i]) {
+			fail("%u not found in hash", i);
+			return;
+		}
+	}
+	pass("Found %u numbers in hash", i);
+}
+
+static void del_vals(struct htable_obj *ht,
+		     const struct obj val[], unsigned int num)
+{
+	unsigned int i;
+
+	for (i = 0; i < num; i++) {
+		if (!htable_obj_delkey(ht, &val[i].key)) {
+			fail("%u not deleted from hash", i);
+			return;
+		}
+	}
+	pass("Deleted %u numbers in hash", i);
+}
+
+static void del_vals_bykey(struct htable_obj *ht,
+			   const struct obj val[], unsigned int num)
+{
+	unsigned int i;
+
+	for (i = 0; i < num; i++) {
+		if (!htable_obj_delkey(ht, &i)) {
+			fail("%u not deleted by key from hash", i);
+			return;
+		}
+	}
+	pass("Deleted %u numbers by key from hash", i);
+}
+
+static bool check_mask(struct htable *ht, const struct obj val[], unsigned num)
+{
+	uint64_t i;
+
+	for (i = 0; i < num; i++) {
+		if (((uintptr_t)&val[i] & ht->common_mask) != ht->common_bits)
+			return false;
+	}
+	return true;
+}
+
+int main(int argc, char *argv[])
+{
+	unsigned int i;
+	struct htable_obj *ht;
+	struct obj val[NUM_VALS];
+	unsigned int dne;
+	void *p;
+	struct htable_obj_iter iter;
+
+	plan_tests(20);
+	for (i = 0; i < NUM_VALS; i++)
+		val[i].key = i;
+	dne = i;
+
+	ht = htable_obj_new();
+	ok1(((struct htable *)ht)->max < (1 << ((struct htable *)ht)->bits));
+	ok1(((struct htable *)ht)->bits == HTABLE_BASE_BITS);
+
+	/* We cannot find an entry which doesn't exist. */
+	ok1(!htable_obj_get(ht, &dne));
+
+	/* Fill it, it should increase in size (once). */
+	add_vals(ht, val, NUM_VALS);
+	ok1(((struct htable *)ht)->bits == HTABLE_BASE_BITS + 1);
+	ok1(((struct htable *)ht)->max < (1 << ((struct htable *)ht)->bits));
+
+	/* Mask should be set. */
+	ok1(((struct htable *)ht)->common_mask != 0);
+	ok1(((struct htable *)ht)->common_mask != -1);
+	ok1(check_mask((struct htable *)ht, val, NUM_VALS));
+
+	/* Find all. */
+	find_vals(ht, val, NUM_VALS);
+	ok1(!htable_obj_get(ht, &dne));
+
+	/* Walk once, should get them all. */
+	i = 0;
+	for (p = htable_obj_first(ht,&iter); p; p = htable_obj_next(ht, &iter))
+		i++;
+	ok1(i == NUM_VALS);
+
+	/* Delete all. */
+	del_vals(ht, val, NUM_VALS);
+	ok1(!htable_obj_get(ht, &val[0].key));
+
+	/* Worst case, a "pointer" which doesn't have any matching bits. */
+	htable_add((struct htable *)ht, 0,
+		   (void *)~(uintptr_t)&val[NUM_VALS-1]);
+	htable_obj_add(ht, &val[NUM_VALS-1]);
+	ok1(((struct htable *)ht)->common_mask == 0);
+	ok1(((struct htable *)ht)->common_bits == 0);
+	/* Delete the bogus one before we trip over it. */
+	htable_del((struct htable *)ht, 0,
+		   (void *)~(uintptr_t)&val[NUM_VALS-1]);
+
+	/* Add the rest. */
+	add_vals(ht, val, NUM_VALS-1);
+
+	/* Check we can find them all. */
+	find_vals(ht, val, NUM_VALS);
+	ok1(!htable_obj_get(ht, &dne));
+
+	/* Delete them all by key. */
+	del_vals_bykey(ht, val, NUM_VALS);
+	htable_obj_free(ht);
+
+	return exit_status();
+}
diff --git a/lib/ccan/htable/test/run.c b/lib/ccan/htable/test/run.c
new file mode 100644
index 0000000000..ece46a0fd7
--- /dev/null
+++ b/lib/ccan/htable/test/run.c
@@ -0,0 +1,176 @@
+#include <ccan/htable/htable.h>
+#include <ccan/htable/htable.c>
+#include <ccan/tap/tap.h>
+#include <stdbool.h>
+#include <string.h>
+
+#define NUM_VALS (1 << HTABLE_BASE_BITS)
+
+/* We use the number divided by two as the hash (for lots of
+   collisions), plus set all the higher bits so we can detect if they
+   don't get masked out. */
+static size_t hash(const void *elem, void *unused)
+{
+	size_t h = *(uint64_t *)elem / 2;
+	h |= -1UL << HTABLE_BASE_BITS;
+	return h;
+}
+
+static bool objcmp(const void *htelem, void *cmpdata)
+{
+	return *(uint64_t *)htelem == *(uint64_t *)cmpdata;
+}
+
+static void add_vals(struct htable *ht,
+		     const uint64_t val[], unsigned int num)
+{
+	uint64_t i;
+
+	for (i = 0; i < num; i++) {
+		if (htable_get(ht, hash(&i, NULL), objcmp, &i)) {
+			fail("%llu already in hash", (long long)i);
+			return;
+		}
+		htable_add(ht, hash(&val[i], NULL), &val[i]);
+		if (htable_get(ht, hash(&i, NULL), objcmp, &i) != &val[i]) {
+			fail("%llu not added to hash", (long long)i);
+			return;
+		}
+	}
+	pass("Added %llu numbers to hash", (long long)i);
+}
+
+#if 0
+static void refill_vals(struct htable *ht,
+			const uint64_t val[], unsigned int num)
+{
+	uint64_t i;
+
+	for (i = 0; i < num; i++) {
+		if (htable_get(ht, hash(&i, NULL), objcmp, &i))
+			continue;
+		htable_add(ht, hash(&val[i], NULL), &val[i]);
+	}
+}
+#endif
+
+static void find_vals(struct htable *ht,
+		      const uint64_t val[], unsigned int num)
+{
+	uint64_t i;
+
+	for (i = 0; i < num; i++) {
+		if (htable_get(ht, hash(&i, NULL), objcmp, &i) != &val[i]) {
+			fail("%llu not found in hash", (long long)i);
+			return;
+		}
+	}
+	pass("Found %llu numbers in hash", (long long)i);
+}
+
+static void del_vals(struct htable *ht,
+		     const uint64_t val[], unsigned int num)
+{
+	uint64_t i;
+
+	for (i = 0; i < num; i++) {
+		if (!htable_del(ht, hash(&val[i], NULL), &val[i])) {
+			fail("%llu not deleted from hash", (long long)i);
+			return;
+		}
+	}
+	pass("Deleted %llu numbers in hash", (long long)i);
+}
+
+static bool check_mask(struct htable *ht, uint64_t val[], unsigned num)
+{
+	uint64_t i;
+
+	for (i = 0; i < num; i++) {
+		if (((uintptr_t)&val[i] & ht->common_mask) != ht->common_bits)
+			return false;
+	}
+	return true;
+}
+
+int main(int argc, char *argv[])
+{
+	unsigned int i;
+	uintptr_t perfect_bit;
+	struct htable *ht;
+	uint64_t val[NUM_VALS];
+	uint64_t dne;
+	void *p;
+	struct htable_iter iter;
+
+	plan_tests(23);
+	for (i = 0; i < NUM_VALS; i++)
+		val[i] = i;
+	dne = i;
+
+	ht = htable_new(hash, NULL);
+	ok1(ht->max < (1 << ht->bits));
+	ok1(ht->bits == HTABLE_BASE_BITS);
+
+	/* We cannot find an entry which doesn't exist. */
+	ok1(!htable_get(ht, hash(&dne, NULL), objcmp, &dne));
+
+	/* Fill it, it should increase in size (once). */
+	add_vals(ht, val, NUM_VALS);
+	ok1(ht->bits == HTABLE_BASE_BITS + 1);
+	ok1(ht->max < (1 << ht->bits));
+
+	/* Mask should be set. */
+	ok1(ht->common_mask != 0);
+	ok1(ht->common_mask != -1);
+	ok1(check_mask(ht, val, NUM_VALS));
+
+	/* Find all. */
+	find_vals(ht, val, NUM_VALS);
+	ok1(!htable_get(ht, hash(&dne, NULL), objcmp, &dne));
+
+	/* Walk once, should get them all. */
+	i = 0;
+	for (p = htable_first(ht,&iter); p; p = htable_next(ht, &iter))
+		i++;
+	ok1(i == NUM_VALS);
+
+	/* Delete all. */
+	del_vals(ht, val, NUM_VALS);
+	ok1(!htable_get(ht, hash(&val[0], NULL), objcmp, &val[0]));
+
+	/* Worst case, a "pointer" which doesn't have any matching bits. */
+	htable_add(ht, 0, (void *)~(uintptr_t)&val[NUM_VALS-1]);
+	htable_add(ht, hash(&val[NUM_VALS-1], NULL), &val[NUM_VALS-1]);
+	ok1(ht->common_mask == 0);
+	ok1(ht->common_bits == 0);
+	/* Get rid of bogus pointer before we trip over it! */
+	htable_del(ht, 0, (void *)~(uintptr_t)&val[NUM_VALS-1]);
+
+	/* Add the rest. */
+	add_vals(ht, val, NUM_VALS-1);
+
+	/* Check we can find them all. */
+	find_vals(ht, val, NUM_VALS);
+	ok1(!htable_get(ht, hash(&dne, NULL), objcmp, &dne));
+
+	htable_free(ht);
+
+	/* Corner cases: wipe out the perfect bit using bogus pointer. */
+	ht = htable_new(hash, NULL);
+	htable_add(ht, 0, (void *)((uintptr_t)&val[NUM_VALS-1]));
+	ok1(ht->perfect_bit);
+	perfect_bit = ht->perfect_bit;
+	htable_add(ht, 0, (void *)((uintptr_t)&val[NUM_VALS-1]
+				   | perfect_bit));
+	ok1(ht->perfect_bit == 0);
+	htable_del(ht, 0, (void *)((uintptr_t)&val[NUM_VALS-1] | perfect_bit));
+
+	/* Enlarging should restore it... */
+	add_vals(ht, val, NUM_VALS-1);
+
+	ok1(ht->perfect_bit != 0);
+	htable_free(ht);
+
+	return exit_status();
+}
diff --git a/lib/ccan/htable/tools/Makefile b/lib/ccan/htable/tools/Makefile
new file mode 100644
index 0000000000..001e160b78
--- /dev/null
+++ b/lib/ccan/htable/tools/Makefile
@@ -0,0 +1,5 @@
+CFLAGS=-Wall -Werror -O3 -I../../..
+
+speed: speed.o ../../hash.o
+
+speed.o: speed.c ../htable.h ../htable.c
diff --git a/lib/ccan/htable/tools/speed.c b/lib/ccan/htable/tools/speed.c
new file mode 100644
index 0000000000..26231924a1
--- /dev/null
+++ b/lib/ccan/htable/tools/speed.c
@@ -0,0 +1,377 @@
+/* Simple speed tests for hashtables. */
+#include <ccan/htable/htable_type.h>
+#include <ccan/htable/htable.c>
+#include <ccan/hash/hash.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <time.h>
+#include <unistd.h>
+#include <sys/time.h>
+
+static size_t hashcount;
+struct object {
+	/* The key. */
+	unsigned int key;
+
+	/* Some contents. Doubles as consistency check. */
+	struct object *self;
+};
+
+static const unsigned int *objkey(const struct object *obj)
+{
+	return &obj->key;
+}
+
+static size_t hash_obj(const unsigned int *key)
+{
+	hashcount++;
+	return hashl(key, 1, 0);
+}
+
+static bool cmp(const unsigned int *key1, const unsigned int *key2)
+{
+	return *key1 == *key2;
+}
+
+HTABLE_DEFINE_TYPE(struct object, objkey, hash_obj, cmp, obj);
+
+static unsigned int popcount(unsigned long val)
+{
+#if HAVE_BUILTIN_POPCOUNTL
+	return __builtin_popcountl(val);
+#else
+	if (sizeof(long) == sizeof(u64)) {
+		u64 v = val;
+		v = (v & 0x5555555555555555ULL)
+			+ ((v >> 1) & 0x5555555555555555ULL);
+		v = (v & 0x3333333333333333ULL)
+			+ ((v >> 1) & 0x3333333333333333ULL);
+		v = (v & 0x0F0F0F0F0F0F0F0FULL)
+			+ ((v >> 1) & 0x0F0F0F0F0F0F0F0FULL);
+		v = (v & 0x00FF00FF00FF00FFULL)
+			+ ((v >> 1) & 0x00FF00FF00FF00FFULL);
+		v = (v & 0x0000FFFF0000FFFFULL)
+			+ ((v >> 1) & 0x0000FFFF0000FFFFULL);
+		v = (v & 0x00000000FFFFFFFFULL)
+			+ ((v >> 1) & 0x00000000FFFFFFFFULL);
+		return v;
+	}
+	val = (val & 0x55555555ULL) + ((val >> 1) & 0x55555555ULL);
+	val = (val & 0x33333333ULL) + ((val >> 1) & 0x33333333ULL);
+	val = (val & 0x0F0F0F0FULL) + ((val >> 1) & 0x0F0F0F0FULL);
+	val = (val & 0x00FF00FFULL) + ((val >> 1) & 0x00FF00FFULL);
+	val = (val & 0x0000FFFFULL) + ((val >> 1) & 0x0000FFFFULL);
+	return val;
+#endif
+}
+
+static size_t perfect(const struct htable *ht)
+{
+	size_t i, placed_perfect = 0;
+
+	for (i = 0; i < ((size_t)1 << ht->bits); i++) {
+		if (!entry_is_valid(ht->table[i]))
+			continue;
+		if (hash_bucket(ht, ht->rehash(get_raw_ptr(ht, ht->table[i]),
+					       ht->priv)) == i) {
+			assert((ht->table[i] & ht->perfect_bit)
+			       == ht->perfect_bit);
+			placed_perfect++;
+		}
+	}
+	return placed_perfect;
+}
+
+static size_t count_deleted(const struct htable *ht)
+{
+	size_t i, delete_markers = 0;
+
+	for (i = 0; i < ((size_t)1 << ht->bits); i++) {
+		if (ht->table[i] == HTABLE_DELETED)
+			delete_markers++;
+	}
+	return delete_markers;
+}
+
+/* Nanoseconds per operation */
+static size_t normalize(const struct timeval *start,
+			const struct timeval *stop,
+			unsigned int num)
+{
+	struct timeval diff;
+
+	timersub(stop, start, &diff);
+
+	/* Floating point is more accurate here. */
+	return (double)(diff.tv_sec * 1000000 + diff.tv_usec)
+		/ num * 1000;
+}
+
+static size_t worst_run(struct htable *ht, size_t *deleted)
+{
+	size_t longest = 0, len = 0, this_del = 0, i;
+
+	*deleted = 0;
+	/* This doesn't take into account end-wrap, but gives an idea. */
+	for (i = 0; i < ((size_t)1 << ht->bits); i++) {
+		if (ht->table[i]) {
+			len++;
+			if (ht->table[i] == HTABLE_DELETED)
+				this_del++;
+		} else {
+			if (len > longest) {
+				longest = len;
+				*deleted = this_del;
+			}
+			len = 0;
+			this_del = 0;
+		}
+	}
+	return longest;
+}
+
+int main(int argc, char *argv[])
+{
+	struct object *objs;
+	size_t i, j, num, deleted;
+	struct timeval start, stop;
+	struct htable_obj *ht;
+	struct htable *htr;
+	bool make_dumb = false;
+
+	if (argv[1] && strcmp(argv[1], "--dumb") == 0) {
+		argv++;
+		make_dumb = true;
+	}
+	num = argv[1] ? atoi(argv[1]) : 1000000;
+	objs = calloc(num, sizeof(objs[0]));
+
+	for (i = 0; i < num; i++) {
+		objs[i].key = i;
+		objs[i].self = &objs[i];
+	}
+
+	ht = htable_obj_new();
+	htr = (void *)ht;
+
+	printf("Initial insert: ");
+	fflush(stdout);
+	gettimeofday(&start, NULL);
+	for (i = 0; i < num; i++)
+		htable_obj_add(ht, objs[i].self);
+	gettimeofday(&stop, NULL);
+	printf(" %zu ns\n", normalize(&start, &stop, num));
+	printf("Details: hash size %u, mask bits %u, perfect %.0f%%\n",
+	       1U << htr->bits, popcount(htr->common_mask),
+	       perfect(htr) * 100.0 / htr->elems);
+
+	if (make_dumb) {
+		/* Screw with mask, to hobble us. */
+		update_common(htr, (void *)~htr->common_bits);
+		printf("Details: DUMB MODE: mask bits %u\n",
+		       popcount(htr->common_mask));
+	}
+
+	printf("Initial lookup (match): ");
+	fflush(stdout);
+	gettimeofday(&start, NULL);
+	for (i = 0; i < num; i++)
+		if (htable_obj_get(ht, &i)->self != objs[i].self)
+			abort();
+	gettimeofday(&stop, NULL);
+	printf(" %zu ns\n", normalize(&start, &stop, num));
+
+	printf("Initial lookup (miss): ");
+	fflush(stdout);
+	gettimeofday(&start, NULL);
+	for (i = 0; i < num; i++) {
+		unsigned int n = i + num;
+		if (htable_obj_get(ht, &n))
+			abort();
+	}
+	gettimeofday(&stop, NULL);
+	printf(" %zu ns\n", normalize(&start, &stop, num));
+
+	/* Lookups in order are very cache-friendly for judy; try random */
+	printf("Initial lookup (random): ");
+	fflush(stdout);
+	gettimeofday(&start, NULL);
+	for (i = 0, j = 0; i < num; i++, j = (j + 10007) % num)
+		if (htable_obj_get(ht, &j)->self != &objs[j])
+			abort();
+	gettimeofday(&stop, NULL);
+	printf(" %zu ns\n", normalize(&start, &stop, num));
+
+	hashcount = 0;
+	printf("Initial delete all: ");
+	fflush(stdout);
+	gettimeofday(&start, NULL);
+	for (i = 0; i < num; i++)
+		if (!htable_obj_del(ht, objs[i].self))
+			abort();
+	gettimeofday(&stop, NULL);
+	printf(" %zu ns\n", normalize(&start, &stop, num));
+	printf("Details: rehashes %zu\n", hashcount);
+
+	printf("Initial re-inserting: ");
+	fflush(stdout);
+	gettimeofday(&start, NULL);
+	for (i = 0; i < num; i++)
+		htable_obj_add(ht, objs[i].self);
+	gettimeofday(&stop, NULL);
+	printf(" %zu ns\n", normalize(&start, &stop, num));
+
+	hashcount = 0;
+	printf("Deleting first half: ");
+	fflush(stdout);
+	gettimeofday(&start, NULL);
+	for (i = 0; i < num; i+=2)
+		if (!htable_obj_del(ht, objs[i].self))
+			abort();
+	gettimeofday(&stop, NULL);
+	printf(" %zu ns\n", normalize(&start, &stop, num));
+
+	printf("Details: rehashes %zu, delete markers %zu\n",
+	       hashcount, count_deleted(htr));
+
+	printf("Adding (a different) half: ");
+	fflush(stdout);
+
+	for (i = 0; i < num; i+=2)
+		objs[i].key = num+i;
+
+	gettimeofday(&start, NULL);
+	for (i = 0; i < num; i+=2)
+		htable_obj_add(ht, objs[i].self);
+	gettimeofday(&stop, NULL);
+	printf(" %zu ns\n", normalize(&start, &stop, num));
+
+	printf("Details: delete markers %zu, perfect %.0f%%\n",
+	       count_deleted(htr), perfect(htr) * 100.0 / htr->elems);
+
+	printf("Lookup after half-change (match): ");
+	fflush(stdout);
+	gettimeofday(&start, NULL);
+	for (i = 1; i < num; i+=2)
+		if (htable_obj_get(ht, &i)->self != objs[i].self)
+			abort();
+	for (i = 0; i < num; i+=2) {
+		unsigned int n = i + num;
+		if (htable_obj_get(ht, &n)->self != objs[i].self)
+			abort();
+	}
+	gettimeofday(&stop, NULL);
+	printf(" %zu ns\n", normalize(&start, &stop, num));
+
+	printf("Lookup after half-change (miss): ");
+	fflush(stdout);
+	gettimeofday(&start, NULL);
+	for (i = 0; i < num; i++) {
+		unsigned int n = i + num * 2;
+		if (htable_obj_get(ht, &n))
+			abort();
+	}
+	gettimeofday(&stop, NULL);
+	printf(" %zu ns\n", normalize(&start, &stop, num));
+
+	/* Hashtables with delete markers can fill with markers over time.
+	 * so do some changes to see how it operates in long-term. */
+	for (i = 0; i < 5; i++) {
+		if (i == 0) {
+			/* We don't measure this: jmap is different. */
+			printf("Details: initial churn\n");
+		} else {
+			printf("Churning %s time: ",
+			       i == 1 ? "second"
+			       : i == 2 ? "third"
+			       : i == 3 ? "fourth"
+			       : "fifth");
+			fflush(stdout);
+		}
+		gettimeofday(&start, NULL);
+		for (j = 0; j < num; j++) {
+			if (!htable_obj_del(ht, &objs[j]))
+				abort();
+			objs[j].key = num*i+j;
+			if (!htable_obj_add(ht, &objs[j]))
+				abort();
+		}
+		gettimeofday(&stop, NULL);
+		if (i != 0)
+			printf(" %zu ns\n", normalize(&start, &stop, num));
+	}
+
+	/* Spread out the keys more to try to make it harder. */
+	printf("Details: reinserting with spread\n");
+	for (i = 0; i < num; i++) {
+		if (!htable_obj_del(ht, objs[i].self))
+			abort();
+		objs[i].key = num * 5 + i * 9;
+		if (!htable_obj_add(ht, objs[i].self))
+			abort();
+	}
+	printf("Details: delete markers %zu, perfect %.0f%%\n",
+	       count_deleted(htr), perfect(htr) * 100.0 / htr->elems);
+	i = worst_run(htr, &deleted);
+	printf("Details: worst run %zu (%zu deleted)\n", i, deleted);
+
+	printf("Lookup after churn & spread (match): ");
+	fflush(stdout);
+	gettimeofday(&start, NULL);
+	for (i = 0; i < num; i++) {
+		unsigned int n = num * 5 + i * 9;
+		if (htable_obj_get(ht, &n)->self != objs[i].self)
+			abort();
+	}
+	gettimeofday(&stop, NULL);
+	printf(" %zu ns\n", normalize(&start, &stop, num));
+
+	printf("Lookup after churn & spread (miss): ");
+	fflush(stdout);
+	gettimeofday(&start, NULL);
+	for (i = 0; i < num; i++) {
+		unsigned int n = num * (5 + 9) + i * 9;
+		if (htable_obj_get(ht, &n))
+			abort();
+	}
+	gettimeofday(&stop, NULL);
+	printf(" %zu ns\n", normalize(&start, &stop, num));
+
+	printf("Lookup after churn & spread (random): ");
+	fflush(stdout);
+	gettimeofday(&start, NULL);
+	for (i = 0, j = 0; i < num; i++, j = (j + 10007) % num) {
+		unsigned int n = num * 5 + j * 9;
+		if (htable_obj_get(ht, &n)->self != &objs[j])
+			abort();
+	}
+	gettimeofday(&stop, NULL);
+	printf(" %zu ns\n", normalize(&start, &stop, num));
+
+	hashcount = 0;
+	printf("Deleting half after churn & spread: ");
+	fflush(stdout);
+	gettimeofday(&start, NULL);
+	for (i = 0; i < num; i+=2)
+		if (!htable_obj_del(ht, objs[i].self))
+			abort();
+	gettimeofday(&stop, NULL);
+	printf(" %zu ns\n", normalize(&start, &stop, num));
+
+	printf("Adding (a different) half after churn & spread: ");
+	fflush(stdout);
+
+	for (i = 0; i < num; i+=2)
+		objs[i].key = num*6+i*9;
+
+	gettimeofday(&start, NULL);
+	for (i = 0; i < num; i+=2)
+		htable_obj_add(ht, objs[i].self);
+	gettimeofday(&stop, NULL);
+	printf(" %zu ns\n", normalize(&start, &stop, num));
+
+	printf("Details: delete markers %zu, perfect %.0f%%\n",
+	       count_deleted(htr), perfect(htr) * 100.0 / htr->elems);
+
+	return 0;
+}