From 0868b7c77d42efd5f361f605bfc0d8d46841db95 Mon Sep 17 00:00:00 2001 From: Andrew Tridgell Date: Fri, 16 Sep 2005 03:52:42 +0000 Subject: r10253: a fairly large tdb cleanup and re-organise. Nearly all of this change just involves splitting up the core tdb.c code into separate files on logical boundaries, but there are some minor functional changes as well: - move the 'struct tdb_context' into tdb_private.h, hiding it from users. This was done to allow the structure to change without breaking code that uses tdb. - added accessor functions tdb_fd(), tdb_name(), and tdb_log_fn() to access the elements of struct tdb_context that were used by external code but are no longer visible - simplied tdb_append() to use tdb_fetch()/tdb_store(), which is just as good due to the way tdb locks work - changed some of the types (such as tdb_off to tdb_off_t) to make syntax highlighting work better - removed the old optional spinlock code. It was a bad idea. - fixed a bug in tdb_reopen_all() that caused tdbtorture to sometimes fail or report nasty looking errors. This is the only real bug fixed in this commit. Jeremy/Jerry, you might like to pickup this change for Samba3, as that could definately affect smbd in Samba3. The aim of all of these changes is to make the tdb transactions/journaling code I am working on easier to write. I started to write it on top of the existing tdb.c code and it got very messy. Splitting up the code makes it much easier to follow. There are more cleanups we could do in tdb, such as using uint32_t instead of u32 (suggested by metze). I'll leave those for another day. (This used to be commit 4673cdd0d261614e707b72a7a348bb0e7dbb2482) --- source4/lib/tdb/common/io.c | 374 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 374 insertions(+) create mode 100644 source4/lib/tdb/common/io.c (limited to 'source4/lib/tdb/common/io.c') diff --git a/source4/lib/tdb/common/io.c b/source4/lib/tdb/common/io.c new file mode 100644 index 0000000000..dbcc83119c --- /dev/null +++ b/source4/lib/tdb/common/io.c @@ -0,0 +1,374 @@ + /* + Unix SMB/CIFS implementation. + + trivial database library + + Copyright (C) Andrew Tridgell 1999-2005 + Copyright (C) Paul `Rusty' Russell 2000 + Copyright (C) Jeremy Allison 2000-2003 + + ** NOTE! The following LGPL license applies to the tdb + ** library. This does NOT imply that all of Samba is released + ** under the LGPL + + This library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2 of the License, or (at your option) any later version. + + This library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with this library; if not, write to the Free Software + Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +*/ + + +#include "tdb_private.h" + +#ifndef HAVE_PREAD + static ssize_t pread(int fd, void *buf, size_t count, off_t offset) +{ + if (lseek(fd, offset, SEEK_SET) != offset) { + errno = EIO; + return -1; + } + return read(fd, buf, count); +} +#endif + +#ifndef HAVE_PWRITE + static ssize_t pwrite(int fd, const void *buf, size_t count, off_t offset) +{ + if (lseek(fd, offset, SEEK_SET) != offset) { + errno = EIO; + return -1; + } + return write(fd, buf, count); +} +#endif + +/* check for an out of bounds access - if it is out of bounds then + see if the database has been expanded by someone else and expand + if necessary + note that "len" is the minimum length needed for the db +*/ +int tdb_oob(struct tdb_context *tdb, tdb_off_t len, int probe) +{ + struct stat st; + if (len <= tdb->map_size) + return 0; + if (tdb->flags & TDB_INTERNAL) { + if (!probe) { + /* Ensure ecode is set for log fn. */ + tdb->ecode = TDB_ERR_IO; + TDB_LOG((tdb, 0,"tdb_oob len %d beyond internal malloc size %d\n", + (int)len, (int)tdb->map_size)); + } + return TDB_ERRCODE(TDB_ERR_IO, -1); + } + + if (fstat(tdb->fd, &st) == -1) { + return TDB_ERRCODE(TDB_ERR_IO, -1); + } + + if (st.st_size < (size_t)len) { + if (!probe) { + /* Ensure ecode is set for log fn. */ + tdb->ecode = TDB_ERR_IO; + TDB_LOG((tdb, 0,"tdb_oob len %d beyond eof at %d\n", + (int)len, (int)st.st_size)); + } + return TDB_ERRCODE(TDB_ERR_IO, -1); + } + + /* Unmap, update size, remap */ + if (tdb_munmap(tdb) == -1) + return TDB_ERRCODE(TDB_ERR_IO, -1); + tdb->map_size = st.st_size; + tdb_mmap(tdb); + return 0; +} + +/* write a lump of data at a specified offset */ +int tdb_write(struct tdb_context *tdb, tdb_off_t off, void *buf, tdb_len_t len) +{ + if (tdb_oob(tdb, off + len, 0) != 0) + return -1; + + if (tdb->map_ptr) { + memcpy(off + (char *)tdb->map_ptr, buf, len); + } else if (pwrite(tdb->fd, buf, len, off) != (ssize_t)len) { + /* Ensure ecode is set for log fn. */ + tdb->ecode = TDB_ERR_IO; + TDB_LOG((tdb, 0,"tdb_write failed at %d len=%d (%s)\n", + off, len, strerror(errno))); + return TDB_ERRCODE(TDB_ERR_IO, -1); + } + return 0; +} + +/* Endian conversion: we only ever deal with 4 byte quantities */ +void *tdb_convert(void *buf, u32 size) +{ + u32 i, *p = buf; + for (i = 0; i < size / 4; i++) + p[i] = TDB_BYTEREV(p[i]); + return buf; +} + + +/* read a lump of data at a specified offset, maybe convert */ +int tdb_read(struct tdb_context *tdb, tdb_off_t off, void *buf, tdb_len_t len, int cv) +{ + ssize_t ret; + + if (tdb_oob(tdb, off + len, 0) != 0) { + return -1; + } + + if (tdb->map_ptr) { + memcpy(buf, off + (char *)tdb->map_ptr, len); + } else { + ssize_t ret = pread(tdb->fd, buf, len, off); + if (ret != (ssize_t)len) { + /* Ensure ecode is set for log fn. */ + tdb->ecode = TDB_ERR_IO; + TDB_LOG((tdb, 0,"tdb_read failed at %d len=%d ret=%d (%s) map_size=%d\n", + off, len, ret, strerror(errno), tdb->map_size)); + return TDB_ERRCODE(TDB_ERR_IO, -1); + } + } + if (cv) + tdb_convert(buf, len); + return 0; +} + + + +/* + do an unlocked scan of the hash table heads to find the next non-zero head. The value + will then be confirmed with the lock held +*/ +void tdb_next_hash_chain(struct tdb_context *tdb, u32 *chain) +{ + u32 h = *chain; + if (tdb->map_ptr) { + for (;h < tdb->header.hash_size;h++) { + if (0 != *(u32 *)(TDB_HASH_TOP(h) + (unsigned char *)tdb->map_ptr)) { + break; + } + } + } else { + u32 off; + for (;h < tdb->header.hash_size;h++) { + if (tdb_ofs_read(tdb, TDB_HASH_TOP(h), &off) != 0 || off != 0) { + break; + } + } + } + (*chain) = h; +} + + +int tdb_munmap(struct tdb_context *tdb) +{ + if (tdb->flags & TDB_INTERNAL) + return 0; + +#ifdef HAVE_MMAP + if (tdb->map_ptr) { + int ret = munmap(tdb->map_ptr, tdb->map_size); + if (ret != 0) + return ret; + } +#endif + tdb->map_ptr = NULL; + return 0; +} + +void tdb_mmap(struct tdb_context *tdb) +{ + if (tdb->flags & TDB_INTERNAL) + return; + +#ifdef HAVE_MMAP + if (!(tdb->flags & TDB_NOMMAP)) { + tdb->map_ptr = mmap(NULL, tdb->map_size, + PROT_READ|(tdb->read_only? 0:PROT_WRITE), + MAP_SHARED|MAP_FILE, tdb->fd, 0); + + /* + * NB. When mmap fails it returns MAP_FAILED *NOT* NULL !!!! + */ + + if (tdb->map_ptr == MAP_FAILED) { + tdb->map_ptr = NULL; + TDB_LOG((tdb, 2, "tdb_mmap failed for size %d (%s)\n", + tdb->map_size, strerror(errno))); + } + } else { + tdb->map_ptr = NULL; + } +#else + tdb->map_ptr = NULL; +#endif +} + +/* expand a file. we prefer to use ftruncate, as that is what posix + says to use for mmap expansion */ +static int tdb_expand_file(struct tdb_context *tdb, tdb_off_t size, tdb_off_t addition) +{ + char buf[1024]; + if (ftruncate(tdb->fd, size+addition) == -1) { + char b = 0; + if (pwrite(tdb->fd, &b, 1, (size+addition) - 1) != 1) { + TDB_LOG((tdb, 0, "expand_file to %d failed (%s)\n", + size+addition, strerror(errno))); + return -1; + } + } + + /* now fill the file with something. This ensures that the file isn't sparse, which would be + very bad if we ran out of disk. This must be done with write, not via mmap */ + memset(buf, 0x42, sizeof(buf)); + while (addition) { + int n = addition>sizeof(buf)?sizeof(buf):addition; + int ret = pwrite(tdb->fd, buf, n, size); + if (ret != n) { + TDB_LOG((tdb, 0, "expand_file write of %d failed (%s)\n", + n, strerror(errno))); + return -1; + } + addition -= n; + size += n; + } + return 0; +} + + +/* expand the database at least size bytes by expanding the underlying + file and doing the mmap again if necessary */ +int tdb_expand(struct tdb_context *tdb, tdb_off_t size) +{ + struct list_struct rec; + tdb_off_t offset; + + if (tdb_lock(tdb, -1, F_WRLCK) == -1) { + TDB_LOG((tdb, 0, "lock failed in tdb_expand\n")); + return -1; + } + + /* must know about any previous expansions by another process */ + tdb_oob(tdb, tdb->map_size + 1, 1); + + /* always make room for at least 10 more records, and round + the database up to a multiple of TDB_PAGE_SIZE */ + size = TDB_ALIGN(tdb->map_size + size*10, TDB_PAGE_SIZE) - tdb->map_size; + + if (!(tdb->flags & TDB_INTERNAL)) + tdb_munmap(tdb); + + /* + * We must ensure the file is unmapped before doing this + * to ensure consistency with systems like OpenBSD where + * writes and mmaps are not consistent. + */ + + /* expand the file itself */ + if (!(tdb->flags & TDB_INTERNAL)) { + if (tdb_expand_file(tdb, tdb->map_size, size) != 0) + goto fail; + } + + tdb->map_size += size; + + if (tdb->flags & TDB_INTERNAL) { + char *new_map_ptr = realloc(tdb->map_ptr, tdb->map_size); + if (!new_map_ptr) { + tdb->map_size -= size; + goto fail; + } + tdb->map_ptr = new_map_ptr; + } else { + /* + * We must ensure the file is remapped before adding the space + * to ensure consistency with systems like OpenBSD where + * writes and mmaps are not consistent. + */ + + /* We're ok if the mmap fails as we'll fallback to read/write */ + tdb_mmap(tdb); + } + + /* form a new freelist record */ + memset(&rec,'\0',sizeof(rec)); + rec.rec_len = size - sizeof(rec); + + /* link it into the free list */ + offset = tdb->map_size - size; + if (tdb_free(tdb, offset, &rec) == -1) + goto fail; + + tdb_unlock(tdb, -1, F_WRLCK); + return 0; + fail: + tdb_unlock(tdb, -1, F_WRLCK); + return -1; +} + +/* read/write a tdb_off_t */ +int tdb_ofs_read(struct tdb_context *tdb, tdb_off_t offset, tdb_off_t *d) +{ + return tdb_read(tdb, offset, (char*)d, sizeof(*d), DOCONV()); +} + +int tdb_ofs_write(struct tdb_context *tdb, tdb_off_t offset, tdb_off_t *d) +{ + tdb_off_t off = *d; + return tdb_write(tdb, offset, CONVERT(off), sizeof(*d)); +} + + +/* read a lump of data, allocating the space for it */ +unsigned char *tdb_alloc_read(struct tdb_context *tdb, tdb_off_t offset, tdb_len_t len) +{ + unsigned char *buf; + + if (!(buf = malloc(len))) { + /* Ensure ecode is set for log fn. */ + tdb->ecode = TDB_ERR_OOM; + TDB_LOG((tdb, 0,"tdb_alloc_read malloc failed len=%d (%s)\n", + len, strerror(errno))); + return TDB_ERRCODE(TDB_ERR_OOM, buf); + } + if (tdb_read(tdb, offset, buf, len, 0) == -1) { + SAFE_FREE(buf); + return NULL; + } + return buf; +} + +/* read/write a record */ +int tdb_rec_read(struct tdb_context *tdb, tdb_off_t offset, struct list_struct *rec) +{ + if (tdb_read(tdb, offset, rec, sizeof(*rec),DOCONV()) == -1) + return -1; + if (TDB_BAD_MAGIC(rec)) { + /* Ensure ecode is set for log fn. */ + tdb->ecode = TDB_ERR_CORRUPT; + TDB_LOG((tdb, 0,"tdb_rec_read bad magic 0x%x at offset=%d\n", rec->magic, offset)); + return TDB_ERRCODE(TDB_ERR_CORRUPT, -1); + } + return tdb_oob(tdb, rec->next+sizeof(*rec), 0); +} + +int tdb_rec_write(struct tdb_context *tdb, tdb_off_t offset, struct list_struct *rec) +{ + struct list_struct r = *rec; + return tdb_write(tdb, offset, CONVERT(r), sizeof(r)); +} + -- cgit From 48b513fd2388840454d257cd110b576b08fb2abd Mon Sep 17 00:00:00 2001 From: Tim Potter Date: Fri, 16 Sep 2005 07:24:36 +0000 Subject: r10258: Fix an unused/duplicate local variable. (This used to be commit 360be7028c2244391cdd9ed6d156cec8dab9e9e9) --- source4/lib/tdb/common/io.c | 2 -- 1 file changed, 2 deletions(-) (limited to 'source4/lib/tdb/common/io.c') diff --git a/source4/lib/tdb/common/io.c b/source4/lib/tdb/common/io.c index dbcc83119c..97da809bc6 100644 --- a/source4/lib/tdb/common/io.c +++ b/source4/lib/tdb/common/io.c @@ -124,8 +124,6 @@ void *tdb_convert(void *buf, u32 size) /* read a lump of data at a specified offset, maybe convert */ int tdb_read(struct tdb_context *tdb, tdb_off_t off, void *buf, tdb_len_t len, int cv) { - ssize_t ret; - if (tdb_oob(tdb, off + len, 0) != 0) { return -1; } -- cgit From ede8415d61b6791114c65de1c283a4e8c11f1585 Mon Sep 17 00:00:00 2001 From: Andrew Tridgell Date: Thu, 22 Sep 2005 03:56:41 +0000 Subject: r10405: added transactions into tdb, and hook them into ldb. See my samba-technical posting for more details on the transactions design. This also adds a number of command line arguments to tdbtorture, making it more flexible, and fixes some lock deadlock conditions in the tdbtorture code. (This used to be commit 06bd8abba942ec9f1e23f5c5d546cbb71ca3a701) --- source4/lib/tdb/common/io.c | 60 ++++++++++++++++++++++++++++++--------------- 1 file changed, 40 insertions(+), 20 deletions(-) (limited to 'source4/lib/tdb/common/io.c') diff --git a/source4/lib/tdb/common/io.c b/source4/lib/tdb/common/io.c index 97da809bc6..f02cd1a0e1 100644 --- a/source4/lib/tdb/common/io.c +++ b/source4/lib/tdb/common/io.c @@ -56,7 +56,7 @@ if necessary note that "len" is the minimum length needed for the db */ -int tdb_oob(struct tdb_context *tdb, tdb_off_t len, int probe) +static int tdb_oob(struct tdb_context *tdb, tdb_off_t len, int probe) { struct stat st; if (len <= tdb->map_size) @@ -94,9 +94,10 @@ int tdb_oob(struct tdb_context *tdb, tdb_off_t len, int probe) } /* write a lump of data at a specified offset */ -int tdb_write(struct tdb_context *tdb, tdb_off_t off, void *buf, tdb_len_t len) +static int tdb_write(struct tdb_context *tdb, tdb_off_t off, + const void *buf, tdb_len_t len) { - if (tdb_oob(tdb, off + len, 0) != 0) + if (tdb->methods->tdb_oob(tdb, off + len, 0) != 0) return -1; if (tdb->map_ptr) { @@ -122,9 +123,10 @@ void *tdb_convert(void *buf, u32 size) /* read a lump of data at a specified offset, maybe convert */ -int tdb_read(struct tdb_context *tdb, tdb_off_t off, void *buf, tdb_len_t len, int cv) +static int tdb_read(struct tdb_context *tdb, tdb_off_t off, void *buf, + tdb_len_t len, int cv) { - if (tdb_oob(tdb, off + len, 0) != 0) { + if (tdb->methods->tdb_oob(tdb, off + len, 0) != 0) { return -1; } @@ -140,8 +142,9 @@ int tdb_read(struct tdb_context *tdb, tdb_off_t off, void *buf, tdb_len_t len, i return TDB_ERRCODE(TDB_ERR_IO, -1); } } - if (cv) + if (cv) { tdb_convert(buf, len); + } return 0; } @@ -151,7 +154,7 @@ int tdb_read(struct tdb_context *tdb, tdb_off_t off, void *buf, tdb_len_t len, i do an unlocked scan of the hash table heads to find the next non-zero head. The value will then be confirmed with the lock held */ -void tdb_next_hash_chain(struct tdb_context *tdb, u32 *chain) +static void tdb_next_hash_chain(struct tdb_context *tdb, u32 *chain) { u32 h = *chain; if (tdb->map_ptr) { @@ -230,9 +233,10 @@ static int tdb_expand_file(struct tdb_context *tdb, tdb_off_t size, tdb_off_t ad } } - /* now fill the file with something. This ensures that the file isn't sparse, which would be - very bad if we ran out of disk. This must be done with write, not via mmap */ - memset(buf, 0x42, sizeof(buf)); + /* now fill the file with something. This ensures that the + file isn't sparse, which would be very bad if we ran out of + disk. This must be done with write, not via mmap */ + memset(buf, TDB_PAD_BYTE, sizeof(buf)); while (addition) { int n = addition>sizeof(buf)?sizeof(buf):addition; int ret = pwrite(tdb->fd, buf, n, size); @@ -261,11 +265,11 @@ int tdb_expand(struct tdb_context *tdb, tdb_off_t size) } /* must know about any previous expansions by another process */ - tdb_oob(tdb, tdb->map_size + 1, 1); + tdb->methods->tdb_oob(tdb, tdb->map_size + 1, 1); /* always make room for at least 10 more records, and round - the database up to a multiple of TDB_PAGE_SIZE */ - size = TDB_ALIGN(tdb->map_size + size*10, TDB_PAGE_SIZE) - tdb->map_size; + the database up to a multiple of the page size */ + size = TDB_ALIGN(tdb->map_size + size*10, tdb->page_size) - tdb->map_size; if (!(tdb->flags & TDB_INTERNAL)) tdb_munmap(tdb); @@ -278,7 +282,7 @@ int tdb_expand(struct tdb_context *tdb, tdb_off_t size) /* expand the file itself */ if (!(tdb->flags & TDB_INTERNAL)) { - if (tdb_expand_file(tdb, tdb->map_size, size) != 0) + if (tdb->methods->tdb_expand_file(tdb, tdb->map_size, size) != 0) goto fail; } @@ -321,13 +325,13 @@ int tdb_expand(struct tdb_context *tdb, tdb_off_t size) /* read/write a tdb_off_t */ int tdb_ofs_read(struct tdb_context *tdb, tdb_off_t offset, tdb_off_t *d) { - return tdb_read(tdb, offset, (char*)d, sizeof(*d), DOCONV()); + return tdb->methods->tdb_read(tdb, offset, (char*)d, sizeof(*d), DOCONV()); } int tdb_ofs_write(struct tdb_context *tdb, tdb_off_t offset, tdb_off_t *d) { tdb_off_t off = *d; - return tdb_write(tdb, offset, CONVERT(off), sizeof(*d)); + return tdb->methods->tdb_write(tdb, offset, CONVERT(off), sizeof(*d)); } @@ -343,7 +347,7 @@ unsigned char *tdb_alloc_read(struct tdb_context *tdb, tdb_off_t offset, tdb_len len, strerror(errno))); return TDB_ERRCODE(TDB_ERR_OOM, buf); } - if (tdb_read(tdb, offset, buf, len, 0) == -1) { + if (tdb->methods->tdb_read(tdb, offset, buf, len, 0) == -1) { SAFE_FREE(buf); return NULL; } @@ -353,7 +357,7 @@ unsigned char *tdb_alloc_read(struct tdb_context *tdb, tdb_off_t offset, tdb_len /* read/write a record */ int tdb_rec_read(struct tdb_context *tdb, tdb_off_t offset, struct list_struct *rec) { - if (tdb_read(tdb, offset, rec, sizeof(*rec),DOCONV()) == -1) + if (tdb->methods->tdb_read(tdb, offset, rec, sizeof(*rec),DOCONV()) == -1) return -1; if (TDB_BAD_MAGIC(rec)) { /* Ensure ecode is set for log fn. */ @@ -361,12 +365,28 @@ int tdb_rec_read(struct tdb_context *tdb, tdb_off_t offset, struct list_struct * TDB_LOG((tdb, 0,"tdb_rec_read bad magic 0x%x at offset=%d\n", rec->magic, offset)); return TDB_ERRCODE(TDB_ERR_CORRUPT, -1); } - return tdb_oob(tdb, rec->next+sizeof(*rec), 0); + return tdb->methods->tdb_oob(tdb, rec->next+sizeof(*rec), 0); } int tdb_rec_write(struct tdb_context *tdb, tdb_off_t offset, struct list_struct *rec) { struct list_struct r = *rec; - return tdb_write(tdb, offset, CONVERT(r), sizeof(r)); + return tdb->methods->tdb_write(tdb, offset, CONVERT(r), sizeof(r)); } +static const struct tdb_methods io_methods = { + .tdb_read = tdb_read, + .tdb_write = tdb_write, + .next_hash_chain = tdb_next_hash_chain, + .tdb_oob = tdb_oob, + .tdb_expand_file = tdb_expand_file, + .tdb_brlock = tdb_brlock +}; + +/* + initialise the default methods table +*/ +void tdb_io_init(struct tdb_context *tdb) +{ + tdb->methods = &io_methods; +} -- cgit From bd310b792509f7305d7dc029eb4bec109322a4bf Mon Sep 17 00:00:00 2001 From: Andrew Tridgell Date: Thu, 22 Sep 2005 13:12:46 +0000 Subject: r10421: following on discussions with simo, I have worked out a way of allowing searches to proceed while another process is in a transaction, then only upgrading the transaction lock to a write lock on commit. The solution is: - split tdb_traverse() into two calls, called tdb_traverse() and tdb_traverse_read(). The _read() version only gets read locks, and will fail any write operations made in the callback from the traverse. - the normal tdb_traverse() call allows for read or write operations in the callback, but gets the transaction lock, preventing transastions from starting inside the traverse In addition we enforce the following rule that you may not start a transaction within a traverse callback, although you can start a traverse within a transaction With these rules in place I believe all the deadlock possibilities are removed, and we can now allow for searches to happen in parallel with transactions (This used to be commit 7dd31288a701d772e45b1960ac4ce4cc1be782ed) --- source4/lib/tdb/common/io.c | 11 +++++++++++ 1 file changed, 11 insertions(+) (limited to 'source4/lib/tdb/common/io.c') diff --git a/source4/lib/tdb/common/io.c b/source4/lib/tdb/common/io.c index f02cd1a0e1..c1f6f55bc4 100644 --- a/source4/lib/tdb/common/io.c +++ b/source4/lib/tdb/common/io.c @@ -97,6 +97,11 @@ static int tdb_oob(struct tdb_context *tdb, tdb_off_t len, int probe) static int tdb_write(struct tdb_context *tdb, tdb_off_t off, const void *buf, tdb_len_t len) { + if (tdb->read_only) { + tdb->ecode = TDB_ERR_RDONLY; + return -1; + } + if (tdb->methods->tdb_oob(tdb, off + len, 0) != 0) return -1; @@ -224,6 +229,12 @@ void tdb_mmap(struct tdb_context *tdb) static int tdb_expand_file(struct tdb_context *tdb, tdb_off_t size, tdb_off_t addition) { char buf[1024]; + + if (tdb->read_only) { + tdb->ecode = TDB_ERR_RDONLY; + return -1; + } + if (ftruncate(tdb->fd, size+addition) == -1) { char b = 0; if (pwrite(tdb->fd, &b, 1, (size+addition) - 1) != 1) { -- cgit From 5860aef9cd53da572bef1b86a62a3a5e86da84b0 Mon Sep 17 00:00:00 2001 From: Andrew Tridgell Date: Sat, 24 Sep 2005 03:43:02 +0000 Subject: r10465: separate out a read_only db from a read-only traversal to ensure we don't end up doing a mmap read only (This used to be commit 294ccfd46a0c4e1af9365d028acdabec03c41ad3) --- source4/lib/tdb/common/io.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'source4/lib/tdb/common/io.c') diff --git a/source4/lib/tdb/common/io.c b/source4/lib/tdb/common/io.c index c1f6f55bc4..a276a8c291 100644 --- a/source4/lib/tdb/common/io.c +++ b/source4/lib/tdb/common/io.c @@ -97,7 +97,7 @@ static int tdb_oob(struct tdb_context *tdb, tdb_off_t len, int probe) static int tdb_write(struct tdb_context *tdb, tdb_off_t off, const void *buf, tdb_len_t len) { - if (tdb->read_only) { + if (tdb->read_only || tdb->traverse_read) { tdb->ecode = TDB_ERR_RDONLY; return -1; } @@ -230,7 +230,7 @@ static int tdb_expand_file(struct tdb_context *tdb, tdb_off_t size, tdb_off_t ad { char buf[1024]; - if (tdb->read_only) { + if (tdb->read_only || tdb->traverse_read) { tdb->ecode = TDB_ERR_RDONLY; return -1; } -- cgit From 6e4ebbed8928d618f56c06bfc0802891c3d939ab Mon Sep 17 00:00:00 2001 From: Andrew Tridgell Date: Sat, 24 Sep 2005 05:02:02 +0000 Subject: r10467: aix doesn't like zero length malloc :( (This used to be commit 0177c6ca3e9baa223b5f7b29adc12d862dcb19d8) --- source4/lib/tdb/common/io.c | 5 +++++ 1 file changed, 5 insertions(+) (limited to 'source4/lib/tdb/common/io.c') diff --git a/source4/lib/tdb/common/io.c b/source4/lib/tdb/common/io.c index a276a8c291..e9d4e87ac8 100644 --- a/source4/lib/tdb/common/io.c +++ b/source4/lib/tdb/common/io.c @@ -351,6 +351,11 @@ unsigned char *tdb_alloc_read(struct tdb_context *tdb, tdb_off_t offset, tdb_len { unsigned char *buf; + /* some systems don't like zero length malloc */ + if (len == 0) { + len = 1; + } + if (!(buf = malloc(len))) { /* Ensure ecode is set for log fn. */ tdb->ecode = TDB_ERR_OOM; -- cgit From 3bcfc68e0d158b272e0ef87bf585f79a570136ba Mon Sep 17 00:00:00 2001 From: Andrew Tridgell Date: Sat, 24 Sep 2005 06:49:28 +0000 Subject: r10468: - terminate tdbtorture quickly when an error is detected - more workarounds for aix not handling malloc of size 0 (This used to be commit c2b1739c6389503854f55fa8407c55071781eff4) --- source4/lib/tdb/common/io.c | 4 ++++ 1 file changed, 4 insertions(+) (limited to 'source4/lib/tdb/common/io.c') diff --git a/source4/lib/tdb/common/io.c b/source4/lib/tdb/common/io.c index e9d4e87ac8..355f76962f 100644 --- a/source4/lib/tdb/common/io.c +++ b/source4/lib/tdb/common/io.c @@ -97,6 +97,10 @@ static int tdb_oob(struct tdb_context *tdb, tdb_off_t len, int probe) static int tdb_write(struct tdb_context *tdb, tdb_off_t off, const void *buf, tdb_len_t len) { + if (len == 0) { + return 0; + } + if (tdb->read_only || tdb->traverse_read) { tdb->ecode = TDB_ERR_RDONLY; return -1; -- cgit From ddf733fc58f29d7315045d6d886ebebe32fd4ee7 Mon Sep 17 00:00:00 2001 From: Andrew Tridgell Date: Sat, 24 Sep 2005 06:53:24 +0000 Subject: r10469: use the older style of structure initialisation for tdb to make it more portable. tdb is used in more than just Samba4, and I think the portability/readability balance is a bit different (This used to be commit fc692dc61f06d61cb9126d2a8ccc240cecd11da6) --- source4/lib/tdb/common/io.c | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) (limited to 'source4/lib/tdb/common/io.c') diff --git a/source4/lib/tdb/common/io.c b/source4/lib/tdb/common/io.c index 355f76962f..0a910270c9 100644 --- a/source4/lib/tdb/common/io.c +++ b/source4/lib/tdb/common/io.c @@ -395,12 +395,12 @@ int tdb_rec_write(struct tdb_context *tdb, tdb_off_t offset, struct list_struct } static const struct tdb_methods io_methods = { - .tdb_read = tdb_read, - .tdb_write = tdb_write, - .next_hash_chain = tdb_next_hash_chain, - .tdb_oob = tdb_oob, - .tdb_expand_file = tdb_expand_file, - .tdb_brlock = tdb_brlock + tdb_read, + tdb_write, + tdb_next_hash_chain, + tdb_oob, + tdb_expand_file, + tdb_brlock }; /* -- cgit From 4b6cec88ae28952095298fb4b4f720e29b37699b Mon Sep 17 00:00:00 2001 From: Andrew Tridgell Date: Sat, 8 Apr 2006 02:29:48 +0000 Subject: r14972: fix an uninitialised warning from ibm checker (This used to be commit 9b867d2acffdfdcd5dc1dd5341db19daf11b5e96) --- source4/lib/tdb/common/io.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'source4/lib/tdb/common/io.c') diff --git a/source4/lib/tdb/common/io.c b/source4/lib/tdb/common/io.c index 0a910270c9..f4f866a4c4 100644 --- a/source4/lib/tdb/common/io.c +++ b/source4/lib/tdb/common/io.c @@ -173,7 +173,7 @@ static void tdb_next_hash_chain(struct tdb_context *tdb, u32 *chain) } } } else { - u32 off; + u32 off=0; for (;h < tdb->header.hash_size;h++) { if (tdb_ofs_read(tdb, TDB_HASH_TOP(h), &off) != 0 || off != 0) { break; -- cgit From d3fee429aee87e9c05a4a606fbf0b60b16dac782 Mon Sep 17 00:00:00 2001 From: Andrew Bartlett Date: Mon, 3 Jul 2006 06:40:56 +0000 Subject: r16774: This patch modifies the tdb API to allow the logging function to be used as part of ldb. This allows tdb failures to be passed all the way up to Samba's DEBUG system, which allowed easier debugging. Unfortunately I had to extend the tdb API, as the logging function didn't have a context pointer. I've worked over the 'debug levels' in TDB. Most of them were 0, which didn't seem right, as some were trace-like messages. We didn't see any of these previously, except when accessing TDB directly. Andrew Bartlett (This used to be commit 58898092c1ce043f6d698db5065f372b79109e22) --- source4/lib/tdb/common/io.c | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) (limited to 'source4/lib/tdb/common/io.c') diff --git a/source4/lib/tdb/common/io.c b/source4/lib/tdb/common/io.c index f4f866a4c4..21d591b67d 100644 --- a/source4/lib/tdb/common/io.c +++ b/source4/lib/tdb/common/io.c @@ -65,7 +65,7 @@ static int tdb_oob(struct tdb_context *tdb, tdb_off_t len, int probe) if (!probe) { /* Ensure ecode is set for log fn. */ tdb->ecode = TDB_ERR_IO; - TDB_LOG((tdb, 0,"tdb_oob len %d beyond internal malloc size %d\n", + TDB_LOG((tdb, TDB_DEBUG_FATAL,"tdb_oob len %d beyond internal malloc size %d\n", (int)len, (int)tdb->map_size)); } return TDB_ERRCODE(TDB_ERR_IO, -1); @@ -79,7 +79,7 @@ static int tdb_oob(struct tdb_context *tdb, tdb_off_t len, int probe) if (!probe) { /* Ensure ecode is set for log fn. */ tdb->ecode = TDB_ERR_IO; - TDB_LOG((tdb, 0,"tdb_oob len %d beyond eof at %d\n", + TDB_LOG((tdb, TDB_DEBUG_FATAL,"tdb_oob len %d beyond eof at %d\n", (int)len, (int)st.st_size)); } return TDB_ERRCODE(TDB_ERR_IO, -1); @@ -114,7 +114,7 @@ static int tdb_write(struct tdb_context *tdb, tdb_off_t off, } else if (pwrite(tdb->fd, buf, len, off) != (ssize_t)len) { /* Ensure ecode is set for log fn. */ tdb->ecode = TDB_ERR_IO; - TDB_LOG((tdb, 0,"tdb_write failed at %d len=%d (%s)\n", + TDB_LOG((tdb, TDB_DEBUG_FATAL,"tdb_write failed at %d len=%d (%s)\n", off, len, strerror(errno))); return TDB_ERRCODE(TDB_ERR_IO, -1); } @@ -146,7 +146,7 @@ static int tdb_read(struct tdb_context *tdb, tdb_off_t off, void *buf, if (ret != (ssize_t)len) { /* Ensure ecode is set for log fn. */ tdb->ecode = TDB_ERR_IO; - TDB_LOG((tdb, 0,"tdb_read failed at %d len=%d ret=%d (%s) map_size=%d\n", + TDB_LOG((tdb, TDB_DEBUG_FATAL,"tdb_read failed at %d len=%d ret=%d (%s) map_size=%d\n", off, len, ret, strerror(errno), tdb->map_size)); return TDB_ERRCODE(TDB_ERR_IO, -1); } @@ -217,7 +217,7 @@ void tdb_mmap(struct tdb_context *tdb) if (tdb->map_ptr == MAP_FAILED) { tdb->map_ptr = NULL; - TDB_LOG((tdb, 2, "tdb_mmap failed for size %d (%s)\n", + TDB_LOG((tdb, TDB_DEBUG_WARNING, "tdb_mmap failed for size %d (%s)\n", tdb->map_size, strerror(errno))); } } else { @@ -242,7 +242,7 @@ static int tdb_expand_file(struct tdb_context *tdb, tdb_off_t size, tdb_off_t ad if (ftruncate(tdb->fd, size+addition) == -1) { char b = 0; if (pwrite(tdb->fd, &b, 1, (size+addition) - 1) != 1) { - TDB_LOG((tdb, 0, "expand_file to %d failed (%s)\n", + TDB_LOG((tdb, TDB_DEBUG_FATAL, "expand_file to %d failed (%s)\n", size+addition, strerror(errno))); return -1; } @@ -256,7 +256,7 @@ static int tdb_expand_file(struct tdb_context *tdb, tdb_off_t size, tdb_off_t ad int n = addition>sizeof(buf)?sizeof(buf):addition; int ret = pwrite(tdb->fd, buf, n, size); if (ret != n) { - TDB_LOG((tdb, 0, "expand_file write of %d failed (%s)\n", + TDB_LOG((tdb, TDB_DEBUG_FATAL, "expand_file write of %d failed (%s)\n", n, strerror(errno))); return -1; } @@ -275,7 +275,7 @@ int tdb_expand(struct tdb_context *tdb, tdb_off_t size) tdb_off_t offset; if (tdb_lock(tdb, -1, F_WRLCK) == -1) { - TDB_LOG((tdb, 0, "lock failed in tdb_expand\n")); + TDB_LOG((tdb, TDB_DEBUG_ERROR, "lock failed in tdb_expand\n")); return -1; } @@ -363,7 +363,7 @@ unsigned char *tdb_alloc_read(struct tdb_context *tdb, tdb_off_t offset, tdb_len if (!(buf = malloc(len))) { /* Ensure ecode is set for log fn. */ tdb->ecode = TDB_ERR_OOM; - TDB_LOG((tdb, 0,"tdb_alloc_read malloc failed len=%d (%s)\n", + TDB_LOG((tdb, TDB_DEBUG_ERROR,"tdb_alloc_read malloc failed len=%d (%s)\n", len, strerror(errno))); return TDB_ERRCODE(TDB_ERR_OOM, buf); } @@ -382,7 +382,7 @@ int tdb_rec_read(struct tdb_context *tdb, tdb_off_t offset, struct list_struct * if (TDB_BAD_MAGIC(rec)) { /* Ensure ecode is set for log fn. */ tdb->ecode = TDB_ERR_CORRUPT; - TDB_LOG((tdb, 0,"tdb_rec_read bad magic 0x%x at offset=%d\n", rec->magic, offset)); + TDB_LOG((tdb, TDB_DEBUG_FATAL,"tdb_rec_read bad magic 0x%x at offset=%d\n", rec->magic, offset)); return TDB_ERRCODE(TDB_ERR_CORRUPT, -1); } return tdb->methods->tdb_oob(tdb, rec->next+sizeof(*rec), 0); -- cgit From 3ca73facc59ed8a97abbc28c1b4bedde87e109a6 Mon Sep 17 00:00:00 2001 From: Andrew Tridgell Date: Wed, 6 Sep 2006 11:31:59 +0000 Subject: r18160: - pread and pwrite replacements need to be non-static - replacing rename() is pointless - all platforms have it (and the #define of rename breaks some code) - use system/locale.h in snprintf.c - fix passwd.h for initgroups - stdlib is in replace.h, not needed elsewhere - fix the initgroups replacement - fix mapping of dl functions to rep_* (This used to be commit 57cd0ca176387d6a3acabf9fedeef4f2a3a3dad7) --- source4/lib/tdb/common/io.c | 22 ---------------------- 1 file changed, 22 deletions(-) (limited to 'source4/lib/tdb/common/io.c') diff --git a/source4/lib/tdb/common/io.c b/source4/lib/tdb/common/io.c index 21d591b67d..d0ef94831d 100644 --- a/source4/lib/tdb/common/io.c +++ b/source4/lib/tdb/common/io.c @@ -29,28 +29,6 @@ #include "tdb_private.h" -#ifndef HAVE_PREAD - static ssize_t pread(int fd, void *buf, size_t count, off_t offset) -{ - if (lseek(fd, offset, SEEK_SET) != offset) { - errno = EIO; - return -1; - } - return read(fd, buf, count); -} -#endif - -#ifndef HAVE_PWRITE - static ssize_t pwrite(int fd, const void *buf, size_t count, off_t offset) -{ - if (lseek(fd, offset, SEEK_SET) != offset) { - errno = EIO; - return -1; - } - return write(fd, buf, count); -} -#endif - /* check for an out of bounds access - if it is out of bounds then see if the database has been expanded by someone else and expand if necessary -- cgit From 30ee8beb9316a99e8a49993306252591106cb349 Mon Sep 17 00:00:00 2001 From: Andrew Tridgell Date: Sat, 9 Sep 2006 10:05:58 +0000 Subject: r18301: I discovered how to load the warnings from a build farm build into emacs compile mode (hint, paste to a file, and compile as "cat filename"). This allowed me to fix nearly all the warnings for a IA_64 SuSE build very quickly. (This used to be commit eba6c84efff735bb0ca941ac4b755ce2b0591667) --- source4/lib/tdb/common/io.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'source4/lib/tdb/common/io.c') diff --git a/source4/lib/tdb/common/io.c b/source4/lib/tdb/common/io.c index d0ef94831d..383023e88a 100644 --- a/source4/lib/tdb/common/io.c +++ b/source4/lib/tdb/common/io.c @@ -125,7 +125,7 @@ static int tdb_read(struct tdb_context *tdb, tdb_off_t off, void *buf, /* Ensure ecode is set for log fn. */ tdb->ecode = TDB_ERR_IO; TDB_LOG((tdb, TDB_DEBUG_FATAL,"tdb_read failed at %d len=%d ret=%d (%s) map_size=%d\n", - off, len, ret, strerror(errno), tdb->map_size)); + off, len, ret, strerror(errno), (int)tdb->map_size)); return TDB_ERRCODE(TDB_ERR_IO, -1); } } -- cgit From 118c064a473562274bff8fb47f37437db904b8fb Mon Sep 17 00:00:00 2001 From: Andrew Tridgell Date: Fri, 20 Oct 2006 08:06:14 +0000 Subject: r19423: merge some tdb changes from SAMBA_3_0 to SAMBA_4_0 this is in preparation of a merge in the other direction (This used to be commit db3211079fd594aa03c3b9bb3eb6ad86bdd32837) --- source4/lib/tdb/common/io.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) (limited to 'source4/lib/tdb/common/io.c') diff --git a/source4/lib/tdb/common/io.c b/source4/lib/tdb/common/io.c index 383023e88a..5d2c5c8e2e 100644 --- a/source4/lib/tdb/common/io.c +++ b/source4/lib/tdb/common/io.c @@ -102,7 +102,7 @@ static int tdb_write(struct tdb_context *tdb, tdb_off_t off, /* Endian conversion: we only ever deal with 4 byte quantities */ void *tdb_convert(void *buf, u32 size) { - u32 i, *p = buf; + u32 i, *p = (u32 *)buf; for (i = 0; i < size / 4; i++) p[i] = TDB_BYTEREV(p[i]); return buf; @@ -282,7 +282,8 @@ int tdb_expand(struct tdb_context *tdb, tdb_off_t size) tdb->map_size += size; if (tdb->flags & TDB_INTERNAL) { - char *new_map_ptr = realloc(tdb->map_ptr, tdb->map_size); + char *new_map_ptr = (char *)realloc(tdb->map_ptr, + tdb->map_size); if (!new_map_ptr) { tdb->map_size -= size; goto fail; -- cgit From eaaf246d4fa42df5e590ee5bfe54e672abd26b02 Mon Sep 17 00:00:00 2001 From: Volker Lendecke Date: Mon, 19 Feb 2007 11:45:33 +0000 Subject: r21445: Apply tdb_parse_record Tridges error return, merge to 3_0_25 and 4_0 (This used to be commit afe7d7855841066b88859976ac748cbf438a9a9f) --- source4/lib/tdb/common/io.c | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) (limited to 'source4/lib/tdb/common/io.c') diff --git a/source4/lib/tdb/common/io.c b/source4/lib/tdb/common/io.c index 5d2c5c8e2e..b3d1c56dc4 100644 --- a/source4/lib/tdb/common/io.c +++ b/source4/lib/tdb/common/io.c @@ -353,6 +353,40 @@ unsigned char *tdb_alloc_read(struct tdb_context *tdb, tdb_off_t offset, tdb_len return buf; } +/* Give a piece of tdb data to a parser */ + +int tdb_parse_data(struct tdb_context *tdb, TDB_DATA key, + tdb_off_t offset, tdb_len_t len, + int (*parser)(TDB_DATA key, TDB_DATA data, + void *private_data), + void *private_data) +{ + TDB_DATA data; + int result; + + data.dsize = len; + + if ((tdb->transaction == NULL) && (tdb->map_ptr != NULL)) { + /* + * Optimize by avoiding the malloc/memcpy/free, point the + * parser directly at the mmap area. + */ + if (tdb->methods->tdb_oob(tdb, offset+len, 0) != 0) { + return -1; + } + data.dptr = offset + (unsigned char *)tdb->map_ptr; + return parser(key, data, private_data); + } + + if (!(data.dptr = tdb_alloc_read(tdb, offset, len))) { + return -1; + } + + result = parser(key, data, private_data); + free(data.dptr); + return result; +} + /* read/write a record */ int tdb_rec_read(struct tdb_context *tdb, tdb_off_t offset, struct list_struct *rec) { -- cgit From 769efdf048d80c4081487d555649de0f31738dd1 Mon Sep 17 00:00:00 2001 From: Stefan Metzmacher Date: Mon, 2 Apr 2007 18:56:25 +0000 Subject: r22041: merge trivial changes from samba3 metze (This used to be commit 902a76ca705f07c61f86a9ef1346583ba9d3157d) --- source4/lib/tdb/common/io.c | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) (limited to 'source4/lib/tdb/common/io.c') diff --git a/source4/lib/tdb/common/io.c b/source4/lib/tdb/common/io.c index b3d1c56dc4..cd06dbb1e3 100644 --- a/source4/lib/tdb/common/io.c +++ b/source4/lib/tdb/common/io.c @@ -124,8 +124,10 @@ static int tdb_read(struct tdb_context *tdb, tdb_off_t off, void *buf, if (ret != (ssize_t)len) { /* Ensure ecode is set for log fn. */ tdb->ecode = TDB_ERR_IO; - TDB_LOG((tdb, TDB_DEBUG_FATAL,"tdb_read failed at %d len=%d ret=%d (%s) map_size=%d\n", - off, len, ret, strerror(errno), (int)tdb->map_size)); + TDB_LOG((tdb, TDB_DEBUG_FATAL,"tdb_read failed at %d " + "len=%d ret=%d (%s) map_size=%d\n", + (int)off, (int)len, (int)ret, strerror(errno), + (int)tdb->map_size)); return TDB_ERRCODE(TDB_ERR_IO, -1); } } @@ -339,7 +341,7 @@ unsigned char *tdb_alloc_read(struct tdb_context *tdb, tdb_off_t offset, tdb_len len = 1; } - if (!(buf = malloc(len))) { + if (!(buf = (unsigned char *)malloc(len))) { /* Ensure ecode is set for log fn. */ tdb->ecode = TDB_ERR_OOM; TDB_LOG((tdb, TDB_DEBUG_ERROR,"tdb_alloc_read malloc failed len=%d (%s)\n", -- cgit From b8d69a7ea2505b706ff7c74d7c97bc89d82dfa07 Mon Sep 17 00:00:00 2001 From: Andrew Tridgell Date: Tue, 10 Jul 2007 02:46:15 +0000 Subject: r23795: more v2->v3 conversion (This used to be commit 84b468b2f8f2dffda89593f816e8bc6a8b6d42ac) --- source4/lib/tdb/common/io.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'source4/lib/tdb/common/io.c') diff --git a/source4/lib/tdb/common/io.c b/source4/lib/tdb/common/io.c index cd06dbb1e3..6dbcae2d1c 100644 --- a/source4/lib/tdb/common/io.c +++ b/source4/lib/tdb/common/io.c @@ -14,7 +14,7 @@ This library is free software; you can redistribute it and/or modify it under the terms of the GNU Lesser General Public License as published by the Free Software Foundation; either - version 2 of the License, or (at your option) any later version. + version 3 of the License, or (at your option) any later version. This library is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of -- cgit From 6c973f4e8ccbcb6c9275f8a54e26abb19df7e15a Mon Sep 17 00:00:00 2001 From: Andrew Tridgell Date: Tue, 10 Jul 2007 03:42:26 +0000 Subject: r23798: updated old Temple Place FSF addresses to new URL (This used to be commit 40c0919aaa9c1b14bbaebb95ecce53eb0380fdbb) --- source4/lib/tdb/common/io.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) (limited to 'source4/lib/tdb/common/io.c') diff --git a/source4/lib/tdb/common/io.c b/source4/lib/tdb/common/io.c index 6dbcae2d1c..cc66b85b33 100644 --- a/source4/lib/tdb/common/io.c +++ b/source4/lib/tdb/common/io.c @@ -22,8 +22,7 @@ Lesser General Public License for more details. You should have received a copy of the GNU Lesser General Public - License along with this library; if not, write to the Free Software - Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + License along with this library; if not, see . */ -- cgit From d568e2b1d3ae65d2f7a7fd2c6e7e317b396ebbcf Mon Sep 17 00:00:00 2001 From: Michael Adam Date: Thu, 19 Jul 2007 13:46:26 +0000 Subject: r23972: Fix a bug in pwrite error detection in tdb_expand_file(): The proper error condition is (ret == -1) instead of (ret != number_of_byte_told_to_write). Michael (This used to be commit 4c3c6363f860ec01d3c789ef8ee2aa3eb77000dc) --- source4/lib/tdb/common/io.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'source4/lib/tdb/common/io.c') diff --git a/source4/lib/tdb/common/io.c b/source4/lib/tdb/common/io.c index cc66b85b33..ea925f3895 100644 --- a/source4/lib/tdb/common/io.c +++ b/source4/lib/tdb/common/io.c @@ -234,13 +234,13 @@ static int tdb_expand_file(struct tdb_context *tdb, tdb_off_t size, tdb_off_t ad while (addition) { int n = addition>sizeof(buf)?sizeof(buf):addition; int ret = pwrite(tdb->fd, buf, n, size); - if (ret != n) { + if (ret == -1) { TDB_LOG((tdb, TDB_DEBUG_FATAL, "expand_file write of %d failed (%s)\n", n, strerror(errno))); return -1; } - addition -= n; - size += n; + addition -= ret; + size += ret; } return 0; } -- cgit From af8432e692eeb94bce80f155c6b9ab158258fff8 Mon Sep 17 00:00:00 2001 From: Michael Adam Date: Fri, 20 Jul 2007 14:23:12 +0000 Subject: r23977: Im prove the pwrite-patch to tdb_expand_file of r23972: * prevent infinite loops due to 0 bytes written: try once more. if we still get 0 as return, set errno to ENOSPC and return -1 (error) * replace int by correct types (ssize_t and size_t). * print a warning log message in case "written < requested to write" usually this means, that the next call to pwrite will fail with return value -1 and set errno accordingly. Note that the former error condition "written != requested to write" is not a correct error condition of write/pwrite. If this is due to an error, a subsequent call to (p)write will reveal the cause (typically "no space left on device" - ENOSPC). Michael (This used to be commit 7f415d12239fc67eb2c7894c6359b9507fe122c6) --- source4/lib/tdb/common/io.c | 27 ++++++++++++++++++++------- 1 file changed, 20 insertions(+), 7 deletions(-) (limited to 'source4/lib/tdb/common/io.c') diff --git a/source4/lib/tdb/common/io.c b/source4/lib/tdb/common/io.c index ea925f3895..a24b5e24a2 100644 --- a/source4/lib/tdb/common/io.c +++ b/source4/lib/tdb/common/io.c @@ -232,15 +232,28 @@ static int tdb_expand_file(struct tdb_context *tdb, tdb_off_t size, tdb_off_t ad disk. This must be done with write, not via mmap */ memset(buf, TDB_PAD_BYTE, sizeof(buf)); while (addition) { - int n = addition>sizeof(buf)?sizeof(buf):addition; - int ret = pwrite(tdb->fd, buf, n, size); - if (ret == -1) { - TDB_LOG((tdb, TDB_DEBUG_FATAL, "expand_file write of %d failed (%s)\n", - n, strerror(errno))); + size_t n = addition>sizeof(buf)?sizeof(buf):addition; + ssize_t written = pwrite(tdb->fd, buf, n, size); + if (written == 0) { + /* prevent infinite loops: try _once_ more */ + written = pwrite(tdb->fd, buf, n, size); + } + if (written == 0) { + /* give up, trying to provide a useful errno */ + TDB_LOG((tdb, TDB_DEBUG_FATAL, "expand_file write " + "returned 0 twice: giving up!\n")); + errno = ENOSPC; + return -1; + } else if (written == -1) { + TDB_LOG((tdb, TDB_DEBUG_FATAL, "expand_file write of " + "%d bytes failed (%s)\n", n, strerror(errno))); return -1; + } else if (written != n) { + TDB_LOG((tdb, TDB_DEBUG_WARNING, "expand_file: wrote " + "only %d of %d bytes - retrying\n", written,n)); } - addition -= ret; - size += ret; + addition -= written; + size += written; } return 0; } -- cgit From 0673a182771878892b584efc5316e4029a8fee47 Mon Sep 17 00:00:00 2001 From: Michael Adam Date: Fri, 20 Jul 2007 16:31:32 +0000 Subject: r23980: Fix one more use of pwrite in expand_file. Michael (This used to be commit b97acdc67b1a55529e69bb7b2b78a317a34b1eba) --- source4/lib/tdb/common/io.c | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) (limited to 'source4/lib/tdb/common/io.c') diff --git a/source4/lib/tdb/common/io.c b/source4/lib/tdb/common/io.c index a24b5e24a2..df924a9dcd 100644 --- a/source4/lib/tdb/common/io.c +++ b/source4/lib/tdb/common/io.c @@ -220,7 +220,16 @@ static int tdb_expand_file(struct tdb_context *tdb, tdb_off_t size, tdb_off_t ad if (ftruncate(tdb->fd, size+addition) == -1) { char b = 0; - if (pwrite(tdb->fd, &b, 1, (size+addition) - 1) != 1) { + ssize_t written = pwrite(tdb->fd, &b, 1, (size+addition) - 1); + if (written == 0) { + /* try once more, potentially revealing errno */ + written = pwrite(tdb->fd, &b, 1, (size+addition) - 1); + } + if (written == 0) { + /* again - give up, guessing errno */ + errno = ENOSPC; + } + if (written != 1) { TDB_LOG((tdb, TDB_DEBUG_FATAL, "expand_file to %d failed (%s)\n", size+addition, strerror(errno))); return -1; -- cgit From f3e13632813c543583fe1d04203825743aa99111 Mon Sep 17 00:00:00 2001 From: Jelmer Vernooij Date: Sat, 11 Aug 2007 21:19:24 +0000 Subject: r24336: Use standard data type uint32_t rather than tdb-specific u32. (This used to be commit f90a698387c53508862eb6359bd4d1fba1d2b4b0) --- source4/lib/tdb/common/io.c | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) (limited to 'source4/lib/tdb/common/io.c') diff --git a/source4/lib/tdb/common/io.c b/source4/lib/tdb/common/io.c index df924a9dcd..86001dfdae 100644 --- a/source4/lib/tdb/common/io.c +++ b/source4/lib/tdb/common/io.c @@ -99,9 +99,9 @@ static int tdb_write(struct tdb_context *tdb, tdb_off_t off, } /* Endian conversion: we only ever deal with 4 byte quantities */ -void *tdb_convert(void *buf, u32 size) +void *tdb_convert(void *buf, uint32_t size) { - u32 i, *p = (u32 *)buf; + uint32_t i, *p = (uint32_t *)buf; for (i = 0; i < size / 4; i++) p[i] = TDB_BYTEREV(p[i]); return buf; @@ -142,17 +142,17 @@ static int tdb_read(struct tdb_context *tdb, tdb_off_t off, void *buf, do an unlocked scan of the hash table heads to find the next non-zero head. The value will then be confirmed with the lock held */ -static void tdb_next_hash_chain(struct tdb_context *tdb, u32 *chain) +static void tdb_next_hash_chain(struct tdb_context *tdb, uint32_t *chain) { - u32 h = *chain; + uint32_t h = *chain; if (tdb->map_ptr) { for (;h < tdb->header.hash_size;h++) { - if (0 != *(u32 *)(TDB_HASH_TOP(h) + (unsigned char *)tdb->map_ptr)) { + if (0 != *(uint32_t *)(TDB_HASH_TOP(h) + (unsigned char *)tdb->map_ptr)) { break; } } } else { - u32 off=0; + uint32_t off=0; for (;h < tdb->header.hash_size;h++) { if (tdb_ofs_read(tdb, TDB_HASH_TOP(h), &off) != 0 || off != 0) { break; -- cgit From 8277256cee6328a130e3a714579a4bc766d66b9b Mon Sep 17 00:00:00 2001 From: Michael Adam Date: Tue, 28 Aug 2007 14:25:46 +0000 Subject: r24738: Fix one more use of pwrite in tdb code in the spirit of r23972 and r23977. Michael (This used to be commit 7b2cabea55cebb98e0fcee17066a0871667cd83f) --- source4/lib/tdb/common/io.c | 31 +++++++++++++++++++++++++------ 1 file changed, 25 insertions(+), 6 deletions(-) (limited to 'source4/lib/tdb/common/io.c') diff --git a/source4/lib/tdb/common/io.c b/source4/lib/tdb/common/io.c index 86001dfdae..3b7e712cfe 100644 --- a/source4/lib/tdb/common/io.c +++ b/source4/lib/tdb/common/io.c @@ -88,12 +88,31 @@ static int tdb_write(struct tdb_context *tdb, tdb_off_t off, if (tdb->map_ptr) { memcpy(off + (char *)tdb->map_ptr, buf, len); - } else if (pwrite(tdb->fd, buf, len, off) != (ssize_t)len) { - /* Ensure ecode is set for log fn. */ - tdb->ecode = TDB_ERR_IO; - TDB_LOG((tdb, TDB_DEBUG_FATAL,"tdb_write failed at %d len=%d (%s)\n", - off, len, strerror(errno))); - return TDB_ERRCODE(TDB_ERR_IO, -1); + } else { + ssize_t written = pwrite(tdb->fd, buf, len, off); + if ((written != (ssize_t)len) && (written != -1)) { + /* try once more */ + TDB_LOG((tdb, TDB_DEBUG_FATAL, "tdb_write: wrote only " + "%d of %d bytes at %d, trying once more\n", + written, len, off)); + errno = ENOSPC; + written = pwrite(tdb->fd, (void *)((char *)buf+written), + len-written, + off+written); + } + if (written == -1) { + /* Ensure ecode is set for log fn. */ + tdb->ecode = TDB_ERR_IO; + TDB_LOG((tdb, TDB_DEBUG_FATAL,"tdb_write failed at %d " + "len=%d (%s)\n", off, len, strerror(errno))); + return TDB_ERRCODE(TDB_ERR_IO, -1); + } else if (written != (ssize_t)len) { + TDB_LOG((tdb, TDB_DEBUG_FATAL, "tdb_write: failed to " + "write %d bytes at %d in two attempts\n", + len, off)); + errno = ENOSPC; + return TDB_ERRCODE(TDB_ERR_IO, -1); + } } return 0; } -- cgit From 2cb22d93ae0eef86a4bcf0fe5f7fb138e43db880 Mon Sep 17 00:00:00 2001 From: Jeremy Allison Date: Wed, 7 Nov 2007 06:59:02 +0100 Subject: r25892: Keep the tdb code in sync between 3.2.x and 4.0. Add in the alarm fix to allow locks to exit on alarm signal. Sync up the changes in tools. Jeremy. (This used to be commit cb6c663fa8818f49cc36f196bb5f4dea47edd69e) --- source4/lib/tdb/common/io.c | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) (limited to 'source4/lib/tdb/common/io.c') diff --git a/source4/lib/tdb/common/io.c b/source4/lib/tdb/common/io.c index 3b7e712cfe..c963e66ad4 100644 --- a/source4/lib/tdb/common/io.c +++ b/source4/lib/tdb/common/io.c @@ -94,7 +94,7 @@ static int tdb_write(struct tdb_context *tdb, tdb_off_t off, /* try once more */ TDB_LOG((tdb, TDB_DEBUG_FATAL, "tdb_write: wrote only " "%d of %d bytes at %d, trying once more\n", - written, len, off)); + (int)written, len, off)); errno = ENOSPC; written = pwrite(tdb->fd, (void *)((char *)buf+written), len-written, @@ -274,11 +274,13 @@ static int tdb_expand_file(struct tdb_context *tdb, tdb_off_t size, tdb_off_t ad return -1; } else if (written == -1) { TDB_LOG((tdb, TDB_DEBUG_FATAL, "expand_file write of " - "%d bytes failed (%s)\n", n, strerror(errno))); + "%d bytes failed (%s)\n", (int)n, + strerror(errno))); return -1; } else if (written != n) { TDB_LOG((tdb, TDB_DEBUG_WARNING, "expand_file: wrote " - "only %d of %d bytes - retrying\n", written,n)); + "only %d of %d bytes - retrying\n", (int)written, + (int)n)); } addition -= written; size += written; -- cgit From 523445cdaa118fd9cb61d63d1a71a0762b98cdcf Mon Sep 17 00:00:00 2001 From: Jeremy Allison Date: Wed, 7 Nov 2007 07:13:44 +0100 Subject: r25893: Fix ldb, tdb builds (and one warning). Jeremy. (This used to be commit 52b26645b04a9c5fb70e7b869b60c9157f821d50) --- source4/lib/tdb/common/io.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'source4/lib/tdb/common/io.c') diff --git a/source4/lib/tdb/common/io.c b/source4/lib/tdb/common/io.c index c963e66ad4..8ab0768883 100644 --- a/source4/lib/tdb/common/io.c +++ b/source4/lib/tdb/common/io.c @@ -96,7 +96,7 @@ static int tdb_write(struct tdb_context *tdb, tdb_off_t off, "%d of %d bytes at %d, trying once more\n", (int)written, len, off)); errno = ENOSPC; - written = pwrite(tdb->fd, (void *)((char *)buf+written), + written = pwrite(tdb->fd, (const void *)((const char *)buf+written), len-written, off+written); } -- cgit From 9170998427ebbb7abfd9b482fb6e0d051bca5205 Mon Sep 17 00:00:00 2001 From: Andrew Tridgell Date: Tue, 15 Jan 2008 14:05:47 +1100 Subject: merged tdb from ctdb bzr tree (This used to be commit ed0c3a0f74c305b3b8554b05c3f97cf79db8296a) --- source4/lib/tdb/common/io.c | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) (limited to 'source4/lib/tdb/common/io.c') diff --git a/source4/lib/tdb/common/io.c b/source4/lib/tdb/common/io.c index 8ab0768883..172ab69d8c 100644 --- a/source4/lib/tdb/common/io.c +++ b/source4/lib/tdb/common/io.c @@ -101,8 +101,8 @@ static int tdb_write(struct tdb_context *tdb, tdb_off_t off, off+written); } if (written == -1) { - /* Ensure ecode is set for log fn. */ - tdb->ecode = TDB_ERR_IO; + /* Ensure ecode is set for log fn. */ + tdb->ecode = TDB_ERR_IO; TDB_LOG((tdb, TDB_DEBUG_FATAL,"tdb_write failed at %d " "len=%d (%s)\n", off, len, strerror(errno))); return TDB_ERRCODE(TDB_ERR_IO, -1); @@ -111,8 +111,8 @@ static int tdb_write(struct tdb_context *tdb, tdb_off_t off, "write %d bytes at %d in two attempts\n", len, off)); errno = ENOSPC; - return TDB_ERRCODE(TDB_ERR_IO, -1); - } + return TDB_ERRCODE(TDB_ERR_IO, -1); + } } return 0; } @@ -230,7 +230,7 @@ void tdb_mmap(struct tdb_context *tdb) says to use for mmap expansion */ static int tdb_expand_file(struct tdb_context *tdb, tdb_off_t size, tdb_off_t addition) { - char buf[1024]; + char buf[8192]; if (tdb->read_only || tdb->traverse_read) { tdb->ecode = TDB_ERR_RDONLY; @@ -294,7 +294,7 @@ static int tdb_expand_file(struct tdb_context *tdb, tdb_off_t size, tdb_off_t ad int tdb_expand(struct tdb_context *tdb, tdb_off_t size) { struct list_struct rec; - tdb_off_t offset; + tdb_off_t offset, new_size; if (tdb_lock(tdb, -1, F_WRLCK) == -1) { TDB_LOG((tdb, TDB_DEBUG_ERROR, "lock failed in tdb_expand\n")); @@ -304,9 +304,11 @@ int tdb_expand(struct tdb_context *tdb, tdb_off_t size) /* must know about any previous expansions by another process */ tdb->methods->tdb_oob(tdb, tdb->map_size + 1, 1); - /* always make room for at least 10 more records, and round - the database up to a multiple of the page size */ - size = TDB_ALIGN(tdb->map_size + size*10, tdb->page_size) - tdb->map_size; + /* always make room for at least 100 more records, and at + least 25% more space. Round the database up to a multiple + of the page size */ + new_size = MAX(tdb->map_size + size*100, tdb->map_size * 1.25); + size = TDB_ALIGN(new_size, tdb->page_size) - tdb->map_size; if (!(tdb->flags & TDB_INTERNAL)) tdb_munmap(tdb); -- cgit From b39e332bd7803dd4ce8267f750c497b5deb837ee Mon Sep 17 00:00:00 2001 From: Simo Sorce Date: Wed, 3 Sep 2008 10:44:09 -0400 Subject: The msync manpage reports that msync *must* be called before munmap. Failure to do so may result in lost data. Fix an ifdef check, I really think we meant to check HAVE_MMAP here. (cherry picked from commit 74c8575b3f3b90ea21ae6aa7ccd95947838af956) (This used to be commit 8fd54bb55f0c23bd025d1719abcbe75c6a2ea8ac) --- source4/lib/tdb/common/io.c | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) (limited to 'source4/lib/tdb/common/io.c') diff --git a/source4/lib/tdb/common/io.c b/source4/lib/tdb/common/io.c index 172ab69d8c..4ec18de48e 100644 --- a/source4/lib/tdb/common/io.c +++ b/source4/lib/tdb/common/io.c @@ -189,7 +189,13 @@ int tdb_munmap(struct tdb_context *tdb) #ifdef HAVE_MMAP if (tdb->map_ptr) { - int ret = munmap(tdb->map_ptr, tdb->map_size); + int ret; + + ret = msync(tdb->map_ptr, tdb->map_size, MS_SYNC); + if (ret != 0) + return ret; + + ret = munmap(tdb->map_ptr, tdb->map_size); if (ret != 0) return ret; } -- cgit From a1cf3ad5d655cf5a847df6d6299b3af1a30ec1e3 Mon Sep 17 00:00:00 2001 From: Jeremy Allison Date: Fri, 5 Sep 2008 15:32:22 -0700 Subject: Remove unecessary msync. Jeremy. (cherry picked from commit 0bae1ef3de8fda5e1e2d641b14a408e627396912) (This used to be commit db2acaf46fdc38078b6b28b68909e289f6c9e0ec) --- source4/lib/tdb/common/io.c | 4 ---- 1 file changed, 4 deletions(-) (limited to 'source4/lib/tdb/common/io.c') diff --git a/source4/lib/tdb/common/io.c b/source4/lib/tdb/common/io.c index 4ec18de48e..661f761489 100644 --- a/source4/lib/tdb/common/io.c +++ b/source4/lib/tdb/common/io.c @@ -191,10 +191,6 @@ int tdb_munmap(struct tdb_context *tdb) if (tdb->map_ptr) { int ret; - ret = msync(tdb->map_ptr, tdb->map_size, MS_SYNC); - if (ret != 0) - return ret; - ret = munmap(tdb->map_ptr, tdb->map_size); if (ret != 0) return ret; -- cgit