From 5a9ceee7a524293c67d4c32edde9da32a877ecce Mon Sep 17 00:00:00 2001 From: Stefan Metzmacher Date: Fri, 25 Mar 2005 13:40:17 +0000 Subject: r6061: add start of compression support in our rpc code this is not complete cuurently... but I want other people to test it and help me on finishing it. (try to change the #if 0 in torture/rpc/drsuapi.c into #if 1) metze (This used to be commit 335adef37082a78e0426decb715629bd778e6582) --- source4/build/pidl/ndr_parser.pm | 138 ++++++++++++++++++++++++++- source4/configure.in | 1 + source4/librpc/config.m4 | 35 +++++++ source4/librpc/config.mk | 3 +- source4/librpc/idl/drsuapi.idl | 8 +- source4/librpc/ndr/libndr.h | 5 + source4/librpc/ndr/ndr_compression.c | 177 +++++++++++++++++++++++++++++++++++ 7 files changed, 359 insertions(+), 8 deletions(-) create mode 100644 source4/librpc/config.m4 create mode 100644 source4/librpc/ndr/ndr_compression.c diff --git a/source4/build/pidl/ndr_parser.pm b/source4/build/pidl/ndr_parser.pm index a689a8a849..6f8029d338 100644 --- a/source4/build/pidl/ndr_parser.pm +++ b/source4/build/pidl/ndr_parser.pm @@ -470,6 +470,92 @@ sub ParseArrayPull($$$$) } } +sub compression_alg($) +{ + my $e = shift; + my $compression = util::has_property($e, "compression"); + my ($alg, $clen, $dlen) = split(/ /, $compression); + + return $alg; +} + +sub compression_clen($) +{ + my $e = shift; + my $compression = util::has_property($e, "compression"); + my ($alg, $clen, $dlen) = split(/ /, $compression); + + return ParseExpr($e, $clen, "r->"); +} + +sub compression_dlen($) +{ + my $e = shift; + my $compression = util::has_property($e, "compression"); + my ($alg, $clen, $dlen) = split(/ /, $compression); + + return ParseExpr($e, $dlen, "r->"); +} + +sub ParseCompressionPushStart($$$) +{ + my $e = shift; + my $subndr = shift; + my $ndr_flags = shift; + my $comndr = $subndr."_compressed"; + + pidl "{"; + indent; + pidl "struct ndr_push *$comndr;"; + pidl ""; + pidl "$comndr = ndr_push_init_ctx($subndr);"; + pidl "if (!$comndr) return NT_STATUS_NO_MEMORY;"; + pidl "$comndr->flags = $subndr->flags;"; + pidl ""; + + return $comndr; +} + +sub ParseCompressionPushEnd($$) +{ + my $e = shift; + my $subndr = shift; + my $comndr = $subndr."_compressed"; + my $alg = compression_alg($e); + + pidl "NDR_CHECK(ndr_push_compression($subndr, $comndr, $alg));"; + deindent; + pidl "}"; +} + +sub ParseCompressionPullStart($$$) +{ + my $e = shift; + my $subndr = shift; + my $ndr_flags = shift; + my $comndr = $subndr."_compressed"; + my $alg = compression_alg($e); + my $dlen = compression_dlen($e); + + pidl "{"; + indent; + pidl "struct ndr_pull *$comndr;"; + pidl "NDR_ALLOC($subndr, $comndr);"; + pidl "NDR_CHECK(ndr_pull_compression($subndr, $comndr, $alg, $dlen));"; + + return $comndr; +} + +sub ParseCompressionPullEnd($$) +{ + my $e = shift; + my $subndr = shift; + my $comndr = $subndr."_compressed"; + + deindent; + pidl "}"; +} + sub ParseSubcontextPushStart($$) { my $e = shift; @@ -551,7 +637,10 @@ sub ParseElementPushScalar($$$) my $cprefix = c_push_prefix($e); my $ptr_prefix = c_ptr_prefix($e); my $sub_size = util::has_property($e, "subcontext"); + my $compression = util::has_property($e, "compression"); my $ndr = "ndr"; + my $subndr = undef; + my $comndr = undef; start_flags($e); @@ -560,7 +649,12 @@ sub ParseElementPushScalar($$$) } if (defined $sub_size and $e->{POINTERS} == 0) { - $ndr = ParseSubcontextPushStart($e, "NDR_SCALARS"); + $subndr = ParseSubcontextPushStart($e, "NDR_SCALARS"); + $ndr = $subndr; + if (defined $compression) { + $comndr = ParseCompressionPushStart($e, $subndr, "NDR_SCALARS"); + $ndr = $comndr; + } } if (Ndr::need_wire_pointer($e)) { @@ -578,6 +672,9 @@ sub ParseElementPushScalar($$$) } if (defined $sub_size and $e->{POINTERS} == 0) { + if (defined $compression) { + ParseCompressionPushEnd($e, $subndr); + } ParseSubcontextPushEnd($e); } @@ -698,13 +795,21 @@ sub ParseElementPullScalar($$$) my $cprefix = c_pull_prefix($e); my $ptr_prefix = c_ptr_prefix($e); my $sub_size = util::has_property($e, "subcontext"); + my $compression = util::has_property($e, "compression"); my $ndr = "ndr"; + my $subndr = undef; + my $comndr = undef; start_flags($e); if (defined $sub_size && $e->{POINTERS} == 0) { - $ndr = ParseSubcontextPullStart($e, $ndr_flags); + $subndr = ParseSubcontextPullStart($e, $ndr_flags); + $ndr = $subndr; $ndr_flags = "NDR_SCALARS|NDR_BUFFERS"; + if (defined $compression) { + $comndr = ParseCompressionPullStart($e, $subndr, $ndr_flags); + $ndr = $comndr; + } } if (Ndr::is_inline_array($e)) { @@ -728,6 +833,9 @@ sub ParseElementPullScalar($$$) } if (defined $sub_size && $e->{POINTERS} == 0) { + if (defined $compression) { + ParseCompressionPullEnd($e, $subndr); + } ParseSubcontextPullEnd($e); } @@ -766,7 +874,10 @@ sub ParseElementPushBuffer($$) my($var_prefix) = shift; my $cprefix = c_push_prefix($e); my $sub_size = util::has_property($e, "subcontext"); + my $compression = util::has_property($e, "compression"); my $ndr = "ndr"; + my $subndr = undef; + my $comndr = undef; return unless (need_buffers_section($e)); @@ -793,8 +904,13 @@ sub ParseElementPushBuffer($$) } if (defined $sub_size) { - $ndr = ParseSubcontextPushStart($e, $ndr_flags); + $subndr = ParseSubcontextPushStart($e, $ndr_flags); + $ndr = $subndr; $ndr_flags = "NDR_SCALARS|NDR_BUFFERS"; + if (defined $compression) { + $comndr = ParseCompressionPushStart($e, $subndr, $ndr_flags); + $ndr = $comndr; + } } if (util::array_size($e)) { @@ -808,6 +924,9 @@ sub ParseElementPushBuffer($$) } if (defined $sub_size) { + if (defined $compression) { + ParseCompressionPushEnd($e, $subndr); + } ParseSubcontextPushEnd($e); } @@ -827,7 +946,10 @@ sub ParseElementPullBuffer($$) my($var_prefix) = shift; my $cprefix = c_pull_prefix($e); my $sub_size = util::has_property($e, "subcontext"); + my $compression = util::has_property($e, "compression"); my $ndr = "ndr"; + my $subndr = undef; + my $comndr = undef; return unless (need_buffers_section($e)); @@ -856,8 +978,13 @@ sub ParseElementPullBuffer($$) } if (defined $sub_size) { - $ndr = ParseSubcontextPullStart($e, $ndr_flags); + $subndr = ParseSubcontextPullStart($e, $ndr_flags); + $ndr = $subndr; $ndr_flags = "NDR_SCALARS|NDR_BUFFERS"; + if (defined $compression) { + $comndr = ParseCompressionPullStart($e, $subndr, $ndr_flags); + $ndr = $comndr; + } } if (util::array_size($e)) { @@ -871,6 +998,9 @@ sub ParseElementPullBuffer($$) } if (defined $sub_size) { + if (defined $compression) { + ParseCompressionPullEnd($e, $subndr); + } ParseSubcontextPullEnd($e); } diff --git a/source4/configure.in b/source4/configure.in index 5791a57adf..366f038030 100644 --- a/source4/configure.in +++ b/source4/configure.in @@ -22,6 +22,7 @@ SMB_INCLUDE_M4(lib/cmdline/config.m4) SMB_INCLUDE_M4(param/config.m4) SMB_INCLUDE_M4(libcli/auth/gensec.m4) SMB_INCLUDE_M4(libcli/config.m4) +SMB_INCLUDE_M4(librpc/config.m4) SMB_INCLUDE_M4(smbd/process_model.m4) SMB_INCLUDE_M4(lib/registry/config.m4) SMB_INCLUDE_M4(scripting/config.m4) diff --git a/source4/librpc/config.m4 b/source4/librpc/config.m4 new file mode 100644 index 0000000000..d67e880a38 --- /dev/null +++ b/source4/librpc/config.m4 @@ -0,0 +1,35 @@ +######################################################## +# Compile with compression support + +with_zlib_support=auto +ZLIB_LIBS="" +ZLIB_CFLAGS="" +ZLIB_CPPFLAGS="" +ZLIB_LDFLAGS="" + +AC_CHECK_HEADERS(zlib.h) +if test x"$ac_cv_header_zlib_h" != x"yes"; then + with_zlib_support=no +fi + +if test x"$with_zlib_support" != x"no"; then + AC_CHECK_LIB_EXT(z, ZLIB_LIBS, inflate) + + if test x"$ac_cv_lib_ext_z_inflate" = x"yes"; then + AC_DEFINE(HAVE_ZLIB,1,[Whether zlib is available]) + with_zlib_support=yes + SMB_EXT_LIB_ENABLE(ZLIB,YES) + else + ZLIB_LIBS="" + with_zlib_support=no + fi + LIBS=$ac_save_LIBS +fi +AC_MSG_CHECKING(whether ZLIB support is available) +AC_MSG_RESULT($with_zlib_support) + +# for now enable this always but maybe all fields are empty +# TODO: move compression methods to seperate files each +SMB_EXT_LIB_ENABLE(ZLIB,YES) + +SMB_EXT_LIB(ZLIB,[${ZLIB_LIBS}],[${ZLIB_CFLAGS}],[${ZLIB_CPPFLAGS}],[${ZLIB_LDFLAGS}]) diff --git a/source4/librpc/config.mk b/source4/librpc/config.mk index f6b05a41b5..d73f6bf96f 100644 --- a/source4/librpc/config.mk +++ b/source4/librpc/config.mk @@ -5,8 +5,9 @@ INIT_OBJ_FILES = \ librpc/ndr/ndr.o ADD_OBJ_FILES = \ librpc/ndr/ndr_basic.o \ + librpc/ndr/ndr_compression.o \ librpc/ndr/ndr_misc.o -REQUIRED_SUBSYSTEMS = LIBCLI_UTILS +REQUIRED_SUBSYSTEMS = LIBCLI_UTILS EXT_LIB_ZLIB # End SUBSYSTEM LIBNDR_RAW ################################################ diff --git a/source4/librpc/idl/drsuapi.idl b/source4/librpc/idl/drsuapi.idl index 330165830c..1b81dd6297 100644 --- a/source4/librpc/idl/drsuapi.idl +++ b/source4/librpc/idl/drsuapi.idl @@ -283,7 +283,8 @@ interface drsuapi } drsuapi_DsReplicaCoursor2Ctr2; typedef [v1_enum] enum { - DRSUAPI_OBJECTCLASS_top = 0x0 + DRSUAPI_OBJECTCLASS_top = 0x00010000, + DRSUAPI_OBJECTCLASS_test = 0x00000000 } drsuapi_DsObjectClassId; typedef [v1_enum,public] enum { @@ -497,9 +498,10 @@ interface drsuapi } drsuapi_DsGetNCChangesCtr1; typedef struct { - uint32 unknown1;/* decompressed_length ? */ + uint32 decompressed_length; uint32 compressed_length; - DATA_BLOB *compressed; + [subcontext(4),compression(NDR_COMPRESSION_ZLIB,compressed_length,decompressed_length),flag(NDR_REMAINING)] DATA_BLOB *decompressed; + /*[subcontext(4),compression(NDR_COMPRESSION_ZLIB,compressed_length,decompressed_length)] drsuapi_DsGetNCChangesCtr1 *ctr1;*/ } drsuapi_DsGetNCChangesCompressedInfo; typedef struct { diff --git a/source4/librpc/ndr/libndr.h b/source4/librpc/ndr/libndr.h index 4df3e7f248..3cc25b0ef2 100644 --- a/source4/librpc/ndr/libndr.h +++ b/source4/librpc/ndr/libndr.h @@ -152,6 +152,7 @@ enum ndr_err_code { NDR_ERR_CHARCNV, NDR_ERR_LENGTH, NDR_ERR_SUBCONTEXT, + NDR_ERR_COMPRESSION, NDR_ERR_STRING, NDR_ERR_VALIDATE, NDR_ERR_BUFSIZE, @@ -160,6 +161,10 @@ enum ndr_err_code { NDR_ERR_TOKEN }; +enum ndr_compression_alg { + NDR_COMPRESSION_ZLIB +}; + /* flags passed to control parse flow */ diff --git a/source4/librpc/ndr/ndr_compression.c b/source4/librpc/ndr/ndr_compression.c new file mode 100644 index 0000000000..d4f1a2a927 --- /dev/null +++ b/source4/librpc/ndr/ndr_compression.c @@ -0,0 +1,177 @@ +/* + Unix SMB/CIFS implementation. + + libndr compression support + + Copyright (C) Stefan Metzmacher 2005 + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. +*/ + +#include "includes.h" + +#ifdef HAVE_ZLIB +#include + +static NTSTATUS ndr_pull_compression_zlib(struct ndr_pull *subndr, + struct ndr_pull *comndr, + ssize_t decompressed_len) +{ + DATA_BLOB inbuf; + DATA_BLOB outbuf = data_blob_talloc(comndr, NULL, decompressed_len); + uint32_t outbuf_len = outbuf.length; + struct z_stream_s zs; + int ret; + + ZERO_STRUCT(zs); + + if (subndr->data_size < 10) { + return ndr_pull_error(subndr, NDR_ERR_COMPRESSION, "Bad ZLIB compressed header (PULL) subcontext size %d", + subndr->data_size); + } + + inbuf.data = subndr->data+10; + inbuf.length = subndr->data_size-10; + + zs.avail_in = inbuf.length; + zs.next_in = inbuf.data; + zs.next_out = outbuf.data; + zs.avail_out = outbuf.length; + + ret = inflateInit2(&zs, 15); + if (ret != Z_OK) { + return ndr_pull_error(subndr, NDR_ERR_COMPRESSION, "Bad ZLIB (PULL) inflateInit2 error %d", + ret); + } + + while(1) { + ret = inflate(&zs, Z_SYNC_FLUSH); + if (ret == Z_STREAM_END) { + + DEBUG(0,("inbuf.length: %d avail_in: %d, avail_out: %d\n", inbuf.length, zs.avail_in, zs.avail_out)); + break; + } + if (ret != Z_OK) { + return ndr_pull_error(subndr, NDR_ERR_COMPRESSION, "Bad ZLIB (PULL) inflate error %d", + ret); + } + } + + inflateEnd(&zs); + + /* TODO: check if the decompressed_len == outbuf_len */ + outbuf.length = outbuf_len - zs.avail_out; + + if (outbuf.length < 16) { + return ndr_pull_error(subndr, NDR_ERR_COMPRESSION, "Bad ZLIB uncompressed header (PULL) uncompressed size %d", + outbuf.length); + } + + outbuf.data += 16; + outbuf.length -= 16; + + /* TODO: really decompress the data here */ + *comndr = *subndr; + comndr->data = outbuf.data; + comndr->data_size = outbuf.length; + comndr->offset = 0; + + return NT_STATUS_OK; +} + +static NTSTATUS ndr_push_compression_zlib(struct ndr_push *subndr, + struct ndr_push *comndr) +{ + DATA_BLOB inbuf; + DATA_BLOB outbuf = data_blob_talloc(comndr, NULL, comndr->offset + 10); + struct z_stream_s zs; + int ret; + + ZERO_STRUCT(zs); + + inbuf = ndr_push_blob(comndr); + + zs.avail_in = inbuf.length; + zs.next_in = inbuf.data; + zs.next_out = outbuf.data+10; + zs.avail_out = outbuf.length-10; + + ret = deflateInit(&zs, Z_DEFAULT_COMPRESSION); + if (ret != Z_OK) { + return ndr_push_error(subndr, NDR_ERR_COMPRESSION, "Bad ZLIB (PUSH) deflateInit2 error %d", + ret); + } + + ret = deflate(&zs, Z_SYNC_FLUSH); + + if (ret != Z_OK && ret != Z_STREAM_END) { + return ndr_push_error(subndr, NDR_ERR_COMPRESSION, "Bad ZLIB (PULL) deflate error %d", + ret); + } + + deflateEnd(&zs); + + /* TODO: push the header here */ + + + NDR_CHECK(ndr_push_bytes(subndr, outbuf.data, outbuf.length)); + + return NT_STATUS_OK; +} +#endif + +/* + handle compressed subcontext buffers, which in midl land are user-marshalled, but + we use magic in pidl to make them easier to cope with +*/ +NTSTATUS ndr_pull_compression(struct ndr_pull *subndr, + struct ndr_pull *comndr, + enum ndr_compression_alg compression_alg, + ssize_t decompressed_len) +{ + comndr->flags = subndr->flags; + + switch (compression_alg) { +#ifdef HAVE_ZLIB + case NDR_COMPRESSION_ZLIB: + return ndr_pull_compression_zlib(subndr, comndr, decompressed_len); +#endif + default: + return ndr_pull_error(subndr, NDR_ERR_COMPRESSION, "Bad compression algorithm %d (PULL)", + compression_alg); + } + return NT_STATUS_OK; +} + +/* + push a compressed subcontext +*/ +NTSTATUS ndr_push_compression(struct ndr_push *subndr, + struct ndr_push *comndr, + enum ndr_compression_alg compression_alg) +{ + comndr->flags = subndr->flags; + + switch (compression_alg) { +#ifdef HAVE_ZLIB + case NDR_COMPRESSION_ZLIB: + return ndr_push_compression_zlib(subndr, comndr); +#endif + default: + return ndr_push_error(subndr, NDR_ERR_COMPRESSION, "Bad compression algorithm %d (PUSH)", + compression_alg); + } + return NT_STATUS_OK; +} -- cgit