r6061: add start of compression support in our rpc code

this is not complete cuurently...

but I want other people to test it and help me on finishing it.
(try to change the #if 0 in torture/rpc/drsuapi.c into #if 1)

metze
(This used to be commit 335adef37082a78e0426decb715629bd778e6582)

7 files changed, 359 insertions, 8 deletions

diff --git a/source4/build/pidl/ndr_parser.pm b/source4/build/pidl/ndr_parser.pm
index a689a8a849..6f8029d338 100644
--- a/source4/build/pidl/ndr_parser.pm
+++ b/source4/build/pidl/ndr_parser.pm
@@ -470,6 +470,92 @@ sub ParseArrayPull($$$$)
 	}
 }
 
+sub compression_alg($)
+{
+	my $e = shift;
+	my $compression = util::has_property($e, "compression");
+	my ($alg, $clen, $dlen) = split(/ /, $compression);
+
+	return $alg;
+}
+
+sub compression_clen($)
+{
+	my $e = shift;
+	my $compression = util::has_property($e, "compression");
+	my ($alg, $clen, $dlen) = split(/ /, $compression);
+
+	return ParseExpr($e, $clen, "r->");
+}
+
+sub compression_dlen($)
+{
+	my $e = shift;
+	my $compression = util::has_property($e, "compression");
+	my ($alg, $clen, $dlen) = split(/ /, $compression);
+
+	return ParseExpr($e, $dlen, "r->");
+}
+
+sub ParseCompressionPushStart($$$)
+{
+	my $e = shift;
+	my $subndr = shift;
+	my $ndr_flags = shift;
+	my $comndr = $subndr."_compressed";
+
+	pidl "{";
+	indent;
+	pidl "struct ndr_push *$comndr;";
+	pidl "";
+	pidl "$comndr = ndr_push_init_ctx($subndr);";
+	pidl "if (!$comndr) return NT_STATUS_NO_MEMORY;";
+	pidl "$comndr->flags = $subndr->flags;";
+	pidl "";
+	
+	return $comndr;
+}
+
+sub ParseCompressionPushEnd($$)
+{
+	my $e = shift;
+	my $subndr = shift;
+	my $comndr = $subndr."_compressed";
+	my $alg = compression_alg($e);
+
+	pidl "NDR_CHECK(ndr_push_compression($subndr, $comndr, $alg));";
+	deindent;
+	pidl "}";
+}
+
+sub ParseCompressionPullStart($$$)
+{
+	my $e = shift;
+	my $subndr = shift;
+	my $ndr_flags = shift;
+	my $comndr = $subndr."_compressed";
+	my $alg = compression_alg($e);
+	my $dlen = compression_dlen($e);
+
+	pidl "{";
+	indent;
+	pidl "struct ndr_pull *$comndr;";
+	pidl "NDR_ALLOC($subndr, $comndr);";
+	pidl "NDR_CHECK(ndr_pull_compression($subndr, $comndr, $alg, $dlen));";
+
+	return $comndr;
+}
+
+sub ParseCompressionPullEnd($$)
+{
+	my $e = shift;
+	my $subndr = shift;
+	my $comndr = $subndr."_compressed";
+
+	deindent;
+	pidl "}";
+}
+
 sub ParseSubcontextPushStart($$)
 {
 	my $e = shift;
@@ -551,7 +637,10 @@ sub ParseElementPushScalar($$$)
 	my $cprefix = c_push_prefix($e);
 	my $ptr_prefix = c_ptr_prefix($e);
 	my $sub_size = util::has_property($e, "subcontext");
+	my $compression = util::has_property($e, "compression");
 	my $ndr = "ndr";
+	my $subndr = undef;
+	my $comndr = undef;
 
 	start_flags($e);
 
@@ -560,7 +649,12 @@ sub ParseElementPushScalar($$$)
 	}
 
 	if (defined $sub_size and $e->{POINTERS} == 0) {
-		$ndr = ParseSubcontextPushStart($e, "NDR_SCALARS");
+		$subndr = ParseSubcontextPushStart($e, "NDR_SCALARS");
+		$ndr = $subndr;
+		if (defined $compression) {
+			$comndr = ParseCompressionPushStart($e, $subndr, "NDR_SCALARS");
+			$ndr = $comndr;
+		}
 	}
 
 	if (Ndr::need_wire_pointer($e)) {
@@ -578,6 +672,9 @@ sub ParseElementPushScalar($$$)
 	}
 
 	if (defined $sub_size and $e->{POINTERS} == 0) {
+		if (defined $compression) {
+			ParseCompressionPushEnd($e, $subndr);
+		}
 		ParseSubcontextPushEnd($e);
 	}
 
@@ -698,13 +795,21 @@ sub ParseElementPullScalar($$$)
 	my $cprefix = c_pull_prefix($e);
 	my $ptr_prefix = c_ptr_prefix($e);
 	my $sub_size = util::has_property($e, "subcontext");
+	my $compression = util::has_property($e, "compression");
 	my $ndr = "ndr";
+	my $subndr = undef;
+	my $comndr = undef;
 
 	start_flags($e);
 
 	if (defined $sub_size && $e->{POINTERS} == 0) {
-		$ndr = ParseSubcontextPullStart($e, $ndr_flags);
+		$subndr = ParseSubcontextPullStart($e, $ndr_flags);
+		$ndr = $subndr;
 		$ndr_flags = "NDR_SCALARS|NDR_BUFFERS";
+		if (defined $compression) {
+			$comndr = ParseCompressionPullStart($e, $subndr, $ndr_flags);
+			$ndr = $comndr;
+		}
 	}
 
 	if (Ndr::is_inline_array($e)) {
@@ -728,6 +833,9 @@ sub ParseElementPullScalar($$$)
 	}
 
 	if (defined $sub_size && $e->{POINTERS} == 0) {
+		if (defined $compression) {
+			ParseCompressionPullEnd($e, $subndr);
+		}
 		ParseSubcontextPullEnd($e);
 	}
 
@@ -766,7 +874,10 @@ sub ParseElementPushBuffer($$)
 	my($var_prefix) = shift;
 	my $cprefix = c_push_prefix($e);
 	my $sub_size = util::has_property($e, "subcontext");
+	my $compression = util::has_property($e, "compression");
 	my $ndr = "ndr";
+	my $subndr = undef;
+	my $comndr = undef;
 
 	return unless (need_buffers_section($e));
 
@@ -793,8 +904,13 @@ sub ParseElementPushBuffer($$)
 	}
 
 	if (defined $sub_size) {
-		$ndr = ParseSubcontextPushStart($e, $ndr_flags);
+		$subndr = ParseSubcontextPushStart($e, $ndr_flags);
+		$ndr = $subndr;
 		$ndr_flags = "NDR_SCALARS|NDR_BUFFERS";
+		if (defined $compression) {
+			$comndr = ParseCompressionPushStart($e, $subndr, $ndr_flags);
+			$ndr = $comndr;
+		}
 	}
 
 	if (util::array_size($e)) {
@@ -808,6 +924,9 @@ sub ParseElementPushBuffer($$)
 	}
 
 	if (defined $sub_size) {
+		if (defined $compression) {
+			ParseCompressionPushEnd($e, $subndr);
+		}
 		ParseSubcontextPushEnd($e);
 	}
 
@@ -827,7 +946,10 @@ sub ParseElementPullBuffer($$)
 	my($var_prefix) = shift;
 	my $cprefix = c_pull_prefix($e);
 	my $sub_size = util::has_property($e, "subcontext");
+	my $compression = util::has_property($e, "compression");
 	my $ndr = "ndr";
+	my $subndr = undef;
+	my $comndr = undef;
 
 	return unless (need_buffers_section($e));
 
@@ -856,8 +978,13 @@ sub ParseElementPullBuffer($$)
 	}
 
 	if (defined $sub_size) {
-		$ndr = ParseSubcontextPullStart($e, $ndr_flags);
+		$subndr = ParseSubcontextPullStart($e, $ndr_flags);
+		$ndr = $subndr;
 		$ndr_flags = "NDR_SCALARS|NDR_BUFFERS";
+		if (defined $compression) {
+			$comndr = ParseCompressionPullStart($e, $subndr, $ndr_flags);
+			$ndr = $comndr;
+		}
 	}
 
 	if (util::array_size($e)) {
@@ -871,6 +998,9 @@ sub ParseElementPullBuffer($$)
 	}
 
 	if (defined $sub_size) {
+		if (defined $compression) {
+			ParseCompressionPullEnd($e, $subndr);
+		}
 		ParseSubcontextPullEnd($e);
 	}
 
diff --git a/source4/configure.in b/source4/configure.in
index 5791a57adf..366f038030 100644
--- a/source4/configure.in
+++ b/source4/configure.in
@@ -22,6 +22,7 @@ SMB_INCLUDE_M4(lib/cmdline/config.m4)
 SMB_INCLUDE_M4(param/config.m4)
 SMB_INCLUDE_M4(libcli/auth/gensec.m4)
 SMB_INCLUDE_M4(libcli/config.m4)
+SMB_INCLUDE_M4(librpc/config.m4)
 SMB_INCLUDE_M4(smbd/process_model.m4)
 SMB_INCLUDE_M4(lib/registry/config.m4)
 SMB_INCLUDE_M4(scripting/config.m4)
diff --git a/source4/librpc/config.m4 b/source4/librpc/config.m4
new file mode 100644
index 0000000000..d67e880a38
--- /dev/null
+++ b/source4/librpc/config.m4
@@ -0,0 +1,35 @@
+########################################################
+# Compile with compression support
+
+with_zlib_support=auto
+ZLIB_LIBS=""
+ZLIB_CFLAGS=""
+ZLIB_CPPFLAGS=""
+ZLIB_LDFLAGS=""
+
+AC_CHECK_HEADERS(zlib.h)  
+if test x"$ac_cv_header_zlib_h" != x"yes"; then
+	with_zlib_support=no
+fi
+
+if test x"$with_zlib_support" != x"no"; then
+  AC_CHECK_LIB_EXT(z, ZLIB_LIBS, inflate)
+
+  if test x"$ac_cv_lib_ext_z_inflate" = x"yes"; then
+    AC_DEFINE(HAVE_ZLIB,1,[Whether zlib is available])
+    with_zlib_support=yes
+    SMB_EXT_LIB_ENABLE(ZLIB,YES)
+  else
+    ZLIB_LIBS=""
+    with_zlib_support=no
+  fi
+  LIBS=$ac_save_LIBS
+fi
+AC_MSG_CHECKING(whether ZLIB support is available)
+AC_MSG_RESULT($with_zlib_support)
+
+# for now enable this always but maybe all fields are empty
+# TODO: move compression methods to seperate files each
+SMB_EXT_LIB_ENABLE(ZLIB,YES)
+
+SMB_EXT_LIB(ZLIB,[${ZLIB_LIBS}],[${ZLIB_CFLAGS}],[${ZLIB_CPPFLAGS}],[${ZLIB_LDFLAGS}])
diff --git a/source4/librpc/config.mk b/source4/librpc/config.mk
index f6b05a41b5..d73f6bf96f 100644
--- a/source4/librpc/config.mk
+++ b/source4/librpc/config.mk
@@ -5,8 +5,9 @@ INIT_OBJ_FILES = \
 		librpc/ndr/ndr.o
 ADD_OBJ_FILES = \
 		librpc/ndr/ndr_basic.o \
+		librpc/ndr/ndr_compression.o \
 		librpc/ndr/ndr_misc.o
-REQUIRED_SUBSYSTEMS = LIBCLI_UTILS
+REQUIRED_SUBSYSTEMS = LIBCLI_UTILS EXT_LIB_ZLIB
 # End SUBSYSTEM LIBNDR_RAW
 ################################################
 
diff --git a/source4/librpc/idl/drsuapi.idl b/source4/librpc/idl/drsuapi.idl
index 330165830c..1b81dd6297 100644
--- a/source4/librpc/idl/drsuapi.idl
+++ b/source4/librpc/idl/drsuapi.idl
@@ -283,7 +283,8 @@ interface drsuapi
 	} drsuapi_DsReplicaCoursor2Ctr2;
 
 	typedef [v1_enum] enum {
-		DRSUAPI_OBJECTCLASS_top		= 0x0
+		DRSUAPI_OBJECTCLASS_top		= 0x00010000,
+		DRSUAPI_OBJECTCLASS_test	= 0x00000000
 	} drsuapi_DsObjectClassId;
 
 	typedef [v1_enum,public] enum {
@@ -497,9 +498,10 @@ interface drsuapi
 	} drsuapi_DsGetNCChangesCtr1;
 
 	typedef struct {
-		uint32 unknown1;/* decompressed_length ? */
+		uint32 decompressed_length;
 		uint32 compressed_length;
-		DATA_BLOB *compressed;
+		[subcontext(4),compression(NDR_COMPRESSION_ZLIB,compressed_length,decompressed_length),flag(NDR_REMAINING)] DATA_BLOB *decompressed;
+		/*[subcontext(4),compression(NDR_COMPRESSION_ZLIB,compressed_length,decompressed_length)] drsuapi_DsGetNCChangesCtr1 *ctr1;*/
 	} drsuapi_DsGetNCChangesCompressedInfo;
 
 	typedef struct {
diff --git a/source4/librpc/ndr/libndr.h b/source4/librpc/ndr/libndr.h
index 4df3e7f248..3cc25b0ef2 100644
--- a/source4/librpc/ndr/libndr.h
+++ b/source4/librpc/ndr/libndr.h
@@ -152,6 +152,7 @@ enum ndr_err_code {
 	NDR_ERR_CHARCNV,
 	NDR_ERR_LENGTH,
 	NDR_ERR_SUBCONTEXT,
+	NDR_ERR_COMPRESSION,
 	NDR_ERR_STRING,
 	NDR_ERR_VALIDATE,
 	NDR_ERR_BUFSIZE,
@@ -160,6 +161,10 @@ enum ndr_err_code {
 	NDR_ERR_TOKEN
 };
 
+enum ndr_compression_alg {
+	NDR_COMPRESSION_ZLIB
+};
+
 /*
   flags passed to control parse flow
 */
diff --git a/source4/librpc/ndr/ndr_compression.c b/source4/librpc/ndr/ndr_compression.c
new file mode 100644
index 0000000000..d4f1a2a927
--- /dev/null
+++ b/source4/librpc/ndr/ndr_compression.c
@@ -0,0 +1,177 @@
+/* 
+   Unix SMB/CIFS implementation.
+
+   libndr compression support
+
+   Copyright (C) Stefan Metzmacher 2005
+   
+   This program is free software; you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation; either version 2 of the License, or
+   (at your option) any later version.
+   
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+   
+   You should have received a copy of the GNU General Public License
+   along with this program; if not, write to the Free Software
+   Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+*/
+
+#include "includes.h"
+
+#ifdef HAVE_ZLIB
+#include <zlib.h>
+
+static NTSTATUS ndr_pull_compression_zlib(struct ndr_pull *subndr,
+					  struct ndr_pull *comndr,
+					  ssize_t decompressed_len)
+{
+	DATA_BLOB inbuf;
+	DATA_BLOB outbuf = data_blob_talloc(comndr, NULL, decompressed_len);
+	uint32_t outbuf_len = outbuf.length;
+	struct z_stream_s zs;
+	int ret;
+
+	ZERO_STRUCT(zs);
+
+	if (subndr->data_size < 10) {
+		return ndr_pull_error(subndr, NDR_ERR_COMPRESSION, "Bad ZLIB compressed header (PULL) subcontext size %d", 
+				      subndr->data_size);
+	}
+
+	inbuf.data = subndr->data+10;
+	inbuf.length = subndr->data_size-10;
+
+	zs.avail_in = inbuf.length;
+	zs.next_in = inbuf.data;
+	zs.next_out = outbuf.data;
+	zs.avail_out = outbuf.length;
+
+	ret = inflateInit2(&zs, 15);
+	if (ret != Z_OK) {
+		return ndr_pull_error(subndr, NDR_ERR_COMPRESSION, "Bad ZLIB (PULL) inflateInit2 error %d", 
+				      ret);
+	}
+
+	while(1) {
+		ret = inflate(&zs, Z_SYNC_FLUSH);
+		if (ret == Z_STREAM_END) {
+			
+			DEBUG(0,("inbuf.length: %d avail_in: %d, avail_out: %d\n", inbuf.length, zs.avail_in, zs.avail_out));
+			break;
+		}
+		if (ret != Z_OK) {
+			return ndr_pull_error(subndr, NDR_ERR_COMPRESSION, "Bad ZLIB (PULL) inflate error %d", 
+				      ret);
+		}
+	}
+
+	inflateEnd(&zs);
+
+	/* TODO: check if the decompressed_len == outbuf_len */
+	outbuf.length = outbuf_len - zs.avail_out;
+
+	if (outbuf.length < 16) {
+		return ndr_pull_error(subndr, NDR_ERR_COMPRESSION, "Bad ZLIB uncompressed header (PULL) uncompressed size %d", 
+				      outbuf.length);
+	}
+
+	outbuf.data	+= 16;
+	outbuf.length	-= 16;
+
+	/* TODO: really decompress the data here */
+	*comndr = *subndr;
+	comndr->data		= outbuf.data;
+	comndr->data_size	= outbuf.length;
+	comndr->offset		= 0;
+
+	return NT_STATUS_OK;
+}
+
+static NTSTATUS ndr_push_compression_zlib(struct ndr_push *subndr,
+					  struct ndr_push *comndr)
+{
+	DATA_BLOB inbuf;
+	DATA_BLOB outbuf = data_blob_talloc(comndr, NULL, comndr->offset + 10);
+	struct z_stream_s zs;
+	int ret;
+
+	ZERO_STRUCT(zs);
+
+	inbuf = ndr_push_blob(comndr);
+
+	zs.avail_in = inbuf.length;
+	zs.next_in = inbuf.data;
+	zs.next_out = outbuf.data+10;
+	zs.avail_out = outbuf.length-10;
+
+	ret = deflateInit(&zs, Z_DEFAULT_COMPRESSION);
+	if (ret != Z_OK) {
+		return ndr_push_error(subndr, NDR_ERR_COMPRESSION, "Bad ZLIB (PUSH) deflateInit2 error %d", 
+				      ret);
+	}
+
+	ret = deflate(&zs, Z_SYNC_FLUSH);
+
+	if (ret != Z_OK && ret != Z_STREAM_END) {
+		return ndr_push_error(subndr, NDR_ERR_COMPRESSION, "Bad ZLIB (PULL) deflate error %d", 
+				      ret);
+	}
+
+	deflateEnd(&zs);
+
+	/* TODO: push the header here */
+
+
+	NDR_CHECK(ndr_push_bytes(subndr, outbuf.data, outbuf.length));
+
+	return NT_STATUS_OK;
+}
+#endif
+
+/*
+  handle compressed subcontext buffers, which in midl land are user-marshalled, but
+  we use magic in pidl to make them easier to cope with
+*/
+NTSTATUS ndr_pull_compression(struct ndr_pull *subndr,
+			      struct ndr_pull *comndr,
+			      enum ndr_compression_alg compression_alg,
+			      ssize_t decompressed_len)
+{
+	comndr->flags = subndr->flags;
+
+	switch (compression_alg) {
+#ifdef HAVE_ZLIB
+	case NDR_COMPRESSION_ZLIB:
+		return ndr_pull_compression_zlib(subndr, comndr, decompressed_len);
+#endif
+	default:
+		return ndr_pull_error(subndr, NDR_ERR_COMPRESSION, "Bad compression algorithm %d (PULL)", 
+				      compression_alg);
+	}
+	return NT_STATUS_OK;
+}
+
+/*
+  push a compressed subcontext
+*/
+NTSTATUS ndr_push_compression(struct ndr_push *subndr,
+			      struct ndr_push *comndr,
+			      enum ndr_compression_alg compression_alg)
+{
+	comndr->flags = subndr->flags;
+
+	switch (compression_alg) {
+#ifdef HAVE_ZLIB
+	case NDR_COMPRESSION_ZLIB:
+		return ndr_push_compression_zlib(subndr, comndr);
+#endif
+	default:
+		return ndr_push_error(subndr, NDR_ERR_COMPRESSION, "Bad compression algorithm %d (PUSH)", 
+				      compression_alg);
+	}
+	return NT_STATUS_OK;
+}