gsskrb5: try to be compatible with windows for gss_wrap* and cfx

The good thing is that windows and heimdal both use EC=0
in the non DCE_STYLE case, so we need the windows compat hack
only in DCE_STYLE mode.

metze
(This used to be commit 0fa41a94e466d5e11bcf362ccd8ff41b72733d1a)

2 files changed, 39 insertions, 11 deletions

diff --git a/source4/heimdal/lib/gssapi/krb5/cfx.c b/source4/heimdal/lib/gssapi/krb5/cfx.c
index 6452f802ab..bc0d736e81 100755
--- a/source4/heimdal/lib/gssapi/krb5/cfx.c
+++ b/source4/heimdal/lib/gssapi/krb5/cfx.c
@@ -43,7 +43,8 @@ RCSID("$Id: cfx.c 19031 2006-11-13 18:02:57Z lha $");
 #define CFXAcceptorSubkey	(1 << 2)
 
 krb5_error_code
-_gsskrb5cfx_wrap_length_cfx(krb5_context context,
+_gsskrb5cfx_wrap_length_cfx(const gsskrb5_ctx context_handle,
+			    krb5_context context,
 			    krb5_crypto crypto,
 			    int conf_req_flag,
 			    size_t input_length,
@@ -72,7 +73,11 @@ _gsskrb5cfx_wrap_length_cfx(krb5_context context,
 	/* Header is concatenated with data before encryption */
 	input_length += sizeof(gss_cfx_wrap_token_desc);
 
-	ret = krb5_crypto_getpadsize(context, crypto, &padsize);
+	if (IS_DCE_STYLE(context_handle)) {
+		ret = krb5_crypto_getblocksize(context, crypto, &padsize);
+	} else {
+		ret = krb5_crypto_getpadsize(context, crypto, &padsize);
+	}
 	if (ret) {
 	    return ret;
 	}
@@ -258,7 +263,7 @@ OM_uint32 _gssapi_wrap_cfx(OM_uint32 *minor_status,
 	return GSS_S_FAILURE;
     }
 
-    ret = _gsskrb5cfx_wrap_length_cfx(context,
+    ret = _gsskrb5cfx_wrap_length_cfx(context_handle, context,
 				      crypto, conf_req_flag, 
 				      input_message_buffer->length,
 				      &wrapped_len, &cksumsize, &padlength);
@@ -380,7 +385,15 @@ OM_uint32 _gssapi_wrap_cfx(OM_uint32 *minor_status,
 	token->RRC[0] = (rrc >> 8) & 0xFF;  
 	token->RRC[1] = (rrc >> 0) & 0xFF;
 
-	ret = rrc_rotate(cipher.data, cipher.length, rrc, FALSE);
+	/*
+	 * this is really ugly, but needed against windows
+	 * for DCERPC, as windows rotates by EC+RRC.
+	 */
+	if (IS_DCE_STYLE(context_handle)) {
+		ret = rrc_rotate(cipher.data, cipher.length, rrc+padlength, FALSE);
+	} else {
+		ret = rrc_rotate(cipher.data, cipher.length, rrc, FALSE);
+	}
 	if (ret != 0) {
 	    *minor_status = ret;
 	    krb5_crypto_destroy(context, crypto);
@@ -553,14 +566,21 @@ OM_uint32 _gssapi_unwrap_cfx(OM_uint32 *minor_status,
     len = input_message_buffer->length;
     len -= (p - (u_char *)input_message_buffer->value);
 
-    /* Rotate by RRC; bogus to do this in-place XXX */
-    *minor_status = rrc_rotate(p, len, rrc, TRUE);
-    if (*minor_status != 0) {
-	krb5_crypto_destroy(context, crypto);
-	return GSS_S_FAILURE;
-    }
-
     if (token_flags & CFXSealed) {
+	/*
+	 * this is really ugly, but needed against windows
+	 * for DCERPC, as windows rotates by EC+RRC.
+	 */
+	if (IS_DCE_STYLE(context_handle)) {
+		*minor_status = rrc_rotate(p, len, rrc+ec, TRUE);
+	} else {
+		*minor_status = rrc_rotate(p, len, rrc, TRUE);
+	}
+	if (*minor_status != 0) {
+	    krb5_crypto_destroy(context, crypto);
+	    return GSS_S_FAILURE;
+	}
+
 	ret = krb5_decrypt(context, crypto, usage,
 	    p, len, &data);
 	if (ret != 0) {
@@ -594,6 +614,13 @@ OM_uint32 _gssapi_unwrap_cfx(OM_uint32 *minor_status,
     } else {
 	Checksum cksum;
 
+	/* Rotate by RRC; bogus to do this in-place XXX */
+	*minor_status = rrc_rotate(p, len, rrc, TRUE);
+	if (*minor_status != 0) {
+	    krb5_crypto_destroy(context, crypto);
+	    return GSS_S_FAILURE;
+	}
+
 	/* Determine checksum type */
 	ret = krb5_crypto_get_checksum_type(context,
 					    crypto, &cksum.cksumtype);
diff --git a/source4/heimdal/lib/gssapi/krb5/gsskrb5-private.h b/source4/heimdal/lib/gssapi/krb5/gsskrb5-private.h
index 64a0dd36b1..f6edb8b247 100644
--- a/source4/heimdal/lib/gssapi/krb5/gsskrb5-private.h
+++ b/source4/heimdal/lib/gssapi/krb5/gsskrb5-private.h
@@ -661,6 +661,7 @@ _gsskrb5cfx_max_wrap_length_cfx (
 
 krb5_error_code
 _gsskrb5cfx_wrap_length_cfx (
+	const gsskrb5_ctx /*context_handle*/,
 	krb5_context /*context*/,
 	krb5_crypto /*crypto*/,
 	int /*conf_req_flag*/,