gsskrb5: add support for DCE_STYLE and des and des3 keys

Only the des keys are tested as windows doesn't support des3

metze
(This used to be commit 86848dd0f217774faed81af8fbf68618013e20a1)

2 files changed, 64 insertions, 22 deletions

diff --git a/source4/heimdal/lib/gssapi/krb5/unwrap.c b/source4/heimdal/lib/gssapi/krb5/unwrap.c
index eec4078a70..c287469e96 100644
--- a/source4/heimdal/lib/gssapi/krb5/unwrap.c
+++ b/source4/heimdal/lib/gssapi/krb5/unwrap.c
@@ -59,10 +59,17 @@ unwrap_des
   OM_uint32 ret;
   int cstate;
   int cmp;
+  int token_len;
+
+  if (IS_DCE_STYLE(context_handle)) {
+     token_len = 22 + 8 + 15; /* 45 */
+  } else {
+     token_len = input_message_buffer->length;
+  }
 
   p = input_message_buffer->value;
   ret = _gsskrb5_verify_header (&p,
-				   input_message_buffer->length,
+				   token_len,
 				   "\x02\x01",
 				   GSS_KRB5_MECHANISM);
   if (ret)
@@ -105,12 +112,17 @@ unwrap_des
       memset (deskey, 0, sizeof(deskey));
       memset (&schedule, 0, sizeof(schedule));
   }
-  /* check pad */
-  ret = _gssapi_verify_pad(input_message_buffer, 
-			   input_message_buffer->length - len,
-			   &padlength);
-  if (ret)
-      return ret;
+
+  if (IS_DCE_STYLE(context_handle)) {
+    padlength = 0;
+  } else {
+    /* check pad */
+    ret = _gssapi_verify_pad(input_message_buffer,
+			     input_message_buffer->length - len,
+			     &padlength);
+    if (ret)
+        return ret;
+  }
 
   MD5_Init (&md5);
   MD5_Update (&md5, p - 24, 8);
@@ -195,10 +207,17 @@ unwrap_des3
   krb5_crypto crypto;
   Checksum csum;
   int cmp;
+  int token_len;
+
+  if (IS_DCE_STYLE(context_handle)) {
+     token_len = 34 + 8 + 15; /* 57 */
+  } else {
+     token_len = input_message_buffer->length;
+  }
 
   p = input_message_buffer->value;
   ret = _gsskrb5_verify_header (&p,
-				   input_message_buffer->length,
+				   token_len,
 				   "\x02\x01",
 				   GSS_KRB5_MECHANISM);
   if (ret)
@@ -245,12 +264,17 @@ unwrap_des3
       memcpy (p, tmp.data, tmp.length);
       krb5_data_free(&tmp);
   }
-  /* check pad */
-  ret = _gssapi_verify_pad(input_message_buffer, 
-			   input_message_buffer->length - len,
-			   &padlength);
-  if (ret)
-      return ret;
+
+  if (IS_DCE_STYLE(context_handle)) {
+    padlength = 0;
+  } else {
+    /* check pad */
+    ret = _gssapi_verify_pad(input_message_buffer,
+			     input_message_buffer->length - len,
+			     &padlength);
+    if (ret)
+        return ret;
+  }
 
   /* verify sequence number */
   
diff --git a/source4/heimdal/lib/gssapi/krb5/wrap.c b/source4/heimdal/lib/gssapi/krb5/wrap.c
index 6d00f2adcf..bedeace4dd 100644
--- a/source4/heimdal/lib/gssapi/krb5/wrap.c
+++ b/source4/heimdal/lib/gssapi/krb5/wrap.c
@@ -210,10 +210,19 @@ wrap_des
   int32_t seq_number;
   size_t len, total_len, padlength, datalen;
 
-  padlength = 8 - (input_message_buffer->length % 8);
-  datalen = input_message_buffer->length + padlength + 8;
-  len = datalen + 22;
-  _gsskrb5_encap_length (len, &len, &total_len, GSS_KRB5_MECHANISM);
+  if (IS_DCE_STYLE(ctx)) {
+    padlength = 0;
+    datalen = input_message_buffer->length;
+    len = 22 + 8;
+    _gsskrb5_encap_length (len, &len, &total_len, GSS_KRB5_MECHANISM);
+    total_len += datalen;
+    datalen += 8;
+  } else {
+    padlength = 8 - (input_message_buffer->length % 8);
+    datalen = input_message_buffer->length + padlength + 8;
+    len = datalen + 22;
+    _gsskrb5_encap_length (len, &len, &total_len, GSS_KRB5_MECHANISM);
+  }
 
   output_message_buffer->length = total_len;
   output_message_buffer->value  = malloc (total_len);
@@ -336,10 +345,19 @@ wrap_des3
   Checksum cksum;
   krb5_data encdata;
 
-  padlength = 8 - (input_message_buffer->length % 8);
-  datalen = input_message_buffer->length + padlength + 8;
-  len = datalen + 34;
-  _gsskrb5_encap_length (len, &len, &total_len, GSS_KRB5_MECHANISM);
+  if (IS_DCE_STYLE(ctx)) {
+    padlength = 0;
+    datalen = input_message_buffer->length;
+    len = 34 + 8;
+    _gsskrb5_encap_length (len, &len, &total_len, GSS_KRB5_MECHANISM);
+    total_len += datalen;
+    datalen += 8;
+  } else {
+    padlength = 8 - (input_message_buffer->length % 8);
+    datalen = input_message_buffer->length + padlength + 8;
+    len = datalen + 34;
+    _gsskrb5_encap_length (len, &len, &total_len, GSS_KRB5_MECHANISM);
+  }
 
   output_message_buffer->length = total_len;
   output_message_buffer->value  = malloc (total_len);