r19019: Add a parser for javascript literals. This is implemented for parsing JSON

strings, but is generally useful for creating ejs variables from object or
array literals, which aren't supported in the old version of ejs that we're
using.

This parser is implemented in C, but is callable from an ejs script via
literal_to_var().
(This used to be commit 82fdcd9339a8ffb1016b96c06f7d6580aa759174)

3 files changed, 704 insertions, 3 deletions

diff --git a/jsonrpc/json.esp b/jsonrpc/json.esp
index d06a173f0f..ad0e13a6c3 100644
--- a/jsonrpc/json.esp
+++ b/jsonrpc/json.esp
@@ -137,6 +137,9 @@ Json = new Object();
 Json.encode = _encode;
 _encode = null;
 
+/* Json.decode(): decode a string into its object form */
+Json.decode = literal_to_var;
+
 /* Internal stuff, not for external access */
 Json._internal = new Object();
 
@@ -191,7 +194,7 @@ Json._internal.convert['\x1f'] = '\\u001f';
 
 /* Test it */
 libinclude("base.js");
-function testIt()
+function testFormat()
 {
     var test = new Object();
     test.int = 23;
@@ -208,6 +211,33 @@ function testIt()
     test.obj.array[1] = 223;
     printf("%s\n", Json.encode(test));
 }
-//testIt();
+
+function testParse()
+{
+    var s;
+
+    s = '{ "x" : 23 }';
+    obj = Json.decode(s);
+    printf("Decode/encode of\n\t%s\nyielded\n\t%s\n\n", s, Json.encode(obj));
+
+    s = '{ "x" : [ 23, 42] }';
+    obj = Json.decode(s);
+    printf("Decode/encode of\n\t%s\nyielded\n\t%s\n\n", s, Json.encode(obj));
+
+    s = '[ 13, 19, { "x" : [ 23, 42] }, 223 ]';
+    obj = Json.decode(s);
+    printf("Decode/encode of\n\t%s\nyielded\n\t%s\n\n", s, Json.encode(obj));
+
+    s = '{ "x" : [ "hi" ] }';
+    obj = Json.decode(s);
+    printf("Decode/encode of\n\t%s\nyielded\n\t%s\n\n", s, Json.encode(obj));
+
+    s = '[ 13, 19, { "x" : [ 23, 42, { "y":{"a":"hello", "b":"world", "c":[1,2,3]}}] }, 223 ]';
+    obj = Json.decode(s);
+    printf("Decode/encode of\n\t%s\nyielded\n\t%s\n\n", s, Json.encode(obj));
+}
+
+//testFormat();
+testParse();
 
 %>
diff --git a/source4/scripting/ejs/config.mk b/source4/scripting/ejs/config.mk
index ca73ce279a..09ad510a12 100644
--- a/source4/scripting/ejs/config.mk
+++ b/source4/scripting/ejs/config.mk
@@ -72,7 +72,8 @@ OBJ_FILES = \
 		smbcalls_creds.o \
 		smbcalls_param.o \
 		ejsnet.o \
-		mprutil.o
+		mprutil.o \
+		literal.o
 PUBLIC_DEPENDENCIES = \
 		EJS LIBSAMBA-UTIL \
 		EJSRPC MESSAGING \
diff --git a/source4/scripting/ejs/literal.c b/source4/scripting/ejs/literal.c
new file mode 100644
index 0000000000..f7168e2c4e
--- /dev/null
+++ b/source4/scripting/ejs/literal.c
@@ -0,0 +1,670 @@
+/*
+ * Copyright (c) 2004, 2005 Metaparadigm Pte. Ltd.
+ * Michael Clark <michael@metaparadigm.com>
+ * Copyright (c) 2006 Derrell Lipman
+ *
+ * This library is free software; you can redistribute it and/or modify
+ * it under the terms of the MIT license. See COPYING for details.
+ *
+ * Derrell Lipman:
+ * This version is modified from the original.  It has been modified to
+ * natively use EJS variables rather than the original C object interface, and
+ * to use the talloc() family of functions for memory allocation.
+ */
+
+#include "includes.h"
+#include "scripting/ejs/smbcalls.h"
+
+enum json_tokener_error {
+        json_tokener_success,
+        json_tokener_error_oom, /* out of memory */
+        json_tokener_error_parse_unexpected,
+        json_tokener_error_parse_null,
+        json_tokener_error_parse_boolean,
+        json_tokener_error_parse_number,
+        json_tokener_error_parse_array,
+        json_tokener_error_parse_object,
+        json_tokener_error_parse_string,
+        json_tokener_error_parse_comment,
+        json_tokener_error_parse_eof
+};
+
+enum json_tokener_state {
+        json_tokener_state_eatws,               /* 0 */
+        json_tokener_state_start,               /* 1 */
+        json_tokener_state_finish,              /* 2 */
+        json_tokener_state_null,                /* 3 */
+        json_tokener_state_comment_start,       /* 4 */
+        json_tokener_state_comment,             /* 5 */
+        json_tokener_state_comment_eol,         /* 6 */
+        json_tokener_state_comment_end,         /* 7 */
+        json_tokener_state_string,              /* 8 */
+        json_tokener_state_string_escape,       /* 9 */
+        json_tokener_state_escape_unicode,      /* 10 */
+        json_tokener_state_boolean,             /* 11 */
+        json_tokener_state_number,              /* 12 */
+        json_tokener_state_array,               /* 13 */
+        json_tokener_state_array_sep,           /* 14 */
+        json_tokener_state_object,              /* 15 */
+        json_tokener_state_object_field_start,  /* 16 */
+        json_tokener_state_object_field,        /* 17 */
+        json_tokener_state_object_field_end,    /* 18 */
+        json_tokener_state_object_value,        /* 19 */
+        json_tokener_state_object_sep           /* 20 */
+};
+
+struct json_tokener
+{
+        char *source;
+        int pos;
+        void *ctx;
+        void *pb;
+};
+
+static const char *json_number_chars = "0123456789.+-e";
+static const char *json_hex_chars = "0123456789abcdef";
+
+#define hexdigit(x) (((x) <= '9') ? (x) - '0' : ((x) & 7) + 9)
+
+extern struct MprVar json_tokener_parse(char *s);
+static struct MprVar json_tokener_do_parse(struct json_tokener *this,
+                                           enum json_tokener_error *err_p);
+
+/*
+ * literal_to_var() parses a string into an ejs variable.  The ejs
+ * variable is returned.  Upon error, the javascript variable will be
+ * `undefined`.  This was created for parsing JSON, but is generally useful
+ * for parsing the literal forms of objects and arrays, since ejs doesn't
+ * procide that functionality.
+ */
+int literal_to_var(int eid, int argc, char **argv)
+{
+        struct json_tokener tok;
+        struct MprVar obj;
+        enum json_tokener_error err = json_tokener_success;
+        
+        if (argc != 1) {
+		ejsSetErrorMsg(eid,
+                               "literal_to_var() requires one parameter: "
+                               "the string to be parsed.");
+		return -1;
+        }
+
+        tok.source = argv[0];
+        tok.pos = 0;
+        tok.ctx = talloc_new(NULL);
+        if (tok.ctx == NULL) {
+                mpr_Return(eid, mprCreateUndefinedVar());
+                return 0;
+        }
+        tok.pb = talloc_zero_size(tok.ctx, 1);
+        if (tok.pb == NULL) {
+                mpr_Return(eid, mprCreateUndefinedVar());
+                return 0;
+        }
+        obj = json_tokener_do_parse(&tok, &err);
+        talloc_free(tok.pb);
+        if (err != json_tokener_success) {
+                mprDestroyVar(&obj);
+                mpr_Return(eid, mprCreateUndefinedVar());
+                return 0;
+        }
+        mpr_Return(eid, obj);
+        return 0;
+}
+
+static void *append_string(void *ctx,
+                           char *orig,
+                           char *append,
+                           int size)
+{
+    char c;
+    char *end_p = append + size;
+    void *ret;
+
+    /*
+     * We need to null terminate the string to be copied.  Save character at
+     * the size limit of the source string.
+     */
+    c = *end_p;
+
+    /* Temporarily null-terminate it */
+    *end_p = '\0';
+
+    /* Append the requested data */
+    ret = talloc_append_string(ctx, orig, append);
+    
+    /* Restore the original character in place of our temporary null byte */
+    *end_p = c;
+
+    /* Give 'em what they came for */
+    return ret;
+}
+
+
+static struct MprVar json_tokener_do_parse(struct json_tokener *this,
+                                           enum json_tokener_error *err_p)
+{
+        enum json_tokener_state state, saved_state;
+        struct MprVar current = mprCreateUndefinedVar();
+        struct MprVar obj;
+        enum json_tokener_error err = json_tokener_success;
+        char *obj_field_name = NULL;
+        char quote_char;
+        int deemed_double;
+        int start_offset;
+        char c;
+        
+        state = json_tokener_state_eatws;
+        saved_state = json_tokener_state_start;
+
+        
+        do {
+                c = this->source[this->pos];
+                switch(state) {
+                        
+                case json_tokener_state_eatws:
+                        if(isspace(c)) {
+                                this->pos++;
+                        } else if(c == '/') {
+                                state = json_tokener_state_comment_start;
+                                start_offset = this->pos++;
+                        } else {
+                                state = saved_state;
+                        }
+                        break;
+                        
+                case json_tokener_state_start:
+                        switch(c) {
+                        case '{':
+                                state = json_tokener_state_eatws;
+                                saved_state = json_tokener_state_object;
+                                current = mprObject(NULL);
+                                this->pos++;
+                                break;
+                        case '[':
+                                state = json_tokener_state_eatws;
+                                saved_state = json_tokener_state_array;
+                                current = mprArray(NULL);
+                                this->pos++;
+                                break;
+                        case 'N':
+                        case 'n':
+                                state = json_tokener_state_null;
+                                start_offset = this->pos++;
+                                break;
+                        case '"':
+                        case '\'':
+                                quote_char = c;
+                                talloc_free(this->pb);
+                                this->pb = talloc_zero_size(this->ctx, 1);
+                                if (this->pb == NULL) {
+                                        *err_p = json_tokener_error_oom;
+                                        goto out;
+                                }
+                                state = json_tokener_state_string;
+                                start_offset = ++this->pos;
+                                break;
+                        case 'T':
+                        case 't':
+                        case 'F':
+                        case 'f':
+                                state = json_tokener_state_boolean;
+                                start_offset = this->pos++;
+                                break;
+#if defined(__GNUC__)
+                        case '0' ... '9':
+#else
+                        case '0':
+                        case '1':
+                        case '2':
+                        case '3':
+                        case '4':
+                        case '5':
+                        case '6':
+                        case '7':
+                        case '8':
+                        case '9':
+#endif
+                        case '-':
+                                deemed_double = 0;
+                                state = json_tokener_state_number;
+                                start_offset = this->pos++;
+                                break;
+                        default:
+                                err = json_tokener_error_parse_unexpected;
+                                goto out;
+                        }
+                        break;
+                        
+                case json_tokener_state_finish:
+                        goto out;
+                        
+                case json_tokener_state_null:
+                        if(strncasecmp("null", this->source + start_offset,
+                                       this->pos - start_offset)) {
+                                *err_p = json_tokener_error_parse_null;
+                                mprDestroyVar(&current);
+                                return mprCreateUndefinedVar();
+                        }
+                        
+                        if(this->pos - start_offset == 4) {
+                                mprDestroyVar(&current);
+                                current = mprCreateNullVar();
+                                saved_state = json_tokener_state_finish;
+                                state = json_tokener_state_eatws;
+                        } else {
+                                this->pos++;
+                        }
+                        break;
+                        
+                case json_tokener_state_comment_start:
+                        if(c == '*') {
+                                state = json_tokener_state_comment;
+                        } else if(c == '/') {
+                                state = json_tokener_state_comment_eol;
+                        } else {
+                                err = json_tokener_error_parse_comment;
+                                goto out;
+                        }
+                        this->pos++;
+                        break;
+                        
+                case json_tokener_state_comment:
+                        if(c == '*') state = json_tokener_state_comment_end;
+                        this->pos++;
+                        break;
+                        
+                case json_tokener_state_comment_eol:
+                        if(c == '\n') {
+                                state = json_tokener_state_eatws;
+                        }
+                        this->pos++;
+                        break;
+                        
+                case json_tokener_state_comment_end:
+                        if(c == '/') {
+                                state = json_tokener_state_eatws;
+                        } else {
+                                state = json_tokener_state_comment;
+                        }
+                        this->pos++;
+                        break;
+                        
+                case json_tokener_state_string:
+                        if(c == quote_char) {
+                                if ((this->pb = append_string(
+                                             this->ctx,
+                                             this->pb,
+                                             this->source + start_offset,
+                                             this->pos - start_offset)) == NULL) {
+                                        err = json_tokener_error_oom;
+                                        goto out;
+                                }
+                                current = mprString(this->pb);
+                                saved_state = json_tokener_state_finish;
+                                state = json_tokener_state_eatws;
+                        } else if(c == '\\') {
+                                saved_state = json_tokener_state_string;
+                                state = json_tokener_state_string_escape;
+                        }
+                        this->pos++;
+                        break;
+                        
+                case json_tokener_state_string_escape:
+                        switch(c) {
+                        case '"':
+                        case '\\':
+                                if ((this->pb = append_string(
+                                             this->ctx,
+                                             this->pb,
+                                             this->source + start_offset,
+                                             this->pos - start_offset - 1)) == NULL) {
+                                        err = json_tokener_error_oom;
+                                        goto out;
+                                }
+                                start_offset = this->pos++;
+                                state = saved_state;
+                                break;
+                        case 'b':
+                        case 'n':
+                        case 'r':
+                        case 't':
+                                if ((this->pb = append_string(
+                                             this->ctx,
+                                             this->pb,
+                                             this->source + start_offset,
+                                             this->pos - start_offset - 1)) == NULL) {
+                                        err = json_tokener_error_oom;
+                                        goto out;
+                                }
+                                if (c == 'b') {
+                                        /*
+                                         * second param to append_string()
+                                         * gets temporarily modified; can't
+                                         * pass string constant.
+                                         */
+                                        char buf[] = "\b";
+                                        if ((this->pb = append_string(
+                                                     this->ctx,
+                                                     this->pb,
+                                                     buf,
+                                                     1)) == NULL) {
+                                                err = json_tokener_error_oom;
+                                                goto out;
+                                        }
+                                } else if (c == 'n') {
+                                        char buf[] = "\n";
+                                        if ((this->pb = append_string(
+                                                     this->ctx,
+                                                     this->pb,
+                                                     buf,
+                                                     1)) == NULL) {
+                                                err = json_tokener_error_oom;
+                                                goto out;
+                                        }
+                                } else if (c == 'r') {
+                                        char buf[] = "\r";
+                                        if ((this->pb = append_string(
+                                                     this->ctx,
+                                                     this->pb,
+                                                     buf,
+                                                     1)) == NULL) {
+                                                err = json_tokener_error_oom;
+                                                goto out;
+                                        }
+                                } else if (c == 't') {
+                                        char buf[] = "\t";
+                                        if ((this->pb = append_string(
+                                                     this->ctx,
+                                                     this->pb,
+                                                     buf,
+                                                     1)) == NULL) {
+                                                err = json_tokener_error_oom;
+                                                goto out;
+                                        }
+                                }
+                                start_offset = ++this->pos;
+                                state = saved_state;
+                                break;
+                        case 'u':
+                                if ((this->pb = append_string(
+                                             this->ctx,
+                                             this->pb,
+                                             this->source + start_offset,
+                                             this->pos - start_offset - 1)) == NULL) {
+                                        err = json_tokener_error_oom;
+                                        goto out;
+                                }
+                                start_offset = ++this->pos;
+                                state = json_tokener_state_escape_unicode;
+                                break;
+                        default:
+                                err = json_tokener_error_parse_string;
+                                goto out;
+                        }
+                        break;
+                        
+                case json_tokener_state_escape_unicode:
+                        if(strchr(json_hex_chars, c)) {
+                                this->pos++;
+                                if(this->pos - start_offset == 4) {
+                                        unsigned char utf_out[3];
+                                        unsigned int ucs_char =
+                                                (hexdigit(*(this->source + start_offset)) << 12) +
+                                                (hexdigit(*(this->source + start_offset + 1)) << 8) +
+                                                (hexdigit(*(this->source + start_offset + 2)) << 4) +
+                                                hexdigit(*(this->source + start_offset + 3));
+                                        if (ucs_char < 0x80) {
+                                                utf_out[0] = ucs_char;
+                                                if ((this->pb = append_string(
+                                                             this->ctx,
+                                                             this->pb,
+                                                             (char *) utf_out,
+                                                             1)) == NULL) {
+                                                        err = json_tokener_error_oom;
+                                                        goto out;
+                                                }
+                                        } else if (ucs_char < 0x800) {
+                                                utf_out[0] = 0xc0 | (ucs_char >> 6);
+                                                utf_out[1] = 0x80 | (ucs_char & 0x3f);
+                                                if ((this->pb = append_string(
+                                                             this->ctx,
+                                                             this->pb,
+                                                             (char *) utf_out,
+                                                             2)) == NULL) {
+                                                        err = json_tokener_error_oom;
+                                                        goto out;
+                                                }
+                                        } else {
+                                                utf_out[0] = 0xe0 | (ucs_char >> 12);
+                                                utf_out[1] = 0x80 | ((ucs_char >> 6) & 0x3f);
+                                                utf_out[2] = 0x80 | (ucs_char & 0x3f);
+                                                if ((this->pb = append_string(
+                                                             this->ctx,
+                                                             this->pb,
+                                                             (char *) utf_out,
+                                                             3)) == NULL) {
+                                                        err = json_tokener_error_oom;
+                                                        goto out;
+                                                }
+                                        }
+                                        start_offset = this->pos;
+                                        state = saved_state;
+                                }
+                        } else {
+                                err = json_tokener_error_parse_string;
+                                goto out;
+                        }
+                        break;
+                        
+                case json_tokener_state_boolean:
+                        if(strncasecmp("true", this->source + start_offset,
+                                       this->pos - start_offset) == 0) {
+                                if(this->pos - start_offset == 4) {
+                                        current = mprCreateBoolVar(1);
+                                        saved_state = json_tokener_state_finish;
+                                        state = json_tokener_state_eatws;
+                                } else {
+                                        this->pos++;
+                                }
+                        } else if(strncasecmp("false", this->source + start_offset,
+                                              this->pos - start_offset) == 0) {
+                                if(this->pos - start_offset == 5) {
+                                        current = mprCreateBoolVar(0);
+                                        saved_state = json_tokener_state_finish;
+                                        state = json_tokener_state_eatws;
+                                } else {
+                                        this->pos++;
+                                }
+                        } else {
+                                err = json_tokener_error_parse_boolean;
+                                goto out;
+                        }
+                        break;
+                        
+                case json_tokener_state_number:
+                        if(!c || !strchr(json_number_chars, c)) {
+                                int numi;
+                                double numd;
+                                char *tmp = talloc_strndup(
+                                        this->ctx,
+                                        this->source + start_offset,
+                                        this->pos - start_offset);
+                                if (tmp == NULL) {
+                                        err = json_tokener_error_oom;
+                                        goto out;
+                                }
+                                if(!deemed_double && sscanf(tmp, "%d", &numi) == 1) {
+                                        current = mprCreateIntegerVar(numi);
+                                } else if(deemed_double && sscanf(tmp, "%lf", &numd) == 1) {
+                                        current = mprCreateFloatVar(numd);
+                                } else {
+                                        talloc_free(tmp);
+                                        err = json_tokener_error_parse_number;
+                                        goto out;
+                                }
+                                talloc_free(tmp);
+                                saved_state = json_tokener_state_finish;
+                                state = json_tokener_state_eatws;
+                        } else {
+                                if(c == '.' || c == 'e') deemed_double = 1;
+                                this->pos++;
+                        }
+                        break;
+                        
+                case json_tokener_state_array:
+                        if(c == ']') {
+                                this->pos++;
+                                saved_state = json_tokener_state_finish;
+                                state = json_tokener_state_eatws;
+                        } else {
+                                int oldlen;
+                                char idx[16];
+                                
+                                obj = json_tokener_do_parse(this, &err);
+                                if (err != json_tokener_success) {
+                                        goto out;
+                                }
+                                oldlen = mprToInt(mprGetProperty(&current,
+                                                                 "length",
+                                                                 NULL));
+                                mprItoa(oldlen, idx, sizeof(idx));
+                                mprSetVar(&current, idx, obj);
+                                saved_state = json_tokener_state_array_sep;
+                                state = json_tokener_state_eatws;
+                        }
+                        break;
+                        
+                case json_tokener_state_array_sep:
+                        if(c == ']') {
+                                this->pos++;
+                                saved_state = json_tokener_state_finish;
+                                state = json_tokener_state_eatws;
+                        } else if(c == ',') {
+                                this->pos++;
+                                saved_state = json_tokener_state_array;
+                                state = json_tokener_state_eatws;
+                        } else {
+                                *err_p = json_tokener_error_parse_array;
+                                mprDestroyVar(&current);
+                                return mprCreateUndefinedVar();
+                        }
+                        break;
+                        
+                case json_tokener_state_object:
+                        state = json_tokener_state_object_field_start;
+                        start_offset = this->pos;
+                        break;
+                        
+                case json_tokener_state_object_field_start:
+                        if(c == '}') {
+                                this->pos++;
+                                saved_state = json_tokener_state_finish;
+                                state = json_tokener_state_eatws;
+                        } else if (c == '"' || c == '\'') {
+                                quote_char = c;
+                                talloc_free(this->pb);
+                                this->pb = talloc_zero_size(this->ctx, 1);
+                                if (this->pb == NULL) {
+                                        *err_p = json_tokener_error_oom;
+                                        goto out;
+                                }
+                                state = json_tokener_state_object_field;
+                                start_offset = ++this->pos;
+                        } else {
+                                err = json_tokener_error_parse_object;
+                                goto out;
+                        }
+                        break;
+                        
+                case json_tokener_state_object_field:
+                        if(c == quote_char) {
+                                if ((this->pb = append_string(
+                                             this->ctx,
+                                             this->pb,
+                                             this->source + start_offset,
+                                             this->pos - start_offset)) == NULL) {
+                                        err = json_tokener_error_oom;
+                                        goto out;
+                                }
+                                if ((obj_field_name =
+                                     talloc_strdup(this->ctx,
+                                                   this->pb)) == NULL) {
+                                        err = json_tokener_error_oom;
+                                        goto out;
+                                }
+                                saved_state = json_tokener_state_object_field_end;
+                                state = json_tokener_state_eatws;
+                        } else if(c == '\\') {
+                                saved_state = json_tokener_state_object_field;
+                                state = json_tokener_state_string_escape;
+                        }
+                        this->pos++;
+                        break;
+                        
+                case json_tokener_state_object_field_end:
+                        if(c == ':') {
+                                this->pos++;
+                                saved_state = json_tokener_state_object_value;
+                                state = json_tokener_state_eatws;
+                        } else {
+                                *err_p = json_tokener_error_parse_object;
+                                mprDestroyVar(&current);
+                                return mprCreateUndefinedVar();
+                        }
+                        break;
+                        
+                case json_tokener_state_object_value:
+                        obj = json_tokener_do_parse(this, &err);
+                        if (err != json_tokener_success) {
+                                goto out;
+                        }
+                        mprSetVar(&current, obj_field_name, obj);
+                        talloc_free(obj_field_name);
+                        obj_field_name = NULL;
+                        saved_state = json_tokener_state_object_sep;
+                        state = json_tokener_state_eatws;
+                        break;
+                        
+                case json_tokener_state_object_sep:
+                        if(c == '}') {
+                                this->pos++;
+                                saved_state = json_tokener_state_finish;
+                                state = json_tokener_state_eatws;
+                        } else if(c == ',') {
+                                this->pos++;
+                                saved_state = json_tokener_state_object;
+                                state = json_tokener_state_eatws;
+                        } else {
+                                err = json_tokener_error_parse_object;
+                                goto out;
+                        }
+                        break;
+                        
+                }
+        } while(c);
+        
+        if(state != json_tokener_state_finish &&
+           saved_state != json_tokener_state_finish)
+                err = json_tokener_error_parse_eof;
+        
+out:
+        talloc_free(obj_field_name);
+        if(err == json_tokener_success) {
+                return current;
+        }
+        mprDestroyVar(&current);
+        *err_p = err;
+        return mprCreateUndefinedVar();
+}
+
+
+void smb_setup_ejs_literal(void)
+{
+	ejsDefineStringCFunction(-1,
+                                 "literal_to_var",
+                                 literal_to_var,
+                                 NULL,
+                                 MPR_VAR_SCRIPT_HANDLE);
+}