COLLECTION Adding flat traversal & copy

The collection is hearachical. The flattening
of the collection was not implemented before
both for traversal and copying. This patch
introduces functionality to traverse or
iterate through collection as flat set
and also copy collection into another flattening
it and automatically resolving conflicts.
Also imptoved tracability and fixed memory leak
in unbind iterator code.

5 files changed, 598 insertions, 99 deletions

diff --git a/common/collection/collection.c b/common/collection/collection.c
index 298ddf4d..b4a2c8d5 100644
--- a/common/collection/collection.c
+++ b/common/collection/collection.c
@@ -112,7 +112,8 @@ static int col_walk_items(struct collection_item *ci,
                           internal_item_fn traverse_handler,
                           void *traverse_data,
                           col_item_fn user_item_handler,
-                          void *custom_data);
+                          void *custom_data,
+                          unsigned *depth);
 
 /* Function to get sub collection */
 static int col_get_subcollection(const char *property,
@@ -283,6 +284,7 @@ static int col_find_property(struct collection_item *collection,
 {
     struct property_search ps;
     int i = 0;
+    unsigned depth = 0;
 
     TRACE_FLOW_STRING("col_find_property", "Entry.");
 
@@ -307,7 +309,8 @@ static int col_find_property(struct collection_item *collection,
     /* We do not care about error here */
     (void)col_walk_items(collection, COL_TRAVERSE_ONELEVEL,
                          col_parent_traverse_handler,
-                         (void *)parent, NULL, (void *)&ps);
+                         (void *)parent, NULL, (void *)&ps,
+                         &depth);
 
     if (*parent) {
         /* Item is found in the collection */
@@ -1221,13 +1224,14 @@ static void col_delete_path_data(struct path_data *path)
 /* For each item walked it will call traverse handler.
    Traverse handler accepts: current item,
    user provided item handler and user provided custom data. */
-/* See below defferent traverse handlers for different cases */
+/* See below different traverse handlers for different cases */
 static int col_walk_items(struct collection_item *ci,
                           int mode_flags,
                           internal_item_fn traverse_handler,
                           void *traverse_data,
                           col_item_fn user_item_handler,
-                          void *custom_data)
+                          void *custom_data,
+                          unsigned *depth)
 {
     struct collection_item *current;
     struct collection_item *parent = NULL;
@@ -1238,6 +1242,9 @@ static int col_walk_items(struct collection_item *ci,
     TRACE_FLOW_STRING("col_walk_items", "Entry.");
     TRACE_INFO_NUMBER("Mode flags:", mode_flags);
 
+    /* Increase depth */
+    (*depth)++;
+
     current = ci;
 
     while (current) {
@@ -1252,22 +1259,34 @@ static int col_walk_items(struct collection_item *ci,
             if ((mode_flags & COL_TRAVERSE_IGNORE) == 0) {
 
                 TRACE_INFO_STRING("Subcollection is not ignored.", "");
-
                 /* We are not ignoring sub collections */
-                error = traverse_handler(ci, parent, current, traverse_data,
-                                         user_item_handler, custom_data, &stop);
-                if (stop != 0) {
-                    TRACE_INFO_STRING("Traverse handler returned STOP.", "");
-                    error = EINTR_INTERNAL;
-                }
-                /* Check what error we got */
-                if (error == EINTR_INTERNAL) {
-                    TRACE_FLOW_NUMBER("Internal error - means we are stopping.", error);
-                    return error;
-                }
-                else if (error) {
-                    TRACE_ERROR_NUMBER("Traverse handler returned error.", error);
-                    return error;
+
+                if ((mode_flags & COL_TRAVERSE_FLAT) == 0) {
+
+                    TRACE_INFO_STRING("Subcollection is not flattened.", "");
+                    /* We are not flattening sub collections.
+                     * The flattening means that we are not going
+                     * to return reference and headers for sub collections.
+                     * We will also not do special end collection
+                     * invocation for sub collections.
+                     */
+                    error = traverse_handler(ci, parent, current, traverse_data,
+                                             user_item_handler, custom_data, &stop);
+                    if (stop != 0) {
+                        TRACE_INFO_STRING("Traverse handler returned STOP.", "");
+                        error = EINTR_INTERNAL;
+                    }
+                    /* Check what error we got */
+                    if (error == EINTR_INTERNAL) {
+                        TRACE_FLOW_NUMBER("Internal error - means we are stopping.", error);
+                        (*depth)--;
+                        return error;
+                    }
+                    else if (error) {
+                        TRACE_ERROR_NUMBER("Traverse handler returned error.", error);
+                        (*depth)--;
+                        return error;
+                    }
                 }
 
                 if ((mode_flags & COL_TRAVERSE_ONELEVEL) == 0) {
@@ -1278,7 +1297,8 @@ static int col_walk_items(struct collection_item *ci,
                     /* We need to go into sub collections */
                     error = col_walk_items(sub, mode_flags,
                                            traverse_handler, traverse_data,
-                                           user_item_handler, custom_data);
+                                           user_item_handler, custom_data,
+                                           depth);
                     TRACE_INFO_STRING("Returned from sub collection processing", "");
                     TRACE_INFO_STRING("Done processing item:", current->property);
                     TRACE_INFO_NUMBER("Done processing item type:", current->type);
@@ -1286,12 +1306,29 @@ static int col_walk_items(struct collection_item *ci,
                 }
             }
         }
-        else
-            /* Call handler then move on */
-            error = traverse_handler(ci, parent, current,
-                                     traverse_data, user_item_handler,
-                                     custom_data, &stop);
+        else {
+            /* Check if it is a header and we are not on the root level.
+             * If we are flattening collection we need to skip headers
+             * for sub collections.
+             */
+
+            /* Call handler if:
+             * a) It is not collection header
+             * OR
+             * b) It is header we are flattening but we are on top level
+             * OR
+             * c) It is header and we are not flattening.
+             */
+            if ((current->type != COL_TYPE_COLLECTION) ||
+                (((mode_flags & COL_TRAVERSE_FLAT) != 0) && (*depth == 1)) ||
+                ((mode_flags & COL_TRAVERSE_FLAT) == 0)) {
+                /* Call handler then move on */
+                error = traverse_handler(ci, parent, current,
+                                         traverse_data, user_item_handler,
+                                         custom_data, &stop);
 
+            }
+        }
         /* If we are stopped - return EINTR_INTERNAL */
         if (stop != 0) {
             TRACE_INFO_STRING("Traverse handler returned STOP.", "");
@@ -1300,10 +1337,12 @@ static int col_walk_items(struct collection_item *ci,
         /* Check what error we got */
         if (error == EINTR_INTERNAL) {
             TRACE_FLOW_NUMBER("Internal error - means we are stopping.", error);
+            (*depth)--;
             return error;
         }
         else if (error) {
             TRACE_ERROR_NUMBER("Traverse handler returned error.", error);
+            (*depth)--;
             return error;
         }
 
@@ -1314,14 +1353,27 @@ static int col_walk_items(struct collection_item *ci,
 
     TRACE_INFO_STRING("Out of loop", "");
 
+    /* Check if we need to have a special
+     * call at the end of the collection.
+     */
     if ((mode_flags & COL_TRAVERSE_END) != 0) {
-        TRACE_INFO_STRING("About to do the special end collection invocation of handler", "");
-        error = traverse_handler(ci, parent, current,
-                                 traverse_data, user_item_handler,
-                                 custom_data, &stop);
+
+        /* Do this dummy invocation only:
+         * a) If we are flattening and on the root level
+         * b) We are not flattening
+         */
+        if ((((mode_flags & COL_TRAVERSE_FLAT) != 0) && (*depth == 1)) ||
+            ((mode_flags & COL_TRAVERSE_FLAT) == 0)) {
+
+            TRACE_INFO_STRING("About to do the special end collection invocation of handler", "");
+            error = traverse_handler(ci, parent, current,
+                                     traverse_data, user_item_handler,
+                                     custom_data, &stop);
+        }
     }
 
     TRACE_FLOW_NUMBER("col_walk_items. Returns: ", error);
+    (*depth)--;
     return error;
 }
 
@@ -1343,6 +1395,7 @@ static int col_find_item_and_do(struct collection_item *ci,
 
     int error = EOK;
     struct find_name *traverse_data = NULL;
+    unsigned depth = 0;
 
     TRACE_FLOW_STRING("col_find_item_and_do", "Entry.");
 
@@ -1386,7 +1439,8 @@ static int col_find_item_and_do(struct collection_item *ci,
     TRACE_INFO_NUMBER("Traverse flags", mode_flags);
 
     error = col_walk_items(ci, mode_flags, col_act_traverse_handler,
-                           (void *)traverse_data, item_handler, custom_data);
+                           (void *)traverse_data, item_handler, custom_data,
+                           &depth);
 
     if (traverse_data->current_path != NULL) {
         TRACE_INFO_STRING("find_item_and_do",
@@ -1793,6 +1847,79 @@ static int col_copy_traverse_handler(struct collection_item *head,
     return error;
 }
 
+/* Function to copy collection as flat set */
+static int col_copy_collection_flat(struct collection_item *acceptor,
+                                    struct collection_item *donor)
+{
+    int error = EOK;
+    struct collection_iterator *iter = NULL;
+    struct collection_item *item = NULL;
+    struct collection_item *new_item = NULL;
+    struct collection_item *current = NULL;
+
+    TRACE_FLOW_STRING("col_copy_collection_flat", "Entry");
+
+    /* Bind iterator in flat mode */
+    error = col_bind_iterator(&iter, donor, COL_TRAVERSE_FLAT);
+    if (error) {
+        TRACE_ERROR_NUMBER("Failed to bind iterator:", error);
+        return error;
+    }
+
+    /* Skip header */
+    error = col_iterate_collection(iter, &item);
+    if (error) {
+        TRACE_ERROR_NUMBER("Failed to iterate on header:", error);
+        col_unbind_iterator(iter);
+        return error;
+    }
+
+    current = item->next;
+
+    while (current) {
+
+        /* Loop through a collection */
+        error = col_iterate_collection(iter, &item);
+        if (error) {
+            TRACE_ERROR_NUMBER("Failed to iterate:", error);
+            col_unbind_iterator(iter);
+            return error;
+        }
+
+        /* Create new item */
+        error = col_allocate_item(&new_item,
+                                  item->property,
+                                  item->data,
+                                  item->length,
+                                  item->type);
+        if(error) {
+            TRACE_ERROR_NUMBER("Error allocating end item.", error);
+            col_unbind_iterator(iter);
+            return error;
+        }
+
+        /* Insert this item into collection */
+        error = col_insert_item(acceptor,
+                                NULL,
+                                new_item,
+                                COL_DSP_END,
+                                NULL,
+                                0,
+                                COL_INSERT_DUPOVER);
+        if(error) {
+            TRACE_ERROR_NUMBER("Error allocating end item.", error);
+            col_unbind_iterator(iter);
+            col_delete_item(new_item);
+            return error;
+        }
+
+        current = item->next;
+    }
+
+    col_unbind_iterator(iter);
+    TRACE_FLOW_NUMBER("col_copy_collection_flat Exit returning", error);
+    return error;
+}
 
 
 /********************* MAIN INTERFACE FUNCTIONS *****************************/
@@ -1889,6 +2016,7 @@ int col_copy_collection(struct collection_item **collection_copy,
     struct collection_item *new_collection = NULL;
     const char *name;
     struct collection_header *header;
+    unsigned depth = 0;
 
     TRACE_FLOW_STRING("col_copy_collection", "Entry.");
 
@@ -1909,7 +2037,7 @@ int col_copy_collection(struct collection_item **collection_copy,
 
     error = col_walk_items(collection_to_copy, COL_TRAVERSE_ONELEVEL,
                            col_copy_traverse_handler, new_collection,
-                           NULL, NULL);
+                           NULL, NULL, &depth);
 
     if (!error) *collection_copy = new_collection;
     else col_destroy_collection(new_collection);
@@ -2156,6 +2284,15 @@ int col_add_collection_to_collection(struct collection_item *ci,
         TRACE_INFO_NUMBER("Adding property returned:", error);
         break;
 
+    case COL_ADD_MODE_FLAT:
+        TRACE_INFO_STRING("We are flattening the collection.", "");
+
+        error = col_copy_collection_flat(acceptor, collection_to_add);
+
+        TRACE_INFO_NUMBER("Copy collection flat returned:", error);
+        break;
+
+
     default:
         error = EINVAL;
     }
@@ -2175,10 +2312,12 @@ int col_traverse_collection(struct collection_item *ci,
 {
 
     int error = EOK;
+    unsigned depth = 0;
+
     TRACE_FLOW_STRING("col_traverse_collection", "Entry.");
 
     error = col_walk_items(ci, mode_flags, col_simple_traverse_handler,
-                           NULL, item_handler, custom_data);
+                           NULL, item_handler, custom_data, &depth);
 
     if ((error != 0) && (error != EINTR_INTERNAL)) {
         TRACE_ERROR_NUMBER("Error walking tree", error);
@@ -2430,8 +2569,11 @@ int col_bind_iterator(struct collection_iterator **iterator,
     iter->stack = NULL;
     iter->stack_size = 0;
     iter->stack_depth = 0;
+    iter->item_level = 0;
     iter->flags = mode_flags;
 
+    TRACE_INFO_NUMBER("Iterator flags", iter->flags);
+
     /* Allocate memory for stack */
     error = col_grow_stack(iter, 1);
     if(error) {
@@ -2491,13 +2633,31 @@ int col_get_iterator_depth(struct collection_iterator *iterator, int *depth)
         return EINVAL;
     }
 
-    *depth = iterator->stack_depth -1;
+    *depth = iterator->stack_depth - 1;
 
     TRACE_INFO_NUMBER("Stack depth at the end:", iterator->stack_depth);
     TRACE_FLOW_STRING("col_get_iterator_depth","Exit");
     return EOK;
 }
 
+/* What was the level of the last item we got? */
+int col_get_item_depth(struct collection_iterator *iterator, int *depth)
+{
+    TRACE_FLOW_STRING("col_get_item_depth", "Entry");
+
+    if ((iterator == NULL) || (depth == NULL)) {
+        TRACE_ERROR_NUMBER("Invalid parameter.", EINVAL);
+        return EINVAL;
+    }
+
+    *depth = iterator->item_level;
+
+    TRACE_INFO_NUMBER("Item level at the end:", iterator->item_level);
+    TRACE_FLOW_STRING("col_get_item_depth","Exit");
+    return EOK;
+}
+
+
 
 /* Unbind the iterator from the collection */
 void col_unbind_iterator(struct collection_iterator *iterator)
@@ -2505,7 +2665,7 @@ void col_unbind_iterator(struct collection_iterator *iterator)
     TRACE_FLOW_STRING("col_unbind_iterator", "Entry.");
     if (iterator != NULL) {
         col_destroy_collection(iterator->top);
-        free(iterator->end_item);
+        col_delete_item(iterator->end_item);
         free(iterator->stack);
         free(iterator);
     }
@@ -2541,6 +2701,7 @@ int col_iterate_collection(struct collection_iterator *iterator,
 
         /* Is current item available */
         current = iterator->stack[iterator->stack_depth - 1];
+        iterator->item_level = iterator->stack_depth - 1;
 
         /* We are not done so check if we have an item  */
         if (current != NULL) {
@@ -2569,6 +2730,7 @@ int col_iterate_collection(struct collection_iterator *iterator,
                             TRACE_INFO_STRING("Instructed to show header not reference", "");
                             other = *((struct collection_item **)current->data);
                             iterator->stack[iterator->stack_depth] = other->next;
+                            iterator->item_level = iterator->stack_depth;
                             *item = other;
                         }
                         /* Do we need to show both? */
@@ -2576,6 +2738,17 @@ int col_iterate_collection(struct collection_iterator *iterator,
                             TRACE_INFO_STRING("Instructed to show header and reference","");
                             iterator->stack[iterator->stack_depth] = *((struct collection_item **)(current->data));
                             *item = current;
+                            /* Do not need to adjust level here */
+                        }
+                        /* Do not show either */
+                        else if ((iterator->flags & COL_TRAVERSE_FLAT) != 0) {
+                            TRACE_INFO_STRING("Instructed not to show header and reference","");
+                            other = *((struct collection_item **)current->data);
+                            iterator->stack[iterator->stack_depth] = other->next;
+                            iterator->stack[iterator->stack_depth - 1] = current->next;
+                            iterator->stack_depth++;
+                            /* Do not need to adjust level here */
+                            continue;
                         }
                         /* We need to show reference only */
                         else {
@@ -2589,6 +2762,8 @@ int col_iterate_collection(struct collection_iterator *iterator,
                                 TRACE_INFO_NUMBER("Will show this item next time type:", other->next->type);
                             }
                             *item = current;
+                            TRACE_INFO_NUMBER("Level of the reference:", iterator->item_level);
+                            /* Do not need to adjust level here */
                         }
 
                         TRACE_INFO_STRING("We return item:", (*item)->property);
@@ -2616,9 +2791,18 @@ int col_iterate_collection(struct collection_iterator *iterator,
             }
             else {
                 /* Got a normal item - return it and move to the next one */
-                TRACE_INFO_STRING("Simple item", "");
-                *item = current;
-                iterator->stack[iterator->stack_depth - 1] = current->next;
+                if ((current->type == COL_TYPE_COLLECTION) &&
+                    ((iterator->flags & COL_TRAVERSE_FLAT) != 0) &&
+                    (iterator->stack_depth > 1)) {
+                    TRACE_INFO_STRING("Header of the sub collection in flat case ", "");
+                    iterator->stack[iterator->stack_depth - 1] = current->next;
+                    continue;
+                }
+                else {
+                    TRACE_INFO_STRING("Simple item", "");
+                    *item = current;
+                    iterator->stack[iterator->stack_depth - 1] = current->next;
+                }
                 break;
             }
         }
@@ -2626,12 +2810,24 @@ int col_iterate_collection(struct collection_iterator *iterator,
             /* Item is NULL */
             TRACE_INFO_STRING("Finished level", "moving to upper level");
             iterator->stack_depth--;
+            /* Remember that item_level is zero based while depth is size
+             * so we decrease and then assign. */
             TRACE_INFO_NUMBER("Stack depth at the end:", iterator->stack_depth);
             if ((iterator->flags & COL_TRAVERSE_END) != 0) {
-                /* Return dummy entry to indicate the end of the collection */
-                TRACE_INFO_STRING("Finished level", "told to return END");
-                *item = iterator->end_item;
-                break;
+
+                /* Show end element
+                 * a) If we are flattening but at the top
+                 * b) We are not flattening
+                 */
+                if ((((iterator->flags & COL_TRAVERSE_FLAT) != 0) &&
+                     (iterator->stack_depth == 0)) ||
+                    ((iterator->flags & COL_TRAVERSE_FLAT) == 0)) {
+
+                    /* Return dummy entry to indicate the end of the collection */
+                    TRACE_INFO_STRING("Finished level", "told to return END");
+                    *item = iterator->end_item;
+                    break;
+                }
             }
             else {
                 /* Move to next level */
diff --git a/common/collection/collection.h b/common/collection/collection.h
index da785b1b..7de17d2e 100644
--- a/common/collection/collection.h
+++ b/common/collection/collection.h
@@ -59,50 +59,58 @@
 /* The modes that define how one collection can be added to another */
 
 #define COL_ADD_MODE_REFERENCE 0    /* The collection will contain a pointer
-                                       to another */
+                                     * to another */
 #define COL_ADD_MODE_EMBED     1    /* The collection will become part of
-                                       another collection.
-                                       After this operation the handle should
-                                       not be used or freed.
-                                       Can't be done more than once.
-                                       If the collection is referenced by
-                                       another collection, the operation will
-                                       fail. */
+                                     * another collection.
+                                     * After this operation the handle should
+                                     * not be used or freed.
+                                     * Can't be done more than once.
+                                     * If the collection is referenced by
+                                     * another collection, the operation will
+                                     * fail. */
 #define COL_ADD_MODE_CLONE     2    /* Creates a deep copy of a collection with
-                                       its sub collections */
+                                     * its sub collections */
 
+#define COL_ADD_MODE_FLAT      3    /* Creates a deep copy of a collection with
+                                     * its sub collections flattening and resolving
+                                     * duplicates.
+                                     */
 
 /* Modes how the collection is traversed */
 #define COL_TRAVERSE_DEFAULT  0x00000000  /* Traverse all items */
 #define COL_TRAVERSE_ONELEVEL 0x00000001  /* Flag to traverse only top level
-                                             ignored if the IGNORE flag is
-                                             specified */
+                                           * ignored if the IGNORE flag is
+                                           * specified */
 #define COL_TRAVERSE_END      0x00000002  /* Call the handler once more when the
-                                             end of the collection is reached.
-                                             Good for processing nested
-                                             collections.
-                                             Implicit for iterators unless
-                                             the FLAT flag is specified */
+                                           * end of the collection is reached.
+                                           * Good for processing nested
+                                           * collections.
+                                           */
 #define COL_TRAVERSE_IGNORE   0x00000004  /* Ignore sub collections as if none
-                                             is present */
-#define COL_TRAVERSE_FLAT     0x00000008  /* Flatten the collection.
-                                             FIXME: not implemented yet */
+                                           * is present */
+#define COL_TRAVERSE_FLAT     0x00000008  /* Flatten the collection. */
 
 /* Additional iterator flags
  * NOTE: ignored by traverse functions */
 #define COL_TRAVERSE_SHOWSUB  0x00010000  /* Include headers of sub collections
-                                             By default iterators return just
-                                             references and skips headers.
-                                             Ignored if the ONELEVEL flag is
-                                             specified and not ignored.
-                                             Ignored if the FLAT flag is
-                                             specified. */
+                                           * By default iterators return just
+                                           * references and skips headers.
+                                           * Ignored if the ONELEVEL flag is
+                                           * specified and not ignored.
+                                           * Ignored if the FLAT flag is
+                                           * specified. */
 #define COL_TRAVERSE_ONLYSUB  0x00020000  /* Show the header of the sub
-                                             collection instead of reference.
-                                             Ignored if the ONELEVEL flag is
-                                             specified and not ignored.
-                                             Ignored if the FLAT flag is
-                                             specified. */
+                                           * collection instead of reference.
+                                           * Ignored if the ONELEVEL flag is
+                                           * specified and not ignored.
+                                           * Ignored if the FLAT flag is
+                                           * specified. */
+
+/* NOTE COL_TRAVERSE_FLAT, COL_TRAVERSE_SHOWSUB, COL_TRAVERSE_ONLYSUB
+ * are mutually exclusive flags. If combined together
+ * results will be unpredictable.
+ * DO NOT MIX THEM IN ONE ITERATOR.
+ */
 
 /* FIXME: move to event level - this does not belong to collection */
 /* Time stamp property name */
@@ -540,9 +548,33 @@ int col_iterate_collection(struct collection_iterator *iterator,
  */
 int col_iterate_up(struct collection_iterator *iterator, int level);
 
-/* How deep are we relative to the top level.*/
+/* How deep are we relative to the top level.
+ * This function might report depth that might look misleading.
+ * The reason is that traverse flags affect the internal
+ * level we are on at each moment.
+ * For example the default traverse behavior is to show
+ * references to the sub collections.
+ * So when the item reference is returned the
+ * depth automatically adjusted to level inside the sub collection.
+ * So if function is called in this situation the level returned will
+ * denote the level inside collection.
+ * Now imagine that this collection is empty so the attempt to read
+ * element will push you automatically one level up (in absence of the
+ * COL_TRAVERSE_END flag). If in this situation you encounter another
+ * collection the reference will be returned and level automatically
+ * adjusted to level inside the collection.
+ * The point is that the level is reliable only after
+ * a data item was returned.
+ * To avoid this ambiguity another function is introduced.
+ * See below.
+*/
 int col_get_iterator_depth(struct collection_iterator *iterator, int *depth);
 
+/* Returns item depth for the last read item.
+ * Returns 0 if no item has been read yet.
+ */
+int col_get_item_depth(struct collection_iterator *iterator, int *depth);
+
 /* FIXME - Do we need to be able to rewind iterator? */
 
 /* Set collection class */
diff --git a/common/collection/collection_priv.h b/common/collection/collection_priv.h
index fc11a1fa..699479a3 100644
--- a/common/collection/collection_priv.h
+++ b/common/collection/collection_priv.h
@@ -57,6 +57,7 @@ struct collection_iterator {
     struct collection_item **stack;
     unsigned stack_size;
     unsigned stack_depth;
+    unsigned item_level;
     int flags;
     struct collection_item *end_item;
 };
diff --git a/common/collection/collection_tools.c b/common/collection/collection_tools.c
index 43466d7f..c6cf995c 100644
--- a/common/collection/collection_tools.c
+++ b/common/collection/collection_tools.c
@@ -40,17 +40,23 @@ int col_debug_handle(const char *property,
 {
     int i;
     int nest_level;
+    int ignore = 0;
 
     TRACE_FLOW_STRING("col_debug_handle", "Entry.");
 
 
     nest_level = *(int *)(custom_data);
+    if (nest_level == -1) {
+        ignore = 1;
+        nest_level = 1;
+    }
+
     TRACE_INFO_NUMBER("We are getting this pointer:", custom_data);
     TRACE_INFO_NUMBER("Nest level:", nest_level);
 
     switch (type) {
     case COL_TYPE_STRING:
-        printf("%*s %s[%d] str: %s (%d)\n",
+        printf(">%*s%s[%d] str: %s (%d)\n",
                (nest_level -1) * 4, "",
                property,
                length,
@@ -58,7 +64,7 @@ int col_debug_handle(const char *property,
                nest_level);
         break;
     case COL_TYPE_BINARY:
-        printf("%*s %s[%d] bin: ",
+        printf(">%*s%s[%d] bin: ",
                (nest_level -1) * 4, "",
                property,
                length);
@@ -67,7 +73,7 @@ int col_debug_handle(const char *property,
         printf(" (%d)\n", nest_level);
         break;
     case COL_TYPE_INTEGER:
-        printf("%*s %s[%d] int: %d (%d)\n",
+        printf(">%*s%s[%d] int: %d (%d)\n",
                (nest_level -1) * 4, "",
                property,
                length,
@@ -75,7 +81,7 @@ int col_debug_handle(const char *property,
                nest_level);
         break;
     case COL_TYPE_UNSIGNED:
-        printf("%*s %s[%d] uint: %u (%d)\n",
+        printf(">%*s%s[%d] uint: %u (%d)\n",
                (nest_level -1) * 4, "",
                property,
                length,
@@ -83,7 +89,7 @@ int col_debug_handle(const char *property,
                nest_level);
         break;
     case COL_TYPE_LONG:
-        printf("%*s %s[%d] long: %ld (%d)\n",
+        printf(">%*s%s[%d] long: %ld (%d)\n",
                (nest_level -1) * 4, "",
                property,
                length,
@@ -91,7 +97,7 @@ int col_debug_handle(const char *property,
                nest_level);
         break;
     case COL_TYPE_ULONG:
-        printf("%*s %s[%d] ulong: %lu (%d)\n",
+        printf(">%*s%s[%d] ulong: %lu (%d)\n",
                (nest_level -1) * 4, "",
                property,
                length,
@@ -99,7 +105,7 @@ int col_debug_handle(const char *property,
                nest_level);
         break;
     case COL_TYPE_DOUBLE:
-        printf("%*s %s[%d] double: %.4f (%d)\n",
+        printf(">%*s%s[%d] double: %.4f (%d)\n",
                (nest_level -1) * 4, "",
                property,
                length,
@@ -107,7 +113,7 @@ int col_debug_handle(const char *property,
                nest_level);
         break;
     case COL_TYPE_BOOL:
-        printf("%*s %s[%d] bool: %s (%d)\n",
+        printf(">%*s%s[%d] bool: %s (%d)\n",
                (nest_level -1) * 4, "",
                property,
                length,
@@ -115,8 +121,8 @@ int col_debug_handle(const char *property,
                nest_level);
         break;
     case COL_TYPE_COLLECTION:
-        nest_level++;
-        printf("%*s %s[%d] header: count %d, ref_count %d class %d data: ",
+        if (!ignore) nest_level++;
+        printf(">%*s%s[%d] header: count %d, ref_count %d class %d data: ",
                (nest_level -1) * 4, "",
                property,
                length,
@@ -128,7 +134,7 @@ int col_debug_handle(const char *property,
         printf(" (%d)\n", nest_level);
         break;
     case COL_TYPE_COLLECTIONREF:
-        printf("%*s %s[%d] external link: ",
+        printf(">%*s%s[%d] external link: ",
                (nest_level -1) * 4, "",
                property,
                length);
@@ -137,8 +143,10 @@ int col_debug_handle(const char *property,
         printf(" (%d)\n", nest_level);
         break;
     case COL_TYPE_END:
-        nest_level--;
-        /* printf("Reduced nest level\n");*/
+        printf(">%*sEND[N/A] (%d)\n",
+               (nest_level -1) * 4, "",
+               nest_level);
+        if (!ignore) nest_level--;
         break;
     default:
         printf("Not implemented yet.\n");
@@ -154,7 +162,7 @@ int col_debug_handle(const char *property,
 inline int col_debug_item(struct collection_item *item)
 {
     int dummy = 0;
-    int nest_level = 0;
+    int nest_level = -1;
     return col_debug_handle(item->property,
                             item->property_len,
                             item->type,
diff --git a/common/collection/collection_ut.c b/common/collection/collection_ut.c
index 53fc4d70..c0adf348 100644
--- a/common/collection/collection_ut.c
+++ b/common/collection/collection_ut.c
@@ -553,26 +553,35 @@ int mixed_collection_test(void)
 int iterator_test(void)
 {
     struct collection_item *peer;
+    struct collection_item *initial;
+
     struct collection_item *socket1;
     struct collection_item *socket2;
+    struct collection_item *socket3;
     struct collection_iterator *iterator = (struct collection_iterator *)(NULL);
     int error = EOK;
     struct collection_item *item = (struct collection_item *)(NULL);
     char binary_dump[] = { 0, 1, 2, 3, 4, 5, 6, 7, 8 };
     int depth = 0;
+    int idepth = 0;
 
     printf("\n\n==== ITERATOR TEST ====\n\n");
 
-    if ((error = col_create_collection(&peer, "peer", 0)) ||
-        (error = col_add_str_property(peer, NULL, "hostname", "peerhost.mytest.com", 0)) ||
+    if ((error = col_create_collection(&initial, "strater", 0)) ||
+        (error = col_create_collection(&peer, "peer", 0)) ||
+        (error = col_add_str_property(initial, NULL, "hostname", "peerhost.mytest.com", 0)) ||
         /* Expect trailing zero to be truncated */
-        (error = col_add_str_property(peer, NULL, "IPv4", "10.10.10.10", 12)) ||
-        (error = col_add_str_property(peer, NULL, "IPv6", "bla:bla:bla:bla:bla:bla", 0))) {
+        (error = col_add_str_property(initial, NULL, "IPv4", "10.10.10.10", 12)) ||
+        (error = col_add_str_property(initial, NULL, "IPv6", "bla:bla:bla:bla:bla:bla", 0)) ||
+        (error = col_add_collection_to_collection(peer, NULL, NULL, initial, COL_ADD_MODE_FLAT))) {
         printf("Failed to add property. Error %d", error);
         col_destroy_collection(peer);
+        col_destroy_collection(initial);
         return error;
     }
 
+    col_destroy_collection(initial);
+
     if ((error = col_create_collection(&socket1, "socket", 0)) ||
         (error = col_add_int_property(socket1, NULL, "id", 1)) ||
         (error = col_add_long_property(socket1, NULL, "packets", 100000000L)) ||
@@ -594,11 +603,20 @@ int iterator_test(void)
         return error;
     }
 
-    error = col_add_collection_to_collection(peer, NULL, "first", socket1, COL_ADD_MODE_EMBED);
+    if ((error = col_create_collection(&socket3, "socket", 0))) {
+        col_destroy_collection(peer);
+        col_destroy_collection(socket1);
+        col_destroy_collection(socket2);
+        printf("Failed to add property. Error %d\n", error);
+        return error;
+    }
+
+    error = col_add_collection_to_collection(peer, NULL, "first", socket1, COL_ADD_MODE_REFERENCE);
     if (error) {
         col_destroy_collection(peer);
         col_destroy_collection(socket1);
         col_destroy_collection(socket2);
+        col_destroy_collection(socket3);
         printf("Failed to add collection to collection. Error %d\n", error);
         return error;
     }
@@ -606,7 +624,26 @@ int iterator_test(void)
     error = col_add_collection_to_collection(peer, NULL, "second", socket2, COL_ADD_MODE_EMBED);
     if (error) {
         col_destroy_collection(peer);
+        col_destroy_collection(socket1);
         col_destroy_collection(socket2);
+        col_destroy_collection(socket3);
+        printf("Failed to add collection to collection. Error %d\n", error);
+        return error;
+    }
+
+    error = col_add_collection_to_collection(peer, NULL, "third", socket3, COL_ADD_MODE_EMBED);
+    if (error) {
+        col_destroy_collection(peer);
+        col_destroy_collection(socket1);
+        col_destroy_collection(socket3);
+        printf("Failed to add collection to collection. Error %d\n", error);
+        return error;
+    }
+
+    error = col_add_collection_to_collection(peer, NULL, "forth", socket1, COL_ADD_MODE_EMBED);
+    if (error) {
+        col_destroy_collection(peer);
+        col_destroy_collection(socket1);
         printf("Failed to add collection to collection. Error %d\n", error);
         return error;
     }
@@ -619,34 +656,41 @@ int iterator_test(void)
         return error;
     }
 
-    printf("\n\nCollection:\n\n");
+    printf("\n\nCollection (traverse default):\n\n");
     col_debug_collection(peer, COL_TRAVERSE_DEFAULT);
 
-    /* This should also work becuase iterator holds to collection */
-    col_destroy_collection(peer);
+    printf("\n\nCollection (traverse flat):\n\n");
+    col_debug_collection(peer, COL_TRAVERSE_FLAT | COL_TRAVERSE_END);
 
-    printf("\n\nIteration:\n\n");
+    printf("\n\nIteration (1):\n\n");
 
     do {
-        depth = 0;
-        col_get_iterator_depth(iterator, &depth);
+
 
         /* Loop through a collection */
         error = col_iterate_collection(iterator, &item);
         if (error) {
             printf("Error (iterate): %d\n", error);
             col_unbind_iterator(iterator);
+            col_destroy_collection(peer);
             return error;
         }
 
+        depth = 0;
+        col_get_item_depth(iterator, &depth);
+        idepth = 0;
+        col_get_iterator_depth(iterator, &idepth);
+
         /* Are we done ? */
         if (item == (struct collection_item *)(NULL)) break;
 
-        printf("%*s Property (%s), type = %d, data size = %d\n",
+        printf("%*sProperty (%s), type = %d, data size = %d depth = %d idepth = %d\n",
                 depth * 4,  "",
                 col_get_item_property(item, NULL),
                 col_get_item_type(item),
-                col_get_item_length(item));
+                col_get_item_length(item),
+                depth,
+                idepth);
 
         if ((strcmp(col_get_item_property(item, NULL), "id")==0) &&
            (*((int *)(col_get_item_data(item))) == 1)) {
@@ -655,6 +699,7 @@ int iterator_test(void)
             if (!error) {
                 printf("We expected error but got seucces - bad.\n");
                 col_unbind_iterator(iterator);
+                col_destroy_collection(peer);
                 return -1;
             }
             /* This should work! */
@@ -662,6 +707,7 @@ int iterator_test(void)
             if (error) {
                 printf("We expected success but got error %d\n", error);
                 col_unbind_iterator(iterator);
+                col_destroy_collection(peer);
                 return error;
             }
 
@@ -684,14 +730,230 @@ int iterator_test(void)
                 (error = col_modify_double_item(item, "double", -1.1)) ||
                 (error = col_debug_item(item))) {
                 printf("Failed to change property.\n");
+                col_unbind_iterator(iterator);
+                col_destroy_collection(peer);
                 return error;
             }
         }
     }
     while(1);
 
+    col_unbind_iterator(iterator);
+
+    /* Bind iterator again in flat mode */
+    error =  col_bind_iterator(&iterator, peer, COL_TRAVERSE_FLAT);
+    if (error) {
+        printf("Error (bind): %d\n", error);
+        col_destroy_collection(peer);
+        return error;
+    }
+
+    printf("\n\nIteration (2 - flat):\n\n");
+
+    do {
+
+        /* Loop through a collection */
+        error = col_iterate_collection(iterator, &item);
+        if (error) {
+            printf("Error (iterate): %d\n", error);
+            col_destroy_collection(peer);
+            col_unbind_iterator(iterator);
+            return error;
+        }
+
+        /* Are we done ? */
+        if (item == (struct collection_item *)(NULL)) break;
+
+        depth = 0;
+        col_get_item_depth(iterator, &depth);
+        printf("%*s", depth * 4, "");
+        col_debug_item(item);
+
+    }
+    while(1);
+
+    /* Do not forget to unbind iterator - otherwise there will be a leak */
+    col_unbind_iterator(iterator);
+
+    /* Bind iterator again in flat mode */
+    error =  col_bind_iterator(&iterator, peer, COL_TRAVERSE_FLAT | COL_TRAVERSE_END);
+    if (error) {
+        printf("Error (bind): %d\n", error);
+        col_destroy_collection(peer);
+        return error;
+    }
+
+    printf("\n\nIteration (3 flat with end):\n\n");
+
+    do {
+
+        /* Loop through a collection */
+        error = col_iterate_collection(iterator, &item);
+        if (error) {
+            printf("Error (iterate): %d\n", error);
+            col_destroy_collection(peer);
+            col_unbind_iterator(iterator);
+            return error;
+        }
+
+        /* Are we done ? */
+        if (item == (struct collection_item *)(NULL)) break;
+
+        depth = 0;
+        col_get_item_depth(iterator, &depth);
+        printf("%*s", depth * 4, "");
+        col_debug_item(item);
+
+    }
+    while(1);
+
     /* Do not forget to unbind iterator - otherwise there will be a leak */
     col_unbind_iterator(iterator);
+
+    /* Bind iterator again in flat mode */
+    error =  col_bind_iterator(&iterator, peer, COL_TRAVERSE_DEFAULT | COL_TRAVERSE_END);
+    if (error) {
+        printf("Error (bind): %d\n", error);
+        col_destroy_collection(peer);
+        return error;
+    }
+
+    printf("\n\nIteration (4 default with end):\n\n");
+
+    do {
+
+        /* Loop through a collection */
+        error = col_iterate_collection(iterator, &item);
+        if (error) {
+            printf("Error (iterate): %d\n", error);
+            col_destroy_collection(peer);
+            col_unbind_iterator(iterator);
+            return error;
+        }
+
+        /* Are we done ? */
+        if (item == (struct collection_item *)(NULL)) break;
+
+        depth = 0;
+        col_get_item_depth(iterator, &depth);
+        printf("%*s", depth * 4, "");
+        col_debug_item(item);
+
+    }
+    while(1);
+
+    /* Do not forget to unbind iterator - otherwise there will be a leak */
+    col_unbind_iterator(iterator);
+
+    /* Bind iterator again in flat mode */
+    error =  col_bind_iterator(&iterator, peer, COL_TRAVERSE_SHOWSUB | COL_TRAVERSE_END);
+    if (error) {
+        printf("Error (bind): %d\n", error);
+        col_destroy_collection(peer);
+        return error;
+    }
+
+
+    printf("\n\nIteration (5 show headers and references with end):\n\n");
+
+    do {
+
+        /* Loop through a collection */
+        error = col_iterate_collection(iterator, &item);
+        if (error) {
+            printf("Error (iterate): %d\n", error);
+            col_destroy_collection(peer);
+            col_unbind_iterator(iterator);
+            return error;
+        }
+
+        /* Are we done ? */
+        if (item == (struct collection_item *)(NULL)) break;
+
+        depth = 0;
+        col_get_item_depth(iterator, &depth);
+        printf("%*s", depth * 4, "");
+        col_debug_item(item);
+
+    }
+    while(1);
+
+    /* Do not forget to unbind iterator - otherwise there will be a leak */
+    col_unbind_iterator(iterator);
+
+    /* Bind iterator again in flat mode */
+    error =  col_bind_iterator(&iterator, peer, COL_TRAVERSE_SHOWSUB);
+    if (error) {
+        printf("Error (bind): %d\n", error);
+        col_destroy_collection(peer);
+        return error;
+    }
+
+
+    printf("\n\nIteration (6 show headers and references no END):\n\n");
+
+    do {
+
+        /* Loop through a collection */
+        error = col_iterate_collection(iterator, &item);
+        if (error) {
+            printf("Error (iterate): %d\n", error);
+            col_destroy_collection(peer);
+            col_unbind_iterator(iterator);
+            return error;
+        }
+
+        /* Are we done ? */
+        if (item == (struct collection_item *)(NULL)) break;
+
+        depth = 0;
+        col_get_item_depth(iterator, &depth);
+        printf("%*s", depth * 4, "");
+        col_debug_item(item);
+
+    }
+    while(1);
+
+    /* Do not forget to unbind iterator - otherwise there will be a leak */
+    col_unbind_iterator(iterator);
+
+    /* Bind iterator again in flat mode */
+    error =  col_bind_iterator(&iterator, peer, COL_TRAVERSE_ONLYSUB);
+    if (error) {
+        printf("Error (bind): %d\n", error);
+        col_destroy_collection(peer);
+        return error;
+    }
+
+    /* This should also work becuase iterator holds to collection */
+    col_destroy_collection(peer);
+
+    printf("\n\nIteration (7 show headers only no END):\n\n");
+
+    do {
+
+        /* Loop through a collection */
+        error = col_iterate_collection(iterator, &item);
+        if (error) {
+            printf("Error (iterate): %d\n", error);
+            col_unbind_iterator(iterator);
+            return error;
+        }
+
+        /* Are we done ? */
+        if (item == (struct collection_item *)(NULL)) break;
+
+        depth = 0;
+        col_get_item_depth(iterator, &depth);
+        printf("%*s", depth * 4, "");
+        col_debug_item(item);
+
+    }
+    while(1);
+
+    /* Do not forget to unbind iterator - otherwise there will be a leak */
+    col_unbind_iterator(iterator);
+
     return EOK;
 }