Array handling

README.md

@@ -236,7 +236,6 @@ A few notes on how this behaves:
 Similarly to JSON, git2consul can treat YAML documents in your repo as fully formed subtrees.
 
 ```yaml
----
 # file: example.yaml or example.yml
 first_level:
   second_level:
@@ -410,8 +409,104 @@ Usage example :
 
 Let say that you have a file called `user-service-dev.properties` in your repo. This file will be saved on consul as `user-service-dev`.
 
+##### array_format (default: none)
 
-#### Debian packaging
+`array_format` is a repo level option that applies only when expand_keys is set.  It defines what format array values from JSON and Yaml will be written to inside a single key.
+
+Valid values are:
+
+- `none`: do not include the array value
+- `json`: write a JSON array value
+- _separator_: Use the separator as a list delimiter -- e.g. use `,` for a comma-separated list. 
+
+Given the following JSON or Yaml
+
+JSON:
+```json
+{
+  "foods": ["apple", "bannana", "orange", "food \"with\" weird, name"]
+}
+```
+```yaml
+foods:
+  - apple
+  - bananna
+  - orange
+  - 'food "with" weird name'
+```
+
+The following values will be generated for the key `foods`:
+
+- `none` -\> No `foods` key will be written
+- `json` -\> `foods` will be set to `["apple", "bannana", "orange", "food \"with\" weird, name"]`
+- `,` -\> `foods` will be set to `apple,bannana,orange,food \"with\" weird,name` (note the extra, un-escaped comma!)
+
+Notes:
+
+- As the above example shows, no escaping is done for delimited lists.
+
+##### array_key_format (default: null)
+
+`array_key_format` is a repo level option that applies only when expand_keys is set.  It defines the format used to output array key values when outputting one key per array value.
+
+The format uses 2 special values, underscore (`_`) and octothorp (`#`):
+
+- `_` is replaced with the name of the array value.
+- `#` is replaced by the index in the array.
+
+Any value that is not a string with at least 1 character will result in no array keys being written.
+
+Given the following JSON or Yaml
+
+JSON:
+```json
+{
+  "cars": [
+    { "economy": ["ford", "gm"] },
+    { "luxury": ["bmw", "mercedes"] }
+  ]
+}
+```
+```yaml
+cars:
+  - economy
+    - ford
+    - gm
+  - luxury
+    - bmw
+    - mercedes
+```
+
+git2consul will generate the following keys:
+
+For `_/#` (prefix becomes a folder with index keys):
+```
+[..prefix...]/cars/0/economy/0=ford
+[..prefix...]/cars/0/economy/1=gm
+[..prefix...]/cars/1/luxury/0=bmw
+[..prefix...]/cars/1/luxury/1=mercedes
+```
+
+For `_[#]` (Spring-style array indexes):
+```
+[..prefix...]/cars[0]/economy/[0]=ford
+[..prefix...]/cars[0]/economy/[1]=gm
+[..prefix...]/cars[1]/luxury/[0]=bmw
+[..prefix...]/cars[1]/luxury/[1]=mercedes
+```
+
+For `#\__` (escaped underscore):
+```
+[..prefix...]/0_cars/0_economy/=ford
+[..prefix...]/0_cars/1_economy/=gm
+[..prefix...]/1_cars/0_luxury/=bmw
+[..prefix...]/1_cars/1_luxury/=mercedes
+```
+
+Note: as shown in the example above, keys are properly generated even for nested objects.
+
+#_
+# Debian packaging
 
 If you don't have grunt `sudo npm install -g grunt-cli`.
 

lib/array_handler.js

@@ -0,0 +1,104 @@
+var _ = require('underscore');
+var logger = require('./logging.js');
+
+exports.create_formatter = function create_formatter(array_format) {
+  switch (array_format) {
+    case 'none':
+      return false;
+    case 'json':
+      return function (arr) {
+        return JSON.stringify(arr);
+      };
+    default:
+      if (_.isString(array_format)) {
+        return function (arr) {
+          return arr.join(array_format);
+        };
+      }
+      if (!_.isUndefined(array_format)) {
+        logger.warn("Ignoring array_format because it is set to invalid value '%s' for branch %s in repo %s",
+          array_format, this.name, this.repo_name);
+      }
+      return false;
+  }
+};
+
+function parse_format(key_token, index_token, format) {
+  // Parse the format into a list of tokens that can be output
+  var escaping = false;
+  var tokens = [];
+  var currentToken = "";
+  for (var i = 0, formatLength = format.length; i < formatLength; i++) {
+    var c = format.charAt(i);
+    switch (c) {
+      case '\\':
+        if (escaping) {
+          escaping = false;
+          currentToken += c;
+        } else {
+          escaping = true;
+        }
+        break;
+      case '_':
+        if (escaping) {
+          escaping = false;
+          currentToken += c;
+        } else {
+          if (currentToken.length > 0) {
+            tokens.push(currentToken);
+            currentToken = "";
+          }
+          tokens.push(key_token);
+        }
+        break;
+      case '#':
+        if (escaping) {
+          escaping = false;
+          currentToken += c;
+        } else {
+          if (currentToken.length > 0) {
+            tokens.push(currentToken);
+            currentToken = "";
+          }
+          tokens.push(index_token);
+        }
+        break;
+      default:
+        if (escaping) {
+          escaping = false;
+        }
+        currentToken += c;
+        break;
+    }
+  }
+  if (currentToken.length > 0) {
+    tokens.push(currentToken);
+  }
+  return tokens;
+}
+
+exports.create_key_formatter = function create_key_formatter(format) {
+  if (!_.isString(format) || format.length === 0) {
+    return false;
+  }
+  var KEY_TOKEN = {};
+  var INDEX_TOKEN = {};
+  if (_.isString(format)) {
+    var tokens = parse_format(KEY_TOKEN, INDEX_TOKEN, format);
+    return function (key, index) {
+      var formattedKey = "";
+      for (var i = 0; i < tokens.length; i++) {
+        var token = tokens[i];
+        if (token === KEY_TOKEN) {
+          formattedKey += encodeURIComponent(key);
+        } else if (token === INDEX_TOKEN) {
+          formattedKey += index;
+        } else {
+          formattedKey += token;
+        }
+      }
+      return formattedKey;
+    };
+  }
+  return false;
+}

lib/consul/index.js

@@ -10,7 +10,7 @@ var consul = require('consul')({'host': global.endpoint, 'port': global.port, 's
 
 var token = undefined;
 
-const EXPAND_EXTENSIONS = ['json', 'yaml', 'yml', 'properties'];
+var EXPAND_EXTENSIONS = ['json', 'yaml', 'yml', 'properties'];
 
 // This makes life a bit easier for expand_keys mode, allowing us to check for a .json
 // extension with less code per line.
@@ -73,23 +73,55 @@ var create_key_name = function(branch, file, ref) {
   return key_parts.join('/');
 };
 
-
 /**
  * Given an obj, recurse into it, populating the parts array with all of the key->value
  * relationships, prefixed by parent objs.
  *
  * For example, the obj { 'first': { 'second': { 'third' : 'whee' }}} should yield a
  * parts array with a single entry: 'first/second/third' with value 'whee'.
+ *
+ * Array values will be rendered according to the configuration specified by array_format
+ * and array_key_format in the branch configuration.
  */
-var render_obj = function(parts, prefix, obj) {
-
+var render_obj = function(array_formatter, array_key_formatter, parts, prefix, obj) {
   _.mapObject(obj, function(val, key) {
-    if (_.isArray(val)) return;
-
-    if (_.isObject(val)) return render_obj(parts, prefix + '/' + encodeURIComponent(key), val)
-
+    if (_.isArray(val)) {
+      return render_array(array_formatter, array_key_formatter, parts, prefix, key, val);
+    }
+    if (_.isObject(val)) {
+      return render_obj(array_formatter, array_key_formatter, parts, prefix + '/' + encodeURIComponent(key), val);
+    }
     parts.push({'key': prefix + '/' + encodeURIComponent(key), 'value': val});
   });
+
+  logger.debug("PARTS found: " , parts);
+}
+
+/**
+ * Array values will be rendered according to the configuration specified by array_format
+ * and array_key_format in the branch configuration.  The former controls how full array
+ * values should be rendered (and only applies to arrays of "simple" objects); the latter
+ * controls how elements of arrays should be rendered to their own keys.
+ */
+var render_array = function(array_formatter, array_key_formatter, parts, prefix, key, arr) {
+  if (array_formatter && {}.constructor !== arr[0].constructor) {
+    // We have an array formatter, and a simple array, so render the single key.
+    parts.push({'key': prefix + '/' + encodeURIComponent(key), 'value': array_formatter(arr)});
+  }
+  if (array_key_formatter) {
+    // We have a key formatter, so format each element of the array
+    for (var i = 0; i < arr.length; i++) {
+      var arrElementKey = prefix + '/' + array_key_formatter(key, i);
+      var arrElement = arr[i];
+      if ({}.constructor === arrElement.constructor) {
+        // Render the object
+        render_obj(array_formatter, array_key_formatter, parts, arrElementKey, arrElement);
+      } else {
+        // Render the non-object value
+        parts.push({'key': arrElementKey, 'value': arrElement});
+      }
+    }
+  }
 }
 
 /**
@@ -102,7 +134,7 @@ var populate_kvs_from_object = function(branch, prefix, obj, existing_kvs, cb) {
   var delete_kvs = [];
   var candidate_kvs = [];
 
-  render_obj(candidate_kvs, prefix, obj);
+  render_obj(branch.array_formatter, branch.array_key_formatter, candidate_kvs, prefix, obj);
 
   // This avoids unnecessary copying if there are no existing KV records.
   if (existing_kvs.length > 0) {

lib/git/branch.js

@@ -2,7 +2,9 @@ var fs = require('fs');
 var path = require('path');
 var mkdirp = require('mkdirp');
 var rimraf = require('rimraf');
+var _ = require('underscore');
 
+var array_handler = require('../array_handler.js');
 var logger = require('../logging.js');
 
 var consul_broker = require('../consul');
@@ -17,6 +19,10 @@ function Branch(repo_config, name) {
   Object.defineProperty(this, 'branch_directory', {value: this.branch_parent + path.sep + name});
   Object.defineProperty(this, 'expand_keys', { value: repo_config['expand_keys'] === true });
   Object.defineProperty(this, 'expand_keys_diff', { value: repo_config['expand_keys_diff'] === true });
+  Object.defineProperty(this, 'array_formatter', {value: array_handler.create_formatter(repo_config['array_format'])});
+  Object.defineProperty(this, 'array_key_formatter', {
+    value: array_handler.create_key_formatter(repo_config['array_key_format'])
+  });
   Object.defineProperty(this, 'common_properties', { value: repo_config['common_properties']});
   Object.defineProperty(this, 'include_branch_name', {
     // If include_branch_name is not set, assume true.  Otherwise, identity check the value against true.

test/git2consul_array_handler_test.js

@@ -0,0 +1,73 @@
+var _ = require('underscore');
+var should = require('should');
+
+// We want this above any git2consul module to make sure logging gets configured
+require('./git2consul_bootstrap_test.js');
+
+var array_key_formatter = require('../lib/array_handler.js');
+
+describe('Array Formatter', function() {
+  it ('should ignore non-string values', function() {
+    array_key_formatter.create_formatter(false).should.equal(false);
+    array_key_formatter.create_formatter(true).should.equal(false);
+    array_key_formatter.create_formatter(0).should.equal(false);
+    array_key_formatter.create_formatter(NaN).should.equal(false);
+    array_key_formatter.create_formatter('none').should.equal(false);
+  });
+
+  it ('should format json', function() {
+    var format = array_key_formatter.create_formatter('json');
+    _.isFunction(format).should.equal(true);
+    var arr = ['foo', 'bar'];
+    format(arr).should.equal(JSON.stringify(arr));
+  });
+
+  it ('should format comma-separated lists', function() {
+    var format = array_key_formatter.create_formatter(',');
+    _.isFunction(format).should.equal(true);
+    var arr = ['foo', 'bar'];
+    format(arr).should.equal('foo,bar');
+  });
+
+  it ('should format un-separated lists', function() {
+    var format = array_key_formatter.create_formatter('');
+    _.isFunction(format).should.equal(true);
+    var arr = ['foo', 'bar'];
+    format(arr).should.equal('foobar');
+  });
+});
+
+describe('Array Key Formatter', function() {
+
+  it ('should ignore non-format values', function() {
+    array_key_formatter.create_key_formatter(false).should.equal(false);
+    array_key_formatter.create_key_formatter(true).should.equal(false);
+    array_key_formatter.create_key_formatter(0).should.equal(false);
+    array_key_formatter.create_key_formatter(NaN).should.equal(false);
+    array_key_formatter.create_key_formatter('').should.equal(false);
+  });
+
+  it ('should format _#', function() {
+    var format = array_key_formatter.create_key_formatter('_#');
+    _.isFunction(format).should.equal(true);
+    format('foo', 3).should.equal('foo3');
+  });
+
+  it ('should format pre_inner#post', function() {
+    var format = array_key_formatter.create_key_formatter('pre_inner#post');
+    _.isFunction(format).should.equal(true);
+    format('foo', 3).should.equal('prefooinner3post');
+  });
+
+  it ('should format #_#/_#', function() {
+    var format = array_key_formatter.create_key_formatter('#_#/_#');
+    _.isFunction(format).should.equal(true);
+    format('foo', 3).should.equal('3foo3/foo3');
+  });
+
+  it ('should format #_\\#\\_#', function() {
+    var format = array_key_formatter.create_key_formatter('#_\\#\\_#');
+    _.isFunction(format).should.equal(true);
+    format('foo', 3).should.equal('3foo#_3');
+  });
+});