diff --git a/README.md b/README.md index ee68dcf..b608cb4 100644 --- a/README.md +++ b/README.md @@ -236,7 +236,6 @@ A few notes on how this behaves: Similarly to JSON, git2consul can treat YAML documents in your repo as fully formed subtrees. ```yaml ---- # file: example.yaml or example.yml first_level: second_level: @@ -410,8 +409,104 @@ Usage example : Let say that you have a file called `user-service-dev.properties` in your repo. This file will be saved on consul as `user-service-dev`. +##### array_format (default: none) -#### Debian packaging +`array_format` is a repo level option that applies only when expand_keys is set. It defines what format array values from JSON and Yaml will be written to inside a single key. + +Valid values are: + +- `none`: do not include the array value +- `json`: write a JSON array value +- _separator_: Use the separator as a list delimiter -- e.g. use `,` for a comma-separated list. + +Given the following JSON or Yaml + +JSON: +```json +{ + "foods": ["apple", "bannana", "orange", "food \"with\" weird, name"] +} +``` +```yaml +foods: + - apple + - bananna + - orange + - 'food "with" weird name' +``` + +The following values will be generated for the key `foods`: + +- `none` -\> No `foods` key will be written +- `json` -\> `foods` will be set to `["apple", "bannana", "orange", "food \"with\" weird, name"]` +- `,` -\> `foods` will be set to `apple,bannana,orange,food \"with\" weird,name` (note the extra, un-escaped comma!) + +Notes: + +- As the above example shows, no escaping is done for delimited lists. + +##### array_key_format (default: null) + +`array_key_format` is a repo level option that applies only when expand_keys is set. It defines the format used to output array key values when outputting one key per array value. + +The format uses 2 special values, underscore (`_`) and octothorp (`#`): + +- `_` is replaced with the name of the array value. +- `#` is replaced by the index in the array. + +Any value that is not a string with at least 1 character will result in no array keys being written. + +Given the following JSON or Yaml + +JSON: +```json +{ + "cars": [ + { "economy": ["ford", "gm"] }, + { "luxury": ["bmw", "mercedes"] } + ] +} +``` +```yaml +cars: + - economy + - ford + - gm + - luxury + - bmw + - mercedes +``` + +git2consul will generate the following keys: + +For `_/#` (prefix becomes a folder with index keys): +``` +[..prefix...]/cars/0/economy/0=ford +[..prefix...]/cars/0/economy/1=gm +[..prefix...]/cars/1/luxury/0=bmw +[..prefix...]/cars/1/luxury/1=mercedes +``` + +For `_[#]` (Spring-style array indexes): +``` +[..prefix...]/cars[0]/economy/[0]=ford +[..prefix...]/cars[0]/economy/[1]=gm +[..prefix...]/cars[1]/luxury/[0]=bmw +[..prefix...]/cars[1]/luxury/[1]=mercedes +``` + +For `#\__` (escaped underscore): +``` +[..prefix...]/0_cars/0_economy/=ford +[..prefix...]/0_cars/1_economy/=gm +[..prefix...]/1_cars/0_luxury/=bmw +[..prefix...]/1_cars/1_luxury/=mercedes +``` + +Note: as shown in the example above, keys are properly generated even for nested objects. + +#_ +# Debian packaging If you don't have grunt `sudo npm install -g grunt-cli`. diff --git a/lib/array_handler.js b/lib/array_handler.js new file mode 100644 index 0000000..6ffe630 --- /dev/null +++ b/lib/array_handler.js @@ -0,0 +1,104 @@ +var _ = require('underscore'); +var logger = require('./logging.js'); + +exports.create_formatter = function create_formatter(array_format) { + switch (array_format) { + case 'none': + return false; + case 'json': + return function (arr) { + return JSON.stringify(arr); + }; + default: + if (_.isString(array_format)) { + return function (arr) { + return arr.join(array_format); + }; + } + if (!_.isUndefined(array_format)) { + logger.warn("Ignoring array_format because it is set to invalid value '%s' for branch %s in repo %s", + array_format, this.name, this.repo_name); + } + return false; + } +}; + +function parse_format(key_token, index_token, format) { + // Parse the format into a list of tokens that can be output + var escaping = false; + var tokens = []; + var currentToken = ""; + for (var i = 0, formatLength = format.length; i < formatLength; i++) { + var c = format.charAt(i); + switch (c) { + case '\\': + if (escaping) { + escaping = false; + currentToken += c; + } else { + escaping = true; + } + break; + case '_': + if (escaping) { + escaping = false; + currentToken += c; + } else { + if (currentToken.length > 0) { + tokens.push(currentToken); + currentToken = ""; + } + tokens.push(key_token); + } + break; + case '#': + if (escaping) { + escaping = false; + currentToken += c; + } else { + if (currentToken.length > 0) { + tokens.push(currentToken); + currentToken = ""; + } + tokens.push(index_token); + } + break; + default: + if (escaping) { + escaping = false; + } + currentToken += c; + break; + } + } + if (currentToken.length > 0) { + tokens.push(currentToken); + } + return tokens; +} + +exports.create_key_formatter = function create_key_formatter(format) { + if (!_.isString(format) || format.length === 0) { + return false; + } + var KEY_TOKEN = {}; + var INDEX_TOKEN = {}; + if (_.isString(format)) { + var tokens = parse_format(KEY_TOKEN, INDEX_TOKEN, format); + return function (key, index) { + var formattedKey = ""; + for (var i = 0; i < tokens.length; i++) { + var token = tokens[i]; + if (token === KEY_TOKEN) { + formattedKey += encodeURIComponent(key); + } else if (token === INDEX_TOKEN) { + formattedKey += index; + } else { + formattedKey += token; + } + } + return formattedKey; + }; + } + return false; +} \ No newline at end of file diff --git a/lib/consul/index.js b/lib/consul/index.js index 5985a84..c0a86c1 100644 --- a/lib/consul/index.js +++ b/lib/consul/index.js @@ -10,7 +10,7 @@ var consul = require('consul')({'host': global.endpoint, 'port': global.port, 's var token = undefined; -const EXPAND_EXTENSIONS = ['json', 'yaml', 'yml', 'properties']; +var EXPAND_EXTENSIONS = ['json', 'yaml', 'yml', 'properties']; // This makes life a bit easier for expand_keys mode, allowing us to check for a .json // extension with less code per line. @@ -73,23 +73,55 @@ var create_key_name = function(branch, file, ref) { return key_parts.join('/'); }; - /** * Given an obj, recurse into it, populating the parts array with all of the key->value * relationships, prefixed by parent objs. * * For example, the obj { 'first': { 'second': { 'third' : 'whee' }}} should yield a * parts array with a single entry: 'first/second/third' with value 'whee'. + * + * Array values will be rendered according to the configuration specified by array_format + * and array_key_format in the branch configuration. */ -var render_obj = function(parts, prefix, obj) { - +var render_obj = function(array_formatter, array_key_formatter, parts, prefix, obj) { _.mapObject(obj, function(val, key) { - if (_.isArray(val)) return; - - if (_.isObject(val)) return render_obj(parts, prefix + '/' + encodeURIComponent(key), val) - + if (_.isArray(val)) { + return render_array(array_formatter, array_key_formatter, parts, prefix, key, val); + } + if (_.isObject(val)) { + return render_obj(array_formatter, array_key_formatter, parts, prefix + '/' + encodeURIComponent(key), val); + } parts.push({'key': prefix + '/' + encodeURIComponent(key), 'value': val}); }); + + logger.debug("PARTS found: " , parts); +} + +/** + * Array values will be rendered according to the configuration specified by array_format + * and array_key_format in the branch configuration. The former controls how full array + * values should be rendered (and only applies to arrays of "simple" objects); the latter + * controls how elements of arrays should be rendered to their own keys. + */ +var render_array = function(array_formatter, array_key_formatter, parts, prefix, key, arr) { + if (array_formatter && {}.constructor !== arr[0].constructor) { + // We have an array formatter, and a simple array, so render the single key. + parts.push({'key': prefix + '/' + encodeURIComponent(key), 'value': array_formatter(arr)}); + } + if (array_key_formatter) { + // We have a key formatter, so format each element of the array + for (var i = 0; i < arr.length; i++) { + var arrElementKey = prefix + '/' + array_key_formatter(key, i); + var arrElement = arr[i]; + if ({}.constructor === arrElement.constructor) { + // Render the object + render_obj(array_formatter, array_key_formatter, parts, arrElementKey, arrElement); + } else { + // Render the non-object value + parts.push({'key': arrElementKey, 'value': arrElement}); + } + } + } } /** @@ -102,7 +134,7 @@ var populate_kvs_from_object = function(branch, prefix, obj, existing_kvs, cb) { var delete_kvs = []; var candidate_kvs = []; - render_obj(candidate_kvs, prefix, obj); + render_obj(branch.array_formatter, branch.array_key_formatter, candidate_kvs, prefix, obj); // This avoids unnecessary copying if there are no existing KV records. if (existing_kvs.length > 0) { diff --git a/lib/git/branch.js b/lib/git/branch.js index 5bb4e79..ebb5208 100644 --- a/lib/git/branch.js +++ b/lib/git/branch.js @@ -2,7 +2,9 @@ var fs = require('fs'); var path = require('path'); var mkdirp = require('mkdirp'); var rimraf = require('rimraf'); +var _ = require('underscore'); +var array_handler = require('../array_handler.js'); var logger = require('../logging.js'); var consul_broker = require('../consul'); @@ -17,6 +19,10 @@ function Branch(repo_config, name) { Object.defineProperty(this, 'branch_directory', {value: this.branch_parent + path.sep + name}); Object.defineProperty(this, 'expand_keys', { value: repo_config['expand_keys'] === true }); Object.defineProperty(this, 'expand_keys_diff', { value: repo_config['expand_keys_diff'] === true }); + Object.defineProperty(this, 'array_formatter', {value: array_handler.create_formatter(repo_config['array_format'])}); + Object.defineProperty(this, 'array_key_formatter', { + value: array_handler.create_key_formatter(repo_config['array_key_format']) + }); Object.defineProperty(this, 'common_properties', { value: repo_config['common_properties']}); Object.defineProperty(this, 'include_branch_name', { // If include_branch_name is not set, assume true. Otherwise, identity check the value against true. diff --git a/test/git2consul_array_handler_test.js b/test/git2consul_array_handler_test.js new file mode 100644 index 0000000..269d522 --- /dev/null +++ b/test/git2consul_array_handler_test.js @@ -0,0 +1,73 @@ +var _ = require('underscore'); +var should = require('should'); + +// We want this above any git2consul module to make sure logging gets configured +require('./git2consul_bootstrap_test.js'); + +var array_key_formatter = require('../lib/array_handler.js'); + +describe('Array Formatter', function() { + it ('should ignore non-string values', function() { + array_key_formatter.create_formatter(false).should.equal(false); + array_key_formatter.create_formatter(true).should.equal(false); + array_key_formatter.create_formatter(0).should.equal(false); + array_key_formatter.create_formatter(NaN).should.equal(false); + array_key_formatter.create_formatter('none').should.equal(false); + }); + + it ('should format json', function() { + var format = array_key_formatter.create_formatter('json'); + _.isFunction(format).should.equal(true); + var arr = ['foo', 'bar']; + format(arr).should.equal(JSON.stringify(arr)); + }); + + it ('should format comma-separated lists', function() { + var format = array_key_formatter.create_formatter(','); + _.isFunction(format).should.equal(true); + var arr = ['foo', 'bar']; + format(arr).should.equal('foo,bar'); + }); + + it ('should format un-separated lists', function() { + var format = array_key_formatter.create_formatter(''); + _.isFunction(format).should.equal(true); + var arr = ['foo', 'bar']; + format(arr).should.equal('foobar'); + }); +}); + +describe('Array Key Formatter', function() { + + it ('should ignore non-format values', function() { + array_key_formatter.create_key_formatter(false).should.equal(false); + array_key_formatter.create_key_formatter(true).should.equal(false); + array_key_formatter.create_key_formatter(0).should.equal(false); + array_key_formatter.create_key_formatter(NaN).should.equal(false); + array_key_formatter.create_key_formatter('').should.equal(false); + }); + + it ('should format _#', function() { + var format = array_key_formatter.create_key_formatter('_#'); + _.isFunction(format).should.equal(true); + format('foo', 3).should.equal('foo3'); + }); + + it ('should format pre_inner#post', function() { + var format = array_key_formatter.create_key_formatter('pre_inner#post'); + _.isFunction(format).should.equal(true); + format('foo', 3).should.equal('prefooinner3post'); + }); + + it ('should format #_#/_#', function() { + var format = array_key_formatter.create_key_formatter('#_#/_#'); + _.isFunction(format).should.equal(true); + format('foo', 3).should.equal('3foo3/foo3'); + }); + + it ('should format #_\\#\\_#', function() { + var format = array_key_formatter.create_key_formatter('#_\\#\\_#'); + _.isFunction(format).should.equal(true); + format('foo', 3).should.equal('3foo#_3'); + }); +}); diff --git a/test/git2consul_ignore_repo_name_test.js b/test/git2consul_ignore_repo_name_test.js index 850f604..de96306 100644 --- a/test/git2consul_ignore_repo_name_test.js +++ b/test/git2consul_ignore_repo_name_test.js @@ -18,7 +18,7 @@ var git_commands = require('../lib/git/commands.js'); describe('ignore_repo_name', function() { it ('should create folders on consul without the repo name prefix', function(done) { - + // Create a remote git repo. Then, init a Repo object with property file and validate // that keys are in the appropriate place in the Consul KV store without the repo name prefix. git_commands.init(git_utils.TEST_REMOTE_REPO, function(err) { diff --git a/test/git2consul_parsing_yml_test.js b/test/git2consul_parsing_yml_test.js new file mode 100644 index 0000000..25f768b --- /dev/null +++ b/test/git2consul_parsing_yml_test.js @@ -0,0 +1,91 @@ +var should = require('should'); +var _ = require('underscore'); +var fs = require('fs'); + +var mkdirp = require('mkdirp'); + +// We want this above any git2consul module to make sure logging gets configured +require('./git2consul_bootstrap_test.js'); + +var git_utils = require('./utils/git_utils.js'); +var consul_utils = require('./utils/consul_utils.js'); + +describe('Parse YAML', function() { + + // The current copy of the git master branch. This is initialized before each test in the suite. + var branch; + beforeEach(function(done) { + + // Each of these tests needs a working repo instance, so create it here and expose it to the suite + // namespace. These are all tests of expand_keys mode, so set that here. + var repoConfig = git_utils.createRepoConfig(); + // Add array settings + repoConfig.array_format = 'json'; + repoConfig.array_key_format = '_/#'; + git_utils.initRepo(_.extend(repoConfig, {'expand_keys': true}), function(err, repo) { + if (err) return done(err); + + // The default repo created by initRepo has a single branch, master. + branch = repo.branches['master']; + + done(); + }); + }); + + /*YAML*/ + + it('should handle complex YAML files', function(done) { + var sample_key = 'complex_sample.yaml'; + // from: js-yaml / test / samples-load-errors / forbidden-value.yml + var sample_file = "./test/resources/complex_sample.yaml"; + + fs.readFile(sample_file, "utf-8", function(err, new_file_content){ + if (err) return done(err); + + // Add the sample file, call branch.handleRef to sync the commit, then validate that consul contains the correct info. + git_utils.addFileToGitRepo(sample_key, new_file_content, "Add a file.", function(err) { + if (err) return done(err); + branch.handleRefChange(0, function(err) { + if (err) return done(err); + + var values_to_test = [ + { + key:'name', + value:'Kostas D\'vloper' + },{ + key:'job', + value:'Developer' + },{ + key:'skill', + value:'Elite' + },{ + key:'employed', + value:'true' + },{ + key:'foods', + value:'["Apple","Orange","Strawberry","Mango"]' + },{ + key:'languages/perl/certified', + value:'true' + },{ + key:'languages/perl/level/scored', + value:'["high","medium"]' + },{ + key:'languages/perl/level/scored/0', + value:'high' + } + ]; + values_to_test.forEach(function(test_value){ + consul_utils.validateValue("test_repo/master/complex_sample.yaml/"+test_value['key'], test_value['value'], function(err, value) { + if (err) return done(err); + }); + }); + consul_utils.validateValue('test_repo/master/complex_sample.yaml/my.server.path/0/KEY-ENV-VAR', 'true', function(err, value) { + if (err) return done(err); + done(); + }); + }); + }); + }); + }); +}); diff --git a/test/resources/complex_sample.yaml b/test/resources/complex_sample.yaml new file mode 100644 index 0000000..9586aba --- /dev/null +++ b/test/resources/complex_sample.yaml @@ -0,0 +1,26 @@ +name: Kostas D'vloper +job: Developer +skill: Elite +employed: True +foods: + - Apple + - Orange + - Strawberry + - Mango +languages: + perl: + certified: true + level: + scored: + - high + - medium + python: + certified: true + level: + scored: + - high + - medium + pascal: Lame +my.server.path: + - KEY-ENV-VAR: "true" + - OTHER-VAR: "59900" \ No newline at end of file diff --git a/test/utils/consul_utils.js b/test/utils/consul_utils.js index c77e4c5..b5a57fd 100644 --- a/test/utils/consul_utils.js +++ b/test/utils/consul_utils.js @@ -33,6 +33,7 @@ exports.getKeyIndices = function(key, cb) { exports.validateValue = function(key, expected_value, cb) { logger.trace('Looking for key %s with value %s', key, expected_value); exports.getValue(key, function(err, value) { + logger.trace("found key: ",key, value); if (err) return cb(err); if (!expected_value) { (value == undefined).should.equal(true);