diff --git a/README.md b/README.md index cec1a5b..7a6fa8e 100644 --- a/README.md +++ b/README.md @@ -327,6 +327,31 @@ grunt.initConfig({ You could then run `grunt lambda_package lambda_deploy` and it'll automatically create the package and deploy it without having to specify a package name. +##### options.deploy_mode +Type: `String` +Default value: `zip` + +Can be `zip` or `s3`. Determines how the code will be uploaded to lambda - via direct upload or via S3. For larger archives, +S3 may be more suitable. + +##### options.S3bucketName +Type: `String` +Default value: `null` + +Mandatory if `options.deploy_mode='s3'`. S3 bucket to upload code archive to. + +##### options.S3Prefix +Type: `String` +Default value: `` + +Used only for `options.deploy_mode='s3'`. S3 bucket prefix to be used when uploading code archive. + +##### options.S3MultiUploadPartSize +Type: `String` +Default value `5mb` + +For larger archives, determines chunk size for s3 multi-upload. Minimum value is '5mb'. + ##### options.profile Type: `String` Default value: `null` diff --git a/package.json b/package.json index aa9e517..040d154 100644 --- a/package.json +++ b/package.json @@ -1,55 +1,56 @@ { - "name": "grunt-aws-lambda", - "description": "A grunt plugin to help develop AWS Lambda functions.", - "version": "0.13.0", - "homepage": "https://github.com/Tim-B/grunt-aws-lambda", - "author": { - "name": "Tim-B", - "email": "tim@galacticcode.com" - }, - "repository": { - "type": "git", - "url": "git://github.com/Tim-B/grunt-aws-lambda.git" - }, - "bugs": { - "url": "https://github.com/Tim-B/grunt-aws-lambda/issues" - }, - "licenses": [ - { - "type": "MIT", - "url": "https://github.com/Tim-B/grunt-aws-lambda/blob/master/LICENSE-MIT" - } - ], - "engines": { - "node": ">= 0.8.0" - }, - "scripts": { - "test": "grunt" - }, - "dependencies": { - "temporary": "~0.0.8", - "archiver": "~0.14.4", - "mkdirp": "~0.5.0", - "rimraf": "~2.2.8", - "glob": "~4.3.0", - "aws-sdk": "~2.2.32", - "proxy-agent": "latest", - "npm": "^2.10.0", - "q": "^1.4.1" - }, - "devDependencies": { - "grunt-contrib-jshint": "^0.9.2", - "grunt-contrib-clean": "^0.5.0", - "grunt-contrib-nodeunit": "^0.3.3", - "mockery": "^1.4.0", - "grunt": "~0.4.5", - "adm-zip": "~0.4.4", - "sinon": "^1.17.3" - }, - "peerDependencies": { - "grunt": ">=0.4.0" - }, - "keywords": [ - "gruntplugin" - ] + "name": "grunt-aws-lambda", + "description": "A grunt plugin to help develop AWS Lambda functions.", + "version": "0.13.0", + "homepage": "https://github.com/Tim-B/grunt-aws-lambda", + "author": { + "name": "Tim-B", + "email": "tim@galacticcode.com" + }, + "repository": { + "type": "git", + "url": "git://github.com/Tim-B/grunt-aws-lambda.git" + }, + "bugs": { + "url": "https://github.com/Tim-B/grunt-aws-lambda/issues" + }, + "licenses": [ + { + "type": "MIT", + "url": "https://github.com/Tim-B/grunt-aws-lambda/blob/master/LICENSE-MIT" + } + ], + "engines": { + "node": ">= 0.8.0" + }, + "scripts": { + "test": "grunt" + }, + "dependencies": { + "archiver": "~0.14.4", + "aws-sdk": "~2.2.32", + "bytes": "^2.4.0", + "glob": "~4.3.0", + "mkdirp": "~0.5.0", + "npm": "^2.10.0", + "proxy-agent": "latest", + "q": "^1.4.1", + "rimraf": "~2.2.8", + "temporary": "~0.0.8" + }, + "devDependencies": { + "grunt-contrib-jshint": "^0.9.2", + "grunt-contrib-clean": "^0.5.0", + "grunt-contrib-nodeunit": "^0.3.3", + "mockery": "^1.4.0", + "grunt": "~0.4.5", + "adm-zip": "~0.4.4", + "sinon": "^1.17.3" + }, + "peerDependencies": { + "grunt": ">=0.4.0" + }, + "keywords": [ + "gruntplugin" + ] } diff --git a/test/unit/deploy_task_test.js b/test/unit/deploy_task_test.js index d584f18..6aa40b9 100644 --- a/test/unit/deploy_task_test.js +++ b/test/unit/deploy_task_test.js @@ -40,6 +40,7 @@ var deployTaskTest = {}; var awsSDKMock, lambdaAPIMock, + s3APIMock, defaultGruntConfig, proxyAgentMock; @@ -70,6 +71,14 @@ deployTaskTest.setUp = function(done) { updateAlias: sinon.stub().callsArgWithAsync(1, null, {}) }; + s3APIMock = { + ManagedUpload : function(params){ + return { + send : sinon.stub(), + }; + } + }; + awsSDKMock = { SharedIniFileCredentials: sinon.stub(), EC2MetadataCredentials: sinon.stub(), @@ -80,9 +89,10 @@ deployTaskTest.setUp = function(done) { }, Lambda: function(params) { return lambdaAPIMock; - } + }, + S3: s3APIMock }; - + proxyAgentMock = sinon.spy(); fsMock.reset(); @@ -91,7 +101,7 @@ deployTaskTest.setUp = function(done) { fsMock.setFileContent('some-package.zip', 'abc123'); mockery.registerMock('aws-sdk', awsSDKMock); - + mockery.registerMock('proxy-agent', proxyAgentMock); var dateFacadeMock = { diff --git a/utils/deploy_task.js b/utils/deploy_task.js index fd6c675..8f4dc7c 100644 --- a/utils/deploy_task.js +++ b/utils/deploy_task.js @@ -14,6 +14,7 @@ var AWS = require('aws-sdk'); var Q = require('q'); var arnParser = require('./arn_parser'); var dateFacade = require('./date_facade'); +var bytes = require('bytes'); var deployTask = {}; @@ -39,18 +40,22 @@ deployTask.getHandler = function (grunt) { aliases: null, enablePackageVersionAlias: false, subnetIds: null, - securityGroupIds: null + securityGroupIds: null, + deploy_mode: 'zip', + S3bucketName: null, + S3Prefix: '', + S3MultiUploadPartSize: '5mb' }); - + if (options.profile !== null) { var credentials = new AWS.SharedIniFileCredentials({profile: options.profile}); AWS.config.credentials = credentials; } //Adding proxy if exists - if(process.env.https_proxy !== undefined) { + if (process.env.https_proxy !== undefined) { AWS.config.update({ - httpOptions: { agent: proxy(process.env.https_proxy) } + httpOptions: {agent: proxy(process.env.https_proxy)} }); } @@ -83,6 +88,12 @@ deployTask.getHandler = function (grunt) { var package_version = grunt.config.get('lambda_deploy.' + this.target + '.version'); var package_name = grunt.config.get('lambda_deploy.' + this.target + '.package_name'); var archive_name = grunt.config.get('lambda_deploy.' + this.target + '.archive_name'); + var deploy_mode = grunt.config.get('lambda_deploy.' + this.target + '.deploy_mode'); + var s3_bucket_name = grunt.config.get('lambda_deploy.' + this.target + '.S3bucketName'); + var s3_prefix = grunt.config.get('lambda_deploy.' + this.target + '.S3Prefix') || ''; + var s3_part_size = grunt.config.get('lambda_deploy.' + this.target + '.S3MultiUploadPartSize') || '5mb'; + var is_s3_upload = deploy_mode === 's3'; + var s3_key = is_s3_upload ? s3_prefix + path.basename(deploy_package) : undefined; if (deploy_arn === null && deploy_function === null) { grunt.fail.warn('You must specify either an arn or a function name.'); @@ -98,9 +109,8 @@ deployTask.getHandler = function (grunt) { var done = this.async(); - var lambda = new AWS.Lambda({ - apiVersion: '2015-03-31' - }); + options.apiVersion = '2015-03-31'; + var lambda = new AWS.Lambda(options); var getDeploymentDescription = function () { var description = 'Deployed '; @@ -123,163 +133,214 @@ deployTask.getHandler = function (grunt) { lambda.getFunction({FunctionName: deploy_function}, function (err, data) { - if (err) { - if (err.statusCode === 404) { - grunt.fail.warn('Unable to find lambda function ' + deploy_function + ', verify the lambda function name and AWS region are correct.'); - } else { - grunt.log.error('AWS API request failed with ' + err.statusCode + ' - ' + err); - grunt.fail.warn('Check your AWS credentials, region and permissions are correct.'); + if (err) { + if (err.statusCode === 404) { + grunt.fail.warn('Unable to find lambda function ' + deploy_function + ', verify the lambda function name and AWS region are correct.'); + } else { + grunt.log.error('AWS API request failed with ' + err.statusCode + ' - ' + err); + grunt.fail.warn('Check your AWS credentials, region and permissions are correct.'); + } } - } - var current = data.Configuration; - var configParams = {}; - var version = '$LATEST'; + var current = data.Configuration; + var configParams = {}; + var version = '$LATEST'; - if (options.timeout !== null) { - configParams.Timeout = options.timeout; - } - - if (options.memory !== null) { - configParams.MemorySize = options.memory; - } - - if (options.handler !== null) { - configParams.Handler = options.handler; - } + if (options.timeout !== null) { + configParams.Timeout = options.timeout; + } - if (options.subnetIds !== null && options.securityGroupIds !== null) { - configParams.VpcConfig = { - SubnetIds : options.subnetIds, - SecurityGroupIds : options.securityGroupIds - }; - } + if (options.memory !== null) { + configParams.MemorySize = options.memory; + } - var updateConfig = function (func_name, func_options) { - var deferred = Q.defer(); - if (Object.keys(func_options).length > 0) { - func_options.FunctionName = func_name; - lambda.updateFunctionConfiguration(func_options, function (err, data) { - if (err) { - grunt.fail.warn('Could not update config, check that values and permissions are valid'); - deferred.reject(); - } else { - grunt.log.writeln('Config updated.'); - deferred.resolve(); - } - }); - } else { - grunt.log.writeln('No config updates to make.'); - deferred.resolve(); + if (options.handler !== null) { + configParams.Handler = options.handler; } - return deferred.promise; - }; - var createVersion = function (func_name) { - var deferred = Q.defer(); - if (options.enableVersioning) { - lambda.publishVersion({FunctionName: func_name, Description: getDeploymentDescription()}, function (err, data) { - if (err) { - grunt.fail.warn('Publishing version for function ' + func_name + ' failed with message ' + err.message); - deferred.reject(); - } else { - version = data.Version; - grunt.log.writeln('Version ' + version + ' published.'); - deferred.resolve(); - } - }); - } else { - deferred.resolve(); + if (options.subnetIds !== null && options.securityGroupIds !== null) { + configParams.VpcConfig = { + SubnetIds: options.subnetIds, + SecurityGroupIds: options.securityGroupIds + }; } - return deferred.promise; - }; + var updateConfig = function (func_name, func_options) { + var deferred = Q.defer(); + if (Object.keys(func_options).length > 0) { + func_options.FunctionName = func_name; + lambda.updateFunctionConfiguration(func_options, function (err, data) { + if (err) { + grunt.fail.warn('Could not update config, check that values and permissions are valid'); + deferred.reject(); + } else { + grunt.log.writeln('Config updated.'); + deferred.resolve(); + } + }); + } else { + grunt.log.writeln('No config updates to make.'); + deferred.resolve(); + } + return deferred.promise; + }; - var createOrUpdateAlias = function (func_name, set_alias) { - var deferred = Q.defer(); + var createVersion = function (func_name) { + var deferred = Q.defer(); + if (options.enableVersioning) { + lambda.publishVersion({FunctionName: func_name, Description: getDeploymentDescription()}, function (err, data) { + if (err) { + grunt.fail.warn('Publishing version for function ' + func_name + ' failed with message ' + err.message); + deferred.reject(); + } else { + version = data.Version; + grunt.log.writeln('Version ' + version + ' published.'); + deferred.resolve(); + } + }); + } else { + deferred.resolve(); + } - var params = { - FunctionName: func_name, - Name: set_alias + return deferred.promise; }; + var createOrUpdateAlias = function (func_name, set_alias) { + var deferred = Q.defer(); - lambda.getAlias(params, function (err, data) { - params.FunctionVersion = version; - params.Description = getDeploymentDescription(); - var aliasFunction = 'updateAlias'; - if (err) { - if (err.statusCode === 404) { - aliasFunction = 'createAlias'; - } else { - grunt.fail.warn('Listing aliases for ' + func_name + ' failed with message ' + err.message); - deferred.reject(); - return; - } - } - lambda[aliasFunction](params, function (err, data) { + var params = { + FunctionName: func_name, + Name: set_alias + }; + + + lambda.getAlias(params, function (err, data) { + params.FunctionVersion = version; + params.Description = getDeploymentDescription(); + var aliasFunction = 'updateAlias'; if (err) { - grunt.fail.warn(aliasFunction + ' for ' + func_name + ' failed with message ' + err.message); - deferred.reject(); - } else { - grunt.log.writeln('Alias ' + set_alias + ' updated pointing to version ' + version + '.'); - deferred.resolve(); + if (err.statusCode === 404) { + aliasFunction = 'createAlias'; + } else { + grunt.fail.warn('Listing aliases for ' + func_name + ' failed with message ' + err.message); + deferred.reject(); + return; + } } + lambda[aliasFunction](params, function (err, data) { + if (err) { + grunt.fail.warn(aliasFunction + ' for ' + func_name + ' failed with message ' + err.message); + deferred.reject(); + } else { + grunt.log.writeln('Alias ' + set_alias + ' updated pointing to version ' + version + '.'); + deferred.resolve(); + } + }); }); - }); - return deferred.promise; - }; + return deferred.promise; + }; - var setAliases = function (func_name) { - if (options.aliases) { - var promises = []; - options.aliases.forEach(function (alias) { - promises.push(createOrUpdateAlias(func_name, alias)); - }); - return Q.all(promises); - } - }; + var setAliases = function (func_name) { + if (options.aliases) { + var promises = []; + options.aliases.forEach(function (alias) { + promises.push(createOrUpdateAlias(func_name, alias)); + }); + return Q.all(promises); + } + }; - var setPackageVersionAlias = function (func_name) { - if (options.enablePackageVersionAlias && package_version) { - return createOrUpdateAlias(func_name, package_version.replace(/\./g, '-')); - } - }; + var setPackageVersionAlias = function (func_name) { + if (options.enablePackageVersionAlias && package_version) { + return createOrUpdateAlias(func_name, package_version.replace(/\./g, '-')); + } + }; - grunt.log.writeln('Uploading...'); - fs.readFile(deploy_package, function (err, data) { - if (err) { - grunt.fail.warn('Could not read package file (' + deploy_package + '), verify the lambda package ' + - 'location is correct, and that you have already created the package using lambda_package.'); - } + var uploadPackageToS3 = function (package_data) { + var upload_params = { + Bucket: s3_bucket_name, + Key: s3_key, + Body: package_data + }, + managed_upload = new AWS.S3.ManagedUpload({params: upload_params, partSize: bytes(s3_part_size)}), + deferred = Q.defer(); + + if (is_s3_upload) { + managed_upload.send(function (err) { + if (err) { + grunt.fail.warn('S3 Upload failed: ' + err); + deferred.reject(); + } + + grunt.log.writeln('S3 Upload success'); + deferred.resolve(); + }); + } else { + deferred.resolve(); + } + + return deferred.promise; + }; + + var updateFunctionCode = function (code_params) { + var deferred = Q.defer(); + lambda.updateFunctionCode(code_params, function (err, data) { + if (err) { + grunt.fail.warn('Package upload failed, check you have lambda:UpdateFunctionCode permissions and that your package is not too big to upload.'); + deferred.reject(); + } - var codeParams = { - FunctionName: deploy_function, - ZipFile: data + grunt.log.writeln('Package deployed.'); + deferred.resolve(); + }); + return deferred.promise; }; - lambda.updateFunctionCode(codeParams, function (err, data) { + grunt.log.writeln('Uploading...'); + fs.readFile(deploy_package, function (err, data) { if (err) { - grunt.fail.warn('Package upload failed, check you have lambda:UpdateFunctionCode permissions and that your package is not too big to upload.'); + grunt.fail.warn('Could not read package file (' + deploy_package + '), verify the lambda package ' + + 'location is correct, and that you have already created the package using lambda_package.'); } - grunt.log.writeln('Package deployed.'); + var code_params; + + if (is_s3_upload) { + code_params = { + FunctionName: deploy_function, + S3Bucket: s3_bucket_name, + S3Key: s3_key + }; + } else { + code_params = { + FunctionName: deploy_function, + ZipFile: data + }; + } + + uploadPackageToS3(data).then(function () { + return updateFunctionCode(code_params); + }).then(function () { + return updateConfig(deploy_function, configParams); + }).then(function () { + return createVersion(deploy_function); + }).then(function () { + return setAliases(deploy_function); + }).then(function () { + return setPackageVersionAlias(deploy_function); + }).then(function () { + done(true); + }).catch(function (err) { + grunt.fail.warn('Uncaught exception: ' + err.message); + }); - updateConfig(deploy_function, configParams) - .then(function () {return createVersion(deploy_function);}) - .then(function () {return setAliases(deploy_function);}) - .then(function () {return setPackageVersionAlias(deploy_function);}) - .then(function () { - done(true); - }).catch(function (err) { - grunt.fail.warn('Uncaught exception: ' + err.message); - }); }); - }); - }); + } + ); }; }; + module.exports = deployTask; diff --git a/utils/invoke_task.js b/utils/invoke_task.js index a9db0b9..1f43cb3 100644 --- a/utils/invoke_task.js +++ b/utils/invoke_task.js @@ -103,8 +103,13 @@ invokeTask.getHandler = function (grunt) { var lambda = invokeTask.loadFunction(options.file_name); var event = JSON.parse(fs.readFileSync(path.resolve(options.event), "utf8")); + + //set environment variables + process.env['LAMBDA_TASK_ROOT'] = process.env['PWD']; + + //invoke process lambda[options.handler](event, context, callback); }; }; -module.exports = invokeTask; \ No newline at end of file +module.exports = invokeTask;