diff --git a/CHANGELOG.md b/CHANGELOG.md index 2feaf98cde..9a5e6ba42a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -31,6 +31,8 @@ CHANGELOG - Upgrade DCGM to version 4.2.3 (from 3.3.6) for all OSs except AL2. - Upgrade Python to 3.12.11 (from 3.12.8) for all OSs except AL2. - Upgrade Intel MPI Library to 2021.16.0 (from 2021.13.1). +- Upgrade Connexion to ~=2.15.0rc3 (from ~=2.13.0). +- Upgrade Werkzeug to ~=3.1 (from ~=2.0) to address [CVE-2024-34069](https://nvd.nist.gov/vuln/detail/cve-2024-34069). **BUG FIXES** - Fix an issue where Security Group validation failed when a rule contained both IPv4 ranges (IpRanges) and security group references (UserIdGroupPairs). diff --git a/cli/requirements.txt b/cli/requirements.txt index b61c0afb01..e0eafb4ac0 100644 --- a/cli/requirements.txt +++ b/cli/requirements.txt @@ -16,13 +16,13 @@ aws-cdk.core~=1.164 aws_cdk.aws-cloudwatch~=1.164 aws_cdk.aws-lambda~=1.164 boto3>=1.16.14 -connexion~=2.13.0 -flask>=2.2.5,<2.3 jinja2~=3.0 jmespath~=0.10 jsii==1.85.0 marshmallow~=3.10 PyYAML>=5.3.1,!=5.4 tabulate>=0.8.8,<=0.8.10 -werkzeug~=2.0 +connexion~=2.15.0rc3 +werkzeug~=3.1 +flask~=3.0 packaging~=25.0 diff --git a/cli/setup.py b/cli/setup.py index 14f9a9000e..80e7ae352e 100644 --- a/cli/setup.py +++ b/cli/setup.py @@ -47,11 +47,12 @@ def readme(): "aws-cdk.aws-ssm~=" + CDK_VERSION, "aws-cdk.aws-sqs~=" + CDK_VERSION, "aws-cdk.aws-cloudformation~=" + CDK_VERSION, - "werkzeug~=2.0", - "connexion~=2.13.0", - "flask>=2.2.5,<2.3", + "connexion~=2.15.0rc3", "jmespath~=0.10", "jsii==1.85.0", + "werkzeug~=3.1", + "flask~=3.0", + "packaging~=25.0", ] LAMBDA_REQUIRES = [ diff --git a/cli/src/pcluster/api/awslambda/serverless_wsgi.py b/cli/src/pcluster/api/awslambda/serverless_wsgi.py index 37bd594df5..067b03d17d 100644 --- a/cli/src/pcluster/api/awslambda/serverless_wsgi.py +++ b/cli/src/pcluster/api/awslambda/serverless_wsgi.py @@ -17,10 +17,10 @@ import json import os import sys +from urllib.parse import unquote, unquote_plus, urlencode -from werkzeug.datastructures import Headers, MultiDict, iter_multi_items +from werkzeug.datastructures import Headers, iter_multi_items from werkzeug.http import HTTP_STATUS_CODES -from werkzeug.urls import url_encode, url_unquote, url_unquote_plus from werkzeug.wrappers import Response # List of MIME types that should not be base64 encoded. MIME types within `text/*` @@ -95,8 +95,8 @@ def encode_query_string(event): if not params: params = "" if is_alb_event(event): - params = MultiDict((url_unquote_plus(k), url_unquote_plus(v)) for k, v in iter_multi_items(params)) - return url_encode(params) + params = [(unquote_plus(k), unquote_plus(v)) for k, v in iter_multi_items(params)] + return urlencode(params, doseq=True) def get_script_name(headers, request_context): @@ -108,7 +108,7 @@ def get_script_name(headers, request_context): "1", ] - if headers.get("Host", "").endswith(".amazonaws.com") and not strip_stage_path: + if "amazonaws.com" in headers.get("Host", "") and not strip_stage_path: script_name = "/{}".format(request_context.get("stage", "")) else: script_name = "" @@ -138,7 +138,7 @@ def setup_environ_items(environ, headers): def generate_response(response, event): returndict = {"statusCode": response.status_code} - if "multiValueHeaders" in event: + if "multiValueHeaders" in event and event["multiValueHeaders"]: returndict["multiValueHeaders"] = group_headers(response.headers) else: returndict["headers"] = split_headers(response.headers) @@ -164,12 +164,27 @@ def generate_response(response, event): return returndict +def strip_express_gateway_query_params(path): + """Contrary to regular AWS lambda HTTP events, Express Gateway + (https://github.com/ExpressGateway/express-gateway-plugin-lambda) + adds query parameters to the path, which we need to strip. + """ + if "?" in path: + path = path.split("?")[0] + return path + + def handle_request(app, event, context): if event.get("source") in ["aws.events", "serverless-plugin-warmup"]: print("Lambda warming event received, skipping handler") return {} - if event.get("version") is None and event.get("isBase64Encoded") is None and not is_alb_event(event): + if ( + event.get("version") is None + and event.get("isBase64Encoded") is None + and event.get("requestPath") is not None + and not is_alb_event(event) + ): return handle_lambda_integration(app, event, context) if event.get("version") == "2.0": @@ -179,7 +194,7 @@ def handle_request(app, event, context): def handle_payload_v1(app, event, context): - if "multiValueHeaders" in event: + if "multiValueHeaders" in event and event["multiValueHeaders"]: headers = Headers(event["multiValueHeaders"]) else: headers = Headers(event["headers"]) @@ -189,7 +204,7 @@ def handle_payload_v1(app, event, context): # If a user is using a custom domain on API Gateway, they may have a base # path in their URL. This allows us to strip it out via an optional # environment variable. - path_info = event["path"] + path_info = strip_express_gateway_query_params(event["path"]) base_path = os.environ.get("API_GATEWAY_BASE_PATH") if base_path: script_name = "/" + base_path @@ -197,27 +212,27 @@ def handle_payload_v1(app, event, context): if path_info.startswith(script_name): path_info = path_info[len(script_name) :] # noqa: E203 - body = event["body"] or "" + body = event.get("body") or "" body = get_body_bytes(event, body) environ = { "CONTENT_LENGTH": str(len(body)), "CONTENT_TYPE": headers.get("Content-Type", ""), - "PATH_INFO": url_unquote(path_info), + "PATH_INFO": unquote(path_info), "QUERY_STRING": encode_query_string(event), "REMOTE_ADDR": event.get("requestContext", {}).get("identity", {}).get("sourceIp", ""), - "REMOTE_USER": event.get("requestContext", {}).get("authorizer", {}).get("principalId", ""), + "REMOTE_USER": (event.get("requestContext", {}).get("authorizer") or {}).get("principalId", ""), "REQUEST_METHOD": event.get("httpMethod", {}), "SCRIPT_NAME": script_name, "SERVER_NAME": headers.get("Host", "lambda"), - "SERVER_PORT": headers.get("X-Forwarded-Port", "80"), + "SERVER_PORT": headers.get("X-Forwarded-Port", "443"), "SERVER_PROTOCOL": "HTTP/1.1", "wsgi.errors": sys.stderr, "wsgi.input": io.BytesIO(body), "wsgi.multiprocess": False, "wsgi.multithread": False, "wsgi.run_once": False, - "wsgi.url_scheme": headers.get("X-Forwarded-Proto", "http"), + "wsgi.url_scheme": headers.get("X-Forwarded-Proto", "https"), "wsgi.version": (1, 0), "serverless.authorizer": event.get("requestContext", {}).get("authorizer"), "serverless.event": event, @@ -237,7 +252,13 @@ def handle_payload_v2(app, event, context): script_name = get_script_name(headers, event.get("requestContext", {})) - path_info = event["rawPath"] + path_info = strip_express_gateway_query_params(event["rawPath"]) + base_path = os.environ.get("API_GATEWAY_BASE_PATH") + if base_path: + script_name = "/" + base_path + + if path_info.startswith(script_name): + path_info = path_info[len(script_name) :] # noqa: E203 body = event.get("body", "") body = get_body_bytes(event, body) @@ -245,23 +266,23 @@ def handle_payload_v2(app, event, context): headers["Cookie"] = "; ".join(event.get("cookies", [])) environ = { - "CONTENT_LENGTH": str(len(body)), + "CONTENT_LENGTH": str(len(body or "")), "CONTENT_TYPE": headers.get("Content-Type", ""), - "PATH_INFO": url_unquote(path_info), + "PATH_INFO": unquote(path_info), "QUERY_STRING": event.get("rawQueryString", ""), "REMOTE_ADDR": event.get("requestContext", {}).get("http", {}).get("sourceIp", ""), "REMOTE_USER": event.get("requestContext", {}).get("authorizer", {}).get("principalId", ""), "REQUEST_METHOD": event.get("requestContext", {}).get("http", {}).get("method", ""), "SCRIPT_NAME": script_name, "SERVER_NAME": headers.get("Host", "lambda"), - "SERVER_PORT": headers.get("X-Forwarded-Port", "80"), + "SERVER_PORT": headers.get("X-Forwarded-Port", "443"), "SERVER_PROTOCOL": "HTTP/1.1", "wsgi.errors": sys.stderr, "wsgi.input": io.BytesIO(body), "wsgi.multiprocess": False, "wsgi.multithread": False, "wsgi.run_once": False, - "wsgi.url_scheme": headers.get("X-Forwarded-Proto", "http"), + "wsgi.url_scheme": headers.get("X-Forwarded-Proto", "https"), "wsgi.version": (1, 0), "serverless.authorizer": event.get("requestContext", {}).get("authorizer"), "serverless.event": event, @@ -282,7 +303,7 @@ def handle_lambda_integration(app, event, context): script_name = get_script_name(headers, event) - path_info = event["requestPath"] + path_info = strip_express_gateway_query_params(event["requestPath"]) for key, value in event.get("path", {}).items(): path_info = path_info.replace("{%s}" % key, value) @@ -293,23 +314,23 @@ def handle_lambda_integration(app, event, context): body = get_body_bytes(event, body) environ = { - "CONTENT_LENGTH": str(len(body)), + "CONTENT_LENGTH": str(len(body or "")), "CONTENT_TYPE": headers.get("Content-Type", ""), - "PATH_INFO": url_unquote(path_info), - "QUERY_STRING": url_encode(event.get("query", {})), + "PATH_INFO": unquote(path_info), + "QUERY_STRING": urlencode(event.get("query", {}), doseq=True), "REMOTE_ADDR": event.get("identity", {}).get("sourceIp", ""), "REMOTE_USER": event.get("principalId", ""), "REQUEST_METHOD": event.get("method", ""), "SCRIPT_NAME": script_name, "SERVER_NAME": headers.get("Host", "lambda"), - "SERVER_PORT": headers.get("X-Forwarded-Port", "80"), + "SERVER_PORT": headers.get("X-Forwarded-Port", "443"), "SERVER_PROTOCOL": "HTTP/1.1", "wsgi.errors": sys.stderr, "wsgi.input": io.BytesIO(body), "wsgi.multiprocess": False, "wsgi.multithread": False, "wsgi.run_once": False, - "wsgi.url_scheme": headers.get("X-Forwarded-Proto", "http"), + "wsgi.url_scheme": headers.get("X-Forwarded-Proto", "https"), "wsgi.version": (1, 0), "serverless.authorizer": event.get("enhancedAuthContext"), "serverless.event": event, diff --git a/cli/src/pcluster/api/encoder.py b/cli/src/pcluster/api/encoder.py index 750d3d0a28..8b60cc737a 100644 --- a/cli/src/pcluster/api/encoder.py +++ b/cli/src/pcluster/api/encoder.py @@ -9,15 +9,16 @@ # Generated by OpenAPI Generator (python-flask) import datetime +import json import six -from connexion.apps.flask_app import FlaskJSONEncoder +from flask.json.provider import DefaultJSONProvider from pcluster.api.models.base_model_ import Model from pcluster.utils import to_iso_timestr -class JSONEncoder(FlaskJSONEncoder): +class JSONEncoder(json.JSONEncoder): """Make the model objects JSON serializable.""" include_nulls = False @@ -35,4 +36,25 @@ def default(self, obj): # pylint: disable=arguments-renamed return dikt elif isinstance(obj, datetime.date): return to_iso_timestr(obj) - return FlaskJSONEncoder.default(self, obj) + return json.JSONEncoder.default(self, obj) + + +class FlaskJSONEncoder(DefaultJSONProvider): + """Make the model objects JSON serializable.""" + + include_nulls = False + + def default(self, obj): # pylint: disable=arguments-renamed + """Override the base method to add support for model objects serialization.""" + if isinstance(obj, Model): + dikt = {} + for attr, _ in six.iteritems(obj.openapi_types): + value = getattr(obj, attr) + if value is None and not self.include_nulls: + continue + attr = obj.attribute_map[attr] + dikt[attr] = value + return dikt + elif isinstance(obj, datetime.date): + return to_iso_timestr(obj) + return super().default(obj) diff --git a/cli/src/pcluster/api/errors.py b/cli/src/pcluster/api/errors.py index 88c42866c9..a30370d9f8 100644 --- a/cli/src/pcluster/api/errors.py +++ b/cli/src/pcluster/api/errors.py @@ -6,7 +6,7 @@ # OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions and # limitations under the License. -from connexion import ProblemException +from connexion.exceptions import ProblemException from werkzeug.exceptions import HTTPException from pcluster.api.models import ( diff --git a/cli/src/pcluster/api/flask_app.py b/cli/src/pcluster/api/flask_app.py index 0c1e23cca4..be39542223 100644 --- a/cli/src/pcluster/api/flask_app.py +++ b/cli/src/pcluster/api/flask_app.py @@ -8,9 +8,9 @@ import functools import logging -import connexion -from connexion import ProblemException +from connexion.apps.flask_app import FlaskApp from connexion.decorators.validation import ParameterValidator +from connexion.exceptions import ProblemException from flask import Response, jsonify, request from werkzeug.exceptions import HTTPException @@ -74,9 +74,10 @@ def __init__(self, swagger_ui: bool = False, validate_responses=False): assert_valid_node_js() options = {"swagger_ui": swagger_ui} - self.app = connexion.FlaskApp(__name__, specification_dir="openapi/", skip_error_handlers=True) + self.app = FlaskApp(__name__, specification_dir="openapi/", skip_error_handlers=True) self.flask_app = self.app.app - self.flask_app.json_encoder = encoder.JSONEncoder + self.flask_app.json_provider_class = encoder.FlaskJSONEncoder + self.flask_app.json = encoder.FlaskJSONEncoder(self.flask_app) self.app.add_api( "openapi.yaml", arguments={"title": "ParallelCluster"},