From eb8a0f384007b7318adbf9e77f1e4f460388a12f Mon Sep 17 00:00:00 2001 From: Eka Winata Date: Wed, 31 Jul 2024 13:58:38 +0700 Subject: [PATCH 1/7] Add CEL string, math and binding library --- .../GrpcResponseCelPayloadEvaluator.java | 3 +-- .../firehose/proto/ProtoToMetadataMapper.java | 3 +-- .../com/gotocompany/firehose/utils/CelUtils.java | 15 +++++++++++++++ 3 files changed, 17 insertions(+), 4 deletions(-) diff --git a/src/main/java/com/gotocompany/firehose/evaluator/GrpcResponseCelPayloadEvaluator.java b/src/main/java/com/gotocompany/firehose/evaluator/GrpcResponseCelPayloadEvaluator.java index b234455a1..b6925fe86 100644 --- a/src/main/java/com/gotocompany/firehose/evaluator/GrpcResponseCelPayloadEvaluator.java +++ b/src/main/java/com/gotocompany/firehose/evaluator/GrpcResponseCelPayloadEvaluator.java @@ -52,8 +52,7 @@ public boolean evaluate(Message payload) { */ private void buildCelEnvironment(String celExpression) { CelCompiler celCompiler = CelUtils.initializeCelCompiler(this.descriptor); - CelRuntime celRuntime = CelRuntimeFactory.standardCelRuntimeBuilder() - .build(); + CelRuntime celRuntime = CelUtils.initializeCelRuntime(); this.celProgram = CelUtils.initializeCelProgram(celExpression, celRuntime, celCompiler, celType -> celType.kind().equals(CelKind.BOOL)); } diff --git a/src/main/java/com/gotocompany/firehose/proto/ProtoToMetadataMapper.java b/src/main/java/com/gotocompany/firehose/proto/ProtoToMetadataMapper.java index 45f708e60..e3b538901 100644 --- a/src/main/java/com/gotocompany/firehose/proto/ProtoToMetadataMapper.java +++ b/src/main/java/com/gotocompany/firehose/proto/ProtoToMetadataMapper.java @@ -8,7 +8,6 @@ import com.gotocompany.firehose.utils.CelUtils; import dev.cel.compiler.CelCompiler; import dev.cel.runtime.CelRuntime; -import dev.cel.runtime.CelRuntimeFactory; import io.grpc.Metadata; import org.apache.commons.collections.MapUtils; @@ -102,7 +101,7 @@ private Object evaluateExpression(String input, Message message) { * @return a map of CEL expressions to their corresponding programs */ private Map initializeCelPrograms() { - CelRuntime celRuntime = CelRuntimeFactory.standardCelRuntimeBuilder().build(); + CelRuntime celRuntime = CelUtils.initializeCelRuntime(); CelCompiler celCompiler = CelUtils.initializeCelCompiler(this.descriptor); return this.metadataTemplate.entrySet() .stream() diff --git a/src/main/java/com/gotocompany/firehose/utils/CelUtils.java b/src/main/java/com/gotocompany/firehose/utils/CelUtils.java index 8d676edb6..7e022dbdd 100644 --- a/src/main/java/com/gotocompany/firehose/utils/CelUtils.java +++ b/src/main/java/com/gotocompany/firehose/utils/CelUtils.java @@ -3,14 +3,17 @@ import com.google.protobuf.Descriptors; import com.google.protobuf.Message; import dev.cel.common.CelAbstractSyntaxTree; +import dev.cel.common.CelOptions; import dev.cel.common.CelValidationException; import dev.cel.common.types.CelType; import dev.cel.common.types.StructTypeReference; import dev.cel.compiler.CelCompiler; import dev.cel.compiler.CelCompilerFactory; +import dev.cel.extensions.CelExtensions; import dev.cel.parser.CelStandardMacro; import dev.cel.runtime.CelEvaluationException; import dev.cel.runtime.CelRuntime; +import dev.cel.runtime.CelRuntimeFactory; import org.aeonbits.owner.util.Collections; import java.util.function.Predicate; @@ -43,10 +46,22 @@ public static CelCompiler initializeCelCompiler(Descriptors.Descriptor descripto return CelCompilerFactory.standardCelCompilerBuilder() .setStandardMacros(CelStandardMacro.values()) .addVar(descriptor.getFullName(), StructTypeReference.create(descriptor.getFullName())) + .addLibraries(CelExtensions.strings(), CelExtensions.bindings(), CelExtensions.math(CelOptions.DEFAULT), CelExtensions.protos()) .addMessageTypes(descriptor) .build(); } + /** + * Initializes the CEL runtime with standard libraries. + * + * @return the initialized CEL runtime + */ + public static CelRuntime initializeCelRuntime() { + return CelRuntimeFactory.standardCelRuntimeBuilder() + .addLibraries(CelExtensions.strings(), CelExtensions.math(CelOptions.DEFAULT)) + .build(); + } + /** * Initializes a CEL program for a given expression. * From dac6eea58238ab5b873bb23766e66cd02a701ff5 Mon Sep 17 00:00:00 2001 From: Eka Winata Date: Wed, 31 Jul 2024 14:08:30 +0700 Subject: [PATCH 2/7] Add CEL string, math and binding library --- build.gradle | 2 +- .../firehose/proto/ProtoToMetadataMapperTest.java | 11 ++++++++++- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/build.gradle b/build.gradle index 7bb8cb063..6f03712aa 100644 --- a/build.gradle +++ b/build.gradle @@ -33,7 +33,7 @@ lombok { } group 'com.gotocompany' -version '0.10.5' +version '0.10.6' def projName = "firehose" diff --git a/src/test/java/com/gotocompany/firehose/proto/ProtoToMetadataMapperTest.java b/src/test/java/com/gotocompany/firehose/proto/ProtoToMetadataMapperTest.java index f77da5264..e5138cea1 100644 --- a/src/test/java/com/gotocompany/firehose/proto/ProtoToMetadataMapperTest.java +++ b/src/test/java/com/gotocompany/firehose/proto/ProtoToMetadataMapperTest.java @@ -18,10 +18,13 @@ public class ProtoToMetadataMapperTest { public void setup() { Map template = new HashMap<>(); template.put("$GenericResponse.detail", "$GenericResponse.success"); + template.put("detail", "$GenericResponse.detail.lowerAscii()"); template.put("someField", "someValue"); template.put("$GenericResponse.success", "staticValue"); template.put("staticKey", "$(GenericResponse.errors[0].cause + '-' + GenericResponse.errors[0].code + '-' + string(GenericResponse.code))"); template.put("entity", "$GenericResponse.errors[0].entity"); + template.put("binding", "$cel.bind(code, GenericResponse.code, code + 100)"); + template.put("math", "$math.greatest(GenericResponse.code, 200)"); this.protoToMetadataMapper = new ProtoToMetadataMapper( GenericResponse.getDescriptor(), template @@ -32,7 +35,7 @@ public void setup() { public void shouldBuildDynamicMetadataWithCorrectPlaceholders() { GenericResponse payload = GenericResponse.newBuilder() .setSuccess(false) - .setDetail("detail_of_error") + .setDetail("Detail_Of_Error") .setCode(100) .addErrors(GenericError.newBuilder() .setCode("404") @@ -44,6 +47,8 @@ public void shouldBuildDynamicMetadataWithCorrectPlaceholders() { Assertions.assertTrue(metadata.containsKey(Metadata.Key.of("detail_of_error", Metadata.ASCII_STRING_MARSHALLER))); Assertions.assertEquals("false", metadata.get(Metadata.Key.of("detail_of_error", Metadata.ASCII_STRING_MARSHALLER))); + Assertions.assertTrue(metadata.containsKey(Metadata.Key.of("detail", Metadata.ASCII_STRING_MARSHALLER))); + Assertions.assertEquals("detail_of_error", metadata.get(Metadata.Key.of("detail", Metadata.ASCII_STRING_MARSHALLER))); Assertions.assertTrue(metadata.containsKey(Metadata.Key.of("statickey", Metadata.ASCII_STRING_MARSHALLER))); Assertions.assertEquals("not_found-404-100", metadata.get(Metadata.Key.of("statickey", Metadata.ASCII_STRING_MARSHALLER))); Assertions.assertTrue(metadata.containsKey(Metadata.Key.of("somefield", Metadata.ASCII_STRING_MARSHALLER))); @@ -52,6 +57,10 @@ public void shouldBuildDynamicMetadataWithCorrectPlaceholders() { Assertions.assertEquals("", metadata.get(Metadata.Key.of("entity", Metadata.ASCII_STRING_MARSHALLER))); Assertions.assertTrue(metadata.containsKey(Metadata.Key.of("false", Metadata.ASCII_STRING_MARSHALLER))); Assertions.assertEquals("staticValue", metadata.get(Metadata.Key.of("false", Metadata.ASCII_STRING_MARSHALLER))); + Assertions.assertTrue(metadata.containsKey(Metadata.Key.of("binding", Metadata.ASCII_STRING_MARSHALLER))); + Assertions.assertEquals("200", metadata.get(Metadata.Key.of("binding", Metadata.ASCII_STRING_MARSHALLER))); + Assertions.assertTrue(metadata.containsKey(Metadata.Key.of("math", Metadata.ASCII_STRING_MARSHALLER))); + Assertions.assertEquals("200", metadata.get(Metadata.Key.of("math", Metadata.ASCII_STRING_MARSHALLER))); } @Test From b25d7c5f43d93eb3486d67af575797f109c6e215 Mon Sep 17 00:00:00 2001 From: Eka Winata Date: Wed, 31 Jul 2024 14:09:29 +0700 Subject: [PATCH 3/7] Checkstyle --- .../firehose/evaluator/GrpcResponseCelPayloadEvaluator.java | 1 - 1 file changed, 1 deletion(-) diff --git a/src/main/java/com/gotocompany/firehose/evaluator/GrpcResponseCelPayloadEvaluator.java b/src/main/java/com/gotocompany/firehose/evaluator/GrpcResponseCelPayloadEvaluator.java index b6925fe86..d05a8a5b0 100644 --- a/src/main/java/com/gotocompany/firehose/evaluator/GrpcResponseCelPayloadEvaluator.java +++ b/src/main/java/com/gotocompany/firehose/evaluator/GrpcResponseCelPayloadEvaluator.java @@ -6,7 +6,6 @@ import dev.cel.common.types.CelKind; import dev.cel.compiler.CelCompiler; import dev.cel.runtime.CelRuntime; -import dev.cel.runtime.CelRuntimeFactory; import lombok.extern.slf4j.Slf4j; /** From 120aa95f730b7fc3269a2a535982f8920b6dd8d1 Mon Sep 17 00:00:00 2001 From: Eka Winata Date: Wed, 31 Jul 2024 14:17:51 +0700 Subject: [PATCH 4/7] Update comments --- src/main/java/com/gotocompany/firehose/utils/CelUtils.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/com/gotocompany/firehose/utils/CelUtils.java b/src/main/java/com/gotocompany/firehose/utils/CelUtils.java index 7e022dbdd..71b7415dc 100644 --- a/src/main/java/com/gotocompany/firehose/utils/CelUtils.java +++ b/src/main/java/com/gotocompany/firehose/utils/CelUtils.java @@ -52,7 +52,7 @@ public static CelCompiler initializeCelCompiler(Descriptors.Descriptor descripto } /** - * Initializes the CEL runtime with standard libraries. + * Initializes the CEL runtime with extended libraries. * * @return the initialized CEL runtime */ From 66533c5fd22e5368623ce7e21cf20b1648e2f278 Mon Sep 17 00:00:00 2001 From: Eka Winata Date: Wed, 31 Jul 2024 14:28:50 +0700 Subject: [PATCH 5/7] Remove proto extensions --- src/main/java/com/gotocompany/firehose/utils/CelUtils.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/com/gotocompany/firehose/utils/CelUtils.java b/src/main/java/com/gotocompany/firehose/utils/CelUtils.java index 71b7415dc..82657c86f 100644 --- a/src/main/java/com/gotocompany/firehose/utils/CelUtils.java +++ b/src/main/java/com/gotocompany/firehose/utils/CelUtils.java @@ -46,7 +46,7 @@ public static CelCompiler initializeCelCompiler(Descriptors.Descriptor descripto return CelCompilerFactory.standardCelCompilerBuilder() .setStandardMacros(CelStandardMacro.values()) .addVar(descriptor.getFullName(), StructTypeReference.create(descriptor.getFullName())) - .addLibraries(CelExtensions.strings(), CelExtensions.bindings(), CelExtensions.math(CelOptions.DEFAULT), CelExtensions.protos()) + .addLibraries(CelExtensions.strings(), CelExtensions.bindings(), CelExtensions.math(CelOptions.DEFAULT)) .addMessageTypes(descriptor) .build(); } From cb8203c2779d58161c2879d8bd73413de2de8f5f Mon Sep 17 00:00:00 2001 From: Eka Winata Date: Wed, 31 Jul 2024 14:36:12 +0700 Subject: [PATCH 6/7] use correct map --- src/main/java/com/gotocompany/firehose/utils/CelUtils.java | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/main/java/com/gotocompany/firehose/utils/CelUtils.java b/src/main/java/com/gotocompany/firehose/utils/CelUtils.java index 82657c86f..231993c86 100644 --- a/src/main/java/com/gotocompany/firehose/utils/CelUtils.java +++ b/src/main/java/com/gotocompany/firehose/utils/CelUtils.java @@ -14,8 +14,7 @@ import dev.cel.runtime.CelEvaluationException; import dev.cel.runtime.CelRuntime; import dev.cel.runtime.CelRuntimeFactory; -import org.aeonbits.owner.util.Collections; - +import java.util.Collections; import java.util.function.Predicate; /** @@ -31,7 +30,7 @@ public class CelUtils { */ public static Object evaluate(CelRuntime.Program program, Message payload) { try { - return program.eval(Collections.map(payload.getDescriptorForType().getFullName(), payload)); + return program.eval(Collections.singletonMap(payload.getDescriptorForType().getFullName(), payload)); } catch (CelEvaluationException e) { throw new IllegalArgumentException("Could not evaluate Cel expression", e); } From 66d6f937a2c1a23325680eb65620f75503a3f6e0 Mon Sep 17 00:00:00 2001 From: Eka Winata Date: Thu, 1 Aug 2024 11:04:47 +0700 Subject: [PATCH 7/7] Update docs --- docs/docs/sinks/grpc-sink.md | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/docs/docs/sinks/grpc-sink.md b/docs/docs/sinks/grpc-sink.md index 25910ac28..f82ce2ca1 100644 --- a/docs/docs/sinks/grpc-sink.md +++ b/docs/docs/sinks/grpc-sink.md @@ -42,14 +42,23 @@ Note - final metadata will be generated with merging metadata and the kafka reco message GenericPayload { string field = "field_name"; - string field_two = "field_two"; + string field_two = "FIELD_two"; string id = "123"; int code = 400; } ``` - Example config : `$com.goto.company.GenericPayload.field: $(com.goto.company.GenericPayload.field_two + '_' + string(com.goto.company.GenericPayload.code))` + Example config : `$com.goto.company.GenericPayload.field: $(com.goto.company.GenericPayload.field_two.lowerAscii() + '_' + string(com.goto.company.GenericPayload.code))` This would result in : `field_name:field_two_400` - +- CEL Extended Libraries : + - String operations (https://github.com/google/cel-java/blob/main/extensions/src/main/java/dev/cel/extensions/README.md#strings) + - Add the capabilities of performing typical string operations such as up/lower casting, trimming, replacement, etc. + - Example : `com.goto.company.GenericPayload.field_two.lowerAscii()`, `com.goto.company.GenericPayload.field_two.upperAscii()` + - Math Operations (https://github.com/google/cel-java/blob/main/extensions/src/main/java/dev/cel/extensions/README.md#math) + - Currently only support finding greatest and least of numerical inputs + - Example : `math.greatest(com.goto.company.GenericPayload.code, 999), math.least(com.goto.company.GenericPayload.code, 999)` + - Binding Operations (https://github.com/google/cel-java/blob/main/extensions/src/main/java/dev/cel/extensions/README.md#celbind) + - Add the capabilities of binding a CEL Expression into variables and reusing them in the subsequent expression + - Example : `cel.bind(stringifiedCode, string(com.goto.company.GenericPayload.code), stringifiedCode + '_' + com.goto.company.GenericPayload.id)` ### `SINK_GRPC_RESPONSE_SCHEMA_PROTO_CLASS`