Skip to content

Commit e89b342

Browse files
elishererelisherer
committed
java-1.3.0 js-1.9.0 js-core-1.9.0 js-monaco-1.9.0 - new inline function tokenizer (now supports nesting functions), improve parsing and fix some related issues
1 parent 78535ba commit e89b342

29 files changed

+872
-203
lines changed

‎.github/workflows/docs-deploy.yml‎

Lines changed: 12 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -26,34 +26,34 @@ jobs:
2626
with:
2727
fetch-depth: 0
2828

29-
- name: setup json-transform build
29+
- name: setup json-transform-core build
3030
uses: actions/setup-node@v4
3131
with:
3232
node-version: 20
3333
cache: npm
34-
cache-dependency-path: ./javascript/json-transform/package-lock.json
34+
cache-dependency-path: ./javascript/json-transform-core/package-lock.json
3535

36-
- name: install json-transform dependencies
37-
working-directory: ./javascript/json-transform
36+
- name: install json-transform-core dependencies
37+
working-directory: ./javascript/json-transform-core
3838
run: npm ci
3939

40-
- name: build json-transform
41-
working-directory: ./javascript/json-transform
40+
- name: build json-transform-core
41+
working-directory: ./javascript/json-transform-core
4242
run: npm run build
4343

44-
- name: setup json-transform-core build
44+
- name: setup json-transform build
4545
uses: actions/setup-node@v4
4646
with:
4747
node-version: 20
4848
cache: npm
49-
cache-dependency-path: ./javascript/json-transform-core/package-lock.json
49+
cache-dependency-path: ./javascript/json-transform/package-lock.json
5050

51-
- name: install json-transform-core dependencies
52-
working-directory: ./javascript/json-transform-core
51+
- name: install json-transform dependencies
52+
working-directory: ./javascript/json-transform
5353
run: npm ci
5454

55-
- name: build json-transform-core
56-
working-directory: ./javascript/json-transform-core
55+
- name: build json-transform
56+
working-directory: ./javascript/json-transform
5757
run: npm run build
5858

5959
- name: setup monaco-json-transform build

‎.github/workflows/docs-test.yml‎

Lines changed: 12 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -25,34 +25,34 @@ jobs:
2525
with:
2626
fetch-depth: 0
2727

28-
- name: setup json-transform build
28+
- name: setup json-transform-core build
2929
uses: actions/setup-node@v4
3030
with:
3131
node-version: 20
3232
cache: npm
33-
cache-dependency-path: ./javascript/json-transform/package-lock.json
33+
cache-dependency-path: ./javascript/json-transform-core/package-lock.json
3434

35-
- name: install json-transform dependencies
36-
working-directory: ./javascript/json-transform
35+
- name: install json-transform-core dependencies
36+
working-directory: ./javascript/json-transform-core
3737
run: npm ci
3838

39-
- name: build json-transform
40-
working-directory: ./javascript/json-transform
39+
- name: build json-transform-core
40+
working-directory: ./javascript/json-transform-core
4141
run: npm run build
4242

43-
- name: setup json-transform-core build
43+
- name: setup json-transform build
4444
uses: actions/setup-node@v4
4545
with:
4646
node-version: 20
4747
cache: npm
48-
cache-dependency-path: ./javascript/json-transform-core/package-lock.json
48+
cache-dependency-path: ./javascript/json-transform/package-lock.json
4949

50-
- name: install json-transform-core dependencies
51-
working-directory: ./javascript/json-transform-core
50+
- name: install json-transform dependencies
51+
working-directory: ./javascript/json-transform
5252
run: npm ci
5353

54-
- name: build json-transform-core
55-
working-directory: ./javascript/json-transform-core
54+
- name: build json-transform
55+
working-directory: ./javascript/json-transform
5656
run: npm run build
5757

5858
- name: setup monaco-json-transform build

‎.github/workflows/javascript-json-transform-test.yml‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,11 +5,13 @@ on:
55
branches:
66
- main
77
paths:
8+
- javascript/json-transform-core/**
89
- javascript/json-transform/**
910
pull_request:
1011
branches:
1112
- main
1213
paths:
14+
- javascript/json-transform-core/**
1315
- javascript/json-transform/**
1416

1517
# cancel previous tests if new commit is pushed to PR branch
@@ -23,6 +25,21 @@ jobs:
2325
steps:
2426
- uses: actions/checkout@v4
2527

28+
- name: setup json-transform-core build
29+
uses: actions/setup-node@v4
30+
with:
31+
node-version: 20
32+
cache: npm
33+
cache-dependency-path: ./javascript/json-transform-core/package-lock.json
34+
35+
- name: install json-transform-core dependencies
36+
working-directory: ./javascript/json-transform-core
37+
run: npm ci
38+
39+
- name: build json-transform-core
40+
working-directory: ./javascript/json-transform-core
41+
run: npm run build
42+
2643
- name: setup json-transform build
2744
uses: actions/setup-node@v4
2845
with:

‎java/json-transform/build.gradle‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@ plugins {
99
}
1010

1111
group 'co.nlighten'
12-
version = '1.2.2'
12+
version = '1.3.0'
1313

1414
ext {
1515
gsonVersion = "2.10.1"

‎java/json-transform/src/main/java/co/nlighten/jsontransform/TransformerFunctions.java‎

Lines changed: 19 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -3,14 +3,13 @@
33
import co.nlighten.jsontransform.adapters.JsonAdapter;
44
import co.nlighten.jsontransform.functions.*;
55
import co.nlighten.jsontransform.functions.common.InlineFunctionContext;
6+
import co.nlighten.jsontransform.functions.common.InlineFunctionTokenizer;
67
import co.nlighten.jsontransform.functions.common.ObjectFunctionContext;
78
import co.nlighten.jsontransform.functions.common.TransformerFunction;
89
import org.slf4j.Logger;
910
import org.slf4j.LoggerFactory;
1011

11-
import java.util.ArrayList;
12-
import java.util.Arrays;
13-
import java.util.Map;
12+
import java.util.*;
1413
import java.util.regex.Pattern;
1514
import java.util.stream.Stream;
1615

@@ -171,40 +170,33 @@ public FunctionMatchResult<Object> matchObject(JsonAdapter<?,?,?> adapter, Strin
171170

172171
private InlineFunctionContext tryParseInlineFunction(JsonAdapter<?,?,?> adapter, String path, String value, co.nlighten.jsontransform.ParameterResolver resolver,
173172
JsonTransformerFunction transformer) {
174-
var matcher = inlineFunctionRegex.matcher(value);
175-
if (matcher.find()) {
176-
var functionKey = matcher.group(1);
173+
var match = InlineFunctionTokenizer.tokenize(value);
174+
if (match != null) {
175+
var functionKey = match.name();
177176
if (functions.containsKey(functionKey)) {
178177
var function = functions.get(functionKey);
179-
var argsString = matcher.group(3);
180178
var args = new ArrayList<>();
181-
if (argsString != null && !argsString.isEmpty()) {
182-
var argMatcher = inlineFunctionArgsRegex.matcher(argsString);
183-
while (argMatcher.find() &&
184-
(argMatcher.start() != argsString.length() || argsString.endsWith(","))) {
185-
var arg = argMatcher.group(1);
186-
var trimmed = argMatcher.group(1).trim();
187-
if (trimmed.startsWith(QUOTE_APOS) && trimmed.endsWith(QUOTE_APOS) && trimmed.length() > 1) {
188-
arg = adapter.getAsString(adapter.parse(trimmed));
189-
//otherwise, take the whole argument as-is
190-
}
191-
args.add(arg);
192-
}
179+
if (match.args() != null && !match.args().isEmpty()) {
180+
match.args().forEach(x -> {
181+
args.add(x.value());
182+
});
193183
}
194-
var matchEndIndex = matcher.end();
195-
String input;
196-
if (value.charAt(matchEndIndex - 1) != ':') { // if not ends with ':' then no input value specified
197-
input = null;
198-
} else {
199-
input = value.substring(matchEndIndex);
184+
for (var i = args.size() - 1; i >= 0; i--) {
185+
if (args.get(i) != null) {
186+
break;
187+
}
188+
args.remove(args.size() - 1); // remove trailing nulls
200189
}
190+
201191
return new InlineFunctionContext(
202192
path + "/" + FUNCTION_KEY_PREFIX + functionKey,
203-
input, args,
193+
match.input() != null ? match.input().value() : null,
194+
args,
204195
adapter,
205196
functionKey,
206197
function,
207-
resolver, transformer);
198+
resolver,
199+
transformer);
208200
}
209201
}
210202
return null;

‎java/json-transform/src/main/java/co/nlighten/jsontransform/functions/TransformerFunctionJoin.java‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@
44

55
import java.util.Map;
66
import java.util.Objects;
7+
import java.util.Optional;
78
import java.util.stream.Collectors;
89

910
public class TransformerFunctionJoin extends TransformerFunction {
@@ -26,8 +27,8 @@ public Object apply(FunctionContext context) {
2627
if (delimiter == null) {
2728
delimiter = context.getString("delimiter");
2829
}
29-
var prefix = context.getString("prefix");
30-
var suffix = context.getString("suffix");
30+
var prefix = Optional.ofNullable(context.getString("prefix")).orElse("");
31+
var suffix = Optional.ofNullable(context.getString("suffix")).orElse("");
3132
var stream = arr.stream().map(context::getAsString);
3233
if (!context.getBoolean("keep_nulls")) {
3334
stream = stream.filter(Objects::nonNull);

‎java/json-transform/src/main/java/co/nlighten/jsontransform/functions/TransformerFunctionWrap.java‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@
33
import co.nlighten.jsontransform.functions.common.*;
44

55
import java.util.Map;
6+
import java.util.Optional;
67

78
public class TransformerFunctionWrap extends TransformerFunction {
89
public TransformerFunctionWrap() {
@@ -18,6 +19,8 @@ public Object apply(FunctionContext context) {
1819
var res = context.getString(null);
1920
if (res == null)
2021
return null;
21-
return context.getString("prefix") + res + context.getString("suffix");
22+
return Optional.ofNullable(context.getString("prefix")).orElse("")
23+
+ res
24+
+ Optional.ofNullable(context.getString("suffix")).orElse("");
2225
}
2326
}

‎java/json-transform/src/main/java/co/nlighten/jsontransform/functions/common/InlineFunctionTokenizer.java‎

Lines changed: 126 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,126 @@
1+
package co.nlighten.jsontransform.functions.common;
2+
3+
import java.util.ArrayList;
4+
import java.util.List;
5+
import java.util.Map;
6+
7+
public class InlineFunctionTokenizer {
8+
9+
public record InlineFunctionArgToken(String value, int index, int length) {
10+
}
11+
12+
public record TokenizedInlineFunction(String name, int keyLength, List<InlineFunctionArgToken> args,
13+
InlineFunctionArgToken input) {
14+
}
15+
16+
private static final Map<Character, String> JSON_ESCAPE_CHARACTERS = Map.of(
17+
'b', "\b",
18+
'f', "\f",
19+
'n', "\n",
20+
'r', "\r",
21+
't', "\t",
22+
'v', "\u000B" // vertical tab
23+
);
24+
25+
public static TokenizedInlineFunction tokenize(String input) {
26+
if (input == null || !input.startsWith("$$")) {
27+
return null;
28+
}
29+
30+
int i = 2;
31+
int len = input.length();
32+
33+
int nameStart = i;
34+
while (i < len && Character.toString(input.charAt(i)).matches("[\\w-]")) {
35+
i++;
36+
}
37+
String functionName = input.substring(nameStart, i);
38+
if (functionName.isEmpty()) {
39+
return null; // syntax error, no function name found
40+
}
41+
42+
int keyLength = i;
43+
44+
List<InlineFunctionArgToken> args = null;
45+
46+
if (i < len && input.charAt(i) == '(') {
47+
i++;
48+
args = new ArrayList<>();
49+
StringBuilder current = new StringBuilder();
50+
int currentStart = i;
51+
boolean inQuote = false;
52+
boolean escape = false;
53+
boolean finished = false;
54+
55+
for (; i < len; i++) {
56+
char ch = input.charAt(i);
57+
58+
if (escape) {
59+
String escapedChar = JSON_ESCAPE_CHARACTERS.get(ch);
60+
if (escapedChar == null) {
61+
if (ch == 'u' || ch == 'x') {
62+
try {
63+
int width = ch == 'u' ? 4 : 2;
64+
String hex = input.substring(i + 1, i + 1 + width);
65+
int codePoint = Integer.parseInt(hex, 16);
66+
escapedChar = new String(Character.toChars(codePoint));
67+
i += width;
68+
} catch (Exception e) {
69+
return null;
70+
}
71+
} else {
72+
escapedChar = String.valueOf(ch);
73+
}
74+
}
75+
current.append(escapedChar);
76+
escape = false;
77+
continue;
78+
}
79+
80+
if (ch == '\\' && inQuote) {
81+
escape = true;
82+
continue;
83+
}
84+
85+
if (ch == '\'') {
86+
if (inQuote) {
87+
while (i + 1 < len && input.charAt(i + 1) == ' ') {
88+
i++;
89+
}
90+
} else if (current.toString().trim().isEmpty()) {
91+
current.setLength(0);
92+
}
93+
inQuote = !inQuote;
94+
continue;
95+
}
96+
97+
if (!inQuote && (ch == ',' || ch == ')')) {
98+
int length = i - currentStart;
99+
args.add(new InlineFunctionArgToken(length > 0 ? current.toString() : null, currentStart, length));
100+
current.setLength(0);
101+
currentStart = i + 1;
102+
if (ch == ')') {
103+
i++;
104+
finished = true;
105+
break;
106+
}
107+
} else {
108+
current.append(ch);
109+
}
110+
}
111+
112+
if (!finished || (i < len && input.charAt(i) != ':')) {
113+
return null;
114+
}
115+
}
116+
117+
InlineFunctionArgToken fnInput = null;
118+
if (i < len && input.charAt(i) == ':') {
119+
i++;
120+
String inputValue = input.substring(i);
121+
fnInput = new InlineFunctionArgToken(inputValue, i, input.length() - i);
122+
}
123+
124+
return new TokenizedInlineFunction(functionName, keyLength, args, fnInput);
125+
}
126+
}

0 commit comments

Comments
 (0)