From 1cbb905932660613390185e786436793a4e2d2ad Mon Sep 17 00:00:00 2001 From: Will Reichert Date: Mon, 3 Aug 2026 19:26:20 -0400 Subject: [PATCH] fix jq path to match postgresql lax jsonpath for load-legacy-tests --- .../tools/h5m/cli/LoadLegacyTests.java | 9 +- .../hyperfoil/tools/h5m/svc/NodeService.java | 151 +++++++++++++++++- .../io/hyperfoil/tools/h5m/cli/H5mTest.java | 4 +- .../tools/h5m/cli/LoadLegacyTestsTest.java | 102 +++++++++++- .../tools/h5m/svc/NodeServiceTest.java | 66 ++++++++ 5 files changed, 313 insertions(+), 19 deletions(-) diff --git a/src/main/java/io/hyperfoil/tools/h5m/cli/LoadLegacyTests.java b/src/main/java/io/hyperfoil/tools/h5m/cli/LoadLegacyTests.java index 486a322a..e67a1a44 100644 --- a/src/main/java/io/hyperfoil/tools/h5m/cli/LoadLegacyTests.java +++ b/src/main/java/io/hyperfoil/tools/h5m/cli/LoadLegacyTests.java @@ -150,10 +150,6 @@ public List getLabelNodes(String name){ } - static String jsonpathToJq(String jsonpath) { - return NodeService.jsonpathToJq(jsonpath); - } - @Inject FolderService folderService; @@ -245,6 +241,7 @@ public int hashCode(){ return Objects.hash(name,JsNode.isNullEmptyOrIdentityFunction(function) ? null : function,extractors); } }; + //id,name,labels,calculation public record Variable(long id,String name,List labels,String calculation){}; @@ -262,8 +259,8 @@ public NodeEntity createNodesFromLabel(Label label, NodeEntity source, NodeGroup // Convert jsonpath to jq — sqlall (isArray) wraps in [...] to collect all matches String jqOperation = extractor.isArray - ? NodeService.jsonpathToJqArray(extractor.jsonpath()) - : NodeService.jsonpathToJq(extractor.jsonpath()); + ? NodeService.jsonpathToLaxJqArray(extractor.jsonpath()) + : NodeService.jsonpathToLaxJq(extractor.jsonpath()); NodeEntity node = JqNode.parse(extractorName, jqOperation, nodeTracking::getNodes); if (node == null) { System.err.println("failed to create node for extractor " + extractor); diff --git a/src/main/java/io/hyperfoil/tools/h5m/svc/NodeService.java b/src/main/java/io/hyperfoil/tools/h5m/svc/NodeService.java index 1718f74e..f3e6ab94 100644 --- a/src/main/java/io/hyperfoil/tools/h5m/svc/NodeService.java +++ b/src/main/java/io/hyperfoil/tools/h5m/svc/NodeService.java @@ -1141,8 +1141,14 @@ private static int followingNonSpace(int idx,String input){ * when the path doesn't exist — matching jsonb_path_query_array's behavior * of returning {@code []} for missing paths. */ - public static String jsonpathToJqArray(String jsonpath) { - String jq = jsonpathToJq(jsonpath); + public static String jsonpathToJqArray(String jsonpath){ + return jsonpathToJqArray(jsonpath,false); + } + public static String jsonpathToLaxJqArray(String jsonpath){ + return jsonpathToJqArray(jsonpath,true); + } + private static String jsonpathToJqArray(String jsonpath,boolean lax) { + String jq = jsonpathToJq(jsonpath,lax); if (jq.contains("[]?")) { // Has iterators — [...] naturally produces [] when no matches return "[" + jq + "]"; @@ -1152,8 +1158,13 @@ public static String jsonpathToJqArray(String jsonpath) { return "[" + jq + " // empty]"; } } - - public static String jsonpathToJq(String jsonpath) { + public static String jsonpathToJq(String jsonpath){ + return jsonpathToJq(jsonpath,false); + } + public static String jsonpathToLaxJq(String jsonpath){ + return jsonpathToJq(jsonpath,true); + } + private static String jsonpathToJq(String jsonpath,boolean lax) { if (jsonpath == null || jsonpath.isEmpty()) return "."; String jq = jsonpath; if(jq.equals("$")){ @@ -1175,9 +1186,137 @@ public static String jsonpathToJq(String jsonpath) { if (jq.contains("?")) { jq = convertJsonpathFilters(jq); } + if(lax){ + jq = convertToLaxJqChains(jq); + } return jq; } + public record Range(int start,int stop){}; + public static List findJqKeyChain(String input){ + return NodeService.findJqKeyChain(input,0); + } + //chains have 2 or more keys, otherwise it's just a key access + public static List findJqKeyChain(String input,int start){ + if(input == null || start > input.length()){ + return Collections.emptyList(); + } + //deal with negative numbers + start = Math.max(0,start); + List rtrn = new ArrayList<>(); + boolean inQuote = false; + boolean inChain = false; + int keyCount = 0; + int chainStart = -1; + char quoteChar = ' '; + + for(int i=start; i<".contains(""+c)){//terminating characters + if(inChain ){ + if(keyCount > 1) { + rtrn.add(new Range(chainStart, i)); + } + inChain = false; + keyCount=0; + } + } + } + } + if(inChain && keyCount > 1){ + rtrn.add(new Range(chainStart,input.length())); + } + return rtrn; + } + + public static List splitNotInQuotes(String input,String split){ + boolean inQuote = false; + char quoteChar = ' '; + List rtrn = new ArrayList<>(); + int startIdx = 0; + for(int i=0; i 0){ + rtrn.add(input.substring(startIdx,i)); + } + i+=split.length()-1;//-1 because of the increment in the for loop + startIdx = i+1;//+1 to set to next index + } + } + } + if(startIdx > 0 && startIdx < input.length()){ + rtrn.add(input.substring(startIdx)); + } + return rtrn; + } + + public static String convertToLaxJqChains(String input){ + if(input==null || input.isEmpty()){ + return input; + } + List ranges = findJqKeyChain(input); + if(ranges.isEmpty()){ + return input; + } + StringBuilder sb = new StringBuilder(); + int prevIdx = 0; + for(Range r : ranges){ + if(r.start() > prevIdx){ + sb.append(input, prevIdx, r.start()); + } + List keys = splitNotInQuotes(input.substring(r.start(),r.stop()),"."); + if(sb.length()>0){ + if(sb.charAt(sb.length()-1)!=' ') { + sb.append(" "); + } + int i=sb.length()-1; + while(i > 0 && sb.charAt(i) == ' '){ + i--; + } + if(!"|(".contains(""+sb.charAt(i))) { + sb.append(" | "); + } + } + for(int idx = 0; idx< keys.size()-1; idx++){ + sb.append("if (.KEY | type) == \"array\" then .KEY[] else .KEY end | ".replaceAll("KEY",keys.get(idx))); + } + sb.append("."+keys.get(keys.size()-1)); + prevIdx = r.stop(); + } + if(prevIdx < input.length()){ + sb.append(input, prevIdx, input.length()); + } + return sb.toString(); + } + /** * Replaces occurrences of {@code target} with {@code replacement} only when * outside double-quoted strings. This prevents corrupting quoted keys that @@ -1268,7 +1407,7 @@ static String convertJsonpathFilters(String jq) { } // Convert remaining ."text()" style field access to ["text()"] String output = result.toString(); - output = output.replaceAll("\\.\"([^\"]+)\"", ".[\"$1\"]"); + output = output.replaceAll("\\.\"([^\"]+)\"", ".[\"$1\"]"); return output; } @@ -1380,7 +1519,7 @@ public List calculateJsValues(JsNode node, Map s newValue.sources = node.sources.stream().filter(n->sourceValues.containsKey(n.getId())).map(n -> sourceValues.get(n.getId())).collect(Collectors.toList()); rtrn.add(newValue); }else{ - System.err.println("null data from value "+resolvedValue); + System.err.println("null data from value "+resolvedValue+" from node="+node.name); } }catch (PolyglotException pe){ System.err.println("exception jsNode "+node.name+" sourceValues="+sourceValues+"\n"+pe.getMessage()); diff --git a/src/test/java/io/hyperfoil/tools/h5m/cli/H5mTest.java b/src/test/java/io/hyperfoil/tools/h5m/cli/H5mTest.java index 143f7e99..37853590 100644 --- a/src/test/java/io/hyperfoil/tools/h5m/cli/H5mTest.java +++ b/src/test/java/io/hyperfoil/tools/h5m/cli/H5mTest.java @@ -50,9 +50,9 @@ public void loadLegacyTests(QuarkusMainLauncher launcher){ @Test @Disabled public void loadLegacyRuns(QuarkusMainLauncher launcher){ LaunchResult result = null; - result = launcher.launch("load-legacy-tests","testId=391","username=horreum","password=horreum","url=jdbc:postgresql://0.0.0.0:6000/horreum"); + result = launcher.launch("load-legacy-tests","testId=339","username=horreum","password=horreum","url=jdbc:postgresql://0.0.0.0:6000/horreum"); assertEquals(0,result.exitCode()); - result = launcher.launch("load-legacy-runs","testId=391","username=horreum","password=horreum","url=jdbc:postgresql://0.0.0.0:6000/horreum"); + result = launcher.launch("load-legacy-runs","testId=339","limit=1","username=horreum","password=horreum","url=jdbc:postgresql://0.0.0.0:6000/horreum"); System.out.println("exitCode="+result.exitCode()); assertEquals(0,result.exitCode()); } diff --git a/src/test/java/io/hyperfoil/tools/h5m/cli/LoadLegacyTestsTest.java b/src/test/java/io/hyperfoil/tools/h5m/cli/LoadLegacyTestsTest.java index 6e6c51cb..81ed5578 100644 --- a/src/test/java/io/hyperfoil/tools/h5m/cli/LoadLegacyTestsTest.java +++ b/src/test/java/io/hyperfoil/tools/h5m/cli/LoadLegacyTestsTest.java @@ -1,6 +1,14 @@ package io.hyperfoil.tools.h5m.cli; +import io.hyperfoil.tools.h5m.FreshDb; +import io.hyperfoil.tools.h5m.api.Folder; +import io.hyperfoil.tools.h5m.api.Upload; +import io.hyperfoil.tools.h5m.entity.ValueEntity; +import io.hyperfoil.tools.h5m.svc.ValueService; +import io.hyperfoil.tools.jjq.JqProgram; import io.hyperfoil.tools.jjq.value.JqObject; +import io.hyperfoil.tools.jjq.value.JqString; +import io.hyperfoil.tools.jjq.value.JqValue; import io.hyperfoil.tools.jjq.value.JqValues; import io.hyperfoil.tools.h5m.api.Node; import io.hyperfoil.tools.h5m.entity.FolderEntity; @@ -14,23 +22,107 @@ import io.quarkus.test.junit.QuarkusTest; import jakarta.enterprise.inject.spi.CDI; import jakarta.inject.Inject; +import jakarta.transaction.*; import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; -import java.util.Arrays; -import java.util.Collections; -import java.util.HashSet; -import java.util.List; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.*; +import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; import static org.junit.jupiter.api.Assertions.*; @QuarkusTest -public class LoadLegacyTestsTest { +public class LoadLegacyTestsTest extends FreshDb { @Inject LoadLegacyTests loadLegacyTests; + @Inject + NodeService nodeService; + + @Inject + FolderService folderService; + + @Test + @Transactional + public void extractor_lax_isArray_match_select() throws IOException { + LoadLegacyTests.Extractor extractor = new LoadLegacyTests.Extractor("biz","$.a.b ? (@.c.e==\"two\").c.d",true); + LoadLegacyTests.Label label = new LoadLegacyTests.Label(1,"foo",null, Arrays.asList(extractor)); + HashedSets schemaPaths = new HashedSets<>(); + schemaPaths.put("$.\"$schema\"",label); + + LoadLegacyTests.Test test = new LoadLegacyTests.Test(-1,"test",schemaPaths, Collections.emptyList(),Collections.emptyList(),Collections.emptyList(),Collections.emptyList()); + + FolderEntity folder = loadLegacyTests.createFolder(test).folder(); + folder.id = folderService.create(folder); + folder = folderService.read(folder.id); + assertNotNull(folder); + assertNotNull(folder.group); + assertNotNull(folder.group.sources); + assertEquals(1,folder.group.sources.size()); + NodeEntity node = folder.group.sources.getFirst(); + assertNotNull(node); + assertEquals("foo",node.name,"name should be changed to match label"); + if(node instanceof JqNode jqNode){ + ValueEntity value = new ValueEntity(null,node,JqValues.parse(""" + { "a" : { "b" : [ { "c" : [{ "d" : "one", "e" : "two" }]} ] } } + """)); + List calculated = nodeService.calculateJqValues(jqNode, Map.of(folder.group.root.id,value),1); + assertNotNull(calculated); + assertEquals(1,calculated.size()); + ValueEntity found = calculated.getFirst(); + assertNotNull(found,"entity should not be null"); + assertNotNull(found.data,"value should have data"); + assertTrue(found.data.isArray(),"data should be an array: "+found.data); + assertEquals(1,found.data.length(),"expect to find one entry: "+found.data); + assertEquals(JqString.of("one"),found.data.getElement(0)); + }else{ + fail("node should be jq"); + } + } + + @Test + @Transactional + public void extractor_lax_isArray_match() throws IOException { + LoadLegacyTests.Extractor extractor = new LoadLegacyTests.Extractor("biz","$.a.b.c",true); + LoadLegacyTests.Label label = new LoadLegacyTests.Label(1,"foo",null, Arrays.asList(extractor)); + HashedSets schemaPaths = new HashedSets<>(); + schemaPaths.put("$.\"$schema\"",label); + + LoadLegacyTests.Test test = new LoadLegacyTests.Test(-1,"test",schemaPaths, Collections.emptyList(),Collections.emptyList(),Collections.emptyList(),Collections.emptyList()); + + FolderEntity folder = loadLegacyTests.createFolder(test).folder(); + folder.id = folderService.create(folder); + folder = folderService.read(folder.id); + assertNotNull(folder); + assertNotNull(folder.group); + assertNotNull(folder.group.sources); + assertEquals(1,folder.group.sources.size()); + NodeEntity node = folder.group.sources.getFirst(); + assertNotNull(node); + assertEquals("foo",node.name,"name should be changed to match label"); + if(node instanceof JqNode jqNode){ + ValueEntity value = new ValueEntity(null,node,JqValues.parse(""" + { "a" : { "b" : [{ "c" : "one" }] } } + """)); + List calculated = nodeService.calculateJqValues(jqNode, Map.of(folder.group.root.id,value),1); + assertNotNull(calculated); + assertEquals(1,calculated.size()); + ValueEntity found = calculated.getFirst(); + assertNotNull(found,"entity should not be null"); + assertNotNull(found.data,"value should have data"); + assertTrue(found.data.isArray(),"data should be an array: "+found.data); + assertEquals(1,found.data.length(),"expect to find one entry: "+found.data); + assertEquals(JqString.of("one"),found.data.getElement(0)); + }else{ + fail("node should be jq"); + } + } + @Test public void extractor_equals(){ LoadLegacyTests.Extractor one = new LoadLegacyTests.Extractor("foo","bar",false); diff --git a/src/test/java/io/hyperfoil/tools/h5m/svc/NodeServiceTest.java b/src/test/java/io/hyperfoil/tools/h5m/svc/NodeServiceTest.java index e40f0913..2955d90c 100644 --- a/src/test/java/io/hyperfoil/tools/h5m/svc/NodeServiceTest.java +++ b/src/test/java/io/hyperfoil/tools/h5m/svc/NodeServiceTest.java @@ -2045,6 +2045,72 @@ public void testCalculateRelativeDifferenceValuesOrdered() throws Exception { } + @Test + public void findJqKeyChain(){ + List found; + String input; + //standard chain + found = NodeService.findJqKeyChain(input=".a.b.c"); + assertEquals(1,found.size()); + assertEquals(0,found.getFirst().start()); + assertEquals(input.length(),found.getFirst().stop()); + + found = NodeService.findJqKeyChain(input = ".a.\"b | \t ? [ c\".c | .c"); + assertEquals(1,found.size(),found.toString()); + assertEquals(0,found.getFirst().start()); + assertEquals(".a.\"b | \t ? [ c\".c",input.substring(found.get(0).start(),found.get(0).stop())); + + found = NodeService.findJqKeyChain(input=".a.b | .c"); + assertEquals(1,found.size()); + assertEquals(0,found.getFirst().start()); + assertEquals(".a.b",input.substring(found.get(0).start(),found.get(0).stop())); + + found = NodeService.findJqKeyChain(input=".a.b | .c.d"); + assertEquals(2,found.size()); + assertEquals(".a.b",input.substring(found.get(0).start(),found.get(0).stop())); + assertEquals(".c.d",input.substring(found.get(1).start(),found.get(1).stop())); + + found = NodeService.findJqKeyChain(input=".a.[\"text()\"]"); + assertEquals(0,found.size()); + + found = NodeService.findJqKeyChain(input="$.a.b ? (@.c.e==\"two\").c.d"); + assertEquals(3,found.size()); + assertEquals(".a.b",input.substring(found.get(0).start(),found.get(0).stop())); + assertEquals(".c.e",input.substring(found.get(1).start(),found.get(1).stop())); + assertEquals(".c.d",input.substring(found.get(2).start(),found.get(2).stop())); + } + + @Test + public void splitNotInQuotes(){ + List found; + found = NodeService.splitNotInQuotes(".a.b.c","."); + assertEquals(3,found.size(),found.toString()); + assertEquals("a",found.get(0)); + assertEquals("b",found.get(1)); + assertEquals("c",found.get(2)); + + found = NodeService.splitNotInQuotes(".a","."); + assertEquals(1,found.size(),found.toString()); + assertEquals("a",found.get(0)); + + found = NodeService.splitNotInQuotes(".a.\".'b.c'\".c","."); + assertEquals(3,found.size(),found.toString()); + assertEquals("a",found.get(0)); + assertEquals("\".'b.c'\"",found.get(1)); + assertEquals("c",found.get(2)); + + } + + @Test + public void convertToLaxChains(){ + String response; + assertEquals(".a",NodeService.convertToLaxJqChains(".a"),"no change"); + assertEquals("if (.a | type) == \"array\" then .a[] else .a end | if (.b | type) == \"array\" then .b[] else .b end | .c", + NodeService.convertToLaxJqChains(".a.b.c")); + + } + + @Test public void jsonpathToJq_filter_equals() { assertEquals(