Skip to content

fix jq path to match postgresql lax jsonpath for load-legacy-tests #235

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
stalep merged 1 commit into from
Aug 4, 2026
Merged

fix jq path to match postgresql lax jsonpath for load-legacy-tests #235

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 3 additions & 6 deletions src/main/java/io/hyperfoil/tools/h5m/cli/LoadLegacyTests.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -150,10 +150,6 @@ public List<NodeEntity> getLabelNodes(String name){

}

static String jsonpathToJq(String jsonpath) {
return NodeService.jsonpathToJq(jsonpath);
}

@Inject
FolderService folderService;

Expand Down Expand Up @@ -245,6 +241,7 @@ public int hashCode(){
return Objects.hash(name,JsNode.isNullEmptyOrIdentityFunction(function) ? null : function,extractors);
}
};

//id,name,labels,calculation
public record Variable(long id,String name,List<String> labels,String calculation){};

Expand All @@ -262,8 +259,8 @@ public NodeEntity createNodesFromLabel(Label label, NodeEntity source, NodeGroup

// Convert jsonpath to jq — sqlall (isArray) wraps in [...] to collect all matches
String jqOperation = extractor.isArray
? NodeService.jsonpathToJqArray(extractor.jsonpath())
: NodeService.jsonpathToJq(extractor.jsonpath());
? NodeService.jsonpathToLaxJqArray(extractor.jsonpath())
: NodeService.jsonpathToLaxJq(extractor.jsonpath());
NodeEntity node = JqNode.parse(extractorName, jqOperation, nodeTracking::getNodes);
if (node == null) {
System.err.println("failed to create node for extractor " + extractor);
Expand Down
151 changes: 145 additions & 6 deletions src/main/java/io/hyperfoil/tools/h5m/svc/NodeService.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1141,8 +1141,14 @@ private static int followingNonSpace(int idx,String input){
* when the path doesn't exist — matching jsonb_path_query_array's behavior
* of returning {@code []} for missing paths.
*/
public static String jsonpathToJqArray(String jsonpath) {
String jq = jsonpathToJq(jsonpath);
public static String jsonpathToJqArray(String jsonpath){
return jsonpathToJqArray(jsonpath,false);
}
public static String jsonpathToLaxJqArray(String jsonpath){
return jsonpathToJqArray(jsonpath,true);
}
private static String jsonpathToJqArray(String jsonpath,boolean lax) {
String jq = jsonpathToJq(jsonpath,lax);
if (jq.contains("[]?")) {
// Has iterators — [...] naturally produces [] when no matches
return "[" + jq + "]";
Expand All @@ -1152,8 +1158,13 @@ public static String jsonpathToJqArray(String jsonpath) {
return "[" + jq + " // empty]";
}
}

public static String jsonpathToJq(String jsonpath) {
public static String jsonpathToJq(String jsonpath){
return jsonpathToJq(jsonpath,false);
}
public static String jsonpathToLaxJq(String jsonpath){
return jsonpathToJq(jsonpath,true);
}
private static String jsonpathToJq(String jsonpath,boolean lax) {
if (jsonpath == null || jsonpath.isEmpty()) return ".";
String jq = jsonpath;
if(jq.equals("$")){
Expand All @@ -1175,9 +1186,137 @@ public static String jsonpathToJq(String jsonpath) {
if (jq.contains("?")) {
jq = convertJsonpathFilters(jq);
}
if(lax){
jq = convertToLaxJqChains(jq);
}
return jq;
}

public record Range(int start,int stop){};
public static List<Range> findJqKeyChain(String input){
return NodeService.findJqKeyChain(input,0);
}
//chains have 2 or more keys, otherwise it's just a key access
public static List<Range> findJqKeyChain(String input,int start){
if(input == null || start > input.length()){
return Collections.emptyList();
}
//deal with negative numbers
start = Math.max(0,start);
List<Range> rtrn = new ArrayList<>();
boolean inQuote = false;
boolean inChain = false;
int keyCount = 0;
int chainStart = -1;
char quoteChar = ' ';

for(int i=start; i<input.length(); i++){
char c = input.charAt(i);
if(inQuote){
if (quoteChar == c && (i==start || input.charAt(i-1) != '\\')){
inQuote = false;
}
}else{
if((c == '"' || c == '\'') && (i==start || input.charAt(i-1) != '\\')){
inQuote = true;
quoteChar = c;
}else if(c == '.'){//a new key
if(!inChain){
chainStart = i;
inChain = true;
keyCount = 1;
}else{
if(i == input.length()-1 || !"[".contains(""+input.charAt(i+1))){
keyCount++;
}
}
} else if (" \t|?[=!><".contains(""+c)){//terminating characters
if(inChain ){
if(keyCount > 1) {
rtrn.add(new Range(chainStart, i));
}
inChain = false;
keyCount=0;
}
}
}
}
if(inChain && keyCount > 1){
rtrn.add(new Range(chainStart,input.length()));
}
return rtrn;
}

public static List<String> splitNotInQuotes(String input,String split){
boolean inQuote = false;
char quoteChar = ' ';
List<String> rtrn = new ArrayList<>();
int startIdx = 0;
for(int i=0; i<input.length(); i++){
char c = input.charAt(i);
if(inQuote){
if (quoteChar == c && (i==0 || input.charAt(i-1) != '\\')){
inQuote = false;
}
}else {
if ((c == '"' || c == '\'') && (i == 0 || input.charAt(i - 1) != '\\')) {
inQuote = true;
quoteChar = c;

}else if( input.startsWith(split,i) ){
if( startIdx > 0){
rtrn.add(input.substring(startIdx,i));
}
i+=split.length()-1;//-1 because of the increment in the for loop
startIdx = i+1;//+1 to set to next index
}
}
}
if(startIdx > 0 && startIdx < input.length()){
rtrn.add(input.substring(startIdx));
}
return rtrn;
}

public static String convertToLaxJqChains(String input){
if(input==null || input.isEmpty()){
return input;
}
List<Range> ranges = findJqKeyChain(input);
if(ranges.isEmpty()){
return input;
}
StringBuilder sb = new StringBuilder();
int prevIdx = 0;
for(Range r : ranges){
if(r.start() > prevIdx){
sb.append(input, prevIdx, r.start());
}
List<String> keys = splitNotInQuotes(input.substring(r.start(),r.stop()),".");
if(sb.length()>0){
if(sb.charAt(sb.length()-1)!=' ') {
sb.append(" ");
}
int i=sb.length()-1;
while(i > 0 && sb.charAt(i) == ' '){
i--;
}
if(!"|(".contains(""+sb.charAt(i))) {
sb.append(" | ");
}
}
for(int idx = 0; idx< keys.size()-1; idx++){
sb.append("if (.KEY | type) == \"array\" then .KEY[] else .KEY end | ".replaceAll("KEY",keys.get(idx)));
}
sb.append("."+keys.get(keys.size()-1));
prevIdx = r.stop();
}
if(prevIdx < input.length()){
sb.append(input, prevIdx, input.length());
}
return sb.toString();
}

/**
* Replaces occurrences of {@code target} with {@code replacement} only when
* outside double-quoted strings. This prevents corrupting quoted keys that
Expand Down Expand Up @@ -1268,7 +1407,7 @@ static String convertJsonpathFilters(String jq) {
}
// Convert remaining ."text()" style field access to ["text()"]
String output = result.toString();
output = output.replaceAll("\\.\"([^\"]+)\"", ".[\"$1\"]");
output = output.replaceAll("\\.\"([^\"]+)\"", ".[\"$1\"]");
return output;
}

Expand Down Expand Up @@ -1380,7 +1519,7 @@ public List<ValueEntity> calculateJsValues(JsNode node, Map<Long, ValueEntity> s
newValue.sources = node.sources.stream().filter(n->sourceValues.containsKey(n.getId())).map(n -> sourceValues.get(n.getId())).collect(Collectors.toList());
rtrn.add(newValue);
}else{
System.err.println("null data from value "+resolvedValue);
System.err.println("null data from value "+resolvedValue+" from node="+node.name);
}
}catch (PolyglotException pe){
System.err.println("exception jsNode "+node.name+" sourceValues="+sourceValues+"\n"+pe.getMessage());
Expand Down
4 changes: 2 additions & 2 deletions src/test/java/io/hyperfoil/tools/h5m/cli/H5mTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -50,9 +50,9 @@ public void loadLegacyTests(QuarkusMainLauncher launcher){
@Test @Disabled
public void loadLegacyRuns(QuarkusMainLauncher launcher){
LaunchResult result = null;
result = launcher.launch("load-legacy-tests","testId=391","username=horreum","password=horreum","url=jdbc:postgresql://0.0.0.0:6000/horreum");
result = launcher.launch("load-legacy-tests","testId=339","username=horreum","password=horreum","url=jdbc:postgresql://0.0.0.0:6000/horreum");
assertEquals(0,result.exitCode());
result = launcher.launch("load-legacy-runs","testId=391","username=horreum","password=horreum","url=jdbc:postgresql://0.0.0.0:6000/horreum");
result = launcher.launch("load-legacy-runs","testId=339","limit=1","username=horreum","password=horreum","url=jdbc:postgresql://0.0.0.0:6000/horreum");
System.out.println("exitCode="+result.exitCode());
assertEquals(0,result.exitCode());
}
Expand Down
102 changes: 97 additions & 5 deletions src/test/java/io/hyperfoil/tools/h5m/cli/LoadLegacyTestsTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,14 @@
package io.hyperfoil.tools.h5m.cli;

import io.hyperfoil.tools.h5m.FreshDb;
import io.hyperfoil.tools.h5m.api.Folder;
import io.hyperfoil.tools.h5m.api.Upload;
import io.hyperfoil.tools.h5m.entity.ValueEntity;
import io.hyperfoil.tools.h5m.svc.ValueService;
import io.hyperfoil.tools.jjq.JqProgram;
import io.hyperfoil.tools.jjq.value.JqObject;
import io.hyperfoil.tools.jjq.value.JqString;
import io.hyperfoil.tools.jjq.value.JqValue;
import io.hyperfoil.tools.jjq.value.JqValues;
import io.hyperfoil.tools.h5m.api.Node;
import io.hyperfoil.tools.h5m.entity.FolderEntity;
Expand All @@ -14,23 +22,107 @@
import io.quarkus.test.junit.QuarkusTest;
import jakarta.enterprise.inject.spi.CDI;
import jakarta.inject.Inject;
import jakarta.transaction.*;
import org.junit.jupiter.api.Disabled;
import org.junit.jupiter.api.Test;

import java.util.Arrays;
import java.util.Collections;
import java.util.HashSet;
import java.util.List;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.*;
import java.util.concurrent.TimeUnit;
import java.util.stream.Collectors;

import static org.junit.jupiter.api.Assertions.*;

@QuarkusTest
public class LoadLegacyTestsTest {
public class LoadLegacyTestsTest extends FreshDb {

@Inject
LoadLegacyTests loadLegacyTests;

@Inject
NodeService nodeService;

@Inject
FolderService folderService;

@Test
@Transactional
public void extractor_lax_isArray_match_select() throws IOException {
LoadLegacyTests.Extractor extractor = new LoadLegacyTests.Extractor("biz","$.a.b ? (@.c.e==\"two\").c.d",true);
LoadLegacyTests.Label label = new LoadLegacyTests.Label(1,"foo",null, Arrays.asList(extractor));
HashedSets<String,LoadLegacyTests.Label> schemaPaths = new HashedSets<>();
schemaPaths.put("$.\"$schema\"",label);

LoadLegacyTests.Test test = new LoadLegacyTests.Test(-1,"test",schemaPaths, Collections.emptyList(),Collections.emptyList(),Collections.emptyList(),Collections.emptyList());

FolderEntity folder = loadLegacyTests.createFolder(test).folder();
folder.id = folderService.create(folder);
folder = folderService.read(folder.id);
assertNotNull(folder);
assertNotNull(folder.group);
assertNotNull(folder.group.sources);
assertEquals(1,folder.group.sources.size());
NodeEntity node = folder.group.sources.getFirst();
assertNotNull(node);
assertEquals("foo",node.name,"name should be changed to match label");
if(node instanceof JqNode jqNode){
ValueEntity value = new ValueEntity(null,node,JqValues.parse("""
{ "a" : { "b" : [ { "c" : [{ "d" : "one", "e" : "two" }]} ] } }
"""));
List<ValueEntity> calculated = nodeService.calculateJqValues(jqNode, Map.of(folder.group.root.id,value),1);
assertNotNull(calculated);
assertEquals(1,calculated.size());
ValueEntity found = calculated.getFirst();
assertNotNull(found,"entity should not be null");
assertNotNull(found.data,"value should have data");
assertTrue(found.data.isArray(),"data should be an array: "+found.data);
assertEquals(1,found.data.length(),"expect to find one entry: "+found.data);
assertEquals(JqString.of("one"),found.data.getElement(0));
}else{
fail("node should be jq");
}
}

@Test
@Transactional
public void extractor_lax_isArray_match() throws IOException {
LoadLegacyTests.Extractor extractor = new LoadLegacyTests.Extractor("biz","$.a.b.c",true);
LoadLegacyTests.Label label = new LoadLegacyTests.Label(1,"foo",null, Arrays.asList(extractor));
HashedSets<String,LoadLegacyTests.Label> schemaPaths = new HashedSets<>();
schemaPaths.put("$.\"$schema\"",label);

LoadLegacyTests.Test test = new LoadLegacyTests.Test(-1,"test",schemaPaths, Collections.emptyList(),Collections.emptyList(),Collections.emptyList(),Collections.emptyList());

FolderEntity folder = loadLegacyTests.createFolder(test).folder();
folder.id = folderService.create(folder);
folder = folderService.read(folder.id);
assertNotNull(folder);
assertNotNull(folder.group);
assertNotNull(folder.group.sources);
assertEquals(1,folder.group.sources.size());
NodeEntity node = folder.group.sources.getFirst();
assertNotNull(node);
assertEquals("foo",node.name,"name should be changed to match label");
if(node instanceof JqNode jqNode){
ValueEntity value = new ValueEntity(null,node,JqValues.parse("""
{ "a" : { "b" : [{ "c" : "one" }] } }
"""));
List<ValueEntity> calculated = nodeService.calculateJqValues(jqNode, Map.of(folder.group.root.id,value),1);
assertNotNull(calculated);
assertEquals(1,calculated.size());
ValueEntity found = calculated.getFirst();
assertNotNull(found,"entity should not be null");
assertNotNull(found.data,"value should have data");
assertTrue(found.data.isArray(),"data should be an array: "+found.data);
assertEquals(1,found.data.length(),"expect to find one entry: "+found.data);
assertEquals(JqString.of("one"),found.data.getElement(0));
}else{
fail("node should be jq");
}
}

@Test
public void extractor_equals(){
LoadLegacyTests.Extractor one = new LoadLegacyTests.Extractor("foo","bar",false);
Expand Down
Loading
Loading