Add full transaction CBOR reconstruction via raw segment splicing (returnFullTxCbor) [next]

core/src/main/java/com/bloxbean/cardano/yaci/core/config/YaciConfig.java

@@ -8,12 +8,14 @@ public enum YaciConfig {
 
     private boolean returnBlockCbor;
     private boolean returnTxBodyCbor;
+    private boolean returnFullTxCbor;
 
     private boolean blockFetchCheckRangeExists = false;
 
     YaciConfig() {
         returnBlockCbor = false;
         returnTxBodyCbor = false;
+        returnFullTxCbor = false;
     }
 
     /**
@@ -55,4 +57,20 @@ public void setBlockFetchCheckRangeExists(boolean blockFetchCheckRangeExists) {
     public boolean isBlockFetchCheckRangeExists() {
         return blockFetchCheckRangeExists;
     }
+
+    /**
+     * Returns true if full transaction cbor is returned.
+     * @return
+     */
+    public boolean isReturnFullTxCbor() {
+        return returnFullTxCbor;
+    }
+
+    /**
+     * Set to true to return full transaction cbor.
+     * @param returnFullTxCbor
+     */
+    public void setReturnFullTxCbor(boolean returnFullTxCbor) {
+        this.returnFullTxCbor = returnFullTxCbor;
+    }
 }

core/src/main/java/com/bloxbean/cardano/yaci/core/model/AuxData.java

@@ -9,12 +9,24 @@
 @NoArgsConstructor
 @ToString
 @EqualsAndHashCode
+@Builder(toBuilder = true)
 public class AuxData {
+    private String cbor;
+
     private String metadataCbor;
     private String metadataJson;
 
     private List<NativeScript> nativeScripts;
     private List<PlutusScript> plutusV1Scripts;
     private List<PlutusScript> plutusV2Scripts;
     private List<PlutusScript> plutusV3Scripts;
+
+    public AuxData(String metadataCbor,
+                   String metadataJson,
+                   List<NativeScript> nativeScripts,
+                   List<PlutusScript> plutusV1Scripts,
+                   List<PlutusScript> plutusV2Scripts,
+                   List<PlutusScript> plutusV3Scripts) {
+        this(null, metadataCbor, metadataJson, nativeScripts, plutusV1Scripts, plutusV2Scripts, plutusV3Scripts);
+    }
 }

core/src/main/java/com/bloxbean/cardano/yaci/core/model/Witnesses.java

@@ -10,8 +10,10 @@
 @AllArgsConstructor
 @EqualsAndHashCode
 @ToString
-@Builder
+@Builder(toBuilder = true)
 public class Witnesses {
+    private String cbor;
+
     private List<VkeyWitness> vkeyWitnesses = new ArrayList<>();
 
     private List<NativeScript> nativeScripts = new ArrayList<>();
@@ -27,4 +29,16 @@ public class Witnesses {
 
     private List<PlutusScript> plutusV2Scripts = new ArrayList<>();
     private List<PlutusScript> plutusV3Scripts = new ArrayList<>();
+
+    public Witnesses(List<VkeyWitness> vkeyWitnesses,
+                     List<NativeScript> nativeScripts,
+                     List<BootstrapWitness> bootstrapWitnesses,
+                     List<PlutusScript> plutusV1Scripts,
+                     List<Datum> datums,
+                     List<Redeemer> redeemers,
+                     List<PlutusScript> plutusV2Scripts,
+                     List<PlutusScript> plutusV3Scripts) {
+        this(null, vkeyWitnesses, nativeScripts, bootstrapWitnesses, plutusV1Scripts,
+                datums, redeemers, plutusV2Scripts, plutusV3Scripts);
+    }
 }

core/src/main/java/com/bloxbean/cardano/yaci/core/model/serializers/BlockSerializer.java

@@ -4,6 +4,7 @@
 import com.bloxbean.cardano.yaci.core.common.EraUtil;
 import com.bloxbean.cardano.yaci.core.config.YaciConfig;
 import com.bloxbean.cardano.yaci.core.model.*;
+import com.bloxbean.cardano.yaci.core.model.serializers.util.AuxDataExtractor;
 import com.bloxbean.cardano.yaci.core.model.serializers.util.TransactionBodyExtractor;
 import com.bloxbean.cardano.yaci.core.model.serializers.util.WitnessUtil;
 import com.bloxbean.cardano.yaci.core.protocol.Serializer;
@@ -78,25 +79,62 @@ private Block deserializeBlock(DataItem di, byte[] blockBody) {
             witnessesSet.add(witnesses);
         }
 
+        List<byte[]> transactionWitnessRawBytes = null;
+        if (!witnessesSet.isEmpty()) {
+            try {
+                transactionWitnessRawBytes = WitnessUtil.getWitnessRawData(blockBody);
+                if (YaciConfig.INSTANCE.isReturnFullTxCbor()) {
+                    setWitnessCbor(blockHeader.getHeaderBody().getBlockNumber(), witnessesSet, transactionWitnessRawBytes);
+                }
+            } catch (Exception e) {
+                //If extraction fails due to some reason
+                log.error("Extraction of witness bytes without serialization/deserialization failed for block : "
+                        + blockHeader.getHeaderBody().getBlockNumber(), e);
+            }
+        }
+
         //To fix #37 incorrect redeemer & datum hash due to cbor serialization <--> deserialization issue
         //Get redeemer and datum bytes directly without full deserialization
         try {
-            handleWitnessDatumRedeemer(blockHeader.getHeaderBody().getBlockNumber(), witnessesSet, blockBody);
+            handleWitnessDatumRedeemer(blockHeader.getHeaderBody().getBlockNumber(), witnessesSet, transactionWitnessRawBytes);
         } catch (Exception e) {
-            //If extraction fails due to some reason
-            log.error("Extraction of redeemer and datum bytes without serialization/deserialization failed for block : " + blockHeader.getHeaderBody().getBlockNumber());
+            log.error("Extraction of redeemer and datum bytes without serialization/deserialization failed for block : "
+                    + blockHeader.getHeaderBody().getBlockNumber(), e);
         }
 
         blockBuilder.transactionWitness(witnessesSet);
 
+        java.util.Map<Integer, byte[]> auxDataRawBytes = Collections.emptyMap();
+        if (YaciConfig.INSTANCE.isReturnFullTxCbor()) {
+            try {
+                auxDataRawBytes = AuxDataExtractor.getAuxDataFromBlock(blockBody);
+            } catch (Exception e) {
+                log.error("Extraction of auxiliary data bytes failed for block : "
+                        + blockHeader.getHeaderBody().getBlockNumber(), e);
+            }
+        }
+
         //auxiliary data
         java.util.Map<Integer, AuxData> auxDataMap = new LinkedHashMap<>();
         Map auxDataMapDI = (Map) blockArray.getDataItems().get(3);
         for (DataItem txIdDI: auxDataMapDI.getKeys()) {
             if (txIdDI == SimpleValue.BREAK)
                 continue;
+            int txIndex = toInt(txIdDI);
             AuxData auxData = AuxDataSerializer.INSTANCE.deserializeDI(auxDataMapDI.get(txIdDI));
-            auxDataMap.put(toInt(txIdDI), auxData);
+            if (YaciConfig.INSTANCE.isReturnFullTxCbor()) {
+                byte[] auxBytes = auxDataRawBytes.get(txIndex);
+                if (auxBytes != null && isAuxDataHashValid(blockHeader.getHeaderBody().getBlockNumber(),
+                        txIndex, txnBodies, auxBytes)) {
+                    auxData = auxData.toBuilder()
+                            .cbor(HexUtil.encodeHexString(auxBytes))
+                            .build();
+                } else if (auxBytes == null) {
+                    log.debug("Missing raw auxiliary data bytes for block: {}, tx index: {}",
+                            blockHeader.getHeaderBody().getBlockNumber(), txIndex);
+                }
+            }
+            auxDataMap.put(txIndex, auxData);
         }
         blockBuilder.auxiliaryDataMap(auxDataMap);
 
@@ -125,9 +163,13 @@ private Block deserializeBlock(DataItem di, byte[] blockBody) {
     }
 
     @SneakyThrows
-    private void handleWitnessDatumRedeemer(long block, List<Witnesses> witnesses, byte[] rawBlockBytes) {
+    private void handleWitnessDatumRedeemer(long block, List<Witnesses> witnesses, List<byte[]> transactionWitness) {
         if (witnesses != null && !witnesses.isEmpty()) {
-            final List<byte[]> transactionWitness = WitnessUtil.getWitnessRawData(rawBlockBytes);
+            if (transactionWitness == null || transactionWitness.size() != witnesses.size()) {
+                log.error("block: {} witness set count mismatch. parsed: {}, raw: {}",
+                        block, witnesses.size(), transactionWitness == null ? null : transactionWitness.size());
+                return;
+            }
 
             for (int witnessIndex = 0; witnessIndex < transactionWitness.size(); witnessIndex++) {
 
@@ -280,4 +322,40 @@ private void handleWitnessDatumRedeemer(long block, List<Witnesses> witnesses, b
             }
         }
     }
+
+    private void setWitnessCbor(long block, List<Witnesses> witnesses, List<byte[]> transactionWitness) {
+        if (witnesses == null || transactionWitness == null || witnesses.size() != transactionWitness.size()) {
+            log.error("block: {} witness set count mismatch. full transaction cbor will not be available. parsed: {}, raw: {}",
+                    block, witnesses == null ? null : witnesses.size(),
+                    transactionWitness == null ? null : transactionWitness.size());
+            return;
+        }
+
+        for (int i = 0; i < witnesses.size(); i++) {
+            witnesses.set(i, witnesses.get(i).toBuilder()
+                    .cbor(HexUtil.encodeHexString(transactionWitness.get(i)))
+                    .build());
+        }
+    }
+
+    private boolean isAuxDataHashValid(long block, int txIndex, List<TransactionBody> txnBodies, byte[] auxBytes) {
+        if (txIndex >= txnBodies.size()) {
+            log.debug("Auxiliary data index outside transaction body list. block: {}, tx index: {}", block, txIndex);
+            return false;
+        }
+
+        String expectedHash = txnBodies.get(txIndex).getAuxiliaryDataHash();
+        if (expectedHash == null) {
+            return true;
+        }
+
+        String actualHash = Datum.cborToHash(auxBytes);
+        if (!expectedHash.equals(actualHash)) {
+            log.debug("Auxiliary data hash mismatch : {} - {} - {} - {}",
+                    block, txIndex, expectedHash, actualHash);
+            return false;
+        }
+
+        return true;
+    }
 }

core/src/main/java/com/bloxbean/cardano/yaci/core/model/serializers/TransactionBodySerializer.java

@@ -36,7 +36,7 @@ public TransactionBody deserializeDI(DataItem di, byte[] txBytes) {
         String txHash = TxUtil.calculateTxHash(txBytes);
         transactionBodyBuilder.txHash(txHash);
 
-        if (YaciConfig.INSTANCE.isReturnTxBodyCbor()) {
+        if (YaciConfig.INSTANCE.isReturnTxBodyCbor() || YaciConfig.INSTANCE.isReturnFullTxCbor()) {
             transactionBodyBuilder.cbor(HexUtil.encodeHexString(txBytes));
         }
 

core/src/main/java/com/bloxbean/cardano/yaci/core/model/serializers/util/AuxDataExtractor.java

@@ -0,0 +1,89 @@
+package com.bloxbean.cardano.yaci.core.model.serializers.util;
+
+import co.nstant.in.cbor.CborDecoder;
+import co.nstant.in.cbor.CborException;
+import co.nstant.in.cbor.model.DataItem;
+import co.nstant.in.cbor.model.Special;
+import com.bloxbean.cardano.yaci.core.util.Tuple;
+import lombok.SneakyThrows;
+
+import java.io.ByteArrayInputStream;
+import java.util.LinkedHashMap;
+import java.util.Map;
+
+import static com.bloxbean.cardano.yaci.core.model.serializers.util.TransactionBodyExtractor.getLength;
+import static com.bloxbean.cardano.yaci.core.model.serializers.util.TransactionBodyExtractor.getSymbolBytes;
+import static com.bloxbean.cardano.yaci.core.util.CborSerializationUtil.toInt;
+
+public class AuxDataExtractor {
+
+    @SneakyThrows
+    public static Map<Integer, byte[]> getAuxDataFromBlock(byte[] blockBytes) {
+        Map<Integer, byte[]> auxDataMap = new LinkedHashMap<>();
+        ByteArrayInputStream stream = new ByteArrayInputStream(blockBytes);
+        CborDecoder decoder = new CborDecoder(stream);
+
+        // era and block body array header
+        stream.read();
+        decoder.decodeNext();
+        stream.read();
+
+        decoder.decodeNext(); // header
+        decoder.decodeNext(); // transaction bodies
+        decoder.decodeNext(); // transaction witness sets
+
+        int auxDataMapByte = stream.read();
+        long auxDataEntryCount = getLength(auxDataMapByte,
+                getSymbolBytes(blockBytes.length - stream.available(), blockBytes));
+
+        int extraBytes = WitnessUtil.skipBytes(auxDataMapByte);
+        if (extraBytes > 0) {
+            stream.skip(extraBytes);
+        }
+
+        if (auxDataEntryCount != TransactionBodyExtractor.INFINITY) {
+            for (int i = 0; i < auxDataEntryCount; i++) {
+                Tuple<Integer, byte[]> auxDataEntry = readAuxDataEntry(blockBytes, stream, decoder);
+                auxDataMap.put(auxDataEntry._1, auxDataEntry._2);
+            }
+        } else {
+            for (;;) {
+                DataItem txIndexDI = decoder.decodeNext();
+                if (txIndexDI == null) {
+                    throw new CborException("Unexpected end of stream");
+                }
+
+                if (Special.BREAK.equals(txIndexDI)) {
+                    break;
+                }
+
+                int txIndex = toInt(txIndexDI);
+                byte[] auxDataBytes = readNextRawValue(blockBytes, stream, decoder);
+                auxDataMap.put(txIndex, auxDataBytes);
+            }
+        }
+
+        return auxDataMap;
+    }
+
+    private static Tuple<Integer, byte[]> readAuxDataEntry(byte[] blockBytes,
+                                                           ByteArrayInputStream stream,
+                                                           CborDecoder decoder) throws CborException {
+        DataItem txIndexDI = decoder.decodeNext();
+        int txIndex = toInt(txIndexDI);
+        return new Tuple<>(txIndex, readNextRawValue(blockBytes, stream, decoder));
+    }
+
+    private static byte[] readNextRawValue(byte[] blockBytes,
+                                           ByteArrayInputStream stream,
+                                           CborDecoder decoder) throws CborException {
+        int start = blockBytes.length - stream.available();
+        int previous = stream.available();
+        decoder.decodeNext();
+        int current = stream.available();
+
+        byte[] rawValue = new byte[previous - current];
+        System.arraycopy(blockBytes, start, rawValue, 0, rawValue.length);
+        return rawValue;
+    }
+}

core/src/main/java/com/bloxbean/cardano/yaci/core/model/serializers/util/WitnessUtil.java

@@ -55,15 +55,35 @@ public static List<byte[]> getWitnessRawData(byte[] blockBytes) throws CborExcep
 
         List<byte[]> witnessList = new ArrayList<>();
 
-        for (int i = 0; i < witnessElementCount; i++) {
-            final var start = blockBytes.length - stream.available(); // start cutting position
-            // find witness byte array length
-            final var previous = stream.available();
-            decoder.decodeNext();
-            final var current = stream.available();
-            final byte[] witness = new byte[previous - current];
-            System.arraycopy(blockBytes, start, witness, 0, witness.length);
-            witnessList.add(witness);
+        if (witnessElementCount != TransactionBodyExtractor.INFINITY) {
+            for (int i = 0; i < witnessElementCount; i++) {
+                final var start = blockBytes.length - stream.available(); // start cutting position
+                // find witness byte array length
+                final var previous = stream.available();
+                decoder.decodeNext();
+                final var current = stream.available();
+                final byte[] witness = new byte[previous - current];
+                System.arraycopy(blockBytes, start, witness, 0, witness.length);
+                witnessList.add(witness);
+            }
+        } else {
+            for (;;) {
+                final var start = blockBytes.length - stream.available();
+                final var previous = stream.available();
+                final var dataItem = decoder.decodeNext();
+                if (dataItem == null) {
+                    throw new CborException("Unexpected end of stream");
+                }
+
+                if (Special.BREAK.equals(dataItem)) {
+                    break;
+                }
+
+                final var current = stream.available();
+                final byte[] witness = new byte[previous - current];
+                System.arraycopy(blockBytes, start, witness, 0, witness.length);
+                witnessList.add(witness);
+            }
         }
 
         return witnessList;