From 66e8919f33b62223a26db9073b781821b8d249f4 Mon Sep 17 00:00:00 2001
From: szsteven008 <97944818+szsteven008@users.noreply.github.com>
Date: Tue, 31 Dec 2024 10:55:06 +0800
Subject: [PATCH 01/13] Add files via upload
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

增加了onnx的输出
---
 export.py | 198 ++++++++++++++++++++++++++++++++++++++++++++++++++++++
 sample.py |  26 +++++++
 2 files changed, 224 insertions(+)
 create mode 100644 export.py
 create mode 100644 sample.py

diff --git a/export.py b/export.py
new file mode 100644
index 00000000..c8a65e8f
--- /dev/null
+++ b/export.py
@@ -0,0 +1,198 @@
+import argparse
+import os
+import torch
+import cv2
+import numpy as np
+from PIL import Image
+from typing import Tuple, List
+from torchvision.ops import box_convert
+import onnx
+import onnxruntime as ort
+
+from groundingdino.util.inference import load_model, annotate
+import groundingdino.datasets.transforms as T
+from groundingdino.util.utils import get_phrases_from_posmap
+
+class Model(torch.nn.Module):
+    def __init__(
+        self,
+        model_config_path: str,
+        model_checkpoint_path: str,
+        device: str = "cuda"
+    ):
+        super().__init__()
+        self.model = load_model(
+            model_config_path=model_config_path,
+            model_checkpoint_path=model_checkpoint_path,
+            device=device
+        ).to(device)
+        self.tokenizer = self.model.tokenizer
+
+#    def forward(self, samples: NestedTensor, targets: List = None, **kw):
+    def forward(self, 
+                image: torch.Tensor, 
+                input_ids: torch.Tensor, 
+                box_threshold: torch.Tensor, 
+                text_threshold: torch.Tensor,
+                **kw):
+        token_type_ids = torch.zeros(input_ids.size(), dtype=torch.int32)
+        attention_mask = (input_ids != 0).int()
+        
+        outputs = self.model(image, input_ids, attention_mask, token_type_ids)
+        prediction_logits = outputs["pred_logits"].sigmoid().squeeze(0)
+        prediction_boxes = outputs["pred_boxes"].squeeze(0)
+
+        mask = prediction_logits.max(dim=1)[0] > box_threshold
+        prediction_logits = prediction_logits[mask]
+        prediction_input_ids_mask = prediction_logits > text_threshold
+        prediction_boxes = prediction_boxes[mask]
+
+        return prediction_logits.max(dim=1)[0].unsqueeze(0), prediction_boxes.unsqueeze(0), prediction_input_ids_mask.unsqueeze(0)
+
+def preprocess_image(image_bgr: np.ndarray) -> torch.Tensor:
+    image_bgr = cv2.resize(image_bgr, (800, 800))
+    transform = T.Compose(
+        [
+            T.RandomResize([800], max_size=1333),
+            T.ToTensor(),
+            T.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225]),
+        ]
+    )
+    image_pillow = Image.fromarray(cv2.cvtColor(image_bgr, cv2.COLOR_BGR2RGB))
+    image_transformed, _ = transform(image_pillow, None)
+    return image_transformed
+
+def preprocess_caption(caption: str) -> str:
+    result = caption.lower().strip()
+    if result.endswith("."):
+        return result
+    return result + "."
+
+def export_onnx(model, output_dir):
+    onnx_file = output_dir + "/" + "gdino.onnx"
+    caption = preprocess_caption("watermark")
+    tokenized = model.tokenizer(caption, padding="longest", return_tensors="pt")
+    box_threshold = torch.tensor(0.35, dtype=torch.float32)
+    text_threshold = torch.tensor(0.25, dtype=torch.float32)
+
+    torch.onnx.export(
+        model,
+        args = (
+            torch.rand(1, 3, 800, 800).type(torch.float32).to("cpu"), 
+            tokenized["input_ids"], 
+            box_threshold, 
+            text_threshold),
+        f = onnx_file,
+        input_names = [ "image", "input_ids", "box_threshold", "text_threshold" ],
+        output_names = [ "logits", "boxes", "masks" ], 
+        opset_version = 17, 
+        export_params = True, 
+        do_constant_folding = True,
+        dynamic_axes = {
+            "input_ids": { 1: "token_num" }
+        },
+    )
+
+    print("export onnx ok!")
+
+    onnx_model = onnx.load(onnx_file)
+    onnx.checker.check_model(onnx_model)
+    print("check model ok!")
+
+def inference(model):
+    image = cv2.imread('asset/cat_dog.jpeg')
+    processed_image = preprocess_image(image).unsqueeze(0)
+    caption = preprocess_caption("cat . dog")
+    tokenized = model.tokenizer(caption, padding="longest", return_tensors="pt")
+    box_threshold = torch.tensor(0.35, dtype=torch.float32)
+    text_threshold = torch.tensor(0.25, dtype=torch.float32)
+
+    outputs = model(processed_image, 
+                    tokenized["input_ids"], 
+                    box_threshold, 
+                    text_threshold)
+
+    prediction_logits = outputs[0] 
+    prediction_boxes = outputs[1] 
+    prediction_masks = outputs[2] 
+
+    input_ids = tokenized["input_ids"][0].tolist()
+    phrases = []
+    for mask in prediction_masks[0]:
+        prediction_token_ids = [input_ids[i] for i in mask.nonzero(as_tuple=True)[0].tolist()]
+        phrases.append(model.tokenizer.decode(prediction_token_ids).replace('.', ''))
+
+    with torch.no_grad():
+        image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
+        image = annotate(image, prediction_boxes[0], prediction_logits[0], phrases)
+
+    cv2.imshow("image", image)
+    cv2.waitKey()
+    cv2.destroyAllWindows()
+
+def inference_onnx(output_dir):
+    onnx_file = output_dir + "/" + "gdino.onnx"
+    session = ort.InferenceSession(onnx_file)
+
+    image = cv2.imread('asset/cat_dog.jpeg')
+    processed_image = preprocess_image(image).unsqueeze(0)
+    caption = preprocess_caption("dog.cat")
+    tokenized = model.tokenizer(caption, padding="longest", return_tensors="pt")
+    box_threshold = torch.tensor(0.35, dtype=torch.float32)
+    text_threshold = torch.tensor(0.25, dtype=torch.float32)
+
+    outputs = session.run(None, {
+        "image": processed_image.numpy().astype(np.float32) ,
+        "input_ids": tokenized["input_ids"].numpy().astype(np.int64) , 
+        "box_threshold": box_threshold.numpy().astype(np.float32) ,
+        "text_threshold": text_threshold.numpy().astype(np.float32) 
+    })
+
+    prediction_logits = torch.from_numpy(outputs[0]) 
+    prediction_boxes = torch.from_numpy(outputs[1]) 
+    prediction_masks = torch.from_numpy(outputs[2]) 
+
+    input_ids = tokenized["input_ids"][0].tolist()
+    phrases = []
+    for mask in prediction_masks[0]:
+        prediction_token_ids = [input_ids[i] for i in mask.nonzero(as_tuple=True)[0].tolist()]
+        phrases.append(model.tokenizer.decode(prediction_token_ids).replace('.', ''))
+
+    with torch.no_grad():
+        image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
+        image = annotate(image, prediction_boxes[0], prediction_logits[0], phrases)
+
+    cv2.imshow("image", image)
+    cv2.waitKey()
+    cv2.destroyAllWindows()
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser("Export Grounding DINO Model to IR", add_help=True)
+    parser.add_argument("--test", "-t", help="test onnx model", action="store_true")
+    parser.add_argument("--orig", "-n", help="test model", action="store_true")
+    parser.add_argument("--config_file", "-c", type=str, required=True, help="path to config file")
+    parser.add_argument(
+        "--checkpoint_path", "-p", type=str, required=True, help="path to checkpoint file"
+    )
+    parser.add_argument(
+        "--output_dir", "-o", type=str, default="outputs", required=True, help="output directory"
+    )
+
+    args = parser.parse_args()
+
+    # cfg
+    config_file = args.config_file  # change the path of the model config file
+    checkpoint_path = args.checkpoint_path  # change the path of the model
+    output_dir = args.output_dir
+    
+    # make dir
+    os.makedirs(output_dir, exist_ok=True)
+
+    model = Model(config_file, checkpoint_path, device='cpu')
+
+    if args.test:
+        inference_onnx(output_dir)
+    elif args.orig:
+        inference(model)
+    else:
+        export_onnx(model, output_dir)
diff --git a/sample.py b/sample.py
new file mode 100644
index 00000000..6c680eeb
--- /dev/null
+++ b/sample.py
@@ -0,0 +1,26 @@
+import numpy as np
+import cv2
+import supervision as sv
+
+from groundingdino.util.inference import Model, annotate
+
+image = cv2.imread("asset/cat_dog.jpeg")
+caption = "cat . dog"
+
+model = Model("groundingdino/config/GroundingDINO_SwinT_OGC.py", 
+              "weights/groundingdino_swint_ogc.pth", 
+              "cpu")
+
+detections, phrases = model.predict_with_caption(image, caption)
+
+labels = [ f"{phrase}" for phrase in phrases ]
+
+bbox_annotator = sv.BoxAnnotator(color_lookup=sv.ColorLookup.INDEX)
+label_annotator = sv.LabelAnnotator(color_lookup=sv.ColorLookup.INDEX)
+annotated_frame = cv2.cvtColor(image, cv2.COLOR_RGB2BGR)
+annotated_frame = bbox_annotator.annotate(scene=image, detections=detections)
+annotated_frame = label_annotator.annotate(scene=image, detections=detections, labels=labels)
+
+cv2.imshow("image", annotated_frame)
+cv2.waitKey()
+cv2.destroyAllWindows()

From 7598ea558b1e0e276eaddfe165aa059d12a291c7 Mon Sep 17 00:00:00 2001
From: szsteven008 <97944818+szsteven008@users.noreply.github.com>
Date: Tue, 31 Dec 2024 10:55:51 +0800
Subject: [PATCH 02/13] Add files via upload
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

onnx的配置
---
 groundingdino/config/GroundingDINO_SwinB_cfg.py | 4 ++--
 groundingdino/config/GroundingDINO_SwinT_OGC.py | 4 ++--
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/groundingdino/config/GroundingDINO_SwinB_cfg.py b/groundingdino/config/GroundingDINO_SwinB_cfg.py
index f490c4bb..320bf31f 100644
--- a/groundingdino/config/GroundingDINO_SwinB_cfg.py
+++ b/groundingdino/config/GroundingDINO_SwinB_cfg.py
@@ -34,8 +34,8 @@
 text_encoder_type = "bert-base-uncased"
 use_text_enhancer = True
 use_fusion_layer = True
-use_checkpoint = True
-use_transformer_ckpt = True
+use_checkpoint = False #True
+use_transformer_ckpt = False #True
 use_text_cross_attention = True
 text_dropout = 0.0
 fusion_dropout = 0.0
diff --git a/groundingdino/config/GroundingDINO_SwinT_OGC.py b/groundingdino/config/GroundingDINO_SwinT_OGC.py
index 9158d5f6..a1196a2b 100644
--- a/groundingdino/config/GroundingDINO_SwinT_OGC.py
+++ b/groundingdino/config/GroundingDINO_SwinT_OGC.py
@@ -34,8 +34,8 @@
 text_encoder_type = "bert-base-uncased"
 use_text_enhancer = True
 use_fusion_layer = True
-use_checkpoint = True
-use_transformer_ckpt = True
+use_checkpoint = False #True
+use_transformer_ckpt = False #True
 use_text_cross_attention = True
 text_dropout = 0.0
 fusion_dropout = 0.0

From f5b535e80daa7255ac341fac3f35ecaabd156a8a Mon Sep 17 00:00:00 2001
From: szsteven008 <97944818+szsteven008@users.noreply.github.com>
Date: Tue, 31 Dec 2024 10:56:52 +0800
Subject: [PATCH 03/13] Add files via upload

onnx
---
 groundingdino/util/inference.py | 13 +++++++++++--
 1 file changed, 11 insertions(+), 2 deletions(-)

diff --git a/groundingdino/util/inference.py b/groundingdino/util/inference.py
index 84a962e9..2711371a 100644
--- a/groundingdino/util/inference.py
+++ b/groundingdino/util/inference.py
@@ -64,8 +64,17 @@ def predict(
     model = model.to(device)
     image = image.to(device)
 
+    tokenizer = model.tokenizer
+    tokenized = tokenizer([caption], padding="longest", return_tensors="pt").to(
+        device
+    )
+
     with torch.no_grad():
-        outputs = model(image[None], captions=[caption])
+        outputs = model(image.unsqueeze(0), 
+                        input_ids=tokenized["input_ids"], 
+                        attention_mask=tokenized["attention_mask"], 
+                        token_type_ids=tokenized["token_type_ids"])
+#        outputs = model(image[None], captions=[caption])
 
     prediction_logits = outputs["pred_logits"].cpu().sigmoid()[0]  # prediction_logits.shape = (nq, 256)
     prediction_boxes = outputs["pred_boxes"].cpu()[0]  # prediction_boxes.shape = (nq, 4)
@@ -74,7 +83,7 @@ def predict(
     logits = prediction_logits[mask]  # logits.shape = (n, 256)
     boxes = prediction_boxes[mask]  # boxes.shape = (n, 4)
 
-    tokenizer = model.tokenizer
+#    tokenizer = model.tokenizer
     tokenized = tokenizer(caption)
     
     if remove_combined:

From 8bfaf941c8cab85fac84a65a0bb143316b54d329 Mon Sep 17 00:00:00 2001
From: szsteven008 <97944818+szsteven008@users.noreply.github.com>
Date: Tue, 31 Dec 2024 10:57:37 +0800
Subject: [PATCH 04/13] Add files via upload

onnx
---
 .../models/GroundingDINO/bertwarper.py        |  6 ++++--
 .../models/GroundingDINO/groundingdino.py     | 21 ++++++++++++++++---
 .../models/GroundingDINO/transformer.py       |  3 ++-
 3 files changed, 24 insertions(+), 6 deletions(-)

diff --git a/groundingdino/models/GroundingDINO/bertwarper.py b/groundingdino/models/GroundingDINO/bertwarper.py
index f0cf9779..84e60896 100644
--- a/groundingdino/models/GroundingDINO/bertwarper.py
+++ b/groundingdino/models/GroundingDINO/bertwarper.py
@@ -190,7 +190,8 @@ def generate_masks_with_special_tokens(tokenized, special_tokens_list, tokenizer
     # special_tokens_mask: bs, num_token. 1 for special tokens. 0 for normal tokens
     special_tokens_mask = torch.zeros((bs, num_token), device=input_ids.device).bool()
     for special_token in special_tokens_list:
-        special_tokens_mask |= input_ids == special_token
+        special_tokens_mask = torch.logical_or(special_tokens_mask, input_ids == special_token)
+        #special_tokens_mask |= input_ids == special_token
 
     # idxs: each row is a list of indices of special tokens
     idxs = torch.nonzero(special_tokens_mask)
@@ -234,7 +235,8 @@ def generate_masks_with_special_tokens_and_transfer_map(tokenized, special_token
     # special_tokens_mask: bs, num_token. 1 for special tokens. 0 for normal tokens
     special_tokens_mask = torch.zeros((bs, num_token), device=input_ids.device).bool()
     for special_token in special_tokens_list:
-        special_tokens_mask |= input_ids == special_token
+        special_tokens_mask = torch.logical_or(special_tokens_mask, input_ids == special_token)
+#        special_tokens_mask |= input_ids == special_token
 
     # idxs: each row is a list of indices of special tokens
     idxs = torch.nonzero(special_tokens_mask)
diff --git a/groundingdino/models/GroundingDINO/groundingdino.py b/groundingdino/models/GroundingDINO/groundingdino.py
index cd97028d..078b5399 100644
--- a/groundingdino/models/GroundingDINO/groundingdino.py
+++ b/groundingdino/models/GroundingDINO/groundingdino.py
@@ -19,7 +19,7 @@
 
 import torch
 import torch.nn.functional as F
-from torch import nn
+from torch import nn, Tensor
 from torchvision.ops.boxes import nms
 from transformers import AutoTokenizer, BertModel, BertTokenizer, RobertaModel, RobertaTokenizerFast
 
@@ -224,7 +224,13 @@ def set_image_features(self, features , poss):
     def init_ref_points(self, use_num_queries):
         self.refpoint_embed = nn.Embedding(use_num_queries, self.query_dim)
 
-    def forward(self, samples: NestedTensor, targets: List = None, **kw):
+#    def forward(self, samples: NestedTensor, targets: List = None, **kw):
+    def forward(self, 
+                samples: NestedTensor, 
+                input_ids: Tensor, 
+                attention_mask: Tensor, 
+                token_type_ids: Tensor, 
+                **kw):
         """The forward expects a NestedTensor, which consists of:
            - samples.tensor: batched images, of shape [batch_size x 3 x H x W]
            - samples.mask: a binary mask of shape [batch_size x H x W], containing 1 on padded pixels
@@ -239,6 +245,7 @@ def forward(self, samples: NestedTensor, targets: List = None, **kw):
            - "aux_outputs": Optional, only returned when auxilary losses are activated. It is a list of
                             dictionnaries containing the two above keys for each decoder layer.
         """
+        """
         if targets is None:
             captions = kw["captions"]
         else:
@@ -248,6 +255,13 @@ def forward(self, samples: NestedTensor, targets: List = None, **kw):
         tokenized = self.tokenizer(captions, padding="longest", return_tensors="pt").to(
             samples.device
         )
+        """
+        tokenized = {
+            "input_ids": input_ids,
+            "attention_mask": attention_mask,
+            "token_type_ids": token_type_ids,
+        }
+
         (
             text_self_attention_masks,
             position_ids,
@@ -277,7 +291,8 @@ def forward(self, samples: NestedTensor, targets: List = None, **kw):
         bert_output = self.bert(**tokenized_for_encoder)  # bs, 195, 768
 
         encoded_text = self.feat_map(bert_output["last_hidden_state"])  # bs, 195, d_model
-        text_token_mask = tokenized.attention_mask.bool()  # bs, 195
+        text_token_mask = tokenized["attention_mask"].bool()  # bs, 195
+#        text_token_mask = tokenizedattention_mask.bool()  # bs, 195
         # text_token_mask: True for nomask, False for mask
         # text_self_attention_masks: True for nomask, False for mask
 
diff --git a/groundingdino/models/GroundingDINO/transformer.py b/groundingdino/models/GroundingDINO/transformer.py
index fcb8742d..c04d4409 100644
--- a/groundingdino/models/GroundingDINO/transformer.py
+++ b/groundingdino/models/GroundingDINO/transformer.py
@@ -859,7 +859,8 @@ def with_pos_embed(tensor, pos):
         return tensor if pos is None else tensor + pos
 
     def forward_ffn(self, tgt):
-        with torch.cuda.amp.autocast(enabled=False):
+#        with torch.cuda.amp.autocast(enabled=False):
+        with torch.amp.autocast(str(tgt.device), enabled=False):
             tgt2 = self.linear2(self.dropout3(self.activation(self.linear1(tgt))))
         tgt = tgt + self.dropout4(tgt2)
         tgt = self.norm3(tgt)

From 17f37607dc99a40a2a2841f3c5adf2b42774cf66 Mon Sep 17 00:00:00 2001
From: szsteven008 <97944818+szsteven008@users.noreply.github.com>
Date: Tue, 31 Dec 2024 15:45:30 +0800
Subject: [PATCH 05/13] Add files via upload
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

只支持单个label
---
 export.py | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/export.py b/export.py
index c8a65e8f..99d07d3a 100644
--- a/export.py
+++ b/export.py
@@ -70,7 +70,7 @@ def preprocess_caption(caption: str) -> str:
 
 def export_onnx(model, output_dir):
     onnx_file = output_dir + "/" + "gdino.onnx"
-    caption = preprocess_caption("watermark")
+    caption = preprocess_caption(".")
     tokenized = model.tokenizer(caption, padding="longest", return_tensors="pt")
     box_threshold = torch.tensor(0.35, dtype=torch.float32)
     text_threshold = torch.tensor(0.25, dtype=torch.float32)
@@ -100,9 +100,9 @@ def export_onnx(model, output_dir):
     print("check model ok!")
 
 def inference(model):
-    image = cv2.imread('asset/cat_dog.jpeg')
+    image = cv2.imread('asset/1.jpg')
     processed_image = preprocess_image(image).unsqueeze(0)
-    caption = preprocess_caption("cat . dog")
+    caption = preprocess_caption("watermark")
     tokenized = model.tokenizer(caption, padding="longest", return_tensors="pt")
     box_threshold = torch.tensor(0.35, dtype=torch.float32)
     text_threshold = torch.tensor(0.25, dtype=torch.float32)
@@ -134,9 +134,9 @@ def inference_onnx(output_dir):
     onnx_file = output_dir + "/" + "gdino.onnx"
     session = ort.InferenceSession(onnx_file)
 
-    image = cv2.imread('asset/cat_dog.jpeg')
+    image = cv2.imread('asset/1.jpg')
     processed_image = preprocess_image(image).unsqueeze(0)
-    caption = preprocess_caption("dog.cat")
+    caption = preprocess_caption("watermark")
     tokenized = model.tokenizer(caption, padding="longest", return_tensors="pt")
     box_threshold = torch.tensor(0.35, dtype=torch.float32)
     text_threshold = torch.tensor(0.25, dtype=torch.float32)

From 89b1ad20b9ff4878b0084da45bb48932f2d03fdd Mon Sep 17 00:00:00 2001
From: szsteven008 <97944818+szsteven008@users.noreply.github.com>
Date: Wed, 1 Jan 2025 16:21:57 +0800
Subject: [PATCH 06/13] Add files via upload

update
---
 export.py | 83 ++++++++++++++++++++++++++++++++++++++++++++++---------
 1 file changed, 70 insertions(+), 13 deletions(-)

diff --git a/export.py b/export.py
index 99d07d3a..be4554f8 100644
--- a/export.py
+++ b/export.py
@@ -12,6 +12,7 @@
 from groundingdino.util.inference import load_model, annotate
 import groundingdino.datasets.transforms as T
 from groundingdino.util.utils import get_phrases_from_posmap
+from groundingdino.models.GroundingDINO.bertwarper import generate_masks_with_special_tokens_and_transfer_map
 
 class Model(torch.nn.Module):
     def __init__(
@@ -27,18 +28,21 @@ def __init__(
             device=device
         ).to(device)
         self.tokenizer = self.model.tokenizer
+        self.specical_tokens = self.model.specical_tokens
+        self.max_text_len = self.model.max_text_len
 
 #    def forward(self, samples: NestedTensor, targets: List = None, **kw):
     def forward(self, 
                 image: torch.Tensor, 
                 input_ids: torch.Tensor, 
+                attention_mask: torch.Tensor, 
+                token_type_ids: torch.Tensor, 
+                position_ids: torch.Tensor, 
+                text_self_attention_masks: torch.Tensor, 
                 box_threshold: torch.Tensor, 
                 text_threshold: torch.Tensor,
                 **kw):
-        token_type_ids = torch.zeros(input_ids.size(), dtype=torch.int32)
-        attention_mask = (input_ids != 0).int()
-        
-        outputs = self.model(image, input_ids, attention_mask, token_type_ids)
+        outputs = self.model(image, input_ids, attention_mask, token_type_ids, position_ids, text_self_attention_masks)
         prediction_logits = outputs["pred_logits"].sigmoid().squeeze(0)
         prediction_boxes = outputs["pred_boxes"].squeeze(0)
 
@@ -70,26 +74,43 @@ def preprocess_caption(caption: str) -> str:
 
 def export_onnx(model, output_dir):
     onnx_file = output_dir + "/" + "gdino.onnx"
-    caption = preprocess_caption(".")
+    caption = preprocess_caption("watermark")
     tokenized = model.tokenizer(caption, padding="longest", return_tensors="pt")
     box_threshold = torch.tensor(0.35, dtype=torch.float32)
     text_threshold = torch.tensor(0.25, dtype=torch.float32)
 
+    specical_tokens = model.specical_tokens
+    (
+        text_self_attention_masks,
+        position_ids,
+        _,
+    ) = generate_masks_with_special_tokens_and_transfer_map(
+        tokenized, specical_tokens, model.tokenizer
+    )
+
     torch.onnx.export(
         model,
         args = (
             torch.rand(1, 3, 800, 800).type(torch.float32).to("cpu"), 
-            tokenized["input_ids"], 
+            tokenized["input_ids"].type(torch.int).to("cpu"), 
+            tokenized["attention_mask"].type(torch.uint8).to("cpu"), 
+            tokenized["token_type_ids"].type(torch.int).to("cpu"), 
+            position_ids.type(torch.int).to("cpu"), 
+            text_self_attention_masks.type(torch.bool).to("cpu"), 
             box_threshold, 
             text_threshold),
         f = onnx_file,
-        input_names = [ "image", "input_ids", "box_threshold", "text_threshold" ],
+        input_names = [ "image", "input_ids", "attention_mask", "token_type_ids", "position_ids", "text_self_attention_masks", "box_threshold", "text_threshold" ],
         output_names = [ "logits", "boxes", "masks" ], 
         opset_version = 17, 
         export_params = True, 
         do_constant_folding = True,
         dynamic_axes = {
-            "input_ids": { 1: "token_num" }
+            "input_ids": { 1: "token_num" },
+            "attention_mask": { 1: "token_num" },
+            "token_type_ids": { 1: "token_num" },
+            "position_ids": { 1: "token_num" },
+            "text_self_attention_masks": { 1: "token_num", 2: "token_num" }
         },
     )
 
@@ -100,15 +121,28 @@ def export_onnx(model, output_dir):
     print("check model ok!")
 
 def inference(model):
-    image = cv2.imread('asset/1.jpg')
+    image = cv2.imread('asset/cat_dog.jpeg')
     processed_image = preprocess_image(image).unsqueeze(0)
-    caption = preprocess_caption("watermark")
+    caption = preprocess_caption("cat. dog")
     tokenized = model.tokenizer(caption, padding="longest", return_tensors="pt")
     box_threshold = torch.tensor(0.35, dtype=torch.float32)
     text_threshold = torch.tensor(0.25, dtype=torch.float32)
 
+    specical_tokens = model.specical_tokens
+    (
+        text_self_attention_masks,
+        position_ids,
+        _,
+    ) = generate_masks_with_special_tokens_and_transfer_map(
+        tokenized, specical_tokens, model.tokenizer
+    )
+
     outputs = model(processed_image, 
                     tokenized["input_ids"], 
+                    tokenized["attention_mask"], 
+                    tokenized["token_type_ids"], 
+                    position_ids, 
+                    text_self_attention_masks, 
                     box_threshold, 
                     text_threshold)
 
@@ -130,7 +164,7 @@ def inference(model):
     cv2.waitKey()
     cv2.destroyAllWindows()
 
-def inference_onnx(output_dir):
+def inference_onnx(model, output_dir):
     onnx_file = output_dir + "/" + "gdino.onnx"
     session = ort.InferenceSession(onnx_file)
 
@@ -141,9 +175,32 @@ def inference_onnx(output_dir):
     box_threshold = torch.tensor(0.35, dtype=torch.float32)
     text_threshold = torch.tensor(0.25, dtype=torch.float32)
 
+    specical_tokens = model.specical_tokens
+    (
+        text_self_attention_masks,
+        position_ids,
+        _,
+    ) = generate_masks_with_special_tokens_and_transfer_map(
+        tokenized, specical_tokens, model.tokenizer
+    )
+
+    max_text_len = model.max_text_len
+    if text_self_attention_masks.shape[1] > max_text_len:
+        text_self_attention_masks = text_self_attention_masks[
+            :, : max_text_len, : max_text_len
+        ]
+        position_ids = position_ids[:, : max_text_len]
+        tokenized["input_ids"] = tokenized["input_ids"][:, : max_text_len]
+        tokenized["attention_mask"] = tokenized["attention_mask"][:, : max_text_len]
+        tokenized["token_type_ids"] = tokenized["token_type_ids"][:, : max_text_len]
+
     outputs = session.run(None, {
         "image": processed_image.numpy().astype(np.float32) ,
-        "input_ids": tokenized["input_ids"].numpy().astype(np.int64) , 
+        "input_ids": tokenized["input_ids"].numpy().astype(np.int32) , 
+        "attention_mask": tokenized["attention_mask"].numpy().astype(np.uint8) , 
+        "token_type_ids": tokenized["token_type_ids"].numpy().astype(np.int32) , 
+        "position_ids": position_ids.numpy().astype(np.int32) , 
+        "text_self_attention_masks": text_self_attention_masks.numpy().astype(np.bool) , 
         "box_threshold": box_threshold.numpy().astype(np.float32) ,
         "text_threshold": text_threshold.numpy().astype(np.float32) 
     })
@@ -191,7 +248,7 @@ def inference_onnx(output_dir):
     model = Model(config_file, checkpoint_path, device='cpu')
 
     if args.test:
-        inference_onnx(output_dir)
+        inference_onnx(model, output_dir)
     elif args.orig:
         inference(model)
     else:

From bd929612c4d0073053891dd6bf2cbc7fbfb7c173 Mon Sep 17 00:00:00 2001
From: szsteven008 <97944818+szsteven008@users.noreply.github.com>
Date: Wed, 1 Jan 2025 16:23:11 +0800
Subject: [PATCH 07/13] Add files via upload

update
---
 .../models/GroundingDINO/groundingdino.py     | 36 ++++++++++---------
 1 file changed, 19 insertions(+), 17 deletions(-)

diff --git a/groundingdino/models/GroundingDINO/groundingdino.py b/groundingdino/models/GroundingDINO/groundingdino.py
index 078b5399..6f3a18a1 100644
--- a/groundingdino/models/GroundingDINO/groundingdino.py
+++ b/groundingdino/models/GroundingDINO/groundingdino.py
@@ -230,6 +230,8 @@ def forward(self,
                 input_ids: Tensor, 
                 attention_mask: Tensor, 
                 token_type_ids: Tensor, 
+                position_ids: Tensor, 
+                text_self_attention_masks: Tensor, 
                 **kw):
         """The forward expects a NestedTensor, which consists of:
            - samples.tensor: batched images, of shape [batch_size x 3 x H x W]
@@ -262,22 +264,22 @@ def forward(self,
             "token_type_ids": token_type_ids,
         }
 
-        (
-            text_self_attention_masks,
-            position_ids,
-            cate_to_token_mask_list,
-        ) = generate_masks_with_special_tokens_and_transfer_map(
-            tokenized, self.specical_tokens, self.tokenizer
-        )
-
-        if text_self_attention_masks.shape[1] > self.max_text_len:
-            text_self_attention_masks = text_self_attention_masks[
-                :, : self.max_text_len, : self.max_text_len
-            ]
-            position_ids = position_ids[:, : self.max_text_len]
-            tokenized["input_ids"] = tokenized["input_ids"][:, : self.max_text_len]
-            tokenized["attention_mask"] = tokenized["attention_mask"][:, : self.max_text_len]
-            tokenized["token_type_ids"] = tokenized["token_type_ids"][:, : self.max_text_len]
+#        (
+#            text_self_attention_masks,
+#            position_ids,
+#            cate_to_token_mask_list,
+#        ) = generate_masks_with_special_tokens_and_transfer_map(
+#            tokenized, self.specical_tokens, self.tokenizer
+#        )
+
+#        if text_self_attention_masks.shape[1] > self.max_text_len:
+#            text_self_attention_masks = text_self_attention_masks[
+#                :, : self.max_text_len, : self.max_text_len
+#            ]
+#            position_ids = position_ids[:, : self.max_text_len]
+#            tokenized["input_ids"] = tokenized["input_ids"][:, : self.max_text_len]
+#            tokenized["attention_mask"] = tokenized["attention_mask"][:, : self.max_text_len]
+#            tokenized["token_type_ids"] = tokenized["token_type_ids"][:, : self.max_text_len]
 
         # extract text embeddings
         if self.sub_sentence_present:
@@ -292,7 +294,7 @@ def forward(self,
 
         encoded_text = self.feat_map(bert_output["last_hidden_state"])  # bs, 195, d_model
         text_token_mask = tokenized["attention_mask"].bool()  # bs, 195
-#        text_token_mask = tokenizedattention_mask.bool()  # bs, 195
+#        text_token_mask = tokenized.attention_mask.bool()  # bs, 195
         # text_token_mask: True for nomask, False for mask
         # text_self_attention_masks: True for nomask, False for mask
 

From 1ea8cfae1be68e8fddcea19e9d797e5e3a46f50f Mon Sep 17 00:00:00 2001
From: szsteven008 <97944818+szsteven008@users.noreply.github.com>
Date: Wed, 1 Jan 2025 16:24:09 +0800
Subject: [PATCH 08/13] Add files via upload

update
---
 groundingdino/util/inference.py | 24 +++++++++++++++++++++++-
 1 file changed, 23 insertions(+), 1 deletion(-)

diff --git a/groundingdino/util/inference.py b/groundingdino/util/inference.py
index 2711371a..98e6c6d1 100644
--- a/groundingdino/util/inference.py
+++ b/groundingdino/util/inference.py
@@ -13,6 +13,7 @@
 from groundingdino.util.misc import clean_state_dict
 from groundingdino.util.slconfig import SLConfig
 from groundingdino.util.utils import get_phrases_from_posmap
+from groundingdino.models.GroundingDINO.bertwarper import generate_masks_with_special_tokens_and_transfer_map
 
 # ----------------------------------------------------------------------------------------------------------------------
 # OLD API
@@ -69,11 +70,32 @@ def predict(
         device
     )
 
+    specical_tokens = model.specical_tokens
+    (
+        text_self_attention_masks,
+        position_ids,
+        _,
+    ) = generate_masks_with_special_tokens_and_transfer_map(
+        tokenized, specical_tokens, tokenizer
+    )
+
+    max_text_len = model.max_text_len
+    if text_self_attention_masks.shape[1] > max_text_len:
+        text_self_attention_masks = text_self_attention_masks[
+            :, : max_text_len, : max_text_len
+        ]
+        position_ids = position_ids[:, : max_text_len]
+        tokenized["input_ids"] = tokenized["input_ids"][:, : max_text_len]
+        tokenized["attention_mask"] = tokenized["attention_mask"][:, : max_text_len]
+        tokenized["token_type_ids"] = tokenized["token_type_ids"][:, : max_text_len]
+
     with torch.no_grad():
         outputs = model(image.unsqueeze(0), 
                         input_ids=tokenized["input_ids"], 
                         attention_mask=tokenized["attention_mask"], 
-                        token_type_ids=tokenized["token_type_ids"])
+                        token_type_ids=tokenized["token_type_ids"], 
+                        position_ids = position_ids, 
+                        text_self_attention_masks = text_self_attention_masks)
 #        outputs = model(image[None], captions=[caption])
 
     prediction_logits = outputs["pred_logits"].cpu().sigmoid()[0]  # prediction_logits.shape = (nq, 256)

From dea15770331dad3216a547ea4302d57817070575 Mon Sep 17 00:00:00 2001
From: szsteven008 <97944818+szsteven008@users.noreply.github.com>
Date: Mon, 6 Jan 2025 16:12:54 +0800
Subject: [PATCH 09/13] Add files via upload
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

将onnx分成两个阶段,encode/decode
---
 export.py | 313 +++++++++++++++++++++++++++++-------------------------
 1 file changed, 170 insertions(+), 143 deletions(-)

diff --git a/export.py b/export.py
index be4554f8..a304f0f2 100644
--- a/export.py
+++ b/export.py
@@ -12,37 +12,57 @@
 from groundingdino.util.inference import load_model, annotate
 import groundingdino.datasets.transforms as T
 from groundingdino.util.utils import get_phrases_from_posmap
-from groundingdino.models.GroundingDINO.bertwarper import generate_masks_with_special_tokens_and_transfer_map
-
-class Model(torch.nn.Module):
-    def __init__(
-        self,
-        model_config_path: str,
-        model_checkpoint_path: str,
-        device: str = "cuda"
-    ):
+from groundingdino.models.GroundingDINO.bertwarper import generate_masks_with_special_tokens
+
+def preprocess_caption(caption: str) -> str:
+    result = caption.lower().strip()
+    if result.endswith("."):
+        return result
+    return result + "."
+
+class Encoder(torch.nn.Module):
+    def __init__(self, model):
         super().__init__()
-        self.model = load_model(
-            model_config_path=model_config_path,
-            model_checkpoint_path=model_checkpoint_path,
-            device=device
-        ).to(device)
-        self.tokenizer = self.model.tokenizer
-        self.specical_tokens = self.model.specical_tokens
-        self.max_text_len = self.model.max_text_len
-
-#    def forward(self, samples: NestedTensor, targets: List = None, **kw):
+        self.tokenizer = model.tokenizer
+        self.bert = model.bert
+        self.specical_tokens = model.specical_tokens
+
     def forward(self, 
-                image: torch.Tensor, 
                 input_ids: torch.Tensor, 
-                attention_mask: torch.Tensor, 
                 token_type_ids: torch.Tensor, 
+                text_self_attention_masks: torch.Tensor, 
+                position_ids: torch.Tensor):
+        # extract text embeddings
+        tokenized_for_encoder = {}
+        tokenized_for_encoder["input_ids"] = input_ids
+        tokenized_for_encoder["token_type_ids"] = token_type_ids
+        tokenized_for_encoder["attention_mask"] = text_self_attention_masks.type(torch.bool)
+        tokenized_for_encoder["position_ids"] = position_ids
+
+        bert_output = self.bert(**tokenized_for_encoder)  # bs, 195, 768
+
+        return bert_output["last_hidden_state"]
+
+class Decoder(torch.nn.Module):
+    def __init__(self, model):
+        super().__init__()
+        self.model = model
+        self.tokenizer = model.tokenizer
+        self.specical_tokens = model.specical_tokens
+
+    def forward(self, 
+                image: torch.Tensor, 
+                last_hidden_state: torch.Tensor, 
+                attention_mask: torch.Tensor, 
                 position_ids: torch.Tensor, 
                 text_self_attention_masks: torch.Tensor, 
                 box_threshold: torch.Tensor, 
-                text_threshold: torch.Tensor,
-                **kw):
-        outputs = self.model(image, input_ids, attention_mask, token_type_ids, position_ids, text_self_attention_masks)
+                text_threshold: torch.Tensor):
+        outputs = self.model(image, 
+                             last_hidden_state, 
+                             attention_mask, 
+                             position_ids, 
+                             text_self_attention_masks.type(torch.bool))
         prediction_logits = outputs["pred_logits"].sigmoid().squeeze(0)
         prediction_boxes = outputs["pred_boxes"].squeeze(0)
 
@@ -51,163 +71,164 @@ def forward(self,
         prediction_input_ids_mask = prediction_logits > text_threshold
         prediction_boxes = prediction_boxes[mask]
 
-        return prediction_logits.max(dim=1)[0].unsqueeze(0), prediction_boxes.unsqueeze(0), prediction_input_ids_mask.unsqueeze(0)
+        return (prediction_logits.max(dim=1)[0].unsqueeze(0), 
+                prediction_boxes.unsqueeze(0), 
+                prediction_input_ids_mask.unsqueeze(0))
 
-def preprocess_image(image_bgr: np.ndarray) -> torch.Tensor:
-    image_bgr = cv2.resize(image_bgr, (800, 800))
-    transform = T.Compose(
-        [
-            T.RandomResize([800], max_size=1333),
-            T.ToTensor(),
-            T.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225]),
-        ]
+def export_encoder(model, output):
+    onnx_file = output + "/" + "gdino.encoder.onnx"
+    caption = preprocess_caption("watermark")
+    tokenized = model.tokenizer(caption, padding="longest", return_tensors="pt")
+
+    (
+        text_self_attention_masks, 
+        position_ids
+    ) = generate_masks_with_special_tokens(tokenized, model.specical_tokens, model.tokenizer)
+
+    torch.onnx.export(
+        model,
+        args = (
+            tokenized["input_ids"].type(torch.int).to("cpu"), 
+            tokenized["token_type_ids"].type(torch.int).to("cpu"),
+            text_self_attention_masks.type(torch.uint8).to("cpu"),
+            position_ids.type(torch.int).to("cpu"),
+        ), 
+        f = onnx_file,
+        input_names = [ "input_ids", "token_type_ids", "text_self_attention_masks", "position_ids" ],
+        output_names = [ "last_hidden_state" ], 
+        opset_version = 17, 
+        export_params = True, 
+        do_constant_folding = True,
+        dynamic_axes = {
+            "input_ids": { 1: "token_num" },
+            "token_type_ids": { 1: "token_num" },
+            "text_self_attention_masks": { 1: "token_num", 2: "token_num" },
+            "position_ids": { 1: "token_num" },
+            "last_hidden_state": { 1: "token_num" }
+        },
     )
-    image_pillow = Image.fromarray(cv2.cvtColor(image_bgr, cv2.COLOR_BGR2RGB))
-    image_transformed, _ = transform(image_pillow, None)
-    return image_transformed
 
-def preprocess_caption(caption: str) -> str:
-    result = caption.lower().strip()
-    if result.endswith("."):
-        return result
-    return result + "."
+    print("export gdino.encoder.onnx ok!")
 
-def export_onnx(model, output_dir):
-    onnx_file = output_dir + "/" + "gdino.onnx"
+    onnx_model = onnx.load(onnx_file)
+    onnx.checker.check_model(onnx_model)
+    print("check gdino.encoder.onnx ok!")
+
+def export_decoder(model, output, encoder):
+    onnx_file = output + "/" + "gdino.decoder.onnx"
     caption = preprocess_caption("watermark")
-    tokenized = model.tokenizer(caption, padding="longest", return_tensors="pt")
+
+    tokenized, last_hidden_state = inference_encoder_onnx(encoder, output, caption)
+
     box_threshold = torch.tensor(0.35, dtype=torch.float32)
     text_threshold = torch.tensor(0.25, dtype=torch.float32)
 
-    specical_tokens = model.specical_tokens
-    (
-        text_self_attention_masks,
-        position_ids,
-        _,
-    ) = generate_masks_with_special_tokens_and_transfer_map(
-        tokenized, specical_tokens, model.tokenizer
-    )
-
     torch.onnx.export(
         model,
         args = (
             torch.rand(1, 3, 800, 800).type(torch.float32).to("cpu"), 
-            tokenized["input_ids"].type(torch.int).to("cpu"), 
+            last_hidden_state, 
             tokenized["attention_mask"].type(torch.uint8).to("cpu"), 
-            tokenized["token_type_ids"].type(torch.int).to("cpu"), 
-            position_ids.type(torch.int).to("cpu"), 
-            text_self_attention_masks.type(torch.bool).to("cpu"), 
+            tokenized["position_ids"].type(torch.int).to("cpu"), 
+            tokenized["text_self_attention_masks"].type(torch.uint8).to("cpu"), 
             box_threshold, 
             text_threshold),
         f = onnx_file,
-        input_names = [ "image", "input_ids", "attention_mask", "token_type_ids", "position_ids", "text_self_attention_masks", "box_threshold", "text_threshold" ],
+        input_names = [ "image", "last_hidden_state", "attention_mask", 
+                        "position_ids", "text_self_attention_masks", 
+                        "box_threshold", "text_threshold" ],
         output_names = [ "logits", "boxes", "masks" ], 
         opset_version = 17, 
         export_params = True, 
         do_constant_folding = True,
         dynamic_axes = {
-            "input_ids": { 1: "token_num" },
+            "last_hidden_state": { 1: "token_num" },
             "attention_mask": { 1: "token_num" },
-            "token_type_ids": { 1: "token_num" },
             "position_ids": { 1: "token_num" },
             "text_self_attention_masks": { 1: "token_num", 2: "token_num" }
         },
     )
 
-    print("export onnx ok!")
+    print("export gdino.decoder.onnx ok!")
 
     onnx_model = onnx.load(onnx_file)
     onnx.checker.check_model(onnx_model)
-    print("check model ok!")
+    print("check gdino.decoder.onnx ok!")
 
-def inference(model):
-    image = cv2.imread('asset/cat_dog.jpeg')
-    processed_image = preprocess_image(image).unsqueeze(0)
-    caption = preprocess_caption("cat. dog")
-    tokenized = model.tokenizer(caption, padding="longest", return_tensors="pt")
-    box_threshold = torch.tensor(0.35, dtype=torch.float32)
-    text_threshold = torch.tensor(0.25, dtype=torch.float32)
+def inference_encoder_onnx(model, output, caption: str = None):
+    onnx_file = output + "/" + "gdino.encoder.onnx"
+    session = ort.InferenceSession(onnx_file)
 
-    specical_tokens = model.specical_tokens
-    (
-        text_self_attention_masks,
-        position_ids,
-        _,
-    ) = generate_masks_with_special_tokens_and_transfer_map(
-        tokenized, specical_tokens, model.tokenizer
-    )
+    if caption:
+        proc_caption = preprocess_caption(caption)
+    else:
+        proc_caption = preprocess_caption("watermark. cat. dog")
+    tokenized = model.tokenizer(proc_caption, padding="longest", return_tensors="pt")
 
-    outputs = model(processed_image, 
-                    tokenized["input_ids"], 
-                    tokenized["attention_mask"], 
-                    tokenized["token_type_ids"], 
-                    position_ids, 
-                    text_self_attention_masks, 
-                    box_threshold, 
-                    text_threshold)
+    (
+        text_self_attention_masks, 
+        position_ids
+    ) = generate_masks_with_special_tokens(tokenized, model.specical_tokens, model.tokenizer)
 
-    prediction_logits = outputs[0] 
-    prediction_boxes = outputs[1] 
-    prediction_masks = outputs[2] 
+    tokenized["text_self_attention_masks"] = text_self_attention_masks
+    tokenized["position_ids"] = position_ids
 
-    input_ids = tokenized["input_ids"][0].tolist()
-    phrases = []
-    for mask in prediction_masks[0]:
-        prediction_token_ids = [input_ids[i] for i in mask.nonzero(as_tuple=True)[0].tolist()]
-        phrases.append(model.tokenizer.decode(prediction_token_ids).replace('.', ''))
+    outputs = session.run(None, {
+        "input_ids": tokenized["input_ids"].numpy().astype(np.int32), 
+        "token_type_ids": tokenized["token_type_ids"].numpy().astype(np.int32),
+        "text_self_attention_masks": tokenized["text_self_attention_masks"].numpy().astype(np.uint8),
+        "position_ids": tokenized["position_ids"].numpy().astype(np.int32)
+    })
 
-    with torch.no_grad():
-        image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
-        image = annotate(image, prediction_boxes[0], prediction_logits[0], phrases)
+    if caption == None:
+        print(outputs)
 
-    cv2.imshow("image", image)
-    cv2.waitKey()
-    cv2.destroyAllWindows()
+    last_hidden_state = torch.from_numpy(outputs[0]).type(torch.float32)
+    return tokenized, last_hidden_state
 
-def inference_onnx(model, output_dir):
-    onnx_file = output_dir + "/" + "gdino.onnx"
-    session = ort.InferenceSession(onnx_file)
+def preprocess_image(image_bgr: np.ndarray) -> torch.Tensor:
+    image_bgr = cv2.resize(image_bgr, (800, 800))
+    transform = T.Compose(
+        [
+            T.RandomResize([800], max_size=1333),
+            T.ToTensor(),
+            T.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225]),
+        ]
+    )
+    image_pillow = Image.fromarray(cv2.cvtColor(image_bgr, cv2.COLOR_BGR2RGB))
+    image_transformed, _ = transform(image_pillow, None)
+    return image_transformed
 
+def inference_decoder_onnx(model, output):
     image = cv2.imread('asset/1.jpg')
     processed_image = preprocess_image(image).unsqueeze(0)
-    caption = preprocess_caption("watermark")
-    tokenized = model.tokenizer(caption, padding="longest", return_tensors="pt")
-    box_threshold = torch.tensor(0.35, dtype=torch.float32)
-    text_threshold = torch.tensor(0.25, dtype=torch.float32)
 
-    specical_tokens = model.specical_tokens
-    (
-        text_self_attention_masks,
-        position_ids,
-        _,
-    ) = generate_masks_with_special_tokens_and_transfer_map(
-        tokenized, specical_tokens, model.tokenizer
-    )
+    caption = "watermark. glasses"
 
-    max_text_len = model.max_text_len
-    if text_self_attention_masks.shape[1] > max_text_len:
-        text_self_attention_masks = text_self_attention_masks[
-            :, : max_text_len, : max_text_len
-        ]
-        position_ids = position_ids[:, : max_text_len]
-        tokenized["input_ids"] = tokenized["input_ids"][:, : max_text_len]
-        tokenized["attention_mask"] = tokenized["attention_mask"][:, : max_text_len]
-        tokenized["token_type_ids"] = tokenized["token_type_ids"][:, : max_text_len]
+    tokenized, last_hidden_state = inference_encoder_onnx(model, output, caption)
 
-    outputs = session.run(None, {
-        "image": processed_image.numpy().astype(np.float32) ,
-        "input_ids": tokenized["input_ids"].numpy().astype(np.int32) , 
-        "attention_mask": tokenized["attention_mask"].numpy().astype(np.uint8) , 
-        "token_type_ids": tokenized["token_type_ids"].numpy().astype(np.int32) , 
-        "position_ids": position_ids.numpy().astype(np.int32) , 
-        "text_self_attention_masks": text_self_attention_masks.numpy().astype(np.bool) , 
-        "box_threshold": box_threshold.numpy().astype(np.float32) ,
+    print(tokenized)
+    print(last_hidden_state)
+
+    onnx_file = output + "/" + "gdino.decoder.onnx"
+    session = ort.InferenceSession(onnx_file)
+
+    box_threshold = torch.tensor(0.35, dtype=torch.float32)
+    text_threshold = torch.tensor(0.25, dtype=torch.float32)
+
+    decode_outputs = session.run(None, {
+        "image": processed_image.numpy().astype(np.float32),
+        "last_hidden_state": last_hidden_state.numpy().astype(np.float32), 
+        "attention_mask": tokenized["attention_mask"].numpy().astype(np.uint8), 
+        "position_ids": tokenized["position_ids"].numpy().astype(np.int32), 
+        "text_self_attention_masks": tokenized["text_self_attention_masks"].numpy().astype(np.uint8), 
+        "box_threshold": box_threshold.numpy().astype(np.float32),
         "text_threshold": text_threshold.numpy().astype(np.float32) 
     })
 
-    prediction_logits = torch.from_numpy(outputs[0]) 
-    prediction_boxes = torch.from_numpy(outputs[1]) 
-    prediction_masks = torch.from_numpy(outputs[2]) 
+    prediction_logits = torch.from_numpy(decode_outputs[0]) 
+    prediction_boxes = torch.from_numpy(decode_outputs[1]) 
+    prediction_masks = torch.from_numpy(decode_outputs[2]) 
 
     input_ids = tokenized["input_ids"][0].tolist()
     phrases = []
@@ -224,9 +245,9 @@ def inference_onnx(model, output_dir):
     cv2.destroyAllWindows()
 
 if __name__ == "__main__":
-    parser = argparse.ArgumentParser("Export Grounding DINO Model to IR", add_help=True)
-    parser.add_argument("--test", "-t", help="test onnx model", action="store_true")
-    parser.add_argument("--orig", "-n", help="test model", action="store_true")
+    parser = argparse.ArgumentParser("Export Grounding DINO Model to ONNX", add_help=True)
+    parser.add_argument("--encode", "-e", help="test encoder.onnx model", action="store_true")
+    parser.add_argument("--decode", "-d", help="test decoder.onnx model", action="store_true")
     parser.add_argument("--config_file", "-c", type=str, required=True, help="path to config file")
     parser.add_argument(
         "--checkpoint_path", "-p", type=str, required=True, help="path to checkpoint file"
@@ -245,11 +266,17 @@ def inference_onnx(model, output_dir):
     # make dir
     os.makedirs(output_dir, exist_ok=True)
 
-    model = Model(config_file, checkpoint_path, device='cpu')
+    source_model = load_model(model_config_path = config_file, 
+                              model_checkpoint_path = checkpoint_path, 
+                              device = "cpu").to("cpu")
+
+    encoder = Encoder(source_model)
+    decoder = Decoder(source_model)
 
-    if args.test:
-        inference_onnx(model, output_dir)
-    elif args.orig:
-        inference(model)
+    if args.encode:
+        inference_encoder_onnx(encoder, output_dir)
+    elif args.decode:
+        inference_decoder_onnx(decoder, output_dir)
     else:
-        export_onnx(model, output_dir)
+        export_encoder(encoder, output_dir)
+        export_decoder(decoder, output_dir, encoder)

From aab23aea55294382edacabf61685735644a7efd5 Mon Sep 17 00:00:00 2001
From: szsteven008 <97944818+szsteven008@users.noreply.github.com>
Date: Mon, 6 Jan 2025 16:14:12 +0800
Subject: [PATCH 10/13] Add files via upload
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

两个阶段
---
 groundingdino/util/inference.py | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)

diff --git a/groundingdino/util/inference.py b/groundingdino/util/inference.py
index 98e6c6d1..b4d6ee2d 100644
--- a/groundingdino/util/inference.py
+++ b/groundingdino/util/inference.py
@@ -89,11 +89,17 @@ def predict(
         tokenized["attention_mask"] = tokenized["attention_mask"][:, : max_text_len]
         tokenized["token_type_ids"] = tokenized["token_type_ids"][:, : max_text_len]
 
+    tokenized_for_encoder = {k: v for k, v in tokenized.items() if k != "attention_mask"}
+    tokenized_for_encoder["attention_mask"] = text_self_attention_masks
+    tokenized_for_encoder["position_ids"] = position_ids
+
+    bert = model.bert
+    bert_output = bert(**tokenized_for_encoder)  # bs, 195, 768
+
     with torch.no_grad():
         outputs = model(image.unsqueeze(0), 
-                        input_ids=tokenized["input_ids"], 
+                        last_hidden_state=bert_output["last_hidden_state"], 
                         attention_mask=tokenized["attention_mask"], 
-                        token_type_ids=tokenized["token_type_ids"], 
                         position_ids = position_ids, 
                         text_self_attention_masks = text_self_attention_masks)
 #        outputs = model(image[None], captions=[caption])

From d4a0bf67011c8774843a23d9a436d95ba2211515 Mon Sep 17 00:00:00 2001
From: szsteven008 <97944818+szsteven008@users.noreply.github.com>
Date: Mon, 6 Jan 2025 16:21:54 +0800
Subject: [PATCH 11/13] Add files via upload
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

两个阶段
---
 .../models/GroundingDINO/bertwarper.py        |  8 ++--
 .../models/GroundingDINO/groundingdino.py     | 37 ++++++++++---------
 2 files changed, 23 insertions(+), 22 deletions(-)

diff --git a/groundingdino/models/GroundingDINO/bertwarper.py b/groundingdino/models/GroundingDINO/bertwarper.py
index 84e60896..9dedbf5c 100644
--- a/groundingdino/models/GroundingDINO/bertwarper.py
+++ b/groundingdino/models/GroundingDINO/bertwarper.py
@@ -190,8 +190,8 @@ def generate_masks_with_special_tokens(tokenized, special_tokens_list, tokenizer
     # special_tokens_mask: bs, num_token. 1 for special tokens. 0 for normal tokens
     special_tokens_mask = torch.zeros((bs, num_token), device=input_ids.device).bool()
     for special_token in special_tokens_list:
-        special_tokens_mask = torch.logical_or(special_tokens_mask, input_ids == special_token)
-        #special_tokens_mask |= input_ids == special_token
+#        special_tokens_mask = torch.logical_or(special_tokens_mask, input_ids == special_token)
+        special_tokens_mask |= input_ids == special_token
 
     # idxs: each row is a list of indices of special tokens
     idxs = torch.nonzero(special_tokens_mask)
@@ -235,8 +235,8 @@ def generate_masks_with_special_tokens_and_transfer_map(tokenized, special_token
     # special_tokens_mask: bs, num_token. 1 for special tokens. 0 for normal tokens
     special_tokens_mask = torch.zeros((bs, num_token), device=input_ids.device).bool()
     for special_token in special_tokens_list:
-        special_tokens_mask = torch.logical_or(special_tokens_mask, input_ids == special_token)
-#        special_tokens_mask |= input_ids == special_token
+#        special_tokens_mask = torch.logical_or(special_tokens_mask, input_ids == special_token)
+        special_tokens_mask |= input_ids == special_token
 
     # idxs: each row is a list of indices of special tokens
     idxs = torch.nonzero(special_tokens_mask)
diff --git a/groundingdino/models/GroundingDINO/groundingdino.py b/groundingdino/models/GroundingDINO/groundingdino.py
index 6f3a18a1..37c3b954 100644
--- a/groundingdino/models/GroundingDINO/groundingdino.py
+++ b/groundingdino/models/GroundingDINO/groundingdino.py
@@ -227,9 +227,8 @@ def init_ref_points(self, use_num_queries):
 #    def forward(self, samples: NestedTensor, targets: List = None, **kw):
     def forward(self, 
                 samples: NestedTensor, 
-                input_ids: Tensor, 
+                last_hidden_state: Tensor, 
                 attention_mask: Tensor, 
-                token_type_ids: Tensor, 
                 position_ids: Tensor, 
                 text_self_attention_masks: Tensor, 
                 **kw):
@@ -258,11 +257,11 @@ def forward(self,
             samples.device
         )
         """
-        tokenized = {
-            "input_ids": input_ids,
-            "attention_mask": attention_mask,
-            "token_type_ids": token_type_ids,
-        }
+#        tokenized = {
+#            "input_ids": input_ids,
+#            "attention_mask": attention_mask,
+#            "token_type_ids": token_type_ids,
+#        }
 
 #        (
 #            text_self_attention_masks,
@@ -281,22 +280,24 @@ def forward(self,
 #            tokenized["attention_mask"] = tokenized["attention_mask"][:, : self.max_text_len]
 #            tokenized["token_type_ids"] = tokenized["token_type_ids"][:, : self.max_text_len]
 
-        # extract text embeddings
-        if self.sub_sentence_present:
-            tokenized_for_encoder = {k: v for k, v in tokenized.items() if k != "attention_mask"}
-            tokenized_for_encoder["attention_mask"] = text_self_attention_masks
-            tokenized_for_encoder["position_ids"] = position_ids
-        else:
-            # import ipdb; ipdb.set_trace()
-            tokenized_for_encoder = tokenized
+#        # extract text embeddings
+#        if self.sub_sentence_present:
+#            tokenized_for_encoder = {k: v for k, v in tokenized.items() if k != "attention_mask"}
+#            tokenized_for_encoder["attention_mask"] = text_self_attention_masks
+#            tokenized_for_encoder["position_ids"] = position_ids
+#        else:
+#            # import ipdb; ipdb.set_trace()
+#            tokenized_for_encoder = tokenized
 
-        bert_output = self.bert(**tokenized_for_encoder)  # bs, 195, 768
+#        bert_output = self.bert(**tokenized_for_encoder)  # bs, 195, 768
 
-        encoded_text = self.feat_map(bert_output["last_hidden_state"])  # bs, 195, d_model
-        text_token_mask = tokenized["attention_mask"].bool()  # bs, 195
+#        encoded_text = self.feat_map(bert_output["last_hidden_state"])  # bs, 195, d_model
+#        text_token_mask = tokenized["attention_mask"].bool()  # bs, 195
 #        text_token_mask = tokenized.attention_mask.bool()  # bs, 195
         # text_token_mask: True for nomask, False for mask
         # text_self_attention_masks: True for nomask, False for mask
+        encoded_text = self.feat_map(last_hidden_state)  # bs, 195, d_model
+        text_token_mask = attention_mask.bool()  # bs, 195
 
         if encoded_text.shape[1] > self.max_text_len:
             encoded_text = encoded_text[:, : self.max_text_len, :]

From e914f98e81e3d361e2d393b6c9409d680ae319a6 Mon Sep 17 00:00:00 2001
From: szsteven008 <97944818+szsteven008@users.noreply.github.com>
Date: Mon, 6 Jan 2025 16:34:44 +0800
Subject: [PATCH 12/13] Add files via upload
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

修改路径

From e053e2e26f4031cf259e3e597b7b669f958a3329 Mon Sep 17 00:00:00 2001
From: szsteven008 <97944818+szsteven008@users.noreply.github.com>
Date: Mon, 6 Jan 2025 16:36:37 +0800
Subject: [PATCH 13/13] Add files via upload