From 34ef00dcdf5cadb84c21b59db4fc42a4d4c75047 Mon Sep 17 00:00:00 2001 From: Kenshi Takayama Date: Tue, 4 Mar 2025 14:03:13 +0900 Subject: [PATCH] make it compatible with torch 2.6.0 --- .../models/GroundingDINO/backbone/swin_transformer.py | 4 ++-- groundingdino/models/GroundingDINO/bertwarper.py | 2 +- .../GroundingDINO/csrc/MsDeformAttn/ms_deform_attn_cuda.cu | 4 ++-- groundingdino/models/GroundingDINO/transformer.py | 5 ++++- groundingdino/models/GroundingDINO/utils.py | 1 + groundingdino/util/box_ops.py | 2 +- 6 files changed, 11 insertions(+), 7 deletions(-) diff --git a/groundingdino/models/GroundingDINO/backbone/swin_transformer.py b/groundingdino/models/GroundingDINO/backbone/swin_transformer.py index 1c66194d..fa55f27f 100644 --- a/groundingdino/models/GroundingDINO/backbone/swin_transformer.py +++ b/groundingdino/models/GroundingDINO/backbone/swin_transformer.py @@ -113,7 +113,7 @@ def __init__( # get pair-wise relative position index for each token inside the window coords_h = torch.arange(self.window_size[0]) coords_w = torch.arange(self.window_size[1]) - coords = torch.stack(torch.meshgrid([coords_h, coords_w])) # 2, Wh, Ww + coords = torch.stack(torch.meshgrid([coords_h, coords_w], indexing="ij")) # 2, Wh, Ww coords_flatten = torch.flatten(coords, 1) # 2, Wh*Ww relative_coords = coords_flatten[:, :, None] - coords_flatten[:, None, :] # 2, Wh*Ww, Wh*Ww relative_coords = relative_coords.permute(1, 2, 0).contiguous() # Wh*Ww, Wh*Ww, 2 @@ -445,7 +445,7 @@ def forward(self, x, H, W): for blk in self.blocks: blk.H, blk.W = H, W if self.use_checkpoint: - x = checkpoint.checkpoint(blk, x, attn_mask) + x = checkpoint.checkpoint(blk, x, attn_mask, use_reentrant=True) else: x = blk(x, attn_mask) if self.downsample is not None: diff --git a/groundingdino/models/GroundingDINO/bertwarper.py b/groundingdino/models/GroundingDINO/bertwarper.py index f0cf9779..f9c473e7 100644 --- a/groundingdino/models/GroundingDINO/bertwarper.py +++ b/groundingdino/models/GroundingDINO/bertwarper.py @@ -107,7 +107,7 @@ def forward( # We can provide a self-attention mask of dimensions [batch_size, from_seq_length, to_seq_length] # ourselves in which case we just need to make it broadcastable to all heads. extended_attention_mask: torch.Tensor = self.get_extended_attention_mask( - attention_mask, input_shape, device + attention_mask, input_shape ) # If a 2D or 3D attention mask is provided for the cross-attention diff --git a/groundingdino/models/GroundingDINO/csrc/MsDeformAttn/ms_deform_attn_cuda.cu b/groundingdino/models/GroundingDINO/csrc/MsDeformAttn/ms_deform_attn_cuda.cu index d04fae8a..93a87c36 100644 --- a/groundingdino/models/GroundingDINO/csrc/MsDeformAttn/ms_deform_attn_cuda.cu +++ b/groundingdino/models/GroundingDINO/csrc/MsDeformAttn/ms_deform_attn_cuda.cu @@ -62,7 +62,7 @@ at::Tensor ms_deform_attn_cuda_forward( for (int n = 0; n < batch/im2col_step_; ++n) { auto columns = output_n.select(0, n); - AT_DISPATCH_FLOATING_TYPES(value.type(), "ms_deform_attn_forward_cuda", ([&] { + AT_DISPATCH_FLOATING_TYPES(value.scalar_type(), "ms_deform_attn_forward_cuda", ([&] { ms_deformable_im2col_cuda(at::cuda::getCurrentCUDAStream(), value.data() + n * im2col_step_ * per_value_size, spatial_shapes.data(), @@ -132,7 +132,7 @@ std::vector ms_deform_attn_cuda_backward( for (int n = 0; n < batch/im2col_step_; ++n) { auto grad_output_g = grad_output_n.select(0, n); - AT_DISPATCH_FLOATING_TYPES(value.type(), "ms_deform_attn_backward_cuda", ([&] { + AT_DISPATCH_FLOATING_TYPES(value.scalar_type(), "ms_deform_attn_backward_cuda", ([&] { ms_deformable_col2im_cuda(at::cuda::getCurrentCUDAStream(), grad_output_g.data(), value.data() + n * im2col_step_ * per_value_size, diff --git a/groundingdino/models/GroundingDINO/transformer.py b/groundingdino/models/GroundingDINO/transformer.py index fcb8742d..b86c3885 100644 --- a/groundingdino/models/GroundingDINO/transformer.py +++ b/groundingdino/models/GroundingDINO/transformer.py @@ -470,6 +470,7 @@ def get_reference_points(spatial_shapes, valid_ratios, device): ref_y, ref_x = torch.meshgrid( torch.linspace(0.5, H_ - 0.5, H_, dtype=torch.float32, device=device), torch.linspace(0.5, W_ - 0.5, W_, dtype=torch.float32, device=device), + indexing="ij", ) ref_y = ref_y.reshape(-1)[None] / (valid_ratios[:, None, lvl, 1] * H_) ref_x = ref_x.reshape(-1)[None] / (valid_ratios[:, None, lvl, 0] * W_) @@ -554,6 +555,7 @@ def forward( memory_text, key_padding_mask, text_attention_mask, + use_reentrant=True, ) else: output, memory_text = self.fusion_layers[layer_id]( @@ -581,6 +583,7 @@ def forward( spatial_shapes, level_start_index, key_padding_mask, + use_reentrant=True, ) else: output = layer( @@ -859,7 +862,7 @@ def with_pos_embed(tensor, pos): return tensor if pos is None else tensor + pos def forward_ffn(self, tgt): - with torch.cuda.amp.autocast(enabled=False): + with torch.amp.autocast(device_type='cuda', dtype=torch.float16, enabled=False, cache_enabled=True): tgt2 = self.linear2(self.dropout3(self.activation(self.linear1(tgt)))) tgt = tgt + self.dropout4(tgt2) tgt = self.norm3(tgt) diff --git a/groundingdino/models/GroundingDINO/utils.py b/groundingdino/models/GroundingDINO/utils.py index 5bd18f70..9c5653fc 100644 --- a/groundingdino/models/GroundingDINO/utils.py +++ b/groundingdino/models/GroundingDINO/utils.py @@ -79,6 +79,7 @@ def gen_encoder_output_proposals( grid_y, grid_x = torch.meshgrid( torch.linspace(0, H_ - 1, H_, dtype=torch.float32, device=memory.device), torch.linspace(0, W_ - 1, W_, dtype=torch.float32, device=memory.device), + indexing="ij", ) grid = torch.cat([grid_x.unsqueeze(-1), grid_y.unsqueeze(-1)], -1) # H_, W_, 2 diff --git a/groundingdino/util/box_ops.py b/groundingdino/util/box_ops.py index 781068d2..6159e757 100644 --- a/groundingdino/util/box_ops.py +++ b/groundingdino/util/box_ops.py @@ -118,7 +118,7 @@ def masks_to_boxes(masks): y = torch.arange(0, h, dtype=torch.float) x = torch.arange(0, w, dtype=torch.float) - y, x = torch.meshgrid(y, x) + y, x = torch.meshgrid(y, x, indexing="ij") x_mask = masks * x.unsqueeze(0) x_max = x_mask.flatten(1).max(-1)[0]