update xpose code

requirements.txt

@@ -1,7 +1,6 @@
 --extra-index-url https://download.pytorch.org/whl/cu118
-
-torch==1.12.1
-torchvision==0.13.1
+torch
+torchvision==0.18.0
 torchaudio==2.3.0
 
 numpy==1.26.4

src/utils/dependencies/XPose/config_model/UniPose_SwinT.py

@@ -108,7 +108,7 @@ shuffle_type = None
 use_text_enhancer = True
 use_fusion_layer = True
 
-use_checkpoint = False # True
+use_checkpoint = True
 use_transformer_ckpt = True
 text_encoder_type = 'bert-base-uncased'
 
@@ -122,4 +122,4 @@ binary_query_selection = False
 use_cdn = True
 ffn_extra_layernorm = False
 
-fix_size=False
+fix_size=False

src/utils/dependencies/XPose/models/UniPose/attention.py

@@ -23,19 +23,44 @@ Mostly copy-paste from https://github.com/pytorch/pytorch/blob/master/torch/nn/m
 and https://github.com/pytorch/pytorch/blob/master/torch/nn/functional.py#L4837
 """
 
+import copy
+from typing import Optional, List
+
+import torch
+import torch.nn.functional as F
+from torch import nn, Tensor
+
 import warnings
+from typing import Tuple, Optional
+
 import torch
+from torch import Tensor
 from torch.nn.modules.linear import Linear
+from torch.nn.init import xavier_uniform_
 from torch.nn.init import constant_
+from torch.nn.init import xavier_normal_
+from torch.nn.parameter import Parameter
 from torch.nn.modules.module import Module
-from torch._jit_internal import Optional, Tuple
+from torch.nn import functional as F
+
+import warnings
+import math
+
+from torch._C import _infer_size, _add_docstr
+from torch.nn import _reduction as _Reduction
+from torch.nn.modules import utils
+from torch.nn.modules.utils import _single, _pair, _triple, _list_with_default
+from torch.nn import grad
+from torch import _VF
+from torch._jit_internal import boolean_dispatch, List, Optional, _overload, Tuple
 try:
     from torch.overrides import has_torch_function, handle_torch_function
 except:
     from torch._overrides import has_torch_function, handle_torch_function
-from torch.nn.functional import linear, pad, softmax, dropout
 Tensor = torch.Tensor
 
+from torch.nn.functional import linear, pad, softmax, dropout
+
 class MultiheadAttention(Module):
     r"""Allows the model to jointly attend to information
     from different representation subspaces.

src/utils/dependencies/XPose/models/UniPose/backbone.py

@@ -16,18 +16,20 @@
 Backbone modules.
 """
 
+from typing import Dict, List
+
 import torch
 import torch.nn.functional as F
 import torchvision
 from torch import nn
 from torchvision.models._utils import IntermediateLayerGetter
-from typing import Dict, List
 
-from util.misc import NestedTensor, is_main_process
+from util.misc import NestedTensor, clean_state_dict, is_main_process
 
 from .position_encoding import build_position_encoding
 from .swin_transformer import build_swin_transformer
 
+
 class FrozenBatchNorm2d(torch.nn.Module):
     """
     BatchNorm2d where the batch statistics and the affine parameters are fixed.

src/utils/dependencies/XPose/models/UniPose/deformable_transformer.py

@@ -16,16 +16,21 @@
 # Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved.
 # ------------------------------------------------------------------------
 
-import math
+import math, random
+import os
 import copy
-import torch
-import torch.utils.checkpoint as checkpoint
-from torch import nn, Tensor
 from typing import Optional
+
 from util.misc import inverse_sigmoid
 
+import torch
+from torch import nn, Tensor
+
+import torch.utils.checkpoint as checkpoint
+
 from .transformer_vanilla import TransformerEncoderLayer
 from .fuse_modules import BiAttentionBlock
+
 from .utils import gen_encoder_output_proposals, MLP, _get_activation_fn, gen_sineembed_for_position, get_sine_pos_embed
 from .ops.modules import MSDeformAttn
 
@@ -580,7 +585,7 @@ class TransformerEncoder(nn.Module):
         reference_points_list = []
         for lvl, (H_, W_) in enumerate(spatial_shapes):
             ref_y, ref_x = torch.meshgrid(torch.linspace(0.5, H_ - 0.5, H_, dtype=torch.float32, device=device),
-                                          torch.linspace(0.5, W_ - 0.5, W_, dtype=torch.float32, device=device),)
+                                          torch.linspace(0.5, W_ - 0.5, W_, dtype=torch.float32, device=device))
             ref_y = ref_y.reshape(-1)[None] / (valid_ratios[:, None, lvl, 1] * H_)
             ref_x = ref_x.reshape(-1)[None] / (valid_ratios[:, None, lvl, 0] * W_)
             ref = torch.stack((ref_x, ref_y), -1)
@@ -1228,3 +1233,5 @@ def build_deformable_transformer(args):
         binary_query_selection=binary_query_selection,
         ffn_extra_layernorm=ffn_extra_layernorm,
     )
+
+

src/utils/dependencies/XPose/models/UniPose/fuse_modules.py

@@ -1,9 +1,12 @@
-import torch
+from typing import List
+import torch, os
 import torch.nn as nn
 import torch.nn.functional as F
+import pdb
+import math
+from timm.models.layers import DropPath
 
-# from timm.models.layers import DropPath
-from src.modules.util import DropPath
+from transformers.activations import ACT2FN
 
 class FeatureResizer(nn.Module):
     """
@@ -178,7 +181,7 @@ class BiMultiHeadAttention(nn.Module):
 
         if self.stable_softmax_2d:
             attn_weights = attn_weights - attn_weights.max()
-
+        
         if self.clamp_min_for_underflow:
             attn_weights = torch.clamp(attn_weights, min=-50000) # Do not increase -50000, data type half has quite limited range
         if self.clamp_max_for_overflow:
@@ -261,8 +264,8 @@ class BiAttentionBlock(nn.Module):
 
         # add layer scale for training stability
         self.drop_path = DropPath(drop_path) if drop_path > 0. else nn.Identity()
-        self.gamma_v = nn.Parameter(init_values * torch.ones((v_dim)), requires_grad=False)
-        self.gamma_l = nn.Parameter(init_values * torch.ones((l_dim)), requires_grad=False)
+        self.gamma_v = nn.Parameter(init_values * torch.ones((v_dim)), requires_grad=True)
+        self.gamma_l = nn.Parameter(init_values * torch.ones((l_dim)), requires_grad=True)
 
     def forward(self, v, l, attention_mask_v=None, attention_mask_l=None):
         v = self.layer_norm_v(v)
@@ -272,3 +275,4 @@ class BiAttentionBlock(nn.Module):
         v = v + self.drop_path(self.gamma_v * delta_v)
         l = l + self.drop_path(self.gamma_l * delta_l)
         return v, l
+    

src/utils/dependencies/XPose/models/UniPose/mask_generate.py

@@ -1,4 +1,10 @@
 import torch
+from util.misc import (NestedTensor, nested_tensor_from_tensor_list,
+                       accuracy, get_world_size, interpolate,
+                       is_dist_avail_and_initialized, inverse_sigmoid)
+# from .DABDETR import sigmoid_focal_loss
+from util import box_ops
+import torch.nn.functional as F
 
 
 def prepare_for_mask(kpt_mask):

src/utils/dependencies/XPose/models/UniPose/ops/modules/ms_deform_attn.py

@@ -20,7 +20,10 @@ from torch import nn
 import torch.nn.functional as F
 from torch.nn.init import xavier_uniform_, constant_
 
-from src.utils.dependencies.XPose.models.UniPose.ops.functions.ms_deform_attn_func import MSDeformAttnFunction
+try:
+    from src.utils.dependencies.XPose.models.UniPose.ops.functions.ms_deform_attn_func import MSDeformAttnFunction
+except:
+    warnings.warn('Failed to import MSDeformAttnFunction.')
 
 
 def _is_power_of_2(n):

src/utils/dependencies/XPose/models/UniPose/ops/setup.py

@@ -41,12 +41,10 @@ def get_extensions():
         sources += source_cuda
         define_macros += [("WITH_CUDA", None)]
         extra_compile_args["nvcc"] = [
-            # "-allow-unsupported-compiler",
             "-DCUDA_HAS_FP16=1",
             "-D__CUDA_NO_HALF_OPERATORS__",
             "-D__CUDA_NO_HALF_CONVERSIONS__",
             "-D__CUDA_NO_HALF2_OPERATORS__",
-            # "-std=c++14",
         ]
     else:
         raise NotImplementedError('Cuda is not availabel')
@@ -64,7 +62,6 @@ def get_extensions():
     ]
     return ext_modules
 
-
 setup(
     name="MultiScaleDeformableAttention",
     version="1.0",

src/utils/dependencies/XPose/models/UniPose/ops/src/cuda/ms_deform_attn_cuda.cu

@@ -61,7 +61,7 @@ at::Tensor ms_deform_attn_cuda_forward(
     for (int n = 0; n < batch/im2col_step_; ++n)
     {
         auto columns = output_n.select(0, n);
-        AT_DISPATCH_FLOATING_TYPES(value.scalar_type(), "ms_deform_attn_forward_cuda", ([&] {
+        AT_DISPATCH_FLOATING_TYPES(value.type(), "ms_deform_attn_forward_cuda", ([&] {
             ms_deformable_im2col_cuda(at::cuda::getCurrentCUDAStream(),
                 value.data<scalar_t>() + n * im2col_step_ * per_value_size,
                 spatial_shapes.data<int64_t>(),
@@ -131,7 +131,7 @@ std::vector<at::Tensor> ms_deform_attn_cuda_backward(
     for (int n = 0; n < batch/im2col_step_; ++n)
     {
         auto grad_output_g = grad_output_n.select(0, n);
-        AT_DISPATCH_FLOATING_TYPES(value.scalar_type(), "ms_deform_attn_backward_cuda", ([&] {
+        AT_DISPATCH_FLOATING_TYPES(value.type(), "ms_deform_attn_backward_cuda", ([&] {
             ms_deformable_col2im_cuda(at::cuda::getCurrentCUDAStream(),
                                     grad_output_g.data<scalar_t>(),
                                     value.data<scalar_t>() + n * im2col_step_ * per_value_size,

src/utils/dependencies/XPose/models/UniPose/position_encoding.py

@@ -15,6 +15,7 @@
 Various positional encodings for the transformer.
 """
 import math
+import os
 import torch
 from torch import nn
 

src/utils/dependencies/XPose/models/UniPose/swin_transformer.py

@@ -4,10 +4,8 @@ import torch.nn as nn
 import torch.nn.functional as F
 import torch.utils.checkpoint as checkpoint
 import numpy as np
-
+from timm.models.layers import DropPath, to_2tuple, trunc_normal_
 from util.misc import NestedTensor
-# from timm.models.layers import DropPath, to_2tuple, trunc_normal_
-from src.modules.util import DropPath, to_2tuple, trunc_normal_
 
 
 
@@ -489,8 +487,8 @@ class SwinTransformer(nn.Module):
         self.frozen_stages = frozen_stages
         self.dilation = dilation
 
-        # if use_checkpoint:
-        #     print("use_checkpoint!!!!!!!!!!!!!!!!!!!!!!!!")
+        if use_checkpoint:
+            print("use_checkpoint!!!!!!!!!!!!!!!!!!!!!!!!")
 
         # split image into non-overlapping patches
         self.patch_embed = PatchEmbed(
@@ -634,7 +632,7 @@ class SwinTransformer(nn.Module):
         #   [torch.Size([2, 192, 256, 256]), torch.Size([2, 384, 128, 128]), \
         #       torch.Size([2, 768, 64, 64]), torch.Size([2, 1536, 32, 32])]
 
-        # collect for nesttensors
+        # collect for nesttensors        
         outs_dict = {}
         for idx, out_i in enumerate(outs):
             m = tensor_list.mask
@@ -661,7 +659,7 @@ def build_swin_transformer(modelname, pretrain_img_size, **kw):
             depths=[ 2, 2, 6, 2 ],
             num_heads=[ 3, 6, 12, 24],
             window_size=7
-        ),
+        ),        
         'swin_B_224_22k': dict(
             embed_dim=128,
             depths=[ 2, 2, 18, 2 ],
@@ -698,4 +696,4 @@ if __name__ == "__main__":
     y = model.forward_raw(x)
     import ipdb; ipdb.set_trace()
     x = torch.rand(2, 3, 384, 384)
-    y = model.forward_raw(x)
+    y = model.forward_raw(x)

src/utils/dependencies/XPose/models/UniPose/transformer_deformable.py

@@ -12,15 +12,19 @@
 # ------------------------------------------------------------------------
 
 import copy
+import os
+from typing import Optional, List
 import math
+
 import torch
+import torch.nn.functional as F
 from torch import nn, Tensor
-from torch.nn.init import xavier_uniform_, constant_, normal_
-from typing import Optional
+from torch.nn.init import xavier_uniform_, constant_, uniform_, normal_
 
 from util.misc import inverse_sigmoid
 from .ops.modules import MSDeformAttn
-from .utils import MLP, _get_activation_fn, gen_sineembed_for_position
+
+from .utils import sigmoid_focal_loss, MLP, _get_activation_fn, gen_sineembed_for_position
 
 class DeformableTransformer(nn.Module):
     def __init__(self, d_model=256, nhead=8,
@@ -45,7 +49,7 @@ class DeformableTransformer(nn.Module):
         decoder_layer = DeformableTransformerDecoderLayer(d_model, dim_feedforward,
                                                           dropout, activation,
                                                           num_feature_levels, nhead, dec_n_points)
-        self.decoder = DeformableTransformerDecoder(decoder_layer, num_decoder_layers, return_intermediate_dec,
+        self.decoder = DeformableTransformerDecoder(decoder_layer, num_decoder_layers, return_intermediate_dec, 
                                                             use_dab=use_dab, d_model=d_model, high_dim_query_update=high_dim_query_update, no_sine_embed=no_sine_embed)
 
         self.level_embed = nn.Parameter(torch.Tensor(num_feature_levels, d_model))
@@ -158,7 +162,7 @@ class DeformableTransformer(nn.Module):
             lvl_pos_embed_flatten.append(lvl_pos_embed)
             src_flatten.append(src)
             mask_flatten.append(mask)
-        src_flatten = torch.cat(src_flatten, 1)     # bs, \sum{hxw}, c
+        src_flatten = torch.cat(src_flatten, 1)     # bs, \sum{hxw}, c 
         mask_flatten = torch.cat(mask_flatten, 1)   # bs, \sum{hxw}
         lvl_pos_embed_flatten = torch.cat(lvl_pos_embed_flatten, 1)
         spatial_shapes = torch.as_tensor(spatial_shapes, dtype=torch.long, device=src_flatten.device)
@@ -187,7 +191,7 @@ class DeformableTransformer(nn.Module):
             pos_trans_out = self.pos_trans_norm(self.pos_trans(self.get_proposal_pos_embed(topk_coords_unact)))
             query_embed, tgt = torch.split(pos_trans_out, c, dim=2)
         elif self.use_dab:
-            reference_points = query_embed[..., self.d_model:].sigmoid()
+            reference_points = query_embed[..., self.d_model:].sigmoid() 
             tgt = query_embed[..., :self.d_model]
             tgt = tgt.unsqueeze(0).expand(bs, -1, -1)
             init_reference_out = reference_points
@@ -195,15 +199,15 @@ class DeformableTransformer(nn.Module):
             query_embed, tgt = torch.split(query_embed, c, dim=1)
             query_embed = query_embed.unsqueeze(0).expand(bs, -1, -1)
             tgt = tgt.unsqueeze(0).expand(bs, -1, -1)
-            reference_points = self.reference_points(query_embed).sigmoid()
+            reference_points = self.reference_points(query_embed).sigmoid() 
                 # bs, num_quires, 2
             init_reference_out = reference_points
 
         # decoder
         # import ipdb; ipdb.set_trace()
         hs, inter_references = self.decoder(tgt, reference_points, memory,
-                                            spatial_shapes, level_start_index, valid_ratios,
-                                            query_pos=query_embed if not self.use_dab else None,
+                                            spatial_shapes, level_start_index, valid_ratios, 
+                                            query_pos=query_embed if not self.use_dab else None, 
                                             src_padding_mask=mask_flatten)
 
         inter_references_out = inter_references
@@ -387,7 +391,7 @@ class DeformableTransformerDecoderLayer(nn.Module):
         tgt = self.norm3(tgt)
         return tgt
 
-    def forward_sa(self,
+    def forward_sa(self, 
                 # for tgt
                 tgt: Optional[Tensor],  # nq, bs, d_model
                 tgt_query_pos: Optional[Tensor] = None, # pos for query. MLP(Sine(pos))
@@ -431,9 +435,9 @@ class DeformableTransformerDecoderLayer(nn.Module):
             else:
                 raise NotImplementedError("Unknown decoder_sa_type {}".format(self.decoder_sa_type))
 
-        return tgt
+        return tgt            
 
-    def forward_ca(self,
+    def forward_ca(self, 
                 # for tgt
                 tgt: Optional[Tensor],  # nq, bs, d_model
                 tgt_query_pos: Optional[Tensor] = None, # pos for query. MLP(Sine(pos))
@@ -468,9 +472,9 @@ class DeformableTransformerDecoderLayer(nn.Module):
         tgt = tgt + self.dropout1(tgt2)
         tgt = self.norm1(tgt)
 
-        return tgt
+        return tgt  
 
-    def forward(self,
+    def forward(self, 
                 # for tgt
                 tgt: Optional[Tensor],  # nq, bs, d_model
                 tgt_query_pos: Optional[Tensor] = None, # pos for query. MLP(Sine(pos))
@@ -530,7 +534,7 @@ class DeformableTransformerDecoder(nn.Module):
             self.ref_point_head = MLP(2 * d_model, d_model, d_model, 2)
 
 
-    def forward(self, tgt, reference_points, src, src_spatial_shapes,
+    def forward(self, tgt, reference_points, src, src_spatial_shapes,       
                 src_level_start_index, src_valid_ratios,
                 query_pos=None, src_padding_mask=None):
         output = tgt
@@ -547,14 +551,14 @@ class DeformableTransformerDecoder(nn.Module):
             else:
                 assert reference_points.shape[-1] == 2
                 reference_points_input = reference_points[:, :, None] * src_valid_ratios[:, None]
-
+                
             if self.use_dab:
                 # import ipdb; ipdb.set_trace()
-                query_sine_embed = gen_sineembed_for_position(reference_points_input[:, :, 0, :]) # bs, nq, 256*2
+                query_sine_embed = gen_sineembed_for_position(reference_points_input[:, :, 0, :]) # bs, nq, 256*2 
                 raw_query_pos = self.ref_point_head(query_sine_embed) # bs, nq, 256
                 pos_scale = self.query_scale(output) if layer_id != 0 else 1
                 query_pos = pos_scale * raw_query_pos
-
+        
             output = layer(output, query_pos, reference_points_input, src, src_spatial_shapes, src_level_start_index, src_padding_mask)
 
             # hack implementation for iterative bounding box refinement
@@ -593,3 +597,5 @@ def build_deforamble_transformer(args):
         use_dab=args.ddetr_use_dab,
         high_dim_query_update=args.ddetr_high_dim_query_update,
         no_sine_embed=args.ddetr_no_sine_embed)
+
+

src/utils/dependencies/XPose/models/UniPose/transformer_vanilla.py

@@ -8,11 +8,15 @@ Copy-paste from torch.nn.Transformer with modifications:
     * extra LN at the end of encoder is removed
     * decoder returns a stack of activations from all decoding layers
 """
+import copy
+import os
+from typing import List, Optional
+import pdb
 import torch
+import torch.nn.functional as F
 from torch import Tensor, nn
-from typing import List, Optional
 
-from .utils import  _get_activation_fn, _get_clones
+from .utils import gen_encoder_output_proposals, sigmoid_focal_loss, MLP, _get_activation_fn, gen_sineembed_for_position, _get_clones
 
 
 class TextTransformer(nn.Module):

src/utils/dependencies/XPose/models/UniPose/unipose.py

@@ -6,21 +6,30 @@
 # Modified from Deformable DETR (https://github.com/fundamentalvision/Deformable-DETR)
 # Copyright (c) 2020 SenseTime. All Rights Reserved.
 # ------------------------------------------------------------------------
-import os
 import copy
+import math
+import os
+from typing import List
 import torch
 import torch.nn.functional as F
 from torch import nn
-from typing import List
-
+from torchvision.ops.boxes import nms
 from util.keypoint_ops import keypoint_xyzxyz_to_xyxyzz
-from util.misc import NestedTensor, nested_tensor_from_tensor_list,inverse_sigmoid
-
-from .utils import MLP
+from util import box_ops
+from util.misc import (NestedTensor, nested_tensor_from_tensor_list,
+                       accuracy, get_world_size, interpolate,
+                       is_dist_avail_and_initialized, inverse_sigmoid)
 from .backbone import build_backbone
+from .deformable_transformer import build_deformable_transformer
+from .utils import sigmoid_focal_loss, MLP
+
 from ..registry import MODULE_BUILD_FUNCS
 from .mask_generate import prepare_for_mask, post_process
-from .deformable_transformer import build_deformable_transformer
+import random
+from .utils import sigmoid_focal_loss, MLP, _get_activation_fn, gen_sineembed_for_position
+from pathlib import Path
+import clip
+
 
 
 class UniPose(nn.Module):
@@ -107,12 +116,12 @@ class UniPose(nn.Module):
 
 
         device = "cuda" if torch.cuda.is_available() else "cpu"
-        # model, _ = clip.load("ViT-B/32", device=device)
-        # self.clip_model = model
-        # visual_parameters = list(self.clip_model.visual.parameters())
-        # #
-        # for param in visual_parameters:
-        #     param.requires_grad = False
+        model, _ = clip.load("ViT-B/32", device=device)
+        self.clip_model = model
+        visual_parameters = list(self.clip_model.visual.parameters())
+        #
+        for param in visual_parameters:
+            param.requires_grad = False
 
         self.pos_proj = nn.Linear(hidden_dim, 768)
         self.padding = nn.Embedding(1, 768)
@@ -531,7 +540,7 @@ def build_unipose(args):
         sub_sentence_present = args.sub_sentence_present
     except:
         sub_sentence_present = True
-    # print('********* sub_sentence_present', sub_sentence_present)
+    print('********* sub_sentence_present', sub_sentence_present)
 
     model = UniPose(
         backbone,

src/utils/dependencies/XPose/models/UniPose/utils.py

@@ -345,4 +345,4 @@ class OKSLoss(nn.Module):
             linear=self.linear,
             sigmas=self.sigmas,
             eps=self.eps)
-        return loss
+        return loss

src/utils/dependencies/XPose/transforms.py

@@ -24,6 +24,7 @@ def crop(image, target, region):
         i, j, h, w = region
         id2catname = target["id2catname"]
         caption_list = target["caption_list"]
+        # should we do something wrt the original size?
         target["size"] = torch.tensor([h, w])
 
         fields = ["labels", "area", "iscrowd", "positive_map","keypoints"]

src/utils/dependencies/XPose/util/__init__.py

@@ -0,0 +1 @@
+# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved

src/utils/dependencies/XPose/util/addict.py

@@ -1,159 +0,0 @@
-import copy
-
-
-class Dict(dict):
-
-    def __init__(__self, *args, **kwargs):
-        object.__setattr__(__self, '__parent', kwargs.pop('__parent', None))
-        object.__setattr__(__self, '__key', kwargs.pop('__key', None))
-        object.__setattr__(__self, '__frozen', False)
-        for arg in args:
-            if not arg:
-                continue
-            elif isinstance(arg, dict):
-                for key, val in arg.items():
-                    __self[key] = __self._hook(val)
-            elif isinstance(arg, tuple) and (not isinstance(arg[0], tuple)):
-                __self[arg[0]] = __self._hook(arg[1])
-            else:
-                for key, val in iter(arg):
-                    __self[key] = __self._hook(val)
-
-        for key, val in kwargs.items():
-            __self[key] = __self._hook(val)
-
-    def __setattr__(self, name, value):
-        if hasattr(self.__class__, name):
-            raise AttributeError("'Dict' object attribute "
-                                 "'{0}' is read-only".format(name))
-        else:
-            self[name] = value
-
-    def __setitem__(self, name, value):
-        isFrozen = (hasattr(self, '__frozen') and
-                    object.__getattribute__(self, '__frozen'))
-        if isFrozen and name not in super(Dict, self).keys():
-                raise KeyError(name)
-        super(Dict, self).__setitem__(name, value)
-        try:
-            p = object.__getattribute__(self, '__parent')
-            key = object.__getattribute__(self, '__key')
-        except AttributeError:
-            p = None
-            key = None
-        if p is not None:
-            p[key] = self
-            object.__delattr__(self, '__parent')
-            object.__delattr__(self, '__key')
-
-    def __add__(self, other):
-        if not self.keys():
-            return other
-        else:
-            self_type = type(self).__name__
-            other_type = type(other).__name__
-            msg = "unsupported operand type(s) for +: '{}' and '{}'"
-            raise TypeError(msg.format(self_type, other_type))
-
-    @classmethod
-    def _hook(cls, item):
-        if isinstance(item, dict):
-            return cls(item)
-        elif isinstance(item, (list, tuple)):
-            return type(item)(cls._hook(elem) for elem in item)
-        return item
-
-    def __getattr__(self, item):
-        return self.__getitem__(item)
-
-    def __missing__(self, name):
-        if object.__getattribute__(self, '__frozen'):
-            raise KeyError(name)
-        return self.__class__(__parent=self, __key=name)
-
-    def __delattr__(self, name):
-        del self[name]
-
-    def to_dict(self):
-        base = {}
-        for key, value in self.items():
-            if isinstance(value, type(self)):
-                base[key] = value.to_dict()
-            elif isinstance(value, (list, tuple)):
-                base[key] = type(value)(
-                    item.to_dict() if isinstance(item, type(self)) else
-                    item for item in value)
-            else:
-                base[key] = value
-        return base
-
-    def copy(self):
-        return copy.copy(self)
-
-    def deepcopy(self):
-        return copy.deepcopy(self)
-
-    def __deepcopy__(self, memo):
-        other = self.__class__()
-        memo[id(self)] = other
-        for key, value in self.items():
-            other[copy.deepcopy(key, memo)] = copy.deepcopy(value, memo)
-        return other
-
-    def update(self, *args, **kwargs):
-        other = {}
-        if args:
-            if len(args) > 1:
-                raise TypeError()
-            other.update(args[0])
-        other.update(kwargs)
-        for k, v in other.items():
-            if ((k not in self) or
-                (not isinstance(self[k], dict)) or
-                (not isinstance(v, dict))):
-                self[k] = v
-            else:
-                self[k].update(v)
-
-    def __getnewargs__(self):
-        return tuple(self.items())
-
-    def __getstate__(self):
-        return self
-
-    def __setstate__(self, state):
-        self.update(state)
-
-    def __or__(self, other):
-        if not isinstance(other, (Dict, dict)):
-            return NotImplemented
-        new = Dict(self)
-        new.update(other)
-        return new
-
-    def __ror__(self, other):
-        if not isinstance(other, (Dict, dict)):
-            return NotImplemented
-        new = Dict(other)
-        new.update(self)
-        return new
-
-    def __ior__(self, other):
-        self.update(other)
-        return self
-
-    def setdefault(self, key, default=None):
-        if key in self:
-            return self[key]
-        else:
-            self[key] = default
-            return default
-
-    def freeze(self, shouldFreeze=True):
-        object.__setattr__(self, '__frozen', shouldFreeze)
-        for key, val in self.items():
-            if isinstance(val, Dict):
-                val.freeze(shouldFreeze)
-
-    def unfreeze(self):
-        self.freeze(False)

src/utils/dependencies/XPose/util/box_ops.py

@@ -136,4 +136,4 @@ if __name__ == '__main__':
     x = torch.rand(5, 4)
     y = torch.rand(3, 4)
     iou, union = box_iou(x, y)
-    import ipdb; ipdb.set_trace()
+    import ipdb; ipdb.set_trace()

src/utils/dependencies/XPose/util/config.py

@@ -1,15 +1,17 @@
 # ==========================================================
 # Modified from mmcv
 # ==========================================================
-import sys
+import os, sys
 import os.path as osp
 import ast
 import tempfile
 import shutil
 from importlib import import_module
+
 from argparse import Action
 
-from .addict import Dict
+from addict import Dict
+from yapf.yapflib.yapf_api import FormatCode
 
 BASE_KEY = '_base_'
 DELETE_KEY = '_delete_'
@@ -81,8 +83,6 @@ class Config(object):
                 temp_config_file = tempfile.NamedTemporaryFile(
                     dir=temp_config_dir, suffix='.py')
                 temp_config_name = osp.basename(temp_config_file.name)
-                # close temp file before copy
-                temp_config_file.close()
                 shutil.copyfile(filename,
                                 osp.join(temp_config_dir, temp_config_name))
                 temp_module_name = osp.splitext(temp_config_name)[0]
@@ -97,8 +97,8 @@ class Config(object):
                 }
                 # delete imported module
                 del sys.modules[temp_module_name]
-                
-
+                # close temp file
+                temp_config_file.close()
         elif filename.lower().endswith(('.yml', '.yaml', '.json')):
             from .slio import slload
             cfg_dict = slload(filename)
@@ -304,6 +304,13 @@ class Config(object):
 
         cfg_dict = self._cfg_dict.to_dict()
         text = _format_dict(cfg_dict, outest_level=True)
+        # copied from setup.cfg
+        yapf_style = dict(
+            based_on_style='pep8',
+            blank_line_before_nested_class_or_def=True,
+            split_before_expression_after_opening_paren=True)
+        text, _ = FormatCode(text, style_config=yapf_style, verify=True)
+
         return text
     
 

src/utils/dependencies/XPose/util/get_param_dicts.py

@@ -0,0 +1,61 @@
+import json
+import torch
+import torch.nn as nn
+
+
+def match_name_keywords(n: str, name_keywords: list):
+    out = False
+    for b in name_keywords:
+        if b in n:
+            out = True
+            break
+    return out
+
+
+def get_param_dict(args, model_without_ddp: nn.Module):
+    try:
+        param_dict_type = args.param_dict_type
+    except:
+        param_dict_type = 'default'
+    assert param_dict_type in ['default', 'ddetr_in_mmdet', 'large_wd']
+
+    # by default
+    if param_dict_type == 'default':
+        param_dicts = [
+            {"params": [p for n, p in model_without_ddp.named_parameters() if "backbone" not in n and "bert" not in n and p.requires_grad]},
+            {
+                "params": [p for n, p in model_without_ddp.named_parameters() if "backbone" in n and p.requires_grad],
+                "lr": args.lr_backbone,
+            },
+            {
+                "params": [p for n, p in model_without_ddp.named_parameters() if "bert" in n and p.requires_grad],
+                "lr": args.lr_backbone,
+            }
+        ]
+
+        param_name_dicts = [
+            {"params": [n for n, p in model_without_ddp.named_parameters() if "backbone" not in n and "bert" not in n and p.requires_grad]},
+            {
+                "params": [n for n, p in model_without_ddp.named_parameters() if "backbone" in n and p.requires_grad],
+                "lr": args.lr_backbone,
+            },
+            {
+                "params": [n for n, p in model_without_ddp.named_parameters() if "bert" in n and p.requires_grad],
+                "lr": args.lr_backbone,
+            }
+        ]
+
+        print('param_name_dicts: ', json.dumps(param_name_dicts, indent=2))
+
+        return param_dicts, param_name_dicts
+
+
+
+
+    raise NotImplementedError
+
+
+
+        # print("param_dicts: {}".format(param_dicts))
+
+    return param_dicts, None

src/utils/dependencies/XPose/util/instance.txt

@@ -0,0 +1,863 @@
+AnimalKindom (850 but only providing 402 non-overlapping animal classes): 
+Orange Clownfish
+Fish
+Damsel Fish
+Sterlet Fish
+Trout
+Danube Bleak Fish
+Grayling Fish
+Catfish
+Sea Toad Fish
+Pike Perch Fish
+Sardine
+Archer Fish
+Giant Trevally
+Atlantic Blue Tang Fish
+Trout Young
+Stonefish
+Yellow Watchman Goby Fish
+Salmon
+Danube Salmon
+Mimic Blenny Fish
+Pink Skunk Clownfish
+Barracuda Fish
+Surgeonfish
+Keeltail Needlefish
+Royal Grammas Fish
+Tench Fish
+Clownfish
+Barramundi Fish
+Sergeant Major Fish
+Gray Angelfish
+Butterfly Fish
+Yellow Wrasse Fish
+Lionfish
+Toadfish
+Perch Fish
+Round Face Bat Fish
+Goby Fish
+Horned Adder
+Atheris Hispida Viper
+Fer-De-Lance Snake
+Naja Nivea Snake
+Elegant Bronzeback Snake
+S Viper"]
+Black Necked Spitting Cobra
+Dice Snake
+Boa
+Dispholidus Typus Snake
+Red Spitting Cobra
+Wild Red-Tailed Boa
+S Spitting Cobra"]
+Coronella Austriaca Snake
+Lesser Sunda Pit Viper
+Oriental Whip Snake
+Snake
+Mojave Rattlesnake
+Annulated Tree Boa
+Bushmaster Snake
+Sidewinder Rattlesnake
+Vipera Berus Snake
+Reticulated Python
+King Cobra
+Atheris Squamigera
+Lichanura Trivirgata Snake
+Crotalus Willardi Ridge Nosed Rattlesnake
+Slender Hognosed Pit Viper
+Paradise Tree Snake
+Coral Mimic Snake
+Many Horned Adder
+Dendroaspis Polylepis Black Mamba
+Northern Pacific Rattlesnake
+S Boa"]
+Mozambique Spitting Cobra
+Grass Snake
+Thelotornis Snake
+Black Headed Python
+Metlapilcoatlus Mexicanus Jumping Pit Viper
+Natrix Natrix Snake
+Atheris Nitschei Viper
+Puff Adder
+Rhamnophis Aethiopissa Snake
+Coral Snake
+Montpellier Snake
+Bothriechis
+Lampropeltis Pyromelana Snake
+Lampropeltis Zonata Snake
+Natrix Tessellata Snake
+Thamnophis Cyrtopsis Snake
+Rat Snake
+White Speckled Rattlesnake
+Namaqua Dwarf Adder
+Aesculapian Snake
+Nose-Horned Viper
+Zamenis Longissiumus Snake
+Rhinoceros Viper
+Rattlesnake
+Red Bellied Black Snake
+Eyelash Pit Viper
+Snouted Cobra
+JamesonS Mamba
+Gaboon Viper
+Lampropeltis Splendida Snake
+Laticauda Saintgironsi Sea Krait
+Pituophis Catenifer Snake
+Eastern Montpellier Snake
+Black Mamba
+Javan Spitting Cobra
+Sea Snake
+Gyalopion Canum Snake
+Mojave Rattlesnake Young
+Spectacled Cobra
+Hognosed Pit Viper
+Dog Faced Water Snake
+Cobra
+Tree Snake
+Lampropeltis Getula Snake
+S Mamba"]
+Bothrops Asper
+Rinkhals Snake
+Malabar Pit Viper
+Western Diamondback Young
+Micruroides Euryxanthus Snake
+Western Diamondback
+Python
+Viper
+Horse
+Horse Young
+Texas Brown Tarantula
+Tarantula
+Diving Bell Water Spider
+Turret Spider
+Avicularia Spider
+Golden Orb Spider
+Spider
+Salticidae Jumping Spider
+Salticidae 3 Spider
+Jumping Spider
+Orb Spider
+Latrodectus Hersperus Western Widow Spider
+Redback Spider
+Western Widow Spider
+Nursery Web Spider
+Grass Spider
+Araneus Diadematus Spider
+Tetragnatha Versicolor
+Cyclosa Conica
+Daddy Longlegs Spider
+Portia Jumping Spider
+Portia 3 Spider
+Habronattus Clypeatus
+Chimpanzee
+Gorilla
+Mountain Gorilla
+Stump-Tailed Macaque
+Pig-Tailed Macaque
+Monkey
+Orangutan
+Grey Langur
+Mitred Leaf Monkey
+Gibbon
+Red Ruffed Lemur
+Proboscis Monkey
+Sumatran Orangutan
+Red-Ruffed Lemur
+White-Faced Saki Monkey
+Bornean Orangutan
+Mandrill
+Maroon Macaque
+Rhesus Macaque
+Raffles Banded Langur
+Western Chimpanzee Young
+Diana Monkey
+Lesser Spot Nosed Monkey
+S Monkey"]
+Sooty Mangabey
+King Colobus
+Olive Colobus
+Capuchin Monkey
+Red-Blacked Squirrel Monkey
+Ring-Tailed Lemur
+Iguana
+Marine Iguana
+Rhacodactylus Trachyrhynchus Gecko
+Gecko
+Lizard
+Common Basilisk Lizard
+Basilisk Lizard
+Monitor Lizard
+Strange-Horned Chameleon
+Chameleon
+Leaf-Tailed Gecko
+Clouded Monitor Lizard
+S Chameleon"]
+Indian Chameleon
+Namaqua Dwarf Chameleon
+Malayan Water Monitor Lizard
+Green Iguana
+White Wig Marine Iguana
+Skink
+Black Bearded Draco
+Yellow Striped Tree Skink
+Side Blotched Lizard
+Frilled Neck Lizard
+Leopard Gecko
+Morning Gecko
+Giant Ground Gecko
+Western Dwarf Chameleon
+Whooper Swan
+Goose
+Puffin
+Mallard Duck
+Greylag Goose
+S Duck"]
+African Finfoot
+Wandering Alabatross
+Duck
+Common Goldeneye
+Garganey
+Black Stork
+Black Swan
+Сommon Eider
+Carolina Duck
+Smew
+Lanius Excubitor
+Song Thrush Bird
+Wedge Tailed Eagle
+Great Grey Shrike
+Anthus Pratensis Bird
+Red-Throated Pipit
+Tit Bird
+Hoopoe
+Woodpecker
+Botaurus Stellaris Bird
+Little Crake Bird
+White-Throated Dipper
+Raven
+Citrine Wagtail Bird
+Yellowhammer Young
+Grey Heron
+Alauda Arvensis Bird
+Bird
+Common Cuckoo Bird
+Tringa Erythropus Bird
+White Throated Dipper Bird
+Skylark
+Mistle Thrush
+Robin Bird
+Australian Bowerbird
+Turtle Dove
+Black-Winged Stilt
+Wood Warbler
+Common Crane
+Eurasian Wren Bird
+Common Quail
+Nightingale Bird
+Tawny Owl
+Grebe Bird
+Water Dipper Bird
+Yellowhammer
+Hazel Grouse Bird
+Greater Racket Tail Drongo
+Golden Oriole
+Great Egret
+Turdus Merula Blackbird
+Nuthatch Bird
+Eagle
+Luscinia Luscinia Nightingale Bird
+Singing Nightingale
+Water Rail Bird
+Gull
+Azure Tit Bird
+Numenius Arquata Bird
+Golden Eagle
+Remiz Pendulinus Bird
+Goldfinch
+Common Whitethroat Bird
+Red-Backed Shrike Bird
+Grasshopper Warbler
+Shoebill Bird
+Common Rosefinch Bird
+Owl
+Chaffinch Bird
+Bluethroat
+Green Woodpecker
+Common Snipe
+Whinchat Bird
+Ostrich
+Boreal Owl
+European Robin Bird
+Larus Canus Bird
+Hawk
+Three-Toed Woodpecker
+Thrush Nightingale Bird
+Jack Snipe Bird
+Red Crossbill
+Chiffchaff Bird
+Shorebird
+Bullfinch
+Red-Backed Shrike Bird Young
+Circus Aeruginosus Bird
+Kingfisher
+White-Backed Woodpecker
+Tringa Ochropus Bird
+Stock Dove
+Heron
+Citrine Wagtail
+Vanellus Vanellus Bird
+Tringa Nebularia Bird
+Eurasian Wryneck Bird
+Tachybaptus Ruficollis Bird
+Quail
+Little Egret
+Stork
+Green Mamba
+Pufferfish
+Bullfrog
+Frog
+Corroboree Frog
+Desert Rain Frog
+African Clawed Toad
+Mountain Yellow-Legged Frog
+Tropical Reed Frog
+Mimic Poison Frog
+Red-Eyed Tree Frog
+S Frog"]
+Water Lily Frog
+Amazon Milk Frog
+Toad
+Marbled Rubber Frog
+Sand Frog
+Golden Poison Frog
+Rain Frog
+Monster Frog
+S Warbler Bird"]
+Great Snipe
+European Serin Bird
+Calidris Apina Bird
+Peacock
+Tringa Glareola Bird
+Cuckoo Bird
+Barred Warbler Bird
+Pacman Frog
+Ardea Alba Egret
+Tern
+Motacilla Alba Bird
+Motacilla Flava
+Anas Crecca Bird
+Blue Poison Dart Frog
+Marsh Harrier Bird
+Glass Frog
+Great Reed Warbler Bird
+Banded Rubber Frog
+Tomato Frog
+Hornbill
+Woodlark Bird
+Starling Bird
+Common Buzzard
+Gallinago Gallinago Bird
+White And Gray Wagtail Bird
+Strawberry Poison-Dart Frog
+Corncrake
+Bald Eagle
+S Harrier Young"]
+Charadrius Dubius Bird
+Pelican
+Flamingo Young
+Socotran Cormorant
+Sparrowhawk
+S Harrier"]
+Pygmy Owl
+Philomachus Pugnax Ruff Bird
+Wren
+Common Wood Pigeon
+Grass Warbler Bird
+Whiskered Tern Bird
+Icterine Warbler Bird
+Crowned Eagle
+Crane
+Hummingbird
+Ardeotis Kori Bird
+Guttural Toad
+Crested Grebe Bird
+Reed Bunting Bird
+White Cockatoo Bird
+Sedge Warbler Bird
+Goldcrest Bird
+Montagus Harrier Young
+European Turtle Dove
+Asian Glossy Starling Bird
+Spotted Wood Owl
+Sagittarius Serpentarius Bird
+Parrot
+Anas Platyrhynchos Bird
+Phalacrocorax Carbo Bird
+White-Breasted Waterhen
+African Bullfrog
+Stork-Billed Kingfisher
+Oriental Pied Hornbill
+Flamingo
+Banded Woodpecker
+Foam Nest Frog
+Vulture
+Larus Ridibundus Bird
+
+AP-10K & APT-36K:
+monkey
+elephant
+leopard
+horse
+jaguar
+panda
+marmot
+deer
+noisy night monkey
+orangutan
+sheep
+spider-monkey
+bison
+zebra
+dog
+weasel
+bat
+uakari
+raccoon
+tiger
+rat
+rhino
+chimpanzee
+antelope
+argali sheep
+gorilla
+buffalo
+bobcat
+hippo
+mouse
+moose
+howling-monkey
+black-bear
+wolf
+squirrel
+skunk
+king cheetah
+cheetah
+spider monkey
+hamster
+arctic fox
+polar bear
+rabbit
+panther
+cow
+brown bear
+otter
+beaver
+pig
+fox
+alouatta
+giraffe
+polar-bear
+raccon
+snow leopard
+lion
+cat
+mole
+black bear
+
+Desert Locust
+
+Vinegar Fly
+
+
+CUB-200-2011:
+grebe_body
+gull_body
+kingfisher_body
+sparrow_body
+tern_body
+warbler_body
+woodpecker_body
+wren_body
+
+
+
+Carfusion:
+bus
+car
+suv
+
+
+Deepfashion2:
+short sleeve top
+long sleeve top
+short sleeve outwear
+long sleeve outwear
+vest
+sling
+shorts
+trousers
+skirt
+short sleeve dress
+long sleeve dress
+vest dress
+sling dress
+
+
+
+Keypoint-5:
+bed
+chair
+sofa
+swivelchair
+table
+
+AnimalWeb:
+blackbuck
+small asian mongoose
+common dwarf mongoose
+galapagos sea lion
+margay
+nilgai
+Humboldt penguin
+oryx
+tammar wallaby
+monkey
+swamp wallaby
+muntjac deer
+blue-eyed black lemur
+binturong
+hamadryas baboon
+Adelie penguin
+Australian cattle dog
+howler
+striped hyena
+vole
+zebu
+woodchuck
+proboscis monkey
+whiptail wallaby
+anoa
+hippopotamus
+crested penguin
+addax
+red-bellied squirrel
+suni
+feral cat
+galagos
+banteng
+Weddell seal
+zebra
+Ethiopian wolf
+snow leopard
+common chimpanzee
+giant schnauzer
+lemur
+jaguarundi
+Asian golden cat
+gray wolf
+anteater
+golden jackal
+banded palm civet
+cougar
+Barbary macaque
+giant otter
+agouti
+emperor penguin
+feral horse
+yellow-footed rock wallaby
+raccoon
+topi
+opossum
+central chimpanzee
+pygmy rabbit
+fishing cat
+reedbuck
+Mediterranean monk seal
+domestic cat
+kangaroo
+boar
+rusty-spotted cat
+spider monkey
+echidna
+Chinese goral
+ringtail
+kultarr
+Californian sea lion
+guanaco
+muriqui
+gerbil
+wildebeest
+bison
+Australian terrier
+hyrax
+clouded leopard
+goat
+badger
+beaver
+Przewalski horse
+camel
+beating mongoose
+field mouse
+collared peccary
+tree shrew
+wombat
+titi
+steenbuck steenbok
+Australian sea lion
+buffalo
+chamois
+baikal seal
+brush-tailed rock wallaby
+bongo
+Barbary sheep
+great dane
+cheetah
+long-nosed mongoose
+cape buffalo
+waterbuck
+rhesus monkey
+jungle cat
+black-and-white ruffed lemur
+Japanese serow
+potto
+dall sheep
+indri
+large-spotted genet
+Amur leopard
+Owston's palm civet
+dingo
+gibbons
+Doberman
+giraffe
+fox
+quokka
+Amur tiger
+wild ass
+walrus
+common genet
+bilby
+hamster
+yellow-eyed penguin
+panda
+agile wallaby
+bengal slow loris
+marmoset
+brown hyena
+gorilla
+aardvark
+swift fox
+Magellanic penguin
+bear
+Anatolian shepherd dog
+irish wolfhound
+husky
+kinkajou
+brown rat
+tarsiers
+matschie's tree kangaroo
+black-backed jackal
+ocelot
+grey seal
+bullmastiff
+gentoo penguin
+gerenuk
+bearded seal
+hooded seal
+monte
+leopard cat
+western chimpanzee
+african penguin
+dikdik
+komondor
+coypu
+dalmatian
+armadillo
+marsh mongoose
+rusty-spotted genet
+lion
+bharal
+wolverine
+visayan warty pig
+lutung
+bornean slow loris
+caiman
+aardwolf
+german shepherd dog
+sunda slow loris
+mangabey
+cape gray mongoose
+crowned lemur
+harp seal
+gelada baboon
+wallaroo
+hare
+goodfellow's tree kangaroo
+elk
+muskox
+capybara
+toque macaque
+roe deer
+eastern lesser bamboo lemur
+leopard
+wapiti
+gray fox
+alpaca
+guinea pig
+crabeater seal
+black rhino
+little blue penguin
+bighorn sheep
+caracal
+tamarin
+hawaiian monk seal
+lumholtz's tree kangaroo
+koala
+gundi
+onager
+cacomistle
+red-ruffed lemur
+orangutan
+bobcat
+black-footed cat
+alaskan hare
+debrazza's monkey
+swamp rabbit
+white wolf
+sharpe's grysbok
+urial
+feral goat
+serval
+degu
+golden bamboo lemur
+deer mouse
+coatis
+wildcat
+roan antelope
+dugong
+fennec fox
+southern elephant seal
+saluki
+golden langur
+oribi
+red-tail monkey
+chital
+dormouse
+woolly monkey
+leopard seal
+possum
+arctic wolf
+japanese macaque
+vervet monkey
+bamboo lemur
+aye-aye
+night monkey
+blue monkey
+sand cat
+bull
+cape fox
+klipspringer
+border collie
+mouflon
+chipmunk
+potoroo
+bushbuck
+northern elephant seal
+patagonian mara
+bandicoot
+feral cattle
+babirusa
+harvest mouse
+alaskan malamute
+servaline genet
+olive baboon
+italian greyhound
+white-headed lemur
+chihuahua
+red-necked wallaby
+fallow deer
+pygmy slow loris
+australian shepherd
+eastern chimpanzee
+colobus
+chinstrap penguin
+deer
+common warthog
+dunnart
+wisent
+hedgehog
+douc langur
+tasmanian devil
+colo
+flying squirrel
+canadian lynx
+ferret
+ribbon seal
+platypus
+cotton rat
+oncilla
+geoffroy's cat
+horse
+pardine genet
+slender mongoose
+liger
+mareeba rock wallaby
+olingos
+bonobo
+harbour seal
+pademelon
+domestic dog
+chow chow
+gharial
+quoll
+capuchin monkey
+corsac fox
+dassie
+bolognese dog
+ruddy mongoose
+rhinoceros
+red panda
+king penguin
+dachshund
+common brown lemur
+pekingese dog
+western lesser bamboo lemur
+banded mongoose
+grey langur
+patas monkey
+francois langur
+white-tailed deer
+african wild dog
+collared brown lemur
+weasel
+mexican wolf
+hartebeest
+uakari
+viverrata ngalunga malayan civet
+dhole
+eurasian lynx
+hog deer
+bushbaby
+grizzly bear
+caribou
+german pinscher
+jaguar
+donkey
+duiker
+spotted hyena
+golden retriever
+pantanal cat
+spotted-necked otter
+asian palm civet
+alpine ibex
+jackrabbit
+greater bamboo lemur
+kiang
+common kusimanse
+pallas cat
+stripe-necked mongoose
+parma wallaby
+yak
+balinese cat
+spotted seal
+french bulldog
+zonkey
+arctic fox

src/utils/dependencies/XPose/util/logger.py

@@ -0,0 +1,95 @@
+# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved
+import functools
+import logging
+import os
+import sys
+from termcolor import colored
+
+
+class _ColorfulFormatter(logging.Formatter):
+    def __init__(self, *args, **kwargs):
+        self._root_name = kwargs.pop("root_name") + "."
+        self._abbrev_name = kwargs.pop("abbrev_name", "")
+        if len(self._abbrev_name):
+            self._abbrev_name = self._abbrev_name + "."
+        super(_ColorfulFormatter, self).__init__(*args, **kwargs)
+
+    def formatMessage(self, record):
+        record.name = record.name.replace(self._root_name, self._abbrev_name)
+        log = super(_ColorfulFormatter, self).formatMessage(record)
+        if record.levelno == logging.WARNING:
+            prefix = colored("WARNING", "red", attrs=["blink"])
+        elif record.levelno == logging.ERROR or record.levelno == logging.CRITICAL:
+            prefix = colored("ERROR", "red", attrs=["blink", "underline"])
+        else:
+            return log
+        return prefix + " " + log
+
+
+# so that calling setup_logger multiple times won't add many handlers
+@functools.lru_cache()
+def setup_logger(
+    output=None, distributed_rank=0, *, color=True, name="imagenet", abbrev_name=None
+):
+    """
+    Initialize the detectron2 logger and set its verbosity level to "INFO".
+
+    Args:
+        output (str): a file name or a directory to save log. If None, will not save log file.
+            If ends with ".txt" or ".log", assumed to be a file name.
+            Otherwise, logs will be saved to `output/log.txt`.
+        name (str): the root module name of this logger
+
+    Returns:
+        logging.Logger: a logger
+    """
+    logger = logging.getLogger(name)
+    logger.setLevel(logging.DEBUG)
+    logger.propagate = False
+
+    if abbrev_name is None:
+        abbrev_name = name
+
+    plain_formatter = logging.Formatter(
+        '[%(asctime)s.%(msecs)03d]: %(message)s',
+        datefmt='%m/%d %H:%M:%S'
+    )
+    # stdout logging: master only
+    if distributed_rank == 0:
+        ch = logging.StreamHandler(stream=sys.stdout)
+        ch.setLevel(logging.DEBUG)
+        if color:
+            formatter = _ColorfulFormatter(
+                colored("[%(asctime)s.%(msecs)03d]: ", "green") + "%(message)s",
+                datefmt="%m/%d %H:%M:%S",
+                root_name=name,
+                abbrev_name=str(abbrev_name),
+            )
+        else:
+            formatter = plain_formatter
+        ch.setFormatter(formatter)
+        logger.addHandler(ch)
+
+    # file logging: all workers
+    if output is not None:
+        if output.endswith(".txt") or output.endswith(".log"):
+            filename = output
+        else:
+            filename = os.path.join(output, "log.txt")
+        if distributed_rank > 0:
+            filename = filename + f".rank{distributed_rank}"
+        os.makedirs(os.path.dirname(filename), exist_ok=True)
+
+        fh = logging.StreamHandler(_cached_log_stream(filename))
+        fh.setLevel(logging.DEBUG)
+        fh.setFormatter(plain_formatter)
+        logger.addHandler(fh)
+
+    return logger
+
+
+# cache the opened file object, so that different calls to `setup_logger`
+# with the same file name can safely write to the same file.
+@functools.lru_cache(maxsize=None)
+def _cached_log_stream(filename):
+    return open(filename, "a")

src/utils/dependencies/XPose/util/metrics.py

@@ -0,0 +1,181 @@
+# Copyright (c) Aishwarya Kamath & Nicolas Carion. Licensed under the Apache License 2.0. All Rights Reserved
+"""
+Various utilities related to track and report metrics
+"""
+import datetime
+import time
+from collections import defaultdict, deque
+
+import torch
+import torch.distributed as dist
+
+from util.misc import is_dist_avail_and_initialized
+
+
+class SmoothedValue:
+    """Track a series of values and provide access to smoothed values over a
+    window or the global series average.
+    """
+
+    def __init__(self, window_size=20, fmt=None):
+        if fmt is None:
+            fmt = "{median:.4f} ({global_avg:.4f})"
+        self.deque = deque(maxlen=window_size)
+        self.total = 0.0
+        self.count = 0
+        self.fmt = fmt
+
+    def update(self, value, num=1):
+        self.deque.append(value)
+        self.count += num
+        self.total += value * num
+
+    def synchronize_between_processes(self):
+        """
+        Distributed synchronization of the metric
+        Warning: does not synchronize the deque!
+        """
+        if not is_dist_avail_and_initialized():
+            return
+        t = torch.tensor([self.count, self.total], dtype=torch.float64, device="cuda")
+        dist.barrier()
+        dist.all_reduce(t)
+        t = t.tolist()
+        self.count = int(t[0])
+        self.total = t[1]
+
+    @property
+    def median(self):
+        d = torch.tensor(list(self.deque))
+        return d.median().item()
+
+    @property
+    def avg(self):
+        d = torch.tensor(list(self.deque), dtype=torch.float32)
+        return d.mean().item()
+
+    @property
+    def global_avg(self):
+        return self.total / self.count
+
+    @property
+    def max(self):
+        return max(self.deque)
+
+    @property
+    def value(self):
+        return self.deque[-1]
+
+    def __str__(self):
+        return self.fmt.format(
+            median=self.median, avg=self.avg, global_avg=self.global_avg, max=self.max, value=self.value
+        )
+
+
+class MetricLogger(object):
+    def __init__(self, delimiter="\t"):
+        self.meters = defaultdict(SmoothedValue)
+        self.delimiter = delimiter
+
+    def update(self, **kwargs):
+        for k, v in kwargs.items():
+            if isinstance(v, torch.Tensor):
+                v = v.item()
+            assert isinstance(v, (float, int))
+            self.meters[k].update(v)
+
+    def __getattr__(self, attr):
+        if attr in self.meters:
+            return self.meters[attr]
+        if attr in self.__dict__:
+            return self.__dict__[attr]
+        raise AttributeError("'{}' object has no attribute '{}'".format(type(self).__name__, attr))
+
+    def __str__(self):
+        loss_str = []
+        for name, meter in self.meters.items():
+            loss_str.append("{}: {}".format(name, str(meter)))
+        return self.delimiter.join(loss_str)
+
+    def synchronize_between_processes(self):
+        for meter in self.meters.values():
+            meter.synchronize_between_processes()
+
+    def add_meter(self, name, meter):
+        self.meters[name] = meter
+
+    def log_every(self, iterable, print_freq, header=None):
+        i = 0
+        if not header:
+            header = ""
+        start_time = time.time()
+        end = time.time()
+        iter_time = SmoothedValue(fmt="{avg:.4f}")
+        data_time = SmoothedValue(fmt="{avg:.4f}")
+        space_fmt = ":" + str(len(str(len(iterable)))) + "d"
+        if torch.cuda.is_available():
+            log_msg = self.delimiter.join(
+                [
+                    header,
+                    "[{0" + space_fmt + "}/{1}]",
+                    "eta: {eta}",
+                    "{meters}",
+                    "time: {time}",
+                    "data: {data}",
+                    "max mem: {memory:.0f}",
+                ]
+            )
+        else:
+            log_msg = self.delimiter.join(
+                [header, "[{0" + space_fmt + "}/{1}]", "eta: {eta}", "{meters}", "time: {time}", "data: {data}"]
+            )
+        MB = 1024.0 * 1024.0
+        for obj in iterable:
+            data_time.update(time.time() - end)
+            yield obj
+            iter_time.update(time.time() - end)
+            if i % print_freq == 0 or i == len(iterable) - 1:
+                eta_seconds = iter_time.global_avg * (len(iterable) - i)
+                eta_string = str(datetime.timedelta(seconds=int(eta_seconds)))
+                if torch.cuda.is_available():
+                    print(
+                        log_msg.format(
+                            i,
+                            len(iterable),
+                            eta=eta_string,
+                            meters=str(self),
+                            time=str(iter_time),
+                            data=str(data_time),
+                            memory=torch.cuda.max_memory_allocated() / MB,
+                        )
+                    )
+                else:
+                    print(
+                        log_msg.format(
+                            i, len(iterable), eta=eta_string, meters=str(self), time=str(iter_time), data=str(data_time)
+                        )
+                    )
+            i += 1
+            end = time.time()
+        total_time = time.time() - start_time
+        total_time_str = str(datetime.timedelta(seconds=int(total_time)))
+        print("{} Total time: {} ({:.4f} s / it)".format(header, total_time_str, total_time / len(iterable)))
+
+
+@torch.no_grad()
+def accuracy(output, target, topk=(1,)):
+    """Computes the precision@k for the specified values of k"""
+    if target.numel() == 0:
+        return [torch.zeros([], device=output.device)]
+    maxk = max(topk)
+    batch_size = target.size(0)
+
+    _, pred = output.topk(maxk, 1, True, True)
+    pred = pred.t()
+    correct = pred.eq(target.view(1, -1).expand_as(pred))
+
+    res = []
+    for k in topk:
+        correct_k = correct[:k].view(-1).float().sum(0)
+        res.append(correct_k.mul_(100.0 / batch_size))
+    return res

src/utils/dependencies/XPose/util/optim.py

@@ -0,0 +1,70 @@
+# Copyright (c) Aishwarya Kamath & Nicolas Carion. Licensed under the Apache License 2.0. All Rights Reserved
+"""Collections of utilities related to optimization."""
+from bisect import bisect_right
+import os
+
+import torch
+
+
+def update_ema(model, model_ema, decay):
+    """Apply exponential moving average update.
+
+    The  weights are updated in-place as follow:
+    w_ema = w_ema * decay + (1 - decay) * w
+    Args:
+        model: active model that is being optimized
+        model_ema: running average model
+        decay: exponential decay parameter
+    """
+    with torch.no_grad():
+        if hasattr(model, "module"):
+            # unwrapping DDP
+            model = model.module
+        msd = model.state_dict()
+        for k, ema_v in model_ema.state_dict().items():
+            model_v = msd[k].detach()
+            ema_v.copy_(ema_v * decay + (1.0 - decay) * model_v)
+
+
+def adjust_learning_rate(
+    optimizer,
+    epoch: int,
+    curr_step: int,
+    args,
+):
+    """Adjust the lr according to the schedule.
+
+    Args:
+        Optimizer: torch optimizer to update.
+        epoch(int): number of the current epoch.
+        curr_step(int): number of optimization step taken so far.
+        num_training_step(int): total number of optimization steps.
+        args: additional training dependent args:
+              - lr_drop(int): number of epochs before dropping the learning rate.
+              - fraction_warmup_steps(float) fraction of steps over which the lr will be increased to its peak.
+              - lr(float): base learning rate
+              - lr_backbone(float): learning rate of the backbone
+              - text_encoder_backbone(float): learning rate of the text encoder
+              - schedule(str): the requested learning rate schedule:
+                   "step": all lrs divided by 10 after lr_drop epochs
+                   "multistep": divided by 2 after lr_drop epochs, then by 2 after every 50 epochs
+                   "linear_with_warmup": same as "step" for backbone + transformer, but for the text encoder, linearly
+                                         increase for a fraction of the training, then linearly decrease back to 0.
+                   "all_linear_with_warmup": same as "linear_with_warmup" for all learning rates involved.
+
+    """
+    try:
+        num_warmup_steps = args.num_warmup_steps
+    except:
+        return
+
+    if epoch > 0:
+        return
+    
+    if curr_step > num_warmup_steps:
+        return
+
+    text_encoder_gamma = float(curr_step) / float(max(1, num_warmup_steps))
+    optimizer.param_groups[-1]["lr"] = args.lr_backbone * text_encoder_gamma
+
+

src/utils/dependencies/XPose/util/plot_utils.py

@@ -0,0 +1,112 @@
+"""
+Plotting utilities to visualize training logs.
+"""
+import torch
+import pandas as pd
+import numpy as np
+import seaborn as sns
+import matplotlib.pyplot as plt
+
+from pathlib import Path, PurePath
+
+
+def plot_logs(logs, fields=('class_error', 'loss_bbox_unscaled', 'mAP'), ewm_col=0, log_name='log.txt'):
+    '''
+    Function to plot specific fields from training log(s). Plots both training and test results.
+
+    :: Inputs - logs = list containing Path objects, each pointing to individual dir with a log file
+              - fields = which results to plot from each log file - plots both training and test for each field.
+              - ewm_col = optional, which column to use as the exponential weighted smoothing of the plots
+              - log_name = optional, name of log file if different than default 'log.txt'.
+
+    :: Outputs - matplotlib plots of results in fields, color coded for each log file.
+               - solid lines are training results, dashed lines are test results.
+
+    '''
+    func_name = "plot_utils.py::plot_logs"
+
+    # verify logs is a list of Paths (list[Paths]) or single Pathlib object Path,
+    # convert single Path to list to avoid 'not iterable' error
+
+    if not isinstance(logs, list):
+        if isinstance(logs, PurePath):
+            logs = [logs]
+            print(f"{func_name} info: logs param expects a list argument, converted to list[Path].")
+        else:
+            raise ValueError(f"{func_name} - invalid argument for logs parameter.\n \
+            Expect list[Path] or single Path obj, received {type(logs)}")
+
+    # Quality checks - verify valid dir(s), that every item in list is Path object, and that log_name exists in each dir
+    for i, dir in enumerate(logs):
+        if not isinstance(dir, PurePath):
+            raise ValueError(f"{func_name} - non-Path object in logs argument of {type(dir)}: \n{dir}")
+        if not dir.exists():
+            raise ValueError(f"{func_name} - invalid directory in logs argument:\n{dir}")
+        # verify log_name exists
+        fn = Path(dir / log_name)
+        if not fn.exists():
+            print(f"-> missing {log_name}.  Have you gotten to Epoch 1 in training?")
+            print(f"--> full path of missing log file: {fn}")
+            return
+
+    # load log file(s) and plot
+    dfs = [pd.read_json(Path(p) / log_name, lines=True) for p in logs]
+
+    fig, axs = plt.subplots(ncols=len(fields), figsize=(16, 5))
+
+    for df, color in zip(dfs, sns.color_palette(n_colors=len(logs))):
+        for j, field in enumerate(fields):
+            if field == 'mAP':
+                coco_eval = pd.DataFrame(
+                    np.stack(df.test_coco_eval_bbox.dropna().values)[:, 1]
+                ).ewm(com=ewm_col).mean()
+                axs[j].plot(coco_eval, c=color)
+            else:
+                df.interpolate().ewm(com=ewm_col).mean().plot(
+                    y=[f'train_{field}', f'test_{field}'],
+                    ax=axs[j],
+                    color=[color] * 2,
+                    style=['-', '--']
+                )
+    for ax, field in zip(axs, fields):
+        if field == 'mAP':
+            ax.legend([Path(p).name for p in logs])
+            ax.set_title(field)
+        else:
+            ax.legend([f'train', f'test'])
+            ax.set_title(field)
+
+    return fig, axs
+
+def plot_precision_recall(files, naming_scheme='iter'):
+    if naming_scheme == 'exp_id':
+        # name becomes exp_id
+        names = [f.parts[-3] for f in files]
+    elif naming_scheme == 'iter':
+        names = [f.stem for f in files]
+    else:
+        raise ValueError(f'not supported {naming_scheme}')
+    fig, axs = plt.subplots(ncols=2, figsize=(16, 5))
+    for f, color, name in zip(files, sns.color_palette("Blues", n_colors=len(files)), names):
+        data = torch.load(f)
+        # precision is n_iou, n_points, n_cat, n_area, max_det
+        precision = data['precision']
+        recall = data['params'].recThrs
+        scores = data['scores']
+        # take precision for all classes, all areas and 100 detections
+        precision = precision[0, :, :, 0, -1].mean(1)
+        scores = scores[0, :, :, 0, -1].mean(1)
+        prec = precision.mean()
+        rec = data['recall'][0, :, 0, -1].mean()
+        print(f'{naming_scheme} {name}: mAP@50={prec * 100: 05.1f}, ' +
+              f'score={scores.mean():0.3f}, ' +
+              f'f1={2 * prec * rec / (prec + rec + 1e-8):0.3f}'
+              )
+        axs[0].plot(recall, precision, c=color)
+        axs[1].plot(recall, scores, c=color)
+
+    axs[0].set_title('Precision / Recall')
+    axs[0].legend(names)
+    axs[1].set_title('Scores / Recall')
+    axs[1].legend(names)
+    return fig, axs

src/utils/dependencies/XPose/util/slio.py

@@ -0,0 +1,173 @@
+# ==========================================================
+# Modified from mmcv
+# ==========================================================
+
+import json, pickle, yaml
+try:
+    from yaml import CLoader as Loader, CDumper as Dumper
+except ImportError:
+    from yaml import Loader, Dumper
+
+from pathlib import Path
+from abc import ABCMeta, abstractmethod
+
+# ===========================
+# Rigister handler
+# ===========================
+
+class BaseFileHandler(metaclass=ABCMeta):
+
+    @abstractmethod
+    def load_from_fileobj(self, file, **kwargs):
+        pass
+
+    @abstractmethod
+    def dump_to_fileobj(self, obj, file, **kwargs):
+        pass
+
+    @abstractmethod
+    def dump_to_str(self, obj, **kwargs):
+        pass
+
+    def load_from_path(self, filepath, mode='r', **kwargs):
+        with open(filepath, mode) as f:
+            return self.load_from_fileobj(f, **kwargs)
+
+    def dump_to_path(self, obj, filepath, mode='w', **kwargs):
+        with open(filepath, mode) as f:
+            self.dump_to_fileobj(obj, f, **kwargs)
+
+class JsonHandler(BaseFileHandler):
+
+    def load_from_fileobj(self, file):
+        return json.load(file)
+
+    def dump_to_fileobj(self, obj, file, **kwargs):
+        json.dump(obj, file, **kwargs)
+
+    def dump_to_str(self, obj, **kwargs):
+        return json.dumps(obj, **kwargs)
+
+class PickleHandler(BaseFileHandler):
+
+    def load_from_fileobj(self, file, **kwargs):
+        return pickle.load(file, **kwargs)
+
+    def load_from_path(self, filepath, **kwargs):
+        return super(PickleHandler, self).load_from_path(
+            filepath, mode='rb', **kwargs)
+
+    def dump_to_str(self, obj, **kwargs):
+        kwargs.setdefault('protocol', 2)
+        return pickle.dumps(obj, **kwargs)
+
+    def dump_to_fileobj(self, obj, file, **kwargs):
+        kwargs.setdefault('protocol', 2)
+        pickle.dump(obj, file, **kwargs)
+
+    def dump_to_path(self, obj, filepath, **kwargs):
+        super(PickleHandler, self).dump_to_path(
+            obj, filepath, mode='wb', **kwargs)
+
+class YamlHandler(BaseFileHandler):
+
+    def load_from_fileobj(self, file, **kwargs):
+        kwargs.setdefault('Loader', Loader)
+        return yaml.load(file, **kwargs)
+
+    def dump_to_fileobj(self, obj, file, **kwargs):
+        kwargs.setdefault('Dumper', Dumper)
+        yaml.dump(obj, file, **kwargs)
+
+    def dump_to_str(self, obj, **kwargs):
+        kwargs.setdefault('Dumper', Dumper)
+        return yaml.dump(obj, **kwargs)
+
+file_handlers = {
+    'json': JsonHandler(),
+    'yaml': YamlHandler(),
+    'yml': YamlHandler(),
+    'pickle': PickleHandler(),
+    'pkl': PickleHandler()
+}
+
+# ===========================
+# load and dump
+# ===========================
+
+def is_str(x):
+    """Whether the input is an string instance.
+
+    Note: This method is deprecated since python 2 is no longer supported.
+    """
+    return isinstance(x, str)
+
+def slload(file, file_format=None, **kwargs):
+    """Load data from json/yaml/pickle files.
+
+    This method provides a unified api for loading data from serialized files.
+
+    Args:
+        file (str or :obj:`Path` or file-like object): Filename or a file-like
+            object.
+        file_format (str, optional): If not specified, the file format will be
+            inferred from the file extension, otherwise use the specified one.
+            Currently supported formats include "json", "yaml/yml" and
+            "pickle/pkl".
+
+    Returns:
+        The content from the file.
+    """
+    if isinstance(file, Path):
+        file = str(file)
+    if file_format is None and is_str(file):
+        file_format = file.split('.')[-1]
+    if file_format not in file_handlers:
+        raise TypeError(f'Unsupported format: {file_format}')
+
+    handler = file_handlers[file_format]
+    if is_str(file):
+        obj = handler.load_from_path(file, **kwargs)
+    elif hasattr(file, 'read'):
+        obj = handler.load_from_fileobj(file, **kwargs)
+    else:
+        raise TypeError('"file" must be a filepath str or a file-object')
+    return obj
+
+
+def sldump(obj, file=None, file_format=None, **kwargs):
+    """Dump data to json/yaml/pickle strings or files.
+
+    This method provides a unified api for dumping data as strings or to files,
+    and also supports custom arguments for each file format.
+
+    Args:
+        obj (any): The python object to be dumped.
+        file (str or :obj:`Path` or file-like object, optional): If not
+            specified, then the object is dump to a str, otherwise to a file
+            specified by the filename or file-like object.
+        file_format (str, optional): Same as :func:`load`.
+
+    Returns:
+        bool: True for success, False otherwise.
+    """
+    if isinstance(file, Path):
+        file = str(file)
+    if file_format is None:
+        if is_str(file):
+            file_format = file.split('.')[-1]
+        elif file is None:
+            raise ValueError(
+                'file_format must be specified since file is None')
+    if file_format not in file_handlers:
+        raise TypeError(f'Unsupported format: {file_format}')
+
+    handler = file_handlers[file_format]
+    if file is None:
+        return handler.dump_to_str(obj, **kwargs)
+    elif is_str(file):
+        handler.dump_to_path(obj, file, **kwargs)
+    elif hasattr(file, 'write'):
+        handler.dump_to_fileobj(obj, file, **kwargs)
+    else:
+        raise TypeError('"file" must be a filename str or a file-object')

src/utils/dependencies/XPose/util/time_counter.py

@@ -0,0 +1,60 @@
+import json
+import time
+
+class TimeCounter:
+    def __init__(self) -> None:
+        pass
+    
+    def clear(self):
+        self.timedict = {}
+        self.basetime = time.perf_counter()
+
+    def timeit(self, name):
+        nowtime = time.perf_counter() - self.basetime
+        self.timedict[name] = nowtime
+        self.basetime = time.perf_counter()
+
+
+class TimeHolder:
+    def __init__(self) -> None:
+        self.timedict = {}
+
+    def update(self, _timedict:dict):
+        for k,v in _timedict.items():
+            if k not in self.timedict:
+                self.timedict[k] = AverageMeter(name=k, val_only=True)
+            self.timedict[k].update(val=v)
+
+    def final_res(self):
+        return {k:v.avg for k,v in self.timedict.items()}
+        
+    def __str__(self):
+        return json.dumps(self.final_res(), indent=2)
+
+
+class AverageMeter(object):
+    """Computes and stores the average and current value"""
+    def __init__(self, name, fmt=':f', val_only=False):
+        self.name = name
+        self.fmt = fmt
+        self.val_only = val_only
+        self.reset()
+
+    def reset(self):
+        self.val = 0
+        self.avg = 0
+        self.sum = 0
+        self.count = 0
+
+    def update(self, val, n=1):
+        self.val = val
+        self.sum += val * n
+        self.count += n
+        self.avg = self.sum / self.count
+
+    def __str__(self):
+        if self.val_only:
+            fmtstr = '{name} {val' + self.fmt + '}'
+        else:
+            fmtstr = '{name} {val' + self.fmt + '} ({avg' + self.fmt + '})'
+        return fmtstr.format(**self.__dict__)

src/utils/dependencies/XPose/util/utils.py

@@ -0,0 +1,499 @@
+from collections import OrderedDict
+from copy import deepcopy
+from typing import Any, Dict, Iterable, List
+import json
+import warnings
+
+import torch
+import numpy as np
+
+def slprint(x, name="x"):
+    if isinstance(x, (torch.Tensor, np.ndarray)):
+        print(f"{name}.shape:", x.shape)
+    elif isinstance(x, (tuple, list)):
+        print("type x:", type(x))
+        for i in range(min(10, len(x))):
+            slprint(x[i], f"{name}[{i}]")
+    elif isinstance(x, dict):
+        for k, v in x.items():
+            slprint(v, f"{name}[{k}]")
+    else:
+        print(f"{name}.type:", type(x))
+
+def clean_state_dict(state_dict):
+    new_state_dict = OrderedDict()
+    for k, v in state_dict.items():
+        if k[:7] == 'module.':
+            k = k[7:]  # remove `module.`
+        new_state_dict[k] = v
+    return new_state_dict
+
+def renorm(img: torch.FloatTensor, mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]) \
+        -> torch.FloatTensor:
+    # img: tensor(3,H,W) or tensor(B,3,H,W)
+    # return: same as img
+    assert img.dim() == 3 or img.dim() == 4, "img.dim() should be 3 or 4 but %d" % img.dim() 
+    if img.dim() == 3:
+        assert img.size(0) == 3, 'img.size(0) shoule be 3 but "%d". (%s)' % (img.size(0), str(img.size()))
+        img_perm = img.permute(1,2,0)
+        mean = torch.Tensor(mean)
+        std = torch.Tensor(std)
+        img_res = img_perm * std + mean
+        return img_res.permute(2,0,1)
+    else: # img.dim() == 4
+        assert img.size(1) == 3, 'img.size(1) shoule be 3 but "%d". (%s)' % (img.size(1), str(img.size()))
+        img_perm = img.permute(0,2,3,1)
+        mean = torch.Tensor(mean)
+        std = torch.Tensor(std)
+        img_res = img_perm * std + mean
+        return img_res.permute(0,3,1,2)
+
+
+
+class CocoClassMapper():
+    def __init__(self) -> None:
+        self.category_map_str = {"1": 1, "2": 2, "3": 3, "4": 4, "5": 5, "6": 6, "7": 7, "8": 8, "9": 9, "10": 10, "11": 11, "13": 12, "14": 13, "15": 14, "16": 15, "17": 16, "18": 17, "19": 18, "20": 19, "21": 20, "22": 21, "23": 22, "24": 23, "25": 24, "27": 25, "28": 26, "31": 27, "32": 28, "33": 29, "34": 30, "35": 31, "36": 32, "37": 33, "38": 34, "39": 35, "40": 36, "41": 37, "42": 38, "43": 39, "44": 40, "46": 41, "47": 42, "48": 43, "49": 44, "50": 45, "51": 46, "52": 47, "53": 48, "54": 49, "55": 50, "56": 51, "57": 52, "58": 53, "59": 54, "60": 55, "61": 56, "62": 57, "63": 58, "64": 59, "65": 60, "67": 61, "70": 62, "72": 63, "73": 64, "74": 65, "75": 66, "76": 67, "77": 68, "78": 69, "79": 70, "80": 71, "81": 72, "82": 73, "84": 74, "85": 75, "86": 76, "87": 77, "88": 78, "89": 79, "90": 80}
+        self.origin2compact_mapper = {int(k):v-1 for k,v in self.category_map_str.items()}
+        self.compact2origin_mapper = {int(v-1):int(k) for k,v in self.category_map_str.items()}
+
+    def origin2compact(self, idx):
+        return self.origin2compact_mapper[int(idx)]
+
+    def compact2origin(self, idx):
+        return self.compact2origin_mapper[int(idx)]
+
+def to_device(item, device):
+    if isinstance(item, torch.Tensor):
+        return item.to(device)
+    elif isinstance(item, list):
+        return [to_device(i, device) for i in item]
+    elif isinstance(item, dict):
+        return {k: to_device(v, device) for k,v in item.items()}
+    else:
+        raise NotImplementedError("Call Shilong if you use other containers! type: {}".format(type(item)))
+
+
+
+# 
+def get_gaussian_mean(x, axis, other_axis, softmax=True):
+    """
+
+    Args:
+        x (float): Input images(BxCxHxW)
+        axis (int): The index for weighted mean
+        other_axis (int): The other index
+
+    Returns: weighted index for axis, BxC
+
+    """
+    mat2line = torch.sum(x, axis=other_axis)
+    # mat2line = mat2line / mat2line.mean() * 10
+    if softmax:
+        u = torch.softmax(mat2line, axis=2)
+    else:
+        u = mat2line / (mat2line.sum(2, keepdim=True) + 1e-6)
+    size = x.shape[axis]
+    ind = torch.linspace(0, 1, size).to(x.device)
+    batch = x.shape[0]
+    channel = x.shape[1]
+    index = ind.repeat([batch, channel, 1])
+    mean_position = torch.sum(index * u, dim=2)
+    return mean_position
+
+def get_expected_points_from_map(hm, softmax=True):
+    """get_gaussian_map_from_points
+        B,C,H,W -> B,N,2 float(0, 1) float(0, 1)
+        softargmax function
+
+    Args:
+        hm (float): Input images(BxCxHxW)
+
+    Returns: 
+        weighted index for axis, BxCx2. float between 0 and 1.
+
+    """
+    # hm = 10*hm
+    B,C,H,W = hm.shape
+    y_mean = get_gaussian_mean(hm, 2, 3, softmax=softmax) # B,C
+    x_mean = get_gaussian_mean(hm, 3, 2, softmax=softmax) # B,C
+    # return torch.cat((x_mean.unsqueeze(-1), y_mean.unsqueeze(-1)), 2)
+    return torch.stack([x_mean, y_mean], dim=2)
+
+# Positional encoding (section 5.1)
+# borrow from nerf
+class Embedder:
+    def __init__(self, **kwargs):
+        self.kwargs = kwargs
+        self.create_embedding_fn()
+        
+    def create_embedding_fn(self):
+        embed_fns = []
+        d = self.kwargs['input_dims']
+        out_dim = 0
+        if self.kwargs['include_input']:
+            embed_fns.append(lambda x : x)
+            out_dim += d
+            
+        max_freq = self.kwargs['max_freq_log2']
+        N_freqs = self.kwargs['num_freqs']
+        
+        if self.kwargs['log_sampling']:
+            freq_bands = 2.**torch.linspace(0., max_freq, steps=N_freqs)
+        else:
+            freq_bands = torch.linspace(2.**0., 2.**max_freq, steps=N_freqs)
+            
+        for freq in freq_bands:
+            for p_fn in self.kwargs['periodic_fns']:
+                embed_fns.append(lambda x, p_fn=p_fn, freq=freq : p_fn(x * freq))
+                out_dim += d
+                    
+        self.embed_fns = embed_fns
+        self.out_dim = out_dim
+        
+    def embed(self, inputs):
+        return torch.cat([fn(inputs) for fn in self.embed_fns], -1)
+
+
+def get_embedder(multires, i=0):
+    import torch.nn as nn
+    if i == -1:
+        return nn.Identity(), 3
+    
+    embed_kwargs = {
+                'include_input' : True,
+                'input_dims' : 3,
+                'max_freq_log2' : multires-1,
+                'num_freqs' : multires,
+                'log_sampling' : True,
+                'periodic_fns' : [torch.sin, torch.cos],
+    }
+    
+    embedder_obj = Embedder(**embed_kwargs)
+    embed = lambda x, eo=embedder_obj : eo.embed(x)
+    return embed, embedder_obj.out_dim
+
+class APOPMeter():
+    def __init__(self) -> None:
+        self.tp = 0
+        self.fp = 0
+        self.tn = 0
+        self.fn = 0
+
+    def update(self, pred, gt):
+        """
+        Input:
+            pred, gt: Tensor()
+        """
+        assert pred.shape == gt.shape
+        self.tp += torch.logical_and(pred == 1, gt == 1).sum().item()
+        self.fp += torch.logical_and(pred == 1, gt == 0).sum().item()
+        self.tn += torch.logical_and(pred == 0, gt == 0).sum().item()
+        self.tn += torch.logical_and(pred == 1, gt == 0).sum().item()
+
+    def update_cm(self, tp, fp, tn, fn):
+        self.tp += tp
+        self.fp += fp
+        self.tn += tn
+        self.tn += fn
+
+def inverse_sigmoid(x, eps=1e-5):
+    x = x.clamp(min=0, max=1)
+    x1 = x.clamp(min=eps)
+    x2 = (1 - x).clamp(min=eps)
+    return torch.log(x1/x2)
+
+import argparse
+from util.config import Config
+def get_raw_dict(args):
+    """
+    return the dicf contained in args.
+    
+    e.g:
+        >>> with open(path, 'w') as f:
+                json.dump(get_raw_dict(args), f, indent=2)
+    """
+    if isinstance(args, argparse.Namespace): 
+        return vars(args)   
+    elif isinstance(args, dict):
+        return args
+    elif isinstance(args, Config):
+        return args._cfg_dict
+    else:
+        raise NotImplementedError("Unknown type {}".format(type(args)))
+
+
+def stat_tensors(tensor):
+    assert tensor.dim() == 1
+    tensor_sm = tensor.softmax(0)
+    entropy = (tensor_sm * torch.log(tensor_sm + 1e-9)).sum()
+
+    return {
+        'max': tensor.max(),
+        'min': tensor.min(),
+        'mean': tensor.mean(),
+        'var': tensor.var(),
+        'std': tensor.var() ** 0.5,
+        'entropy': entropy
+    }
+
+
+class NiceRepr:
+    """Inherit from this class and define ``__nice__`` to "nicely" print your
+    objects.
+
+    Defines ``__str__`` and ``__repr__`` in terms of ``__nice__`` function
+    Classes that inherit from :class:`NiceRepr` should redefine ``__nice__``.
+    If the inheriting class has a ``__len__``, method then the default
+    ``__nice__`` method will return its length.
+
+    Example:
+        >>> class Foo(NiceRepr):
+        ...    def __nice__(self):
+        ...        return 'info'
+        >>> foo = Foo()
+        >>> assert str(foo) == '<Foo(info)>'
+        >>> assert repr(foo).startswith('<Foo(info) at ')
+
+    Example:
+        >>> class Bar(NiceRepr):
+        ...    pass
+        >>> bar = Bar()
+        >>> import pytest
+        >>> with pytest.warns(None) as record:
+        >>>     assert 'object at' in str(bar)
+        >>>     assert 'object at' in repr(bar)
+
+    Example:
+        >>> class Baz(NiceRepr):
+        ...    def __len__(self):
+        ...        return 5
+        >>> baz = Baz()
+        >>> assert str(baz) == '<Baz(5)>'
+    """
+
+    def __nice__(self):
+        """str: a "nice" summary string describing this module"""
+        if hasattr(self, '__len__'):
+            # It is a common pattern for objects to use __len__ in __nice__
+            # As a convenience we define a default __nice__ for these objects
+            return str(len(self))
+        else:
+            # In all other cases force the subclass to overload __nice__
+            raise NotImplementedError(
+                f'Define the __nice__ method for {self.__class__!r}')
+
+    def __repr__(self):
+        """str: the string of the module"""
+        try:
+            nice = self.__nice__()
+            classname = self.__class__.__name__
+            return f'<{classname}({nice}) at {hex(id(self))}>'
+        except NotImplementedError as ex:
+            warnings.warn(str(ex), category=RuntimeWarning)
+            return object.__repr__(self)
+
+    def __str__(self):
+        """str: the string of the module"""
+        try:
+            classname = self.__class__.__name__
+            nice = self.__nice__()
+            return f'<{classname}({nice})>'
+        except NotImplementedError as ex:
+            warnings.warn(str(ex), category=RuntimeWarning)
+            return object.__repr__(self)
+
+
+
+def ensure_rng(rng=None):
+    """Coerces input into a random number generator.
+
+    If the input is None, then a global random state is returned.
+
+    If the input is a numeric value, then that is used as a seed to construct a
+    random state. Otherwise the input is returned as-is.
+
+    Adapted from [1]_.
+
+    Args:
+        rng (int | numpy.random.RandomState | None):
+            if None, then defaults to the global rng. Otherwise this can be an
+            integer or a RandomState class
+    Returns:
+        (numpy.random.RandomState) : rng -
+            a numpy random number generator
+
+    References:
+        .. [1] https://gitlab.kitware.com/computer-vision/kwarray/blob/master/kwarray/util_random.py#L270  # noqa: E501
+    """
+
+    if rng is None:
+        rng = np.random.mtrand._rand
+    elif isinstance(rng, int):
+        rng = np.random.RandomState(rng)
+    else:
+        rng = rng
+    return rng
+
+def random_boxes(num=1, scale=1, rng=None):
+    """Simple version of ``kwimage.Boxes.random``
+
+    Returns:
+        Tensor: shape (n, 4) in x1, y1, x2, y2 format.
+
+    References:
+        https://gitlab.kitware.com/computer-vision/kwimage/blob/master/kwimage/structs/boxes.py#L1390
+
+    Example:
+        >>> num = 3
+        >>> scale = 512
+        >>> rng = 0
+        >>> boxes = random_boxes(num, scale, rng)
+        >>> print(boxes)
+        tensor([[280.9925, 278.9802, 308.6148, 366.1769],
+                [216.9113, 330.6978, 224.0446, 456.5878],
+                [405.3632, 196.3221, 493.3953, 270.7942]])
+    """
+    rng = ensure_rng(rng)
+
+    tlbr = rng.rand(num, 4).astype(np.float32)
+
+    tl_x = np.minimum(tlbr[:, 0], tlbr[:, 2])
+    tl_y = np.minimum(tlbr[:, 1], tlbr[:, 3])
+    br_x = np.maximum(tlbr[:, 0], tlbr[:, 2])
+    br_y = np.maximum(tlbr[:, 1], tlbr[:, 3])
+
+    tlbr[:, 0] = tl_x * scale
+    tlbr[:, 1] = tl_y * scale
+    tlbr[:, 2] = br_x * scale
+    tlbr[:, 3] = br_y * scale
+
+    boxes = torch.from_numpy(tlbr)
+    return boxes
+
+
+class ModelEma(torch.nn.Module):
+    def __init__(self, model, decay=0.9997, device=None):
+        super(ModelEma, self).__init__()
+        # make a copy of the model for accumulating moving average of weights
+        self.module = deepcopy(model)
+        self.module.eval()
+
+        # import ipdb; ipdb.set_trace()
+
+        self.decay = decay
+        self.device = device  # perform ema on different device from model if set
+        if self.device is not None:
+            self.module.to(device=device)
+
+    def _update(self, model, update_fn):
+        with torch.no_grad():
+            for ema_v, model_v in zip(self.module.state_dict().values(), model.state_dict().values()):
+                if self.device is not None:
+                    model_v = model_v.to(device=self.device)
+                ema_v.copy_(update_fn(ema_v, model_v))
+
+    def update(self, model):
+        self._update(model, update_fn=lambda e, m: self.decay * e + (1. - self.decay) * m)
+
+    def set(self, model):
+        self._update(model, update_fn=lambda e, m: m)
+
+class BestMetricSingle():
+    def __init__(self, init_res=0.0, better='large') -> None:
+        self.init_res = init_res
+        self.best_res = init_res
+        self.best_ep = -1
+
+        self.better = better
+        assert better in ['large', 'small']
+
+    def isbetter(self, new_res, old_res):
+        if self.better == 'large':
+            return new_res > old_res
+        if self.better == 'small':
+            return new_res < old_res
+
+    def update(self, new_res, ep):
+        if self.isbetter(new_res, self.best_res):
+            self.best_res = new_res
+            self.best_ep = ep
+            return True
+        return False
+
+    def __str__(self) -> str:
+        return "best_res: {}\t best_ep: {}".format(self.best_res, self.best_ep)
+
+    def __repr__(self) -> str:
+        return self.__str__()
+
+    def summary(self) -> dict:
+        return {
+            'best_res': self.best_res,
+            'best_ep': self.best_ep,
+        }
+
+
+class BestMetricHolder():
+    def __init__(self, init_res=0.0, better='large', use_ema=False) -> None:
+        self.best_all = BestMetricSingle(init_res, better)
+        self.use_ema = use_ema
+        if use_ema:
+            self.best_ema = BestMetricSingle(init_res, better)
+            self.best_regular = BestMetricSingle(init_res, better)
+    
+
+    def update(self, new_res, epoch, is_ema=False):
+        """
+        return if the results is the best.
+        """
+        if not self.use_ema:
+            return self.best_all.update(new_res, epoch)
+        else:
+            if is_ema:
+                self.best_ema.update(new_res, epoch)
+                return self.best_all.update(new_res, epoch)
+            else:
+                self.best_regular.update(new_res, epoch)
+                return self.best_all.update(new_res, epoch)
+
+    def summary(self):
+        if not self.use_ema:
+            return self.best_all.summary()
+
+        res = {}
+        res.update({f'all_{k}':v for k,v in self.best_all.summary().items()})
+        res.update({f'regular_{k}':v for k,v in self.best_regular.summary().items()})
+        res.update({f'ema_{k}':v for k,v in self.best_ema.summary().items()})
+        return res
+
+    def __repr__(self) -> str:
+        return json.dumps(self.summary(), indent=2)
+
+    def __str__(self) -> str:
+        return self.__repr__()
+            
+
+def targets_to(targets: List[Dict[str, Any]], device):
+    """Moves the target dicts to the given device."""
+    excluded_keys = [
+        "questionId",
+        "tokens_positive",
+        "strings_positive",
+        "tokens",
+        "dataset_name",
+        "sentence_id",
+        "original_img_id",
+        "nb_eval",
+        "task_id",
+        "original_id",
+        "token_span",
+        "caption",
+        "dataset_type",
+        "caption_list",
+        "id2catname",
+        "valid_kpt_num",
+        "image_id_ref",
+        "image_id_current",
+        "test_id"
+    ]
+    return [{k: v.to(device) if k not in excluded_keys else v for k, v in t.items()} for t in targets]