import math import torch import torch.nn as nn import torch.nn.functional as F import time # FFN def FeedForward(dim, mult=4): inner_dim = int(dim * mult) return nn.Sequential( nn.LayerNorm(dim), nn.Linear(dim, inner_dim, bias=False), nn.GELU(), nn.Linear(inner_dim, dim, bias=False), ) def reshape_tensor(x, heads): bs, length, width = x.shape # (bs, length, width) --> (bs, length, n_heads, dim_per_head) x = x.view(bs, length, heads, -1) # (bs, length, n_heads, dim_per_head) --> (bs, n_heads, length, dim_per_head) x = x.transpose(1, 2) # (bs, n_heads, length, dim_per_head) --> (bs*n_heads, length, dim_per_head) x = x.reshape(bs, heads, length, -1) return x class PerceiverAttentionCA(nn.Module): def __init__(self, *, dim=3072, dim_head=128, heads=16, kv_dim=2048): super().__init__() self.scale = dim_head ** -0.5 self.dim_head = dim_head self.heads = heads inner_dim = dim_head * heads self.norm1 = nn.LayerNorm(dim if kv_dim is None else kv_dim) self.norm2 = nn.LayerNorm(dim) self.to_q = nn.Linear(dim, inner_dim, bias=False) self.to_kv = nn.Linear(dim if kv_dim is None else kv_dim, inner_dim * 2, bias=False) self.to_out = nn.Linear(inner_dim, dim, bias=False) def forward(self, x, latents): """ Args: x (torch.Tensor): image features shape (b, n1, D) latent (torch.Tensor): latent features shape (b, n2, D) """ x = self.norm1(x) latents = self.norm2(latents) b, seq_len, _ = latents.shape q = self.to_q(latents) k, v = self.to_kv(x).chunk(2, dim=-1) q = reshape_tensor(q, self.heads) k = reshape_tensor(k, self.heads) v = reshape_tensor(v, self.heads) # attention scale = 1 / math.sqrt(math.sqrt(self.dim_head)) weight = (q * scale) @ (k * scale).transpose(-2, -1) # More stable with f16 than dividing afterwards weight = torch.softmax(weight.float(), dim=-1).type(weight.dtype) out = weight @ v out = out.permute(0, 2, 1, 3).reshape(b, seq_len, -1) return self.to_out(out) class PerceiverAttention(nn.Module): def __init__(self, *, dim, dim_head=64, heads=8, kv_dim=None): super().__init__() self.scale = dim_head ** -0.5 self.dim_head = dim_head self.heads = heads inner_dim = dim_head * heads self.norm1 = nn.LayerNorm(dim if kv_dim is None else kv_dim) self.norm2 = nn.LayerNorm(dim) self.to_q = nn.Linear(dim, inner_dim, bias=False) self.to_k = nn.Linear(dim if kv_dim is None else kv_dim, inner_dim , bias=False) self.to_v = nn.Linear(dim if kv_dim is None else kv_dim, inner_dim , bias=False) self.to_out = nn.Linear(inner_dim, dim, bias=False) def forward(self, gl_key,gl_value, latents): """ Args: x (torch.Tensor): image features shape (b, n1, D) latent (torch.Tensor): latent features shape (b, n2, D) """ gl_key=self.norm1(gl_key) gl_value=self.norm1(gl_value) latents = self.norm2(latents) b, seq_len, _ = latents.shape q = self.to_q(latents) k=self.to_k(gl_key) v=self.to_v(gl_value) q = reshape_tensor(q, self.heads) k = reshape_tensor(k, self.heads) v = reshape_tensor(v, self.heads) out = F.scaled_dot_product_attention( q, k, v, attn_mask=None, dropout_p=0.0, is_causal=False ) # attention ''' scale = 1 / math.sqrt(math.sqrt(self.dim_head)) weight = (q * scale) @ (k * scale).transpose(-2, -1) # More stable with f16 than dividing afterwards weight = torch.softmax(weight.float(), dim=-1).type(weight.dtype) out = weight @ v ''' out = out.permute(0, 2, 1, 3).reshape(b, seq_len, -1) return self.to_out(out) class global_local_memory(nn.Module): """ - perceiver resampler like arch (compared with previous MLP-like arch) - we concat id embedding (generated by arcface) and query tokens as latents - latents will attend each other and interact with vit features through cross-attention - vit features are multi-scaled and inserted into IDFormer in order, currently, each scale corresponds to two IDFormer layers """ def __init__( self, dim=3072, depth=3, dim_head=96, heads=32, num_queries=32, output_dim=3072, ff_mult=4, ): super().__init__() self.dim = dim self.num_queries = num_queries assert depth % 3 == 0 scale = dim ** -0.5 self.proj_out = nn.Parameter( torch.zeros(dim, output_dim)) self.layers = nn.ModuleList([]) for _ in range(depth): self.layers.append( nn.ModuleList( [ PerceiverAttention(dim=dim, dim_head=dim_head, heads=heads), FeedForward(dim=dim, mult=ff_mult), ] ) ) for i in range(3): setattr( self, f'key_mapping_{i}', nn.Sequential( nn.Linear(512, 1024), nn.LayerNorm(1024), nn.LeakyReLU(), nn.Linear(1024, 2048), nn.LayerNorm(2048), nn.LeakyReLU(), nn.Linear(2048, dim), ), ) setattr( self, f'value_mapping_{i}', nn.Sequential( nn.Linear(512, 1024), nn.LayerNorm(1024), nn.LeakyReLU(), nn.Linear(1024, 2048), nn.LayerNorm(2048), nn.LeakyReLU(), nn.Linear(2048, dim), ), ) def forward(self, global_memory, local_memory,latents): latents = latents.repeat(global_memory.size(0), 1,1) global_local_memory_key=torch.cat((global_memory[:,0],local_memory[:,0]),dim=2) #torch.Size([5, 5110, 512]) global_local_memory_value=torch.cat((global_memory[:,1],local_memory[:,1]),dim=2) #torch.Size([5, 5110, 512]) for i in range(3): global_local_key = getattr(self, f'key_mapping_{i}')(global_local_memory_key[:,i]) global_local_value = getattr(self, f'value_mapping_{i}')(global_local_memory_value[:,i]) attn,ff=self.layers[i] latents = attn(global_local_key.repeat_interleave(latents.shape[0], dim=0), global_local_value.repeat_interleave(latents.shape[0], dim=0), latents) + latents latents = ff(latents) + latents latents = latents @ self.proj_out return latents if __name__=="__main__": global_memory=torch.randn(1,2, 3, 3626, 512) local_memory=torch.randn(1,2,3,1484,512) latents=torch.randn(1,25344,3072) glm=global_local_memory() total_params = sum(p.numel() for p in glm.parameters()) print(total_params) start_time=time.time() latents=glm(global_memory,local_memory,latents) end_time=time.time() print(f"Elapsed time (process_time): {end_time-start_time} seconds")