python<\/p>\n
import math
\nimport torch
\nimport torch.nn as nn
\nfrom LabmL_helpers.module import Module
\nfrom labml_n.utils import clone_module_List
\nfrom typing import Optional, List
\nfrom torch.utils.data import DataLoader, TensorDataset
\nfrom torch import optim
\nimport torch.nn.functional as F<\/p>\n
python<\/p>\n
import numpy as np
\nfrom math import sqrt
\nimport torch
\nfrom torch import nn<\/p>\n
class Self_Attention(nn.Module):
\n # input : batch_size * seq_len * input_dim
\n # q : batch_size * input_dim * dim_k
\n # k : batch_size * input_dim * dim_k
\n # v : batch_size * input_dim * dim_v
\n def __init__(self, input_dim, dim_k, dim_v):
\n super(Self_Attention, self).__init__()
\n self.q = nn.Linear(input_dim, dim_k)
\n self.k = nn.Linear(input_dim, dim_k)
\n self.v = nn.Linear(input_dim, dim_v)
\n self._norm_fact = 1 \/ sqrt(dim_k)<\/p>\n
def forward(self, x):
\n Q = self.q(x) # Q: batch_size * seq_len * dim_k
\n K = self.k(x) # K: batch_size * seq_len * dim_k
\n V = self.v(x) # V: batch_size * seq_len * dim_v
\n # Q * K.T() # batch_size * seq_len * seq_len
\n atten = nn.Softmax(
\n dim=-1)(torch.bmm(Q, K.permute(0, 2, 1))) * self._norm_fact
\n # Q * K.T() * V # batch_size * seq_len * dim_v
\n output = torch.bmm(atten, V)
\n return output<\/p>\n
X = torch.randn(4, 3, 2)
\nprint(X)
\nself_atten = Self_Attention(2, 4, 5) # input_dim:2, k_dim:4, v_dim:5
\nres = self_atten(X)
\nprint(res.shape) # [4,3,5]<\/p>\n
python<\/p>\n
from math import sqrt
\nimport torch
\nimport torch.nn as nn<\/p>\n
class Self_Attention_Muti_Head(nn.Module):
\n # input : batch_size * seq_len * input_dim
\n # q : batch_size * input_dim * dim_k
\n # k : batch_size * input_dim * dim_k
\n # v : batch_size * input_dim * dim_v
\n def __init__(self, input_dim, dim_k, dim_v, nums_head):
\n super(Self_Attention_Muti_Head, self).__init__()
\n assert dim_k % nums_head == 0
\n assert dim_v % nums_head == 0
\n self.q = nn.Linear(input_dim, dim_k)
\n self.k = nn.Linear(input_dim, dim_k)
\n self.v = nn.Linear(input_dim, dim_v)
\n self.nums_head = nums_head
\n self.dim_k = dim_k
\n self.dim_v = dim_v
\n self._norm_fact = 1 \/ sqrt(dim_k)<\/p>\n
def forward(self, x):
\n Q = self.q(x).reshape(-1, x.shape[0], x.shape[1], self.dim_k \/\/
\n self.nums_head)
\n K = self.k(x).reshape(-1, x.shape[0], x.shape[1], self.dim_k \/\/
\n self.nums_head)
\n V = self.v(x).reshape(-1, x.shape[0], x.shape[1], self.dim_v \/\/
\n self.nums_head)
\n print(x.shape)
\n print(Q.size())
\n atten = nn.Softmax(dim=-1)(torch.matmul(Q, K.permute(0, 1, 3, 2))) # Q * K.T() # batch_size * seq_len * seq_len
\n output = torch.matmul(atten, V).reshape(x.shape[0], x.shape[1], -1) # Q * K.T() * V # batch_size * seq_len * dim_v
\n return output<\/p>\n
x = torch.rand(1, 3, 4)
\nprint(x)
\natten = Self_Attention_Muti_Head(4, 4, 4, 2)
\ny = atten(x)
\nprint(y.shape)<\/p>\n
python<\/p>\n
class SELayer(nn.Module):
\n def __init__(self, channel, reduction=16):
\n super(SELayer, self).__init__()
\n self.avg_pool = nn.AdaptiveAvgPool2d(1)
\n self.fc = nn.Sequential(
\n nn.Linear(channel, channel \/\/ reduction, bias=False),
\n nn.ReLU(inplace=True),
\n nn.Linear(channel \/\/ reduction, channel, bias=False),
\n nn.Sigmoid()
\n )<\/p>\n
def forward(self, x):
\n b, c, _, _ = x.size()
\n y = self.avg_pool(x).view(b, c) # \u5bf9\u5e94Squeeze\u64cd\u4f5c
\n y = self.fc(y).view(b, c, 1, 1) # \u5bf9\u5e94Excitation\u64cd\u4f5c
\n return x * y.expand_as(x)<\/p>\n
python<\/p>\n
class GCT(nn.Module):
\n def __init__(self, num_channels, epsilon=1e-5, mode='l2', after_relu=False):
\n super(GCT, self).__init__()
\n self.alpha = nn.Parameter(torch.ones(1, num_channels, 1, 1))
\n self.gamma = nn.Parameter(torch.zeros(1, num_channels, 1, 1))
\n self.beta = nn.Parameter(torch.zeros(1, num_channels, 1, 1))
\n self.epsilon = epsilon
\n self.mode = mode
\n self.after_relu = after_relu<\/p>\n
def forward(self, x):
\n if self.mode == 'l2':
\n embedding = (x.pow(2).sum((2, 3), keepdim=True) +
\n self.epsilon).pow(0.5) * self.alpha
\n norm = self.gamma \/ \\\\
\n (embedding.pow(2).mean(dim=1, keepdim=True) +
\n self.epsilon).pow(0.5)
\n elif self.mode == 'l1':
\n if not self.after_relu:
\n _x = torch.abs(x)
\n else:
\n _x = x
\n embedding = _x.sum((2, 3), keepdim=True) * self.alpha
\n norm = self.gamma \/ \\\\
\n (torch.abs(embedding).mean(dim=1, keepdim=True) + self.epsilon)
\n gate = 1. + torch.tanh(embedding * norm + self.beta)
\n return x * gate<\/p>\n
GCT \u5efa\u8bae\u6dfb\u52a0\u5728 Conv \u5c42\u524d，\u4e00\u822c\u53ef\u4ee5\u5148\u51bb\u7ed3\u539f\u6765\u7684\u6a21\u578b，\u6765\u8bad\u7ec3 GCT，\u7136\u540e\u89e3\u51bb\u518d\u8fdb\u884c\u5fae\u8c03\u3002<\/p>\n","protected":false},"excerpt":{"rendered":"
\u6587\u7ae0\u6d4f\u89c8\u9605\u8bfb994\u6b21\uff0c\u70b9\u8d5e20\u6b21\uff0c\u6536\u85cf8\u6b21\u3002GCT \u5efa\u8bae\u6dfb\u52a0\u5728 Conv \u5c42\u524d\uff0c\u4e00\u822c\u53ef\u4ee5\u5148\u51bb\u7ed3\u539f\u6765\u7684\u6a21\u578b\uff0c\u6765\u8bad\u7ec3 GCT\uff0c\u7136\u540e\u89e3\u51bb\u518d\u8fdb\u884c\u5fae\u8c03\u3002<\/p>\n","protected":false},"author":2,"featured_media":0,"comment_status":"open","ping_status":"open","sticky":false,"template":"","format":"standard","meta":{"footnotes":""},"categories":[1],"tags":[81,152,841,50,207,86,427],"topic":[],"class_list":["post-36928","post","type-post","status-publish","format-standard","hentry","category-server","tag-python","tag-pytorch","tag-transformer","tag-50","tag-207","tag-86","tag-427"],"yoast_head":"\n