使用Transformers实现文本分类
实现 分类 文本 Transformers 使用
2023-09-14 09:08:40 时间
github: https://github.com/haibincoder/NlpSummary/tree/master/torchcode/classification
import copy
from torch import nn
import torch.nn.functional as F
import torch
import math
class Config(object):
"""配置参数"""
def __init__(self, vocab_size, embed_dim, label_num, max_length=32):
self.embedding_pretrained = None
self.num_classes = label_num # 类别数
self.vocab_size = vocab_size # 词表大小,在运行时赋值
self.embed_dim = embed_dim # 字向量维度
self.num_head = 5
self.dropout = 0.1
self.hidden = 512
self.num_encoder = 2
self.max_length = max_length
self.lr = 1e-3
class Model(nn.Module):
def __init__(self, config):
super().__init__()
if config.embedding_pretrained is not None:
self.embedding = nn.Embedding.from_pretrained(config.embedding_pretrained, freeze=False)
else:
self.embedding = nn.Embedding(config.vocab_size, config.embed_dim, padding_idx=config.vocab_size - 1)
self.encoder = Encoder(config.embed_dim, config.num_head, config.hidden, config.dropout)
self.encoders = nn.ModuleList([
copy.deepcopy(self.encoder)
for _ in range(config.num_encoder)])
self.fc1 = nn.Linear(config.max_length * config.embed_dim, config.num_classes)
def forward(self, x):
out = self.embedding(x)
for encoder in self.encoders:
out = encoder(out)
out = out.view(out.size(0), -1)
# out = torch.mean(out, 1)
out = self.fc1(out)
return out
class Encoder(nn.Module):
def __init__(self, embed_dim, num_head, hidden, dropout=0.0):
super().__init__()
self.attention = MultiHeadAttention(embed_dim, num_head, dropout)
self.feed_forward = Position_wise_Feed_Forward(embed_dim, hidden, dropout)
def forward(self, x):
out = self.attention(x)
out = self.feed_forward(out)
return out
class MultiHeadAttention(nn.Module):
def __init__(self, embed_dim, num_head, dropout=0.0):
super().__init__()
self.num_head = num_head
assert embed_dim % num_head == 0, 'head num error'
self.dim_head = embed_dim // num_head
self.fc_q = nn.Linear(embed_dim, embed_dim)
self.fc_k = nn.Linear(embed_dim, embed_dim)
self.fc_v = nn.Linear(embed_dim, embed_dim)
self.attention = Attention()
self.fc = nn.Linear(embed_dim, embed_dim)
self.dropout = nn.Dropout(dropout)
self.layer_norm = nn.LayerNorm(embed_dim)
def forward(self, x):
batch_size = x.size(0)
Q = self.fc_q(x)
K = self.fc_k(x)
V = self.fc_v(x)
Q = Q.view(batch_size * self.num_head, -1, self.dim_head)
K = K.view(batch_size * self.num_head, -1, self.dim_head)
V = V.view(batch_size * self.num_head, -1, self.dim_head)
context = self.attention(Q, K, V)
context = context.view(batch_size, -1, self.dim_head * self.num_head)
out = self.fc(context)
out = self.dropout(out)
# 残差
out = out + x
out = self.layer_norm(out)
return out
'''
attention计算
'''
class Attention(nn.Module):
def __init__(self):
super().__init__()
def forward(self, query, key, value):
k = query.size(-1)
result = torch.matmul(query, key.transpose(-2, -1))
score = result / math.sqrt(k)
softmax_result = torch.softmax(score, dim=-1)
result = torch.matmul(softmax_result, value)
return result
'''
这里对应transformers encoder的Feed forward
'''
class Position_wise_Feed_Forward(nn.Module):
def __init__(self, dim_model, hidden, dropout=0.0):
super().__init__()
self.fc1 = nn.Linear(dim_model, hidden)
self.fc2 = nn.Linear(hidden, dim_model)
self.dropout = nn.Dropout(dropout)
self.layer_norm = nn.LayerNorm(dim_model)
def forward(self, x):
out = self.fc1(x)
out = F.relu(out)
out = self.fc2(out)
out = self.dropout(out)
out = out + x # 残差连接
out = self.layer_norm(out)
return out
相关文章
- IIR数字滤波器的实现(C语言)
- 使用libsvm实现文本分类
- Java实现 蓝桥杯VIP 算法提高 栅格打印问题
- Java实现 蓝桥杯VIP 算法训练 接水问题
- Java实现 洛谷 P1064 金明的预算方案
- SKlearn实现鸢尾花分类
- 利用Jquery实现http长连接(LongPoll) {转}
- 项目:用Pygame实现一个简单的垃圾分类小游戏
- 【STM32F429的DSP教程】第47章 STM32F429的IIR带阻滤波器实现(支持逐个数据的实时滤波)
- ZZNUOJ_用C语言编写程序实现1158:矩阵的最大值(指针专题)(附完整源码)
- ML之shap:基于adult人口普查收入二分类预测数据集(预测年收入是否超过50k)利用Shap值对XGBoost模型实现可解释性案例之详细攻略
- ML之FE:数据处理—特征工程之特征选择常用方法之基于搜索策略的三种分类、基于评价准则划分的三种分类(Filter/Wrapper/Embedded)及其代码实现
- ML之shap:基于adult人口普查收入二分类预测数据集(预测年收入是否超过50k)利用shap决策图结合LightGBM模型实现异常值检测案例之详细攻略
- DL之GD:利用LogisticGD算法(梯度下降)依次基于一次函数和二次函数分布的数据集实现二分类预测(超平面可视化)
- TF之Transformer:基于tensorflow和Keras框架(特征编码+Tokenizer处理文本+保存模型)针对UCI新闻数据集利用Transformer算法实现新闻文本多分类案例
- ML之xgboost:基于xgboost(5f-CrVa)算法对HiggsBoson数据集(Kaggle竞赛)训练(模型保存+可视化)实现二分类预测
- Dropout和网络结构实现数据增广带来的不同影响分析
- ConvNeXt V2实战:使用ConvNeXt V2实现图像分类任务(一)
- MicroNet实战:使用MicroNet实现图像分类(二)
- Python实现贝叶斯优化器(Bayes_opt)优化循环神经网络回归模型(LSTM回归算法)项目实战
- Python实现BP神经网络ANN单隐层分类模型项目实战
- Python实现WOA智能鲸鱼优化算法优化支持向量机分类模型(SVC算法)项目实战
- 【项目实战】Python实现LightGBM分类模型(LGBMClassifier算法)项目实战
- 使用TextCNN实现文本分类
- Android kotlin 用RecyclerView(androidx+BRVAH3.0.6+AnyLayer)实现从顶部弹出下拉横式列表一二级分类菜单对话框功能
- DL之CNN:利用卷积神经网络算法(2→2,基于Keras的API-Functional)利用MNIST(手写数字图片识别)数据集实现多分类预测
- NLP进阶,使用TextRNN和TextRNN_ATT实现文本分类
- 嵌入式linux开发,单网卡绑定多IP,实现多网段的访问