从零开始构建基于textcnn的文本分类模型(上)

沉睡的小卡比兽. 基于LAC分词与gensim的词向量训练，pandas批量中文分词[EB/OL]. https://blog.csdn.net/chen10314/article/details/121996866

from collections import Counterfrom tqdm import tqdm
def get_cutwords_list(line):    sen = lac.run(line) #采用lac进行分词，返回list    return sen
def get_vocab(config):    model_train = pd.read_csv(config['train_file_path'])    model_train['cut'] = ''    cut_df(model_train[['sentence']], get_cutwords_list)
    token_counter = Counter()
    for i in tqdm(model_train['cut'], total=len(model_train['cut']), desc='Counting token'):        token_counter.update(i)
    vocab = set(token for token, _ in token_counter.most_common(config['vocab_size']))
    return vocab
vocab = get_vocab(config)

from gensim.models import KeyedVectorsdef get_embedding(vocab):    token2embedding ={}
    word2vec = KeyedVectors.load('w2v/model/allw2v.model')
    for token in vocab:        if token in word2vec.wv.key_to_index.keys():            token2embedding[token] = word2vec.wv[token]
    meta_info = word2vec.wv[0].shape[0]    token2id = {token: idx for idx, token in enumerate(token2embedding.keys(), 4)}    id2embedding = {token2id[token]: embedding for token, embedding in token2embedding.items()}
    PAD, UNK, BOS, EOS = '<pad>', '<unk>', '<bos>', '<eos>'
    token2id[PAD] = 0    token2id[UNK] = 1    token2id[BOS] = 2    token2id[EOS] = 3
    id2embedding[0] = [.0] * int(meta_info)    id2embedding[1] = [.0] * int(meta_info)    id2embedding[2] = np.random.random(int(meta_info)).tolist()    id2embedding[3] = np.random.random(int(meta_info)).tolist()
    emb_mat = [id2embedding[idx] for idx in range(len(id2embedding))]
    return torch.tensor(emb_mat, dtype=torch.float), token2id, len(vocab)+4

config = {    'train_file_path': 'dataset/train.csv',    'test_file_path': 'dataset/test.csv',    'train_val_ratio': 0.1,  # 10%用作验证集    'vocab_size': 30000,   # 词典 3W    'batch_size': 64,      # batch 大小 64    'num_epochs': 2,      # 2次迭代    'learning_rate': 1e-3, # 学习率    'logging_step': 300,   # 每跑300个batch记录一次    'seed': 2021           # 随机种子}
config['device'] = 'cuda' if torch.cuda.is_available() else 'cpu' # cpu&gpu
import randomimport numpy as np
def seed_everything(seed):    random.seed(seed)    np.random.seed(seed)    torch.manual_seed(seed)    torch.cuda.manual_seed_all(seed)    return seed
seed_everything(config['seed'])

def tokenizer(sent, token2id):    ids = [token2id.get(token, 1) for token in lac.run(sent)]    return ids

import pandas as pdfrom collections import defaultdictdef read_data(config, token2id, mode='train'):    data_df = pd.read_csv(config[f'{mode}_file_path'], sep=',')    if mode == 'train':        X_train, y_train = defaultdict(list), []        X_val, y_val = defaultdict(list), []        num_val = int(config['train_val_ratio'] * len(data_df))
    else:        X_test, y_test = defaultdict(list), []
    for i, row in tqdm(data_df.iterrows(), desc=f'Preprocesing {mode} data', total=len(data_df)):        label=row[1] if mode == 'train' else 0        sentence = row[-1]        inputs = tokenizer(sentence, token2id)        if mode == 'train':            if i < num_val:                X_val['input_ids'].append(inputs)                y_val.append(label)            else:                X_train['input_ids'].append(inputs)                y_train.append(label)
        else:            X_test['input_ids'].append(inputs)            y_test.append(label)
    if mode == 'train':        label2id = {label: i for i, label in enumerate(np.unique(y_train))}        id2label = {i: label for label, i in label2id.items()}
        y_train = torch.tensor([label2id[label] for label in y_train], dtype=torch.long)        y_val = torch.tensor([label2id[label] for label in y_val], dtype=torch.long)
        return X_train, y_train, X_val, y_val, label2id, id2label
    else:        y_test = torch.tensor(y_test, dtype=torch.long)        return X_test, y_test

X_train, y_train, X_val, y_val, label2id, id2label = read_data(config, token2id, mode='train')X_test, y_test = read_data(config, token2id, mode='test')

from torch.utils.data import Datasetclass TNEWSDataset(Dataset):    def __init__(self, X, y):        self.x = X        self.y = y
    def __getitem__(self, idx):        return {            'input_ids': self.x['input_ids'][idx],            'label': self.y[idx]        }
    def __len__(self):        return self.y.size(0)

torch.utils.data — PyTorch 1.7.1 documentation[EB/OL]. https://pytorch.org/docs/1.7.1/data.html?highlight=dataset#torch.utils.data.Dataset

def my_collate_fn(examples):    input_ids_list =[]    labels = []    for example in examples:        input_ids_list.append(example['input_ids'])        labels.append(example['label'])
    # 1.找到 input_ids_list 中最长的句子    max_length = max(len(input_ids) for input_ids in input_ids_list)
    # 2. 定义一个Tensor    input_ids_tensor = torch.zeros((len(labels), max_length), dtype=torch.long)
    for i, input_ids in enumerate(input_ids_list):        # 3.得到当前句子长度        seq_len = len(input_ids)        input_ids_tensor[i, :seq_len] = torch.tensor(input_ids, dtype=torch.long)
    return {        'input_ids': input_ids_tensor,        'label': torch.tensor(labels, dtype=torch.long)    }

from torch.utils.data import DataLoaderdef build_dataloader(config, vocab):    X_train, y_train, X_val, y_val, label2id, id2label = read_data(config, token2id, mode='train')    X_test, y_test = read_data(config, token2id, mode='test')
    train_dataset = TNEWSDataset(X_train, y_train)    val_dataset = TNEWSDataset(X_val, y_val)    test_dataset = TNEWSDataset(X_test, y_test)
    train_dataloader = DataLoader(dataset=train_dataset, batch_size=config['batch_size'], num_workers=4, shuffle=True, collate_fn=my_collate_fn)    val_dataloader = DataLoader(dataset=val_dataset, batch_size=config['batch_size'], num_workers=4, shuffle=False, collate_fn=my_collate_fn)    test_dataloader = DataLoader(dataset=test_dataset, batch_size=config['batch_size'], num_workers=4, shuffle=False, collate_fn=my_collate_fn)
    return id2label, train_dataloader, val_dataloader, test_dataloader

 id2label, train_dataloader, val_dataloader, test_dataloader = build_dataloader(config, vocab)

torch.utils.data — PyTorch 1.7.1 documentation[EB/OL]. https://pytorch.org/docs/1.7.1/data.html?highlight=dataloader#torch.utils.data.DataLoader