""" Adapted from Oliver Atanaszov’s notebook on transformer fine-tuning https://github.com/ben0it8/containerized-transformer-finetuning/blob/develop/research/finetune-transformer-on-imdb5k.ipynb """ from concurrent.futures import ProcessPoolExecutor import multiprocessing import os import numpy as np import pandas as pd import torch from torch.utils.data import TensorDataset, random_split, DataLoader from tqdm import tqdm
n_cpu = multiprocessing.cpu_count() MAX_LENGTH = 256
class TextProcessor: def __init__(self, tokenizer, label2id: dict, max_length: int=512): self.tokenizer = tokenizer self.label2id = label2id self.max_length = max_length self.clf_token = self.tokenizer.vocab[’[CLS]’] self.pad_token = self.tokenizer.vocab[’[PAD]']
def encode(self, input): return list(self.tokenizer.convert_tokens_to_ids(o) for o in input)
def token2id(self, item: Tuple[str, str]): “Convert text (item[0]) to sequence of IDs and label (item[1]) to integer” assert len(item) == 2 # Need a row of text AND labels label, text = item[0], item[1] assert isinstance(text, str) # Need position 1 of input to be of type(str) inputs = self.tokenizer.tokenize(text)
if len(inputs) >= self.max_length: inputs = inputs[:self.max_length - 1] ids = self.encode(inputs) + [self.clf_token] else: pad = [self.pad_token] * (self.max_length - len(inputs) - 1) ids = self.encode(inputs) + [self.clf_token] + pad
return np.array(ids, dtype=‘int64’), self.label2id[label]
def process_row(self, row): “Calls the token2id method of the text processor for passing items to executor” return self.token2id((row[1][LABEL_COL], row[1][TEXT_COL]))
def create_dataloader(self, df: pd.DataFrame, batch_size: int = 32, shuffle: bool = False, valid_pct: float = None): “Process rows in pd.DataFrame using n_cpus and return a DataLoader”
tqdm.pandas() with ProcessPoolExecutor(max_workers=n_cpu) as executor: result = list( tqdm(executor.map(self.process_row, df.iterrows(), chunksize=8192), desc=f"Processing {len(df)} examples on {n_cpu} cores", total=len(df)))
features = [r[0] for r in result] labels = [r[1] for r in result]
dataset = TensorDataset(torch.tensor(features, dtype=torch.long), torch.tensor(labels, dtype=torch.long))
data_loader = DataLoader(dataset, batch_size=batch_size, num_workers=0, shuffle=shuffle, pin_memory=torch.cuda.is_available()) return data_loader