버트를 활용한 영화리뷰 분류

Pre-trained BERT를 활용하여 영화리뷰 감정분류

 

두가지 방법:

1. transformers 의 Trainer를 활용한다.

2. pytorch를 사용한다. 

이 두가지 방법을 다 익혀두는 것이 좋으며, 1번은 수정 혹은 미세조정하기가 조금 까다로워서 2번으로 하는 것을 선호 하는 경우가 많다고 한다. 

 

 

from transformers import AutoConfig, AutoTokenizer, AutoModelForSequenceClassification

config = AutoConfig.from_pretrained('bert-base-uncased')
tokenizer = AutoTokenizer.from_pretrained('bert-base-uncased')
model = AutoModelForSequenceClassification.from_pretrained("bert-base-uncased")

#data 준비
from datasets import load_dataset
raw_datasets = load_dataset("imdb")

 

 

tokenizer.model_max_length= 512

def tokenize_function(examples):
    return tokenizer(examples['text'], padding="max_length", truncation=True)

tokenized_datasets = raw_datasets.map(tokenize_function, batched=True)

small_train_dataset = tokenized_datasets['train'].shuffle(seed=42).select(range(1000))
small_eval_dataset = tokenized_datasets['test'].shuffle(seed=42).select(range(1000))
full_train_dataset = tokenized_datasets['train']
full_eval_dataset = tokenized_datasets['test']

첫번째방법

#Transformers library를 이용한 영화 리뷰 분류기 학습

from transformers import TrainingArguments, Trainer
training_args = TrainingArguments("test_trainer")
# 전체 dataset 학습/평가을 원하시는 분들은 full_train_dataset, full_eval_dataset을 사용하시면 됩니다.
trainer = Trainer(model=model, args=training_args, train_dataset=small_train_dataset, eval_dataset=small_eval_dataset)

trainer.train()

model = BertForSequenceClassification.from_pretrained('finiteautomata/beto-sentiment-analysis')
trainer = Trainer(model=model, args=training_args, train_dataset=small_train_dataset, eval_dataset=small_eval_dataset)

import numpy as np
from datasets import load_metric

metric = load_metric("accuracy")

def compute_metrics(eval_pred):
    logits, labels = eval_pred
    predictions = np.argmax(logits, axis=-1)
    return metric.compute(predictions=predictions, references=labels)
    
trainer = Trainer(model=model,
                  args=training_args,
                  train_dataset=small_train_dataset,
                  eval_dataset=small_eval_dataset,
                  compute_metrics=compute_metrics)
trainer.evaluate()

두번째방법

#Pytorch library를 이용한 영화 리뷰 분류기 학습

from transformers import AdamW

model = BertForSequenceClassification.from_pretrained("bert-base-uncased")
optimizer = AdamW(model.parameters(), lr=5e-5)

tokenized_datasets = tokenized_datasets.remove_columns(["text"])
tokenized_datasets = tokenized_datasets.rename_column("label", "labels")
tokenized_datasets.set_format("torch")

# 마찬가지로 1000개의 학습/평가 데이터셋만을 이용해 진행해보겠습니다.
small_train_dataset = tokenized_datasets["train"].shuffle(seed=42).select(range(1000))
small_eval_dataset = tokenized_datasets["test"].shuffle(seed=42).select(range(1000))

from torch.utils.data import DataLoader

train_dataloader = DataLoader(small_train_dataset, shuffle=True, batch_size=8)
eval_dataloader = DataLoader(small_eval_dataset, batch_size=32)

import torch

num_epochs = 3
num_training_steps = num_epochs * len(train_dataloader)

device = torch.device("cuda") if torch.cuda.is_available() else torch.device("cpu")
model.to(device)

from tqdm.auto import tqdm

progress_bar = tqdm(range(num_training_steps))

model.train()

for epoch in range(num_epochs):
    for input in train_dataloader:
        input = {k: v.to(device) for k, v in input.items()}
        outputs = model(**input)
        loss = outputs.loss
        loss.backward()

        optimizer.step()
        optimizer.zero_grad()
        progress_bar.update()
        
metric = load_metric("accuracy")
model.eval()
all_pred = []
all_ref = []
for input in eval_dataloader:
    input = {k: v.to(device) for k, v in input.items()}
    with torch.no_grad():
        outputs = model(**input)

    logits = outputs.logits
    predictions = torch.argmax(logits, dim=-1)
    all_pred.append(predictions.cpu().detach().numpy())
    all_ref.append(input['labels'].cpu().detach().numpy())
    metric.add_batch(predictions=predictions, references=input['labels'])

metric.compute()