1
import pandas as pd
2
import numpy as np
3
import torch
4
import torch.nn as nn
5
import torch.nn.functional as F
6
from torch.utils.data import TensorDataset, DataLoader
7

8
np.random.seed(2020)
9
torch.manual_seed(2020)
10
USE_CUDA = torch.cuda.is_available()
11
if USE_CUDA:
12
    torch.cuda.manual_seed(2020)
13

14
data=pd.read_csv('./dianping.csv',encoding='utf-8')
15

16
#剔除标点符号,\xa0 空格
17
def pretreatment(comments):
18
    result_comments=[]
19
    punctuation='。，？！：%&~（）、；“”&|,.?!:%&~();""'
20
    for comment in comments:
21
        comment= ''.join([c for c in comment if c not in punctuation])
22
        comment= ''.join(comment.split())   #\xa0
23
        result_comments.append(comment)
24

25
    return result_comments
26

27
result_comments=pretreatment(list(data['comment'].values))
28

29
len(result_comments)

1
result_comments[:1]

1
from transformers import BertTokenizer,BertModel
2

3
tokenizer = BertTokenizer.from_pretrained("./chinese-bert_chinese_wwm_pytorch/data")
4

5
result_comments_id=tokenizer(result_comments,padding=True,truncation=True,max_length=200,return_tensors='pt')
6

7
result_comments_id

1
result_comments_id['input_ids'].shape

1
from sklearn.model_selection import train_test_split

1
X=result_comments_id['input_ids']
2
y=torch.from_numpy(data['sentiment'].values).float()
3

4
X_train,X_test, y_train, y_test =train_test_split(X,y,test_size=0.3,shuffle=True,stratify=y,random_state=2020)
5

6
len(X_train),len(X_test)

1
X_valid,X_test,y_valid,y_test=train_test_split(X_test,y_test,test_size=0.5,shuffle=True,stratify=y_test,random_state=2020)
2

3
len(X_valid),len(X_test)

1
X_train.shape

1
y_train.shape

1
y_train[:1]

1
# create Tensor datasets
2
train_data = TensorDataset(X_train, y_train)
3
valid_data = TensorDataset(X_valid, y_valid)
4
test_data = TensorDataset(X_test,y_test)
5

6
# dataloaders
7
batch_size = 32
8

9
# make sure the SHUFFLE your training data
10
train_loader = DataLoader(train_data, shuffle=True, batch_size=batch_size,drop_last=True)
11
valid_loader = DataLoader(valid_data, shuffle=True, batch_size=batch_size,drop_last=True)
12
test_loader = DataLoader(test_data, shuffle=True, batch_size=batch_size,drop_last=True)

1
if(USE_CUDA):
2
    print('Training on GPU.')
3
else:
4
    print('No GPU available, training on CPU.')

1
class bert_lstm(nn.Module):
2
    def __init__(self, hidden_dim,output_size,n_layers,bidirectional=True, drop_prob=0.5):
3
        super(bert_lstm, self).__init__()
4

5
        self.output_size = output_size
6
        self.n_layers = n_layers
7
        self.hidden_dim = hidden_dim
8
        self.bidirectional = bidirectional
9

10
        #Bert ----------------重点，bert模型需要嵌入到自定义模型里面
11
        self.bert=BertModel.from_pretrained("../chinese-bert_chinese_wwm_pytorch/data")
12
        for param in self.bert.parameters():
13
            param.requires_grad = True
14

15
        # LSTM layers
16
        self.lstm = nn.LSTM(768, hidden_dim, n_layers, batch_first=True,bidirectional=bidirectional)
17

18
        # dropout layer
19
        self.dropout = nn.Dropout(drop_prob)
20

21
        # linear and sigmoid layers
22
        if bidirectional:
23
            self.fc = nn.Linear(hidden_dim*2, output_size)
24
        else:
25
            self.fc = nn.Linear(hidden_dim, output_size)
26

27
        #self.sig = nn.Sigmoid()
28

29
    def forward(self, x, hidden):
30
        batch_size = x.size(0)
31
        #生成bert字向量
32
        x=self.bert(x)[0]     #bert 字向量
33

34
        # lstm_out
35
        #x = x.float()
36
        lstm_out, (hidden_last,cn_last) = self.lstm(x, hidden)
37
        #print(lstm_out.shape)   #[32,100,768]
38
        #print(hidden_last.shape)   #[4, 32, 384]
39
        #print(cn_last.shape)    #[4, 32, 384]
40

41
        #修改 双向的需要单独处理
42
        if self.bidirectional:
43
            #正向最后一层，最后一个时刻
44
            hidden_last_L=hidden_last[-2]
45
            #print(hidden_last_L.shape)  #[32, 384]
46
            #反向最后一层，最后一个时刻
47
            hidden_last_R=hidden_last[-1]
48
            #print(hidden_last_R.shape)   #[32, 384]
49
            #进行拼接
50
            hidden_last_out=torch.cat([hidden_last_L,hidden_last_R],dim=-1)
51
            #print(hidden_last_out.shape,'hidden_last_out')   #[32, 768]
52
        else:
53
            hidden_last_out=hidden_last[-1]   #[32, 384]
54

55

56
        # dropout and fully-connected layer
57
        out = self.dropout(hidden_last_out)
58
        #print(out.shape)    #[32,768]
59
        out = self.fc(out)
60

61
        return out
62

63
    def init_hidden(self, batch_size):
64
        weight = next(self.parameters()).data
65

66
        number = 1
67
        if self.bidirectional:
68
            number = 2
69

70
        if (USE_CUDA):
71
            hidden = (weight.new(self.n_layers*number, batch_size, self.hidden_dim).zero_().float().cuda(),
72
                      weight.new(self.n_layers*number, batch_size, self.hidden_dim).zero_().float().cuda()
73
                     )
74
        else:
75
            hidden = (weight.new(self.n_layers*number, batch_size, self.hidden_dim).zero_().float(),
76
                      weight.new(self.n_layers*number, batch_size, self.hidden_dim).zero_().float()
77
                     )
78

79
        return hidden

1
output_size = 1
2
hidden_dim = 384   #768/2
3
n_layers = 2
4
bidirectional = True  #这里为True，为双向LSTM
5

6
net = bert_lstm(hidden_dim, output_size,n_layers, bidirectional)
7

8
#print(net)

1
# loss and optimization functions
2
lr=2e-5
3
criterion = nn.CrossEntropyLoss()
4
optimizer = torch.optim.Adam(net.parameters(), lr=lr)
5

6
# training params
7
epochs = 10
8
# batch_size=50
9
print_every = 7
10
clip=5 # gradient clipping
11

12
# move model to GPU, if available
13
if(USE_CUDA):
14
    net.cuda()

1
net.train()
2
# train for some number of epochs
3
for e in range(epochs):
4
    # initialize hidden state
5
    h = net.init_hidden(batch_size)
6
    counter = 0
7

8
    # batch loop
9
    for inputs, labels in train_loader:
10
        counter += 1
11

12
        if(USE_CUDA):
13
            inputs, labels = inputs.cuda(), labels.cuda()
14
        h = tuple([each.data for each in h])
15
        net.zero_grad()
16
        output= net(inputs, h)
17
        loss = criterion(output.squeeze(), labels.float())
18
        loss.backward()
19
        optimizer.step()
20

21
        # loss stats
22
        if counter % print_every == 0:
23
            net.eval()
24
            with torch.no_grad():
25
                val_h = net.init_hidden(batch_size)
26
                val_losses = []
27
                for inputs, labels in valid_loader:
28
                    val_h = tuple([each.data for each in val_h])
29

30
                    if(USE_CUDA):
31
                        inputs, labels = inputs.cuda(), labels.cuda()
32

33
                    output = net(inputs, val_h)
34
                    val_loss = criterion(output.squeeze(), labels.float())
35

36
                    val_losses.append(val_loss.item())
37

38
            net.train()
39
            print("Epoch: {}/{}...".format(e+1, epochs),
40
                  "Step: {}...".format(counter),
41
                  "Loss: {:.6f}...".format(loss.item()),
42
                  "Val Loss: {:.6f}".format(np.mean(val_losses)))

1
test_losses = [] # track loss
2
num_correct = 0
3

4
# init hidden state
5
h = net.init_hidden(batch_size)
6

7
net.eval()
8
# iterate over test data
9
for inputs, labels in test_loader:
10
    h = tuple([each.data for each in h])
11
    if(USE_CUDA):
12
        inputs, labels = inputs.cuda(), labels.cuda()
13
    output = net(inputs, h)
14
    test_loss = criterion(output.squeeze(), labels.float())
15
    test_losses.append(test_loss.item())
16
    output=torch.nn.Softmax(dim=1)(output)
17
    pred=torch.max(output, 1)[1]
18

19
    # compare predictions to true label
20
    correct_tensor = pred.eq(labels.float().view_as(pred))
21
    correct = np.squeeze(correct_tensor.numpy()) if not USE_CUDA else np.squeeze(correct_tensor.cpu().numpy())
22
    num_correct += np.sum(correct)
23

24
print("Test loss: {:.3f}".format(np.mean(test_losses)))
25

26
# accuracy over all test data
27
test_acc = num_correct/len(test_loader.dataset)
28
print("Test accuracy: {:.3f}".format(test_acc))

1
def predict(net, test_comments):
2
    result_comments=pretreatment(test_comments)   #预处理去掉标点符号
3

4
    #转换为字id
5
    tokenizer = BertTokenizer.from_pretrained("./chinese-bert_chinese_wwm_pytorch/data")
6
    result_comments_id=tokenizer(result_comments,padding=True,truncation=True,max_length=120,return_tensors='pt')
7
    tokenizer_id=result_comments_id['input_ids']
8
    inputs=tokenizer_id
9
    batch_size = inputs.size(0)
10

11
    # initialize hidden state
12
    h = net.init_hidden(batch_size)
13

14
    if(USE_CUDA):
15
        inputs = inputs.cuda()
16

17
    net.eval()
18
    with torch.no_grad():
19
        # get the output from the model
20
        output = net(inputs, h)
21
        output=torch.nn.Softmax(dim=1)(output)
22
        pred=torch.max(output, 1)[1]
23
        # printing output value, before rounding
24
        print('预测概率为: {:.6f}'.format(output.item()))
25
        if(pred.item()==1):
26
            print("预测结果为:正向")
27
        else:
28
            print("预测结果为:负向")

1
comment1 = ['菜品一般，不好吃！！']
2
predict(net, comment1)

1
comment2 = ['环境不错']
2
predict(net, comment2)

1
comment3 = ['服务员还可以，就是菜有点不好吃']
2
predict(net, comment3)

1
comment4 = ['服务员还可以，就是菜不好吃']
2
predict(net, comment4)

1
# 保存
2
torch.save(net.state_dict(), './大众点评二分类_parameters.pth')

1
output_size = 1
2
hidden_dim = 384   #768/2
3
n_layers = 2
4
bidirectional = True  #这里为True，为双向LSTM
5

6
net = bert_lstm(hidden_dim, output_size,n_layers, bidirectional)

1
net.load_state_dict(torch.load('./大众点评二分类_parameters.pth'))

1
# move model to GPU, if available
2
if(USE_CUDA):
3
    net.cuda()

1
comment1 = ['菜品一般，不好吃！！']
2
predict(net, comment1)