语言模型与新闻的内容预测

# coding=UTF-8
import os
import requests
import re
from bs4 import BeautifulSoup

newsList = []


def finNextPage(newsUrl):
    url = newsUrl.split(".htm")
    url = url[0] + '_2.htm'
    result = requests.get(url)
    if (result.status_code == 200):
        return url


def getListPage(pageUrl):
    reslistnew = requests.get(pageUrl)
    reslistnew.encoding = 'gbk'
    souplistnew = BeautifulSoup(reslistnew.text, 'html.parser')
    for news in souplistnew.select('li'):
        if len(news.select('.pub_time')) > 0:
            newsUrl = news.select('a')[0].attrs['href']
            # getNewsDetail(newsUrl)
            result = re.search('http(.*?)html', newsUrl)
            if result is None:
                # getNewsDetail(newsUrl)
                newsList.append(newsUrl)


# 判断当天新闻是否存在下一页
for month in range(1, 13):
    for day in range(1, 32):
        try:
            mon = str(month)
            da = str(day)
            if(month < 10):
                mon = '0'+mon
            if(day < 10):
                da = '0'+da
            url = 'http://tech.qq.com/l/2019{mon}/scroll_{da}.htm'.format(
                mon=mon, da=da)
            nextUrl = finNextPage(url)
            if nextUrl:
                getListPage(nextUrl)
            getListPage(url)
        except:
            print("Error: "+url)
    print(str(month)+" "+str(len(newsList)))
# 根据上一步获得的Url抓取新闻
if not os.path.exists("吴坎_17341163_新闻数据/"):
    os.mkdir("吴坎_17341163_新闻数据/")
for i in range(len(newsList)):
    try:
        newsUrl = newsList[i]
        resd = requests.get(newsUrl)
        resd.encoding = 'gbk'
        soupd = BeautifulSoup(resd.text, 'html.parser')
        news = {}
        Cnt_Main_Article = soupd.select('.Cnt-Main-Article-QQ')
        Main_P_QQ = soupd.select('Main-P-QQ')
        if soupd.select('.rv-middle'):
            news['content'] = soupd.select('h1')[0].text
        else:
            news['title'] = soupd.select('h1')[0].text
            if Cnt_Main_Article:
                news['content'] = Cnt_Main_Article[0].text
            elif Main_P_QQ:
                news['content'] = '空'
            else:
                news['content'] = '空'
        f = open("吴坎_17341163_新闻数据/"+str(i+1)+".txt", 'w', encoding='utf-8')
        f.write(news['content'])
        f.close()
    except:
        print("Error: "+newsList[i])

# coding=UTF-8
import io
import os
import re
import jieba

if __name__ == '__main__':
    if not os.path.exists("吴坎_17341163_预处理结果/"):
        os.mkdir("吴坎_17341163_预处理结果/")
    for filename in os.listdir("吴坎_17341163_新闻数据/"):
        f = io.open("吴坎_17341163_新闻数据/"+filename, 'r', encoding='utf-8')
        sentences = re.split("(。|！|\!|\.|？|\?)", f.read())
        f.close()
        f = io.open("吴坎_17341163_预处理结果/"+filename, 'w', encoding='utf-8')
        for s in sentences:
            for w in jieba.cut(s):
                is_all_chinese = True
                for _char in w:
                    if not '\u4e00' <= _char <= '\u9fa5':
                        is_all_chinese = False
                        break
                if is_all_chinese:
                    f.write(w+" ")
            f.write("\n")
        f.close()

# coding=UTF-8
import io
import os


def getWordsList():
    sentence_set = []
    word_set = set()
    for filename in os.listdir("吴坎_17341163_预处理结果/"):
        if filename != "wordsList.txt":
            with io.open("吴坎_17341163_预处理结果/"+filename, 'r', encoding='utf-8')as f:
                for sentence in f.readlines():
                    if not sentence.isspace():
                        sentence_set.append(sentence)
                        word_set.update(sentence.split())

    # 构建词典，注意：word_to_ix用于对字词进行编号,ix_to_word用于将模型的输出转化为字词
    # 'EOS': end of sentence
    # 'SOS': start of sentencex
    # 'UNK': unknown token
    word_to_ix = {'SOS': 0, 'EOS': 1, 'UNK': 2}
    ix_to_word = {0: 'SOS', 1: 'EOS', 2: 'UNK'}
    for word in word_set:
        if word not in word_to_ix:
            word_to_ix[word] = len(word_to_ix)
            ix_to_word[len(ix_to_word)] = word
    return sentence_set, word_to_ix, ix_to_word


if __name__ == '__main__':
    _, _, ix_to_word = getWordsList()
    with io.open('吴坎_17341163_预处理结果/wordsList.txt', 'w', encoding='UTF-8') as f:  # 保存词典
        for vocab in ix_to_word.items():
            f.write(str(vocab[0])+' '+str(vocab[1])+'\n')

# coding=UTF-8
import torch
import torch.nn as nn
from torch.autograd import Variable
import torch.nn.functional as F
import torch.optim as optim
from wordsList import getWordsList
import io

CONTEXT_SIZE = 2
EMBEDDING_DIM = 10
EPOCH = 7
Sentence_Num = 31


class NgramModel(nn.Module):
    def __init__(self, vocb_size, context_size, n_dim):
        # super(NgramModel, self)._init_()
        super().__init__()
        self.n_word = vocb_size
        self.embedding = nn.Embedding(self.n_word, n_dim)
        self.linear1 = nn.Linear(context_size*n_dim, 128)
        self.linear2 = nn.Linear(128, self.n_word)

    def forward(self, x):
        # the first step: transmit words and achieve word embedding. eg. transmit two words, and then achieve (2, 100)
        emb = self.embedding(x)
        # the second step: word wmbedding unfold to (1,200)
        emb = emb.view(1, -1)
        # the third step: transmit to linear model, and then use relu, at last, transmit to linear model again
        out = self.linear1(emb)
        out = F.relu(out)
        out = self.linear2(out)
        # the output dim of last step is the number of words, wo can view as a classification problem
        # if we want to predict the max probability of the words, finally we need use log softmax
        log_prob = F.log_softmax(out)
        return log_prob


def getNgram(trigram, word2id, id2word):
    ngrammodel = NgramModel(len(word2id), CONTEXT_SIZE, 100)
    criterion = nn.NLLLoss()
    optimizer = optim.SGD(ngrammodel.parameters(), lr=1e-3)
    for epoch in range(EPOCH):
        print('epoch: {}'.format(epoch+1))
        print('*'*10)
        running_loss = 0
        for data in trigram:
            # we use 'word' to represent the two words forward the predict word, we use 'label' to represent the predict word
            word, label = data                          # attention
            word = Variable(torch.LongTensor([word2id[e] for e in word]))
            label = Variable(torch.LongTensor([word2id[label]]))
            # forward
            out = ngrammodel(word)
            loss = criterion(out, label)
            running_loss += loss.data.item()
            # backward
            optimizer.zero_grad()
            loss.backward()
            optimizer.step()
        print('loss: {:.6f}'.format(running_loss/len(word2id)))
    return ngrammodel


# predict
if __name__ == '__main__':
    sentence_set, word2id, id2word = getWordsList()
    trigram = []
    for j in range(Sentence_Num):
        test_sentence = sentence_set[j].split()
        test_sentence.insert(0, 'SOS')
        test_sentence.append('EOS')
        for i in range(len(test_sentence)-2):
            trigram.append(
                ((test_sentence[i], test_sentence[i+1]), test_sentence[i+2]))

    ngrammodel = torch.load('吴坎_17341163_预测结果/ngrammodel.pkl')
    #ngrammodel = getNgram(trigram, word2id, id2word)
    torch.save(ngrammodel, '吴坎_17341163_预测结果/ngrammodel.pkl')

    fout = io.open("吴坎_17341163_预测结果/prediction.ngram.txt",
                   "w", encoding="utf-8")
    with io.open("吴坎_17341163_预测结果/questions.1.txt", "r", encoding="utf-8") as f:
        for sentence in f.readlines():
            wordss = sentence.split()
            wordss.insert(0, 'SOS')
            wordss.append('EOS')
            for j in range(len(wordss)):
                if j+6 >= len(wordss):
                    word = wordss[j+2], wordss[j+3]
                    break
                if wordss[j+5] == "[MASK]":
                    word = wordss[j+3], wordss[j+4]
                    break
            word = Variable(torch.LongTensor([word2id[i] for i in word]))
            out = ngrammodel(word)
            _, predict_label = torch.max(out, 1)
            fout.write(id2word[out.max(1)[1].item()]+"\n")
    fout.close()

# coding=UTF-8
import torch
import torch.nn as nn
import torch.nn.functional as F
import torch.optim as optim
from torch.autograd import Variable
from torch.utils.data import DataLoader, Dataset
import numpy
import io
from wordsList import getWordsList

# Set Hyper Parameters
sentence_set, word_to_ix, ix_to_word = getWordsList()
Sentence_Num = 300
Embedding_Dim = len(word_to_ix)
LR = 0.005
EPOCH = 31
BATCH_SIZE = 1


class MyDataset(Dataset):
    def __init__(self, words, labels, transform=None, target_transform=None):
        self.words = words
        self.labels = labels
        self.transform = transform
        self.target_transform = target_transform

    def __getitem__(self, index):
        words, labels = self.words[index], self.labels[index]
        if self.transform is not None:
            words = [self.transform(word) for word in words]
        if self.target_transform is not None:
            labels = self.target_transform(labels)
        return words, labels

    def __len__(self):
        return len(self.labels)


def bulid_one_hot(word, word_dict):
    if word in word_dict:
        return torch.LongTensor([word_dict[word]])
    return torch.LongTensor([word_dict['UNK']])


def getTrainDataSetLoader():
    # transforms.ToTensor(), transforms.ToTensor()
    train_words, train_labels, trans, target_trans = [], [], None, None

    for i in range(Sentence_Num):
        words = sentence_set[i].split()
        words.insert(0, 'SOS')
        words.append('EOS')
        words = [bulid_one_hot(word, word_to_ix) for word in words]

        for j in range(len(words)):
            if j+6 >= len(words):
                break
            train_words.append(words[j:j+5])
            train_labels.append(words[j+5])

    return DataLoader(dataset=MyDataset(train_words, train_labels, trans, target_trans), batch_size=BATCH_SIZE)


def getTestDataSetLoader():
    test_words, test_labels, trans, target_trans = [], [], None, None
    # transforms.ToTensor(), transforms.ToTensor()
    fquestion = io.open("吴坎_17341163_预测结果/questions.1.txt",
                        "r", encoding="utf-8")
    fanswer = io.open("吴坎_17341163_预测结果/answer.txt", "r", encoding="utf-8")
    for sentence in fquestion.readlines():
        ans = fanswer.readline().split()[0]
        test_labels.append(bulid_one_hot(ans, word_to_ix))
        wordss = sentence.split()
        wordss.insert(0, 'SOS')
        wordss.append('EOS')
        words = [bulid_one_hot(word, word_to_ix) for word in wordss]
        for j in range(len(words)):
            if j+6 >= len(words):
                test_words.append(words[j-1:j+4])
                break
            if wordss[j+5] == "[MASK]":
                test_words.append(words[j:j+5])
                break
    fquestion.close()
    fanswer.close()
    return DataLoader(dataset=MyDataset(test_words, test_labels, trans, target_trans), batch_size=BATCH_SIZE)


class Encoder(nn.Module):
    def __init__(self, input_size, hidden_size):
        super(Encoder, self).__init__()
        self.hidden_size = hidden_size
        self.embedding = nn.Embedding(
            input_size, hidden_size)     # 将one-hot向量embedding为词向量
        # GRU的hidden layer的size与词向量的size一样，并非必须
        self.gru = nn.GRU(hidden_size, hidden_size)

    def forward(self, input, hidden):
        # RNN的输入格式为 (seq_len, batch, input_size)
        embedded = self.embedding(input).view(1, 1, -1)
        output = embedded
        output, hidden = self.gru(output, hidden)
        return output, hidden

    def initHidden(self):
        # 初始化Encoder的隐状态
        return torch.zeros(1, 1, self.hidden_size)


class Decoder(nn.Module):
    def __init__(self, input_size, hidden_size, output_size):
        super(Decoder, self).__init__()
        self.hidden_size = hidden_size
        self.embedding = nn.Embedding(input_size, hidden_size)
        self.gru = nn.GRU(hidden_size, hidden_size)
        self.out = nn.Linear(hidden_size, output_size)

    def forward(self, input, hidden):
        output = self.embedding(input).view(1, 1, -1)
        output = F.relu(output)
        output, hidden = self.gru(output, hidden)
        output = self.out(output[0])
        return output, hidden

    def initHidden(self):
        return torch.zeros(1, 1, self.hidden_size)


class Seq2Seq(nn.Module):  # Bulid Seq2Seq Model
    def __init__(self, encoder, decoder):
        super(Seq2Seq, self).__init__()
        self.encoder = encoder
        self.decoder = decoder

    def forward(self, inputs):
        encoder_hidden = self.encoder.initHidden()
        if torch.cuda.is_available():
            encoder_hidden = encoder_hidden.cuda()

        # encode
        for word in inputs:
            encoder_out, encoder_hidden = self.encoder(word, encoder_hidden)

        # decode
        decoder_hidden = encoder_hidden
        pred, decoder_hidden = self.decoder(inputs[-1], decoder_hidden)

        return pred


def getSeq2Seq():
    encoder = Encoder(Embedding_Dim, 1000)
    decoder = Decoder(Embedding_Dim, 1000, Embedding_Dim)

    if torch.cuda.is_available():
        encoder = encoder.cuda()
        decoder = decoder.cuda()
    seq2seq = Seq2Seq(encoder, decoder)
    if torch.cuda.is_available():
        seq2seq = seq2seq.cuda()

    # Bulid loss function and optimizer

    loss_func = nn.CrossEntropyLoss()
    #encoder_optimizer = optim.SGD(encoder.parameters(), lr=LR, momentum=0.9)
    #decoder_optimizer = optim.SGD(decoder.parameters(), lr=LR, momentum=0.9)
    seq2seq_optimizer = optim.SGD(seq2seq.parameters(), lr=LR, momentum=0.9)
    # Train Seq2Seq Model

    train_loader = getTrainDataSetLoader()

    for epoch in range(EPOCH):
        loss_sum = 0
        for step, (inputs, labels) in enumerate(train_loader):
            # encoder_hidden = encoder.initHidden()
            label = torch.LongTensor((1,))
            label[0] = int(labels.data.numpy()[0])

            if torch.cuda.is_available():
                inputs = [word.cuda() for word in inputs]
                label = label.cuda()
                # encoder_hidden = encoder_hidden.cuda()

            # forward
            pred = seq2seq(inputs)
            loss = loss_func(pred, label)

            # backward
            seq2seq_optimizer.zero_grad()
            loss.backward()
            seq2seq_optimizer.step()

            '''
            for word in inputs:
                encoder_out, encoder_hidden = encoder(word, encoder_hidden)
            decoder_hidden = encoder_hidden
            decoder_out, decoder_hidden = decoder(inputs[-1], decoder_hidden)

            loss = loss_func(decoder_out,label)

            #backward
            encoder_optimizer.zero_grad()
            decoder_optimizer.zero_grad()
            loss.backward()
            encoder_optimizer.step()
            decoder_optimizer.step()
            '''

            loss_sum += loss.item()
        print('Epoch: %2d train loss: %.4f' % (epoch, loss_sum))
    return seq2seq


if __name__ == '__main__':
    #seq2seq = torch.load('吴坎_17341163_预测结果/seq2seq.pkl')
    seq2seq = getSeq2Seq()
    '''
    torch.save(encoder.state_dict(),'吴坎_17341163_预测结果/encoder_params.pkl')
    torch.save(decoder.state_dict(),'吴坎_17341163_预测结果/decoder_params.pkl')

    torch.save(encoder,'吴坎_17341163_预测结果/encoder.pkl')
    torch.save(decoder,'吴坎_17341163_预测结果/decoder.pkl')
    torch.save(seq2seq.state_dict(), '吴坎_17341163_预测结果/seq2seq_params.pkl')
    '''
    torch.save(seq2seq, '吴坎_17341163_预测结果/seq2seq.pkl')
    test_loader = getTestDataSetLoader()
    with io.open("吴坎_17341163_预测结果/prediction.txt", "w", encoding="utf-8") as f:
        for step, (inputs, labels) in enumerate(test_loader):
            # encoder_hidden = encoder.initHidden()
            label = torch.LongTensor((1,))
            label[0] = int(labels.data.numpy()[0])

            if torch.cuda.is_available():
                inputs = [word.cuda() for word in inputs]
                label = label.cuda()
                # encoder_hidden = encoder_hidden.cuda()

            decoder_output = seq2seq(inputs)

            '''
            # forward
            for word in inputs:
                encoder_out, encoder_hidden = encoder(word, encoder_hidden)
            decoder_hidden = encoder_hidden
            decoder_out, decoder_hidden = decoder(inputs[-1], decoder_hidden)
            '''

            #ans = ix_to_word[int(labels.data.numpy()[0])]
            pred = ix_to_word[numpy.argmax(decoder_output.cpu().data.numpy())]
            f.write(pred+"\n")