word2vecリターンズ！ 品詞分類による精度改善：ディープラーニング習得、次の一歩（1/2 ページ）

import tensorflow as tf
import numpy as np
import matplotlib.pyplot as plt
import csv
import pandas as pd
import random
import numpy.random as nr
import sys
import h5py
import keras
import math

from __future__ import print_function
from keras.layers.core import Activation
from keras.layers.core import Dense
from keras.layers.core import Flatten
from keras.layers.embeddings import Embedding
from keras.models import Sequential
from keras.layers import Input
from keras.models import Model
from keras.callbacks import EarlyStopping
from keras.initializers import glorot_uniform
from keras.initializers import uniform
from keras.optimizers import RMSprop
from keras.utils import np_utils

# 元データ
df1 = csv.reader(open('rampo_separate.csv', 'r'))
df2 = csv.reader(open('rampo_separate2.csv', 'r'))
df3 = csv.reader(open('rampo_separate3.csv', 'r'))

data1 = [ v for v in df1]
data2 = [ v for v in df2]
data3 = [ v for v in df3]

mat1 = np.array(data1)
mat2 = np.array(data2)
mat3 = np.array(data3)

mat = np.r_[mat1[:,0],mat2[:,0],mat3[:,0]]
print(mat.shape)

mat_part_of_speech1 = np.r_[mat1[:,3],mat2[:,3],mat3[:,3]]      # 品詞
mat_part_of_speech2 = np.r_[mat1[:,5],mat2[:,5],mat3[:,5]]      # 品詞詳細

mat_part_of_speech = []
for i in range (0,len(mat_part_of_speech1)) :
  mat_part_of_speech.append( mat_part_of_speech1[i] +'_'+ mat_part_of_speech2[i])

# サンプルとして1つ出力
print(mat[10])
print(mat_part_of_speech[10])

words = sorted(list(set(mat)))
cnt = np.zeros(len(words))

print('total words:', len(words))
word_indices = dict((w, i) for i, w in enumerate(words))    # 単語をキーにインデックス検索
indices_word = dict((i, w) for i, w in enumerate(words))    # インデックスをキーに単語を検索

# 単語の出現数をカウント
for j in range (0, len(mat)):
  cnt[word_indices[mat[j]]] += 1

# 出現頻度の少ない単語を「UNK」で置き換え
words_unk = []                              # 未知語一覧

for k in range(0, len(words)):
  if cnt[k] <= 3 :
    words_unk.append(words[k])
    words[k] = 'UNK'

print('UNK words:', len(words_unk))          # words_unkはunkに変換された単語のリスト

words = sorted(list(set(words)))
print('total words:', len(words))
word_indices = dict((w, i) for i, w in enumerate(words))    # 単語をキーにインデックス検索
indices_word = dict((i, w) for i, w in enumerate(words))    # インデックスをキーに単語を検索

parts = sorted(list(set(mat_part_of_speech)))

print('total parts:', len(parts))
parts_indices = dict((w, i) for i, w in enumerate(parts))    # 単語をキーにインデックス検索
indices_parts = dict((i, w) for i, w in enumerate(parts))    # インデックスをキーに単語を検索

maxlen = 10                              # ウィンドウサイズ

mat_urtext = np.zeros((len(mat), 1), dtype='int16')
for i in range(0, len(mat)):
  # 出現頻度の低い単語のインデックスをunkのそれに置き換え
  if mat[i] in word_indices : 
    mat_urtext[i,0]=word_indices[mat[i]]
  else:
    mat_urtext[i,0]=word_indices['UNK']

mat_parts = np.zeros((len(mat_part_of_speech),1), dtype='int8')
for i in range(0,len(mat_part_of_speech)):
  mat_parts[i,0] = parts_indices[mat_part_of_speech[i]]

len_seq   = len(mat_urtext) - maxlen
x_train   = np.zeros((len_seq-maxlen,1), dtype='int16')
t_train   = np.zeros((len_seq-maxlen,maxlen*2), dtype='int16')
x_p_train = np.zeros((len_seq-maxlen,1), dtype='int8')
t_p_train = np.zeros((len_seq-maxlen,maxlen*2), dtype='int8')

for i in range(maxlen, len_seq):
  # 単語
  x_train[i-maxlen,0] = mat_urtext[i]
  t_train[i-maxlen,0:maxlen] = np.array(mat_urtext[i-maxlen:i]).reshape(1, maxlen)
  t_train[i-maxlen,maxlen:maxlen*2] = np.array(mat_urtext[i+1:i+1+maxlen]).reshape(1, maxlen)

  # 品詞
  x_p_train[i-maxlen,0] = mat_parts[i]
  t_p_train[i-maxlen,0:maxlen] = np.array(mat_parts[i-maxlen:i]).reshape(1, maxlen)
  t_p_train[i-maxlen,maxlen:maxlen*2] = np.array(mat_parts[i+1:i+1+maxlen]).reshape(1, maxlen)

z = list(zip(x_train,t_train,x_p_train,t_p_train))
nr.seed(12345)
nr.shuffle(z)                               # シャッフル
x_train,t_train,x_p_train,t_p_train = zip(*z)

x_train   = np.array(x_train).reshape(len_seq-maxlen,1)
t_train   = np.array(t_train).reshape(len_seq-maxlen, maxlen*2)
x_p_train = np.array(x_p_train).reshape(len_seq-maxlen, 1)
t_p_train = np.array(t_p_train).reshape(len_seq-maxlen, maxlen*2)

print(x_train.shape, t_train.shape, x_p_train.shape, t_p_train.shape)