diff --git a/.DS_Store b/.DS_Store new file mode 100644 index 0000000..feded73 Binary files /dev/null and b/.DS_Store differ diff --git a/code/__pycache__/model.cpython-36.pyc b/code/__pycache__/model.cpython-36.pyc new file mode 100644 index 0000000..99e7412 Binary files /dev/null and b/code/__pycache__/model.cpython-36.pyc differ diff --git a/code/__pycache__/my_layers.cpython-36.pyc b/code/__pycache__/my_layers.cpython-36.pyc new file mode 100644 index 0000000..563ed3c Binary files /dev/null and b/code/__pycache__/my_layers.cpython-36.pyc differ diff --git a/code/__pycache__/optimizers.cpython-36.pyc b/code/__pycache__/optimizers.cpython-36.pyc new file mode 100644 index 0000000..c6d47ba Binary files /dev/null and b/code/__pycache__/optimizers.cpython-36.pyc differ diff --git a/code/__pycache__/reader.cpython-36.pyc b/code/__pycache__/reader.cpython-36.pyc new file mode 100644 index 0000000..828f537 Binary files /dev/null and b/code/__pycache__/reader.cpython-36.pyc differ diff --git a/code/__pycache__/utils.cpython-36.pyc b/code/__pycache__/utils.cpython-36.pyc new file mode 100644 index 0000000..16e9afa Binary files /dev/null and b/code/__pycache__/utils.cpython-36.pyc differ diff --git a/code/__pycache__/w2vEmbReader.cpython-36.pyc b/code/__pycache__/w2vEmbReader.cpython-36.pyc new file mode 100644 index 0000000..34f2877 Binary files /dev/null and b/code/__pycache__/w2vEmbReader.cpython-36.pyc differ diff --git a/code/evaluation.py b/code/evaluation.py index 6b0f334..7d92c35 100644 --- a/code/evaluation.py +++ b/code/evaluation.py @@ -1,146 +1,146 @@ -import argparse -import logging -import numpy as np -from time import time -import utils as U -from sklearn.metrics import classification_report -import codecs - -######### Get hyper-params in order to rebuild the model architecture ########### -# The hyper parameters should be exactly the same as those used for training -parser = argparse.ArgumentParser() -parser.add_argument("-o", "--out-dir", dest="out_dir_path", type=str, metavar='', required=True, help="The path to the output directory") -parser.add_argument("-e", "--embdim", dest="emb_dim", type=int, metavar='', default=200, help="Embeddings dimension (default=200)") -parser.add_argument("-b", "--batch-size", dest="batch_size", type=int, metavar='', default=50, help="Batch size (default=50)") -parser.add_argument("-v", "--vocab-size", dest="vocab_size", type=int, metavar='', default=9000, help="Vocab size. '0' means no limit (default=9000)") -parser.add_argument("-as", "--aspect-size", dest="aspect_size", type=int, metavar='', default=14, help="The number of aspects specified by users (default=14)") -parser.add_argument("--emb", dest="emb_path", type=str, metavar='', help="The path to the word embeddings file") -parser.add_argument("--epochs", dest="epochs", type=int, metavar='', default=15, help="Number of epochs (default=15)") -parser.add_argument("-n", "--neg-size", dest="neg_size", type=int, metavar='', default=20, help="Number of negative instances (default=20)") -parser.add_argument("--maxlen", dest="maxlen", type=int, metavar='', default=0, help="Maximum allowed number of words during training. '0' means no limit (default=0)") -parser.add_argument("--seed", dest="seed", type=int, metavar='', default=1234, help="Random seed (default=1234)") -parser.add_argument("-a", "--algorithm", dest="algorithm", type=str, metavar='', default='adam', help="Optimization algorithm (rmsprop|sgd|adagrad|adadelta|adam|adamax) (default=adam)") -parser.add_argument("--domain", dest="domain", type=str, metavar='', default='restaurant', help="domain of the corpus {restaurant, beer}") -parser.add_argument("--ortho-reg", dest="ortho_reg", type=float, metavar='', default=0.1, help="The weight of orthogonol regularizaiton (default=0.1)") - -args = parser.parse_args() -out_dir = args.out_dir_path + '/' + args.domain -# out_dir = '../pre_trained_model/' + args.domain -U.print_args(args) - -assert args.algorithm in {'rmsprop', 'sgd', 'adagrad', 'adadelta', 'adam', 'adamax'} -assert args.domain in {'restaurant', 'beer'} - -from keras.preprocessing import sequence -import reader as dataset - -###### Get test data ############# -vocab, train_x, test_x, overall_maxlen = dataset.get_data(args.domain, vocab_size=args.vocab_size, maxlen=args.maxlen) -test_x = sequence.pad_sequences(test_x, maxlen=overall_maxlen) - - -############# Build model architecture, same as the model used for training ######### -from model import create_model -import keras.backend as K -from optimizers import get_optimizer - -optimizer = get_optimizer(args) - -def max_margin_loss(y_true, y_pred): - return K.mean(y_pred) -model = create_model(args, overall_maxlen, vocab) - -## Load the save model parameters -model.load_weights(out_dir+'/model_param') -model.compile(optimizer=optimizer, loss=max_margin_loss, metrics=[max_margin_loss]) - - - -################ Evaluation #################################### - -def evaluation(true, predict, domain): - true_label = [] - predict_label = [] - - if domain == 'restaurant': - - for line in predict: - predict_label.append(line.strip()) - - for line in true: - true_label.append(line.strip()) - - print(classification_report(true_label, predict_label, - ['Food', 'Staff', 'Ambience', 'Anecdotes', 'Price', 'Miscellaneous'], digits=3)) - - else: - for line in predict: - label = line.strip() - if label == 'smell' or label == 'taste': - label = 'taste+smell' - predict_label.append(label) - - for line in true: - label = line.strip() - if label == 'smell' or label == 'taste': - label = 'taste+smell' - true_label.append(label) - - print(classification_report(true_label, predict_label, - ['feel', 'taste+smell', 'look', 'overall', 'None'], digits=3)) - - -def prediction(test_labels, aspect_probs, cluster_map, domain): - label_ids = np.argsort(aspect_probs, axis=1)[:,-1] - predict_labels = [cluster_map[label_id] for label_id in label_ids] - evaluation(open(test_labels), predict_labels, domain) - - -## Create a dictionary that map word index to word -vocab_inv = {} -for w, ind in vocab.items(): - vocab_inv[ind] = w - - -test_fn = K.function([model.get_layer('sentence_input').input, K.learning_phase()], - [model.get_layer('att_weights').output, model.get_layer('p_t').output]) -att_weights, aspect_probs = test_fn([test_x, 0]) - - -## Save attention weights on test sentences into a file -att_out = codecs.open(out_dir + '/att_weights', 'w', 'utf-8') -print 'Saving attention weights on test sentences...' -for c in xrange(len(test_x)): - att_out.write('----------------------------------------\n') - att_out.write(str(c) + '\n') - - word_inds = [i for i in test_x[c] if i!=0] - line_len = len(word_inds) - weights = att_weights[c] - weights = weights[(overall_maxlen-line_len):] - - words = [vocab_inv[i] for i in word_inds] - att_out.write(' '.join(words) + '\n') - for j in range(len(words)): - att_out.write(words[j] + ' '+str(round(weights[j], 3)) + '\n') - - - -###################################################### -# Uncomment the below part for F scores -###################################################### - -## cluster_map need to be specified manually according to the top words in each inferred aspect (save in aspect.log) - -# map for the pre-trained restaurant model (under pre_trained_model/restaurant) -# cluster_map = {0: 'Food', 1: 'Miscellaneous', 2: 'Miscellaneous', 3: 'Food', -# 4: 'Miscellaneous', 5: 'Food', 6:'Price', 7: 'Miscellaneous', 8: 'Staff', -# 9: 'Food', 10: 'Food', 11: 'Anecdotes', -# 12: 'Ambience', 13: 'Staff'} - - -# print '--- Results on %s domain ---' % (args.domain) -# test_labels = '../preprocessed_data/%s/test_label.txt' % (args.domain) -# prediction(test_labels, aspect_probs, cluster_map, domain=args.domain) - - +import argparse +import logging +import numpy as np +from time import time +import utils as U +from sklearn.metrics import classification_report +import codecs + +######### Get hyper-params in order to rebuild the model architecture ########### +# The hyper parameters should be exactly the same as those used for training +parser = argparse.ArgumentParser() +parser.add_argument("-o", "--out-dir", dest="out_dir_path", type=str, metavar='', required=True, help="The path to the output directory") +parser.add_argument("-e", "--embdim", dest="emb_dim", type=int, metavar='', default=200, help="Embeddings dimension (default=200)") +parser.add_argument("-b", "--batch-size", dest="batch_size", type=int, metavar='', default=50, help="Batch size (default=50)") +parser.add_argument("-v", "--vocab-size", dest="vocab_size", type=int, metavar='', default=9000, help="Vocab size. '0' means no limit (default=9000)") +parser.add_argument("-as", "--aspect-size", dest="aspect_size", type=int, metavar='', default=14, help="The number of aspects specified by users (default=14)") +parser.add_argument("--emb", dest="emb_path", type=str, metavar='', help="The path to the word embeddings file") +parser.add_argument("--epochs", dest="epochs", type=int, metavar='', default=15, help="Number of epochs (default=15)") +parser.add_argument("-n", "--neg-size", dest="neg_size", type=int, metavar='', default=20, help="Number of negative instances (default=20)") +parser.add_argument("--maxlen", dest="maxlen", type=int, metavar='', default=0, help="Maximum allowed number of words during training. '0' means no limit (default=0)") +parser.add_argument("--seed", dest="seed", type=int, metavar='', default=1234, help="Random seed (default=1234)") +parser.add_argument("-a", "--algorithm", dest="algorithm", type=str, metavar='', default='adam', help="Optimization algorithm (rmsprop|sgd|adagrad|adadelta|adam|adamax) (default=adam)") +parser.add_argument("--domain", dest="domain", type=str, metavar='', default='restaurant', help="domain of the corpus {restaurant, beer}") +parser.add_argument("--ortho-reg", dest="ortho_reg", type=float, metavar='', default=0.1, help="The weight of orthogonol regularizaiton (default=0.1)") + +args = parser.parse_args() +out_dir = args.out_dir_path + '/' + args.domain +# out_dir = '../pre_trained_model/' + args.domain +U.print_args(args) + +assert args.algorithm in {'rmsprop', 'sgd', 'adagrad', 'adadelta', 'adam', 'adamax'} +assert args.domain in {'restaurant', 'beer'} + +from keras.preprocessing import sequence +import reader as dataset + +###### Get test data ############# +vocab, train_x, test_x, overall_maxlen = dataset.get_data(args.domain, vocab_size=args.vocab_size, maxlen=args.maxlen) +test_x = sequence.pad_sequences(test_x, maxlen=overall_maxlen) + + +############# Build model architecture, same as the model used for training ######### +from model import create_model +import keras.backend as K +from optimizers import get_optimizer + +optimizer = get_optimizer(args) + +def max_margin_loss(y_true, y_pred): + return K.mean(y_pred) +model = create_model(args, overall_maxlen, vocab) + +## Load the save model parameters +model.load_weights(out_dir+'/model_param') +model.compile(optimizer=optimizer, loss=max_margin_loss, metrics=[max_margin_loss]) + + + +################ Evaluation #################################### + +def evaluation(true, predict, domain): + true_label = [] + predict_label = [] + + if domain == 'restaurant': + + for line in predict: + predict_label.append(line.strip()) + + for line in true: + true_label.append(line.strip()) + + print(classification_report(true_label, predict_label, + ['Food', 'Staff', 'Ambience', 'Anecdotes', 'Price', 'Miscellaneous'], digits=3)) + + else: + for line in predict: + label = line.strip() + if label == 'smell' or label == 'taste': + label = 'taste+smell' + predict_label.append(label) + + for line in true: + label = line.strip() + if label == 'smell' or label == 'taste': + label = 'taste+smell' + true_label.append(label) + + print(classification_report(true_label, predict_label, + ['feel', 'taste+smell', 'look', 'overall', 'None'], digits=3)) + + +def prediction(test_labels, aspect_probs, cluster_map, domain): + label_ids = np.argsort(aspect_probs, axis=1)[:,-1] + predict_labels = [cluster_map[label_id] for label_id in label_ids] + evaluation(open(test_labels), predict_labels, domain) + + +## Create a dictionary that map word index to word +vocab_inv = {} +for w, ind in vocab.items(): + vocab_inv[ind] = w + + +test_fn = K.function([model.get_layer('sentence_input').input, K.learning_phase()], + [model.get_layer('att_weights').output, model.get_layer('p_t').output]) +att_weights, aspect_probs = test_fn([test_x, 0]) + + +## Save attention weights on test sentences into a file +att_out = codecs.open(out_dir + '/att_weights', 'w', 'utf-8') +print('Saving attention weights on test sentences...') +for c in xrange(len(test_x)): + att_out.write('----------------------------------------\n') + att_out.write(str(c) + '\n') + + word_inds = [i for i in test_x[c] if i!=0] + line_len = len(word_inds) + weights = att_weights[c] + weights = weights[(overall_maxlen-line_len):] + + words = [vocab_inv[i] for i in word_inds] + att_out.write(' '.join(words) + '\n') + for j in range(len(words)): + att_out.write(words[j] + ' '+str(round(weights[j], 3)) + '\n') + + + +###################################################### +# Uncomment the below part for F scores +###################################################### + +## cluster_map need to be specified manually according to the top words in each inferred aspect (save in aspect.log) + +# map for the pre-trained restaurant model (under pre_trained_model/restaurant) +# cluster_map = {0: 'Food', 1: 'Miscellaneous', 2: 'Miscellaneous', 3: 'Food', +# 4: 'Miscellaneous', 5: 'Food', 6:'Price', 7: 'Miscellaneous', 8: 'Staff', +# 9: 'Food', 10: 'Food', 11: 'Anecdotes', +# 12: 'Ambience', 13: 'Staff'} + + +# print('--- Results on %s domain ---' % (args.domain)) +# test_labels = '../preprocessed_data/%s/test_label.txt' % (args.domain) +# prediction(test_labels, aspect_probs, cluster_map, domain=args.domain) + + diff --git a/code/model.py b/code/model.py index 572864e..60920c6 100644 --- a/code/model.py +++ b/code/model.py @@ -1,67 +1,67 @@ -import logging -import keras.backend as K -from keras.layers import Dense, Activation, Embedding, Input -from keras.models import Model -from my_layers import Attention, Average, WeightedSum, WeightedAspectEmb, MaxMargin - - -logging.basicConfig(level=logging.INFO, - format='%(asctime)s %(levelname)s %(message)s') -logger = logging.getLogger(__name__) - -def create_model(args, maxlen, vocab): - - def ortho_reg(weight_matrix): - ### orthogonal regularization for aspect embedding matrix ### - w_n = weight_matrix / K.cast(K.epsilon() + K.sqrt(K.sum(K.square(weight_matrix), axis=-1, keepdims=True)), K.floatx()) - reg = K.sum(K.square(K.dot(w_n, K.transpose(w_n)) - K.eye(w_n.shape[0].eval()))) - return args.ortho_reg*reg - - vocab_size = len(vocab) - - ##### Inputs ##### - sentence_input = Input(shape=(maxlen,), dtype='int32', name='sentence_input') - neg_input = Input(shape=(args.neg_size, maxlen), dtype='int32', name='neg_input') - - ##### Construct word embedding layer ##### - word_emb = Embedding(vocab_size, args.emb_dim, mask_zero=True, name='word_emb') - - ##### Compute sentence representation ##### - e_w = word_emb(sentence_input) - y_s = Average()(e_w) - att_weights = Attention(name='att_weights')([e_w, y_s]) - z_s = WeightedSum()([e_w, att_weights]) - - ##### Compute representations of negative instances ##### - e_neg = word_emb(neg_input) - z_n = Average()(e_neg) - - ##### Reconstruction ##### - p_t = Dense(args.aspect_size)(z_s) - p_t = Activation('softmax', name='p_t')(p_t) - r_s = WeightedAspectEmb(args.aspect_size, args.emb_dim, name='aspect_emb', - W_regularizer=ortho_reg)(p_t) - - ##### Loss ##### - loss = MaxMargin(name='max_margin')([z_s, z_n, r_s]) - model = Model(input=[sentence_input, neg_input], output=loss) - - ### Word embedding and aspect embedding initialization ###### - if args.emb_path: - from w2vEmbReader import W2VEmbReader as EmbReader - emb_reader = EmbReader(args.emb_path, emb_dim=args.emb_dim) - logger.info('Initializing word embedding matrix') - model.get_layer('word_emb').W.set_value(emb_reader.get_emb_matrix_given_vocab(vocab, model.get_layer('word_emb').W.get_value())) - logger.info('Initializing aspect embedding matrix as centroid of kmean clusters') - model.get_layer('aspect_emb').W.set_value(emb_reader.get_aspect_matrix(args.aspect_size)) - - return model - - - - - - - - - +import logging +import keras.backend as K +from keras.layers import Dense, Activation, Embedding, Input +from keras.models import Model +from my_layers import Attention, Average, WeightedSum, WeightedAspectEmb, MaxMargin + + +logging.basicConfig(level=logging.INFO, + format='%(asctime)s %(levelname)s %(message)s') +logger = logging.getLogger(__name__) + +def create_model(args, maxlen, vocab): + + def ortho_reg(weight_matrix): + ### orthogonal regularization for aspect embedding matrix ### + w_n = weight_matrix / K.cast(K.epsilon() + K.sqrt(K.sum(K.square(weight_matrix), axis=-1, keepdims=True)), K.floatx()) + reg = K.sum(K.square(K.dot(w_n, K.transpose(w_n)) - K.eye(w_n.shape[0]))) + return args.ortho_reg*reg + + vocab_size = len(vocab) + + ##### Inputs ##### + sentence_input = Input(shape=(maxlen,), dtype='int32', name='sentence_input') + neg_input = Input(shape=(args.neg_size, maxlen), dtype='int32', name='neg_input') + + ##### Construct word embedding layer ##### + word_emb = Embedding(vocab_size, args.emb_dim, mask_zero=True, name='word_emb') + print(W) + ##### Compute sentence representation ##### + e_w = word_emb(sentence_input) + y_s = Average()(e_w) + att_weights = Attention(name='att_weights')([e_w, y_s]) + z_s = WeightedSum()([e_w, att_weights]) + + ##### Compute representations of negative instances ##### + e_neg = word_emb(neg_input) + z_n = Average()(e_neg) + + ##### Reconstruction ##### + p_t = Dense(args.aspect_size)(z_s) + p_t = Activation('softmax', name='p_t')(p_t) + r_s = WeightedAspectEmb(args.aspect_size, args.emb_dim, name='aspect_emb', + W_regularizer=ortho_reg)(p_t) + + ##### Loss ##### + loss = MaxMargin(name='max_margin')([z_s, z_n, r_s]) + model = Model(input=[sentence_input, neg_input], output=loss) + + ### Word embedding and aspect embedding initialization ###### + if args.emb_path: + from w2vEmbReader import W2VEmbReader as EmbReader + emb_reader = EmbReader(args.emb_path, emb_dim=args.emb_dim) + logger.info('Initializing word embedding matrix') + model.get_layer('word_emb').W.set_value(emb_reader.get_emb_matrix_given_vocab(vocab, model.get_layer('word_emb').W.get_value())) + logger.info('Initializing aspect embedding matrix as centroid of kmean clusters') + model.get_layer('aspect_emb').W.set_value(emb_reader.get_aspect_matrix(args.aspect_size)) + + return model + + + + + + + + + diff --git a/code/my_layers.py b/code/my_layers.py index f76a124..f529a90 100644 --- a/code/my_layers.py +++ b/code/my_layers.py @@ -1,195 +1,199 @@ -import keras.backend as K -from keras.engine.topology import Layer -from keras import initializations -from keras import regularizers -from keras import constraints -import numpy as np -import theano.tensor as T - -class Attention(Layer): - def __init__(self, W_regularizer=None, b_regularizer=None, - W_constraint=None, b_constraint=None, - bias=True, **kwargs): - """ - Keras Layer that implements an Content Attention mechanism. - Supports Masking. - """ - self.supports_masking = True - self.init = initializations.get('glorot_uniform') - - self.W_regularizer = regularizers.get(W_regularizer) - self.b_regularizer = regularizers.get(b_regularizer) - self.W_constraint = constraints.get(W_constraint) - self.b_constraint = constraints.get(b_constraint) - - self.bias = bias - super(Attention, self).__init__(**kwargs) - - def build(self, input_shape): - assert type(input_shape) == list - assert len(input_shape) == 2 - - self.steps = input_shape[0][1] - - self.W = self.add_weight((input_shape[0][-1], input_shape[1][-1]), - initializer=self.init, - name='{}_W'.format(self.name), - regularizer=self.W_regularizer, - constraint=self.W_constraint) - if self.bias: - self.b = self.add_weight((1,), - initializer='zero', - name='{}_b'.format(self.name), - regularizer=self.b_regularizer, - constraint=self.b_constraint) - self.built = True - - def compute_mask(self, input_tensor, mask=None): - return None - - def call(self, input_tensor, mask=None): - x = input_tensor[0] - y = input_tensor[1] - mask = mask[0] - - y = K.transpose(K.dot(self.W, K.transpose(y))) - y = K.expand_dims(y, dim=-2) - y = K.repeat_elements(y, self.steps, axis=1) - eij = K.sum(x*y, axis=-1) - - if self.bias: - b = K.repeat_elements(self.b, self.steps, axis=0) - eij += b - - eij = K.tanh(eij) - a = K.exp(eij) - - if mask is not None: - a *= K.cast(mask, K.floatx()) - - a /= K.cast(K.sum(a, axis=1, keepdims=True) + K.epsilon(), K.floatx()) - return a - - def get_output_shape_for(self, input_shape): - return (input_shape[0][0], input_shape[0][1]) - -class WeightedSum(Layer): - def __init__(self, **kwargs): - self.supports_masking = True - super(WeightedSum, self).__init__(**kwargs) - - def call(self, input_tensor, mask=None): - assert type(input_tensor) == list - assert type(mask) == list - - x = input_tensor[0] - a = input_tensor[1] - - a = K.expand_dims(a) - weighted_input = x * a - - return K.sum(weighted_input, axis=1) - - def get_output_shape_for(self, input_shape): - return (input_shape[0][0], input_shape[0][-1]) - - def compute_mask(self, x, mask=None): - return None - -class WeightedAspectEmb(Layer): - def __init__(self, input_dim, output_dim, - init='uniform', input_length=None, - W_regularizer=None, activity_regularizer=None, - W_constraint=None, - weights=None, dropout=0., **kwargs): - self.input_dim = input_dim - self.output_dim = output_dim - self.init = initializations.get(init) - self.input_length = input_length - self.dropout = dropout - - self.W_constraint = constraints.get(W_constraint) - self.W_regularizer = regularizers.get(W_regularizer) - self.activity_regularizer = regularizers.get(activity_regularizer) - - if 0. < self.dropout < 1.: - self.uses_learning_phase = True - self.initial_weights = weights - kwargs['input_shape'] = (self.input_length,) - kwargs['input_dtype'] = K.floatx() - super(WeightedAspectEmb, self).__init__(**kwargs) - - def build(self, input_shape): - self.W = self.add_weight((self.input_dim, self.output_dim), - initializer=self.init, - name='{}_W'.format(self.name), - regularizer=self.W_regularizer, - constraint=self.W_constraint) - - if self.initial_weights is not None: - self.set_weights(self.initial_weights) - self.built = True - - def compute_mask(self, x, mask=None): - return None - - def get_output_shape_for(self, input_shape): - return (input_shape[0], self.output_dim) - - def call(self, x, mask=None): - return K.dot(x, self.W) - - -class Average(Layer): - def __init__(self, **kwargs): - self.supports_masking = True - super(Average, self).__init__(**kwargs) - - def call(self, x, mask=None): - if mask is not None: - mask = K.cast(mask, K.floatx()) - mask = K.expand_dims(mask) - x = x * mask - return K.sum(x, axis=-2) / K.sum(mask, axis=-2) - - def get_output_shape_for(self, input_shape): - return input_shape[0:-2]+input_shape[-1:] - - def compute_mask(self, x, mask=None): - return None - - -class MaxMargin(Layer): - def __init__(self, **kwargs): - super(MaxMargin, self).__init__(**kwargs) - - def call(self, input_tensor, mask=None): - z_s = input_tensor[0] - z_n = input_tensor[1] - r_s = input_tensor[2] - - z_s = z_s / K.cast(K.epsilon() + K.sqrt(K.sum(K.square(z_s), axis=-1, keepdims=True)), K.floatx()) - z_n = z_n / K.cast(K.epsilon() + K.sqrt(K.sum(K.square(z_n), axis=-1, keepdims=True)), K.floatx()) - r_s = r_s / K.cast(K.epsilon() + K.sqrt(K.sum(K.square(r_s), axis=-1, keepdims=True)), K.floatx()) - - steps = z_n.shape[1] - - pos = K.sum(z_s*r_s, axis=-1, keepdims=True) - pos = K.repeat_elements(pos, steps, axis=-1) - r_s = K.expand_dims(r_s, dim=-2) - r_s = K.repeat_elements(r_s, steps, axis=1) - neg = K.sum(z_n*r_s, axis=-1) - - loss = K.cast(K.sum(T.maximum(0., (1. - pos + neg)), axis=-1, keepdims=True), K.floatx()) - return loss - - def compute_mask(self, input_tensor, mask=None): - return None - - def get_output_shape_for(self, input_shape): - return (input_shape[0][0], 1) - - - - - +import keras.backend as K +from keras.engine.topology import Layer +from keras import initializers +from keras import regularizers +from keras import constraints +import numpy as np +import theano.tensor as T + +class Attention(Layer): + def __init__(self, W_regularizer=None, b_regularizer=None, + W_constraint=None, b_constraint=None, + bias=True, **kwargs): + """ + Keras Layer that implements an Content Attention mechanism. + Supports Masking. + """ + self.supports_masking = True + self.init = initializers.get('glorot_uniform') + + self.W_regularizer = regularizers.get(W_regularizer) + self.b_regularizer = regularizers.get(b_regularizer) + self.W_constraint = constraints.get(W_constraint) + self.b_constraint = constraints.get(b_constraint) + + self.bias = bias + super(Attention, self).__init__(**kwargs) + + def build(self, input_shape): + assert type(input_shape) == list + assert len(input_shape) == 2 + + self.steps = input_shape[0][1] + + self.W = self.add_weight(shape=(input_shape[0][-1], input_shape[1][-1]), + initializer=self.init, + name='{}_W'.format(self.name), + regularizer=self.W_regularizer, + constraint=self.W_constraint) + print(self.W) + + if self.bias: + self.b = self.add_weight(shape=(1,), + initializer='zero', + name='{}_b'.format(self.name), + regularizer=self.b_regularizer, + constraint=self.b_constraint) + self.built = True + #super(Attention, self).build(input_shape) + + def compute_mask(self, input_tensor, mask=None): + return None + + def call(self, input_tensor, mask=None): + x = input_tensor[0] + y = input_tensor[1] + mask = mask[0] + + y = K.transpose(K.dot(self.W, K.transpose(y))) + y = K.expand_dims(y, -2) + y = K.repeat_elements(y, self.steps, axis=1) + eij = K.sum(x*y, axis=-1) + + if self.bias: + b = K.repeat_elements(self.b, self.steps, axis=0) + eij += b + + eij = K.tanh(eij) + a = K.exp(eij) + + if mask is not None: + a *= K.cast(mask, K.floatx()) + + a /= K.cast(K.sum(a, axis=1, keepdims=True) + K.epsilon(), K.floatx()) + return a + + def get_output_shape_for(self, input_shape): + return (input_shape[0][0], input_shape[0][1]) + +class WeightedSum(Layer): + def __init__(self, **kwargs): + self.supports_masking = True + super(WeightedSum, self).__init__(**kwargs) + + def call(self, input_tensor, mask=None): + assert type(input_tensor) == list + assert type(mask) == list + + x = input_tensor[0] + a = input_tensor[1] + + a = K.expand_dims(a) + weighted_input = x * a + + return K.sum(weighted_input, axis=1) + + def get_output_shape_for(self, input_shape): + return (input_shape[0][0], input_shape[0][-1]) + + def compute_mask(self, x, mask=None): + return None + +class WeightedAspectEmb(Layer): + def __init__(self, input_dim, output_dim, + init='uniform', input_length=None, + W_regularizer=None, activity_regularizer=None, + W_constraint=None, + weights=None, dropout=0., **kwargs): + self.input_dim = input_dim + self.output_dim = output_dim + self.init = initializers.get(init) + self.input_length = input_length + self.dropout = dropout + + self.W_constraint = constraints.get(W_constraint) + self.W_regularizer = regularizers.get(W_regularizer) + self.activity_regularizer = regularizers.get(activity_regularizer) + + if 0. < self.dropout < 1.: + self.uses_learning_phase = True + self.initial_weights = weights + kwargs['input_shape'] = (self.input_length,) + kwargs['input_dtype'] = K.floatx() + super(WeightedAspectEmb, self).__init__(**kwargs) + + def build(self, input_shape): + self.W = self.add_weight(shape=(self.input_dim, self.output_dim), + initializer=self.init, + name='{}_W'.format(self.name), + regularizer=self.W_regularizer, + constraint=self.W_constraint) + + if self.initial_weights is not None: + self.set_weights(self.initial_weights) + #super(WeightedAspectEmb, self).build(input_shape) + self.built = True + + def compute_mask(self, x, mask=None): + return None + + def get_output_shape_for(self, input_shape): + return (input_shape[0], self.output_dim) + + def call(self, x, mask=None): + return K.dot(x, self.W) + + +class Average(Layer): + def __init__(self, **kwargs): + self.supports_masking = True + super(Average, self).__init__(**kwargs) + + def call(self, x, mask=None): + if mask is not None: + mask = K.cast(mask, K.floatx()) + mask = K.expand_dims(mask) + x = x * mask + return K.sum(x, axis=-2) / K.sum(mask, axis=-2) + + def get_output_shape_for(self, input_shape): + return input_shape[0:-2]+input_shape[-1:] + + def compute_mask(self, x, mask=None): + return None + + +class MaxMargin(Layer): + def __init__(self, **kwargs): + super(MaxMargin, self).__init__(**kwargs) + + def call(self, input_tensor, mask=None): + z_s = input_tensor[0] + z_n = input_tensor[1] + r_s = input_tensor[2] + + z_s = z_s / K.cast(K.epsilon() + K.sqrt(K.sum(K.square(z_s), axis=-1, keepdims=True)), K.floatx()) + z_n = z_n / K.cast(K.epsilon() + K.sqrt(K.sum(K.square(z_n), axis=-1, keepdims=True)), K.floatx()) + r_s = r_s / K.cast(K.epsilon() + K.sqrt(K.sum(K.square(r_s), axis=-1, keepdims=True)), K.floatx()) + + steps = z_n.shape[1] + + pos = K.sum(z_s*r_s, axis=-1, keepdims=True) + pos = K.repeat_elements(pos, steps, axis=-1) + r_s = K.expand_dims(r_s, -2) + r_s = K.repeat_elements(r_s, steps, axis=1) + neg = K.sum(z_n*r_s, axis=-1) + + loss = K.cast(K.sum(K.maximum(0., (1. - pos + neg)), axis=-1, keepdims=True), K.floatx()) + return loss + + def compute_mask(self, input_tensor, mask=None): + return None + + def get_output_shape_for(self, input_shape): + return (input_shape[0][0], 1) + + + + + diff --git a/code/optimizers.py b/code/optimizers.py old mode 100755 new mode 100644 index 556eb53..6c04d4d --- a/code/optimizers.py +++ b/code/optimizers.py @@ -1,21 +1,21 @@ -import keras.optimizers as opt - -def get_optimizer(args): - - clipvalue = 0 - clipnorm = 10 - - if args.algorithm == 'rmsprop': - optimizer = opt.RMSprop(lr=0.001, rho=0.9, epsilon=1e-06, clipnorm=clipnorm, clipvalue=clipvalue) - elif args.algorithm == 'sgd': - optimizer = opt.SGD(lr=0.01, momentum=0.0, decay=0.0, nesterov=False, clipnorm=clipnorm, clipvalue=clipvalue) - elif args.algorithm == 'adagrad': - optimizer = opt.Adagrad(lr=0.01, epsilon=1e-06, clipnorm=clipnorm, clipvalue=clipvalue) - elif args.algorithm == 'adadelta': - optimizer = opt.Adadelta(lr=1.0, rho=0.95, epsilon=1e-06, clipnorm=clipnorm, clipvalue=clipvalue) - elif args.algorithm == 'adam': - optimizer = opt.Adam(lr=0.001, beta_1=0.9, beta_2=0.999, epsilon=1e-08, clipnorm=clipnorm, clipvalue=clipvalue) - elif args.algorithm == 'adamax': - optimizer = opt.Adamax(lr=0.002, beta_1=0.9, beta_2=0.999, epsilon=1e-08, clipnorm=clipnorm, clipvalue=clipvalue) - - return optimizer +import keras.optimizers as opt + +def get_optimizer(args): + + clipvalue = 0 + clipnorm = 10 + + if args.algorithm == 'rmsprop': + optimizer = opt.RMSprop(lr=0.001, rho=0.9, epsilon=1e-06, clipnorm=clipnorm, clipvalue=clipvalue) + elif args.algorithm == 'sgd': + optimizer = opt.SGD(lr=0.01, momentum=0.0, decay=0.0, nesterov=False, clipnorm=clipnorm, clipvalue=clipvalue) + elif args.algorithm == 'adagrad': + optimizer = opt.Adagrad(lr=0.01, epsilon=1e-06, clipnorm=clipnorm, clipvalue=clipvalue) + elif args.algorithm == 'adadelta': + optimizer = opt.Adadelta(lr=1.0, rho=0.95, epsilon=1e-06, clipnorm=clipnorm, clipvalue=clipvalue) + elif args.algorithm == 'adam': + optimizer = opt.Adam(lr=0.001, beta_1=0.9, beta_2=0.999, epsilon=1e-08, clipnorm=clipnorm, clipvalue=clipvalue) + elif args.algorithm == 'adamax': + optimizer = opt.Adamax(lr=0.002, beta_1=0.9, beta_2=0.999, epsilon=1e-08, clipnorm=clipnorm, clipvalue=clipvalue) + + return optimizer diff --git a/code/preprocess.py b/code/preprocess.py index 6932eb7..aaddc07 100644 --- a/code/preprocess.py +++ b/code/preprocess.py @@ -1,51 +1,51 @@ -from sklearn.feature_extraction.text import CountVectorizer -from nltk.corpus import stopwords -from nltk.stem.wordnet import WordNetLemmatizer -import codecs - -def parseSentence(line): - lmtzr = WordNetLemmatizer() - stop = stopwords.words('english') - text_token = CountVectorizer().build_tokenizer()(line.lower()) - text_rmstop = [i for i in text_token if i not in stop] - text_stem = [lmtzr.lemmatize(w) for w in text_rmstop] - return text_stem - -def preprocess_train(domain): - f = codecs.open('../datasets/'+domain+'/train.txt', 'r', 'utf-8') - out = codecs.open('../preprocessed_data/'+domain+'/train.txt', 'w', 'utf-8') - - for line in f: - tokens = parseSentence(line) - if len(tokens) > 0: - out.write(' '.join(tokens)+'\n') - -def preprocess_test(domain): - # For restaurant domain, only keep sentences with single - # aspect label that in {Food, Staff, Ambience} - - f1 = codecs.open('../datasets/'+domain+'/test.txt', 'r', 'utf-8') - f2 = codecs.open('../datasets/'+domain+'/test_label.txt', 'r', 'utf-8') - out1 = codecs.open('../preprocessed_data/'+domain+'/test.txt', 'w', 'utf-8') - out2 = codecs.open('../preprocessed_data/'+domain+'/test_label.txt', 'w', 'utf-8') - - for text, label in zip(f1, f2): - label = label.strip() - if domain == 'restaurant' and label not in ['Food', 'Staff', 'Ambience']: - continue - tokens = parseSentence(text) - if len(tokens) > 0: - out1.write(' '.join(tokens) + '\n') - out2.write(label+'\n') - -def preprocess(domain): - print '\t'+domain+' train set ...' - preprocess_train(domain) - print '\t'+domain+' test set ...' - preprocess_test(domain) - -print 'Preprocessing raw review sentences ...' -preprocess('restaurant') -preprocess('beer') - - +from sklearn.feature_extraction.text import CountVectorizer +from nltk.corpus import stopwords +from nltk.stem.wordnet import WordNetLemmatizer +import codecs + +def parseSentence(line): + lmtzr = WordNetLemmatizer() + stop = stopwords.words('english') + text_token = CountVectorizer().build_tokenizer()(line.lower()) + text_rmstop = [i for i in text_token if i not in stop] + text_stem = [lmtzr.lemmatize(w) for w in text_rmstop] + return text_stem + +def preprocess_train(domain): + f = codecs.open('../datasets/'+domain+'/train.txt', 'r', 'utf-8') + out = codecs.open('../preprocessed_data/'+domain+'/train.txt', 'w', 'utf-8') + + for line in f: + tokens = parseSentence(line) + if len(tokens) > 0: + out.write(' '.join(tokens)+'\n') + +def preprocess_test(domain): + # For restaurant domain, only keep sentences with single + # aspect label that in {Food, Staff, Ambience} + + f1 = codecs.open('../datasets/'+domain+'/test.txt', 'r', 'utf-8') + f2 = codecs.open('../datasets/'+domain+'/test_label.txt', 'r', 'utf-8') + out1 = codecs.open('../preprocessed_data/'+domain+'/test.txt', 'w', 'utf-8') + out2 = codecs.open('../preprocessed_data/'+domain+'/test_label.txt', 'w', 'utf-8') + + for text, label in zip(f1, f2): + label = label.strip() + if domain == 'restaurant' and label not in ['Food', 'Staff', 'Ambience']: + continue + tokens = parseSentence(text) + if len(tokens) > 0: + out1.write(' '.join(tokens) + '\n') + out2.write(label+'\n') + +def preprocess(domain): + print('\t'+domain+' train set ...') + preprocess_train(domain) + print('\t'+domain+' test set ...') + preprocess_test(domain) + +print('Preprocessing raw review sentences ...') +preprocess('restaurant') +preprocess('beer') + + diff --git a/code/reader.py b/code/reader.py index 9737938..c6f7bdb 100644 --- a/code/reader.py +++ b/code/reader.py @@ -1,113 +1,113 @@ -import codecs -import re -import operator - -num_regex = re.compile('^[+-]?[0-9]+\.?[0-9]*$') - -def is_number(token): - return bool(num_regex.match(token)) - -def create_vocab(domain, maxlen=0, vocab_size=0): - assert domain in {'restaurant', 'beer'} - source = '../preprocessed_data/'+domain+'/train.txt' - - total_words, unique_words = 0, 0 - word_freqs = {} - top = 0 - - fin = codecs.open(source, 'r', 'utf-8') - for line in fin: - words = line.split() - if maxlen > 0 and len(words) > maxlen: - continue - - for w in words: - if not is_number(w): - try: - word_freqs[w] += 1 - except KeyError: - unique_words += 1 - word_freqs[w] = 1 - total_words += 1 - - print (' %i total words, %i unique words' % (total_words, unique_words)) - sorted_word_freqs = sorted(word_freqs.items(), key=operator.itemgetter(1), reverse=True) - - vocab = {'':0, '':1, '':2} - index = len(vocab) - for word, _ in sorted_word_freqs: - vocab[word] = index - index += 1 - if vocab_size > 0 and index > vocab_size + 2: - break - if vocab_size > 0: - print (' keep the top %i words' % vocab_size) - - #Write (vocab, frequence) to a txt file - vocab_file = codecs.open('../preprocessed_data/%s/vocab' % domain, mode='w', encoding='utf8') - sorted_vocab = sorted(vocab.items(), key=operator.itemgetter(1)) - for word, index in sorted_vocab: - if index < 3: - vocab_file.write(word+'\t'+str(0)+'\n') - continue - vocab_file.write(word+'\t'+str(word_freqs[word])+'\n') - vocab_file.close() - - return vocab - -def read_dataset(domain, phase, vocab, maxlen): - assert domain in {'restaurant', 'beer'} - assert phase in {'train', 'test'} - - source = '../preprocessed_data/'+domain+'/'+phase+'.txt' - num_hit, unk_hit, total = 0., 0., 0. - maxlen_x = 0 - data_x = [] - - fin = codecs.open(source, 'r', 'utf-8') - for line in fin: - words = line.strip().split() - if maxlen > 0 and len(words) > maxlen: - continue - - indices = [] - for word in words: - if is_number(word): - indices.append(vocab['']) - num_hit += 1 - elif word in vocab: - indices.append(vocab[word]) - else: - indices.append(vocab['']) - unk_hit += 1 - total += 1 - - data_x.append(indices) - if maxlen_x < len(indices): - maxlen_x = len(indices) - - print ' hit rate: %.2f%%, hit rate: %.2f%%' % (100*num_hit/total, 100*unk_hit/total) - return data_x, maxlen_x - - - -def get_data(domain, vocab_size=0, maxlen=0): - print 'Reading data from', domain - print ' Creating vocab ...' - vocab = create_vocab(domain, maxlen, vocab_size) - print ' Reading dataset ...' - print ' train set' - train_x, train_maxlen = read_dataset(domain, 'train', vocab, maxlen) - print ' test set' - test_x, test_maxlen = read_dataset(domain, 'test', vocab, maxlen) - maxlen = max(train_maxlen, test_maxlen) - return vocab, train_x, test_x, maxlen - - - -if __name__ == "__main__": - vocab, train_x, test_x, maxlen = get_data('restaurant') - print len(train_x) - print len(test_x) - print maxlen - +import codecs +import re +import operator + +num_regex = re.compile('^[+-]?[0-9]+\.?[0-9]*$') + +def is_number(token): + return bool(num_regex.match(token)) + +def create_vocab(domain, maxlen=0, vocab_size=0): + assert domain in {'restaurant', 'beer'} + source = '../preprocessed_data/'+domain+'/train.txt' + + total_words, unique_words = 0, 0 + word_freqs = {} + top = 0 + + fin = codecs.open(source, 'r', 'utf-8') + for line in fin: + words = line.split() + if maxlen > 0 and len(words) > maxlen: + continue + + for w in words: + if not is_number(w): + try: + word_freqs[w] += 1 + except KeyError: + unique_words += 1 + word_freqs[w] = 1 + total_words += 1 + + print(' %i total words, %i unique words' % (total_words, unique_words)) + sorted_word_freqs = sorted(word_freqs.items(), key=operator.itemgetter(1), reverse=True) + + vocab = {'':0, '':1, '':2} + index = len(vocab) + for word, _ in sorted_word_freqs: + vocab[word] = index + index += 1 + if vocab_size > 0 and index > vocab_size + 2: + break + if vocab_size > 0: + print(' keep the top %i words' % vocab_size) + + #Write (vocab, frequence) to a txt file + vocab_file = codecs.open('../preprocessed_data/%s/vocab' % domain, mode='w', encoding='utf8') + sorted_vocab = sorted(vocab.items(), key=operator.itemgetter(1)) + for word, index in sorted_vocab: + if index < 3: + vocab_file.write(word+'\t'+str(0)+'\n') + continue + vocab_file.write(word+'\t'+str(word_freqs[word])+'\n') + vocab_file.close() + + return vocab + +def read_dataset(domain, phase, vocab, maxlen): + assert domain in {'restaurant', 'beer'} + assert phase in {'train', 'test'} + + source = '../preprocessed_data/'+domain+'/'+phase+'.txt' + num_hit, unk_hit, total = 0., 0., 0. + maxlen_x = 0 + data_x = [] + + fin = codecs.open(source, 'r', 'utf-8') + for line in fin: + words = line.strip().split() + if maxlen > 0 and len(words) > maxlen: + continue + + indices = [] + for word in words: + if is_number(word): + indices.append(vocab['']) + num_hit += 1 + elif word in vocab: + indices.append(vocab[word]) + else: + indices.append(vocab['']) + unk_hit += 1 + total += 1 + + data_x.append(indices) + if maxlen_x < len(indices): + maxlen_x = len(indices) + + print(' hit rate: %.2f%%, hit rate: %.2f%%' % (100*num_hit/total, 100*unk_hit/total)) + return data_x, maxlen_x + + + +def get_data(domain, vocab_size=0, maxlen=0): + print('Reading data from', domain) + print(' Creating vocab ...') + vocab = create_vocab(domain, maxlen, vocab_size) + print(' Reading dataset ...') + print(' train set') + train_x, train_maxlen = read_dataset(domain, 'train', vocab, maxlen) + print(' test set') + test_x, test_maxlen = read_dataset(domain, 'test', vocab, maxlen) + maxlen = max(train_maxlen, test_maxlen) + return vocab, train_x, test_x, maxlen + + + +if __name__ == "__main__": + vocab, train_x, test_x, maxlen = get_data('restaurant') + print(len(train_x)) + print(len(test_x)) + print(maxlen) + diff --git a/code/run_script.sh b/code/run_script.sh old mode 100755 new mode 100644 index a614ab2..e17c105 --- a/code/run_script.sh +++ b/code/run_script.sh @@ -1,7 +1,7 @@ - - -THEANO_FLAGS="device=gpu0,floatX=float32" python train.py \ ---emb ../preprocessed_data/restaurant/w2v_embedding \ ---domain restaurant \ --o output_dir \ - + + +THEANO_FLAGS="device=gpu0,floatX=float32" python train.py \ +--emb ../preprocessed_data/restaurant/w2v_embedding \ +--domain restaurant \ +-o output_dir \ + diff --git a/code/train.py b/code/train.py index 6b1fdf0..12b0ede 100644 --- a/code/train.py +++ b/code/train.py @@ -1,175 +1,178 @@ -import argparse -import logging -import numpy as np -from time import time -import utils as U -import codecs - -logging.basicConfig( - #filename='out.log', - level=logging.INFO, - format='%(asctime)s %(levelname)s %(message)s') -logger = logging.getLogger(__name__) - - -############################################################################################################################### -## Parse arguments -# - -parser = argparse.ArgumentParser() -parser.add_argument("-o", "--out-dir", dest="out_dir_path", type=str, metavar='', required=True, help="The path to the output directory") -parser.add_argument("-e", "--embdim", dest="emb_dim", type=int, metavar='', default=200, help="Embeddings dimension (default=200)") -parser.add_argument("-b", "--batch-size", dest="batch_size", type=int, metavar='', default=50, help="Batch size (default=50)") -parser.add_argument("-v", "--vocab-size", dest="vocab_size", type=int, metavar='', default=9000, help="Vocab size. '0' means no limit (default=9000)") -parser.add_argument("-as", "--aspect-size", dest="aspect_size", type=int, metavar='', default=14, help="The number of aspects specified by users (default=14)") -parser.add_argument("--emb", dest="emb_path", type=str, metavar='', help="The path to the word embeddings file") -parser.add_argument("--epochs", dest="epochs", type=int, metavar='', default=15, help="Number of epochs (default=15)") -parser.add_argument("-n", "--neg-size", dest="neg_size", type=int, metavar='', default=20, help="Number of negative instances (default=20)") -parser.add_argument("--maxlen", dest="maxlen", type=int, metavar='', default=0, help="Maximum allowed number of words during training. '0' means no limit (default=0)") -parser.add_argument("--seed", dest="seed", type=int, metavar='', default=1234, help="Random seed (default=1234)") -parser.add_argument("-a", "--algorithm", dest="algorithm", type=str, metavar='', default='adam', help="Optimization algorithm (rmsprop|sgd|adagrad|adadelta|adam|adamax) (default=adam)") -parser.add_argument("--domain", dest="domain", type=str, metavar='', default='restaurant', help="domain of the corpus {restaurant, beer}") -parser.add_argument("--ortho-reg", dest="ortho_reg", type=float, metavar='', default=0.1, help="The weight of orthogonol regularizaiton (default=0.1)") - -args = parser.parse_args() -out_dir = args.out_dir_path + '/' + args.domain -U.mkdir_p(out_dir) -U.print_args(args) - -assert args.algorithm in {'rmsprop', 'sgd', 'adagrad', 'adadelta', 'adam', 'adamax'} -assert args.domain in {'restaurant', 'beer'} - -if args.seed > 0: - np.random.seed(args.seed) - - -# ############################################################################################################################### -# ## Prepare data -# # - -from keras.preprocessing import sequence -import reader as dataset - -vocab, train_x, test_x, overall_maxlen = dataset.get_data(args.domain, vocab_size=args.vocab_size, maxlen=args.maxlen) -train_x = sequence.pad_sequences(train_x, maxlen=overall_maxlen) -test_x = sequence.pad_sequences(test_x, maxlen=overall_maxlen) - -print 'Number of training examples: ', len(train_x) -print 'Length of vocab: ', len(vocab) - -def sentence_batch_generator(data, batch_size): - n_batch = len(data) / batch_size - batch_count = 0 - np.random.shuffle(data) - - while True: - if batch_count == n_batch: - np.random.shuffle(data) - batch_count = 0 - - batch = data[batch_count*batch_size: (batch_count+1)*batch_size] - batch_count += 1 - yield batch - -def negative_batch_generator(data, batch_size, neg_size): - data_len = data.shape[0] - dim = data.shape[1] - - while True: - indices = np.random.choice(data_len, batch_size * neg_size) - samples = data[indices].reshape(batch_size, neg_size, dim) - yield samples - - - -############################################################################################################################### -## Optimizaer algorithm -# - -from optimizers import get_optimizer - -optimizer = get_optimizer(args) - - - -############################################################################################################################### -## Building model - -from model import create_model -import keras.backend as K - -logger.info(' Building model') - - -def max_margin_loss(y_true, y_pred): - return K.mean(y_pred) - -model = create_model(args, overall_maxlen, vocab) -# freeze the word embedding layer -model.get_layer('word_emb').trainable=False -model.compile(optimizer=optimizer, loss=max_margin_loss, metrics=[max_margin_loss]) - - -############################################################################################################################### -## Training -# -from keras.models import load_model -from tqdm import tqdm - -logger.info('--------------------------------------------------------------------------------------------------------------------------') - -vocab_inv = {} -for w, ind in vocab.items(): - vocab_inv[ind] = w - - -sen_gen = sentence_batch_generator(train_x, args.batch_size) -neg_gen = negative_batch_generator(train_x, args.batch_size, args.neg_size) -# batches_per_epoch = len(train_x) / args.batch_size -batches_per_epoch = 1000 - -min_loss = float('inf') -for ii in xrange(args.epochs): - t0 = time() - loss, max_margin_loss = 0., 0. - - for b in tqdm(xrange(batches_per_epoch)): - sen_input = sen_gen.next() - neg_input = neg_gen.next() - - batch_loss, batch_max_margin_loss = model.train_on_batch([sen_input, neg_input], np.ones((args.batch_size, 1))) - loss += batch_loss / batches_per_epoch - max_margin_loss += batch_max_margin_loss / batches_per_epoch - - tr_time = time() - t0 - - if loss < min_loss: - min_loss = loss - word_emb = model.get_layer('word_emb').W.get_value() - aspect_emb = model.get_layer('aspect_emb').W.get_value() - word_emb = word_emb / np.linalg.norm(word_emb, axis=-1, keepdims=True) - aspect_emb = aspect_emb / np.linalg.norm(aspect_emb, axis=-1, keepdims=True) - aspect_file = codecs.open(out_dir+'/aspect.log', 'w', 'utf-8') - model.save_weights(out_dir+'/model_param') - - for ind in range(len(aspect_emb)): - desc = aspect_emb[ind] - sims = word_emb.dot(desc.T) - ordered_words = np.argsort(sims)[::-1] - desc_list = [vocab_inv[w] for w in ordered_words[:100]] - print 'Aspect %d:' % ind - print desc_list - aspect_file.write('Aspect %d:\n' % ind) - aspect_file.write(' '.join(desc_list) + '\n\n') - - logger.info('Epoch %d, train: %is' % (ii, tr_time)) - logger.info('Total loss: %.4f, max_margin_loss: %.4f, ortho_reg: %.4f' % (loss, max_margin_loss, loss-max_margin_loss)) - - - - - - - - - +import argparse +import logging +import numpy as np +from time import time +import utils as U +import codecs +import os +os.environ["MKL_THREADING_LAYER"] = "GNU" + +logging.basicConfig( + #filename='out.log', + level=logging.INFO, + format='%(asctime)s %(levelname)s %(message)s') +logger = logging.getLogger(__name__) + + +############################################################################################################################### +## Parse arguments +# + +parser = argparse.ArgumentParser() +parser.add_argument("-o", "--out-dir", dest="out_dir_path", type=str, metavar='', required=True, help="The path to the output directory") +parser.add_argument("-e", "--embdim", dest="emb_dim", type=int, metavar='', default=200, help="Embeddings dimension (default=200)") +parser.add_argument("-b", "--batch-size", dest="batch_size", type=int, metavar='', default=50, help="Batch size (default=50)") +parser.add_argument("-v", "--vocab-size", dest="vocab_size", type=int, metavar='', default=9000, help="Vocab size. '0' means no limit (default=9000)") +parser.add_argument("-as", "--aspect-size", dest="aspect_size", type=int, metavar='', default=14, help="The number of aspects specified by users (default=14)") +parser.add_argument("--emb", dest="emb_path", type=str, metavar='', help="The path to the word embeddings file") +parser.add_argument("--epochs", dest="epochs", type=int, metavar='', default=15, help="Number of epochs (default=15)") +parser.add_argument("-n", "--neg-size", dest="neg_size", type=int, metavar='', default=20, help="Number of negative instances (default=20)") +parser.add_argument("--maxlen", dest="maxlen", type=int, metavar='', default=0, help="Maximum allowed number of words during training. '0' means no limit (default=0)") +parser.add_argument("--seed", dest="seed", type=int, metavar='', default=1234, help="Random seed (default=1234)") +parser.add_argument("-a", "--algorithm", dest="algorithm", type=str, metavar='', default='adam', help="Optimization algorithm (rmsprop|sgd|adagrad|adadelta|adam|adamax) (default=adam)") +parser.add_argument("--domain", dest="domain", type=str, metavar='', default='restaurant', help="domain of the corpus {restaurant, beer}") +parser.add_argument("--ortho-reg", dest="ortho_reg", type=float, metavar='', default=0.1, help="The weight of orthogonol regularizaiton (default=0.1)") + +args = parser.parse_args() +out_dir = args.out_dir_path + '/' + args.domain +U.mkdir_p(out_dir) +U.print_args(args) + +assert args.algorithm in {'rmsprop', 'sgd', 'adagrad', 'adadelta', 'adam', 'adamax'} +assert args.domain in {'restaurant', 'beer'} + +if args.seed > 0: + np.random.seed(args.seed) + + +# ############################################################################################################################### +# ## Prepare data +# # + +from keras.preprocessing import sequence +import reader as dataset + +vocab, train_x, test_x, overall_maxlen = dataset.get_data(args.domain, vocab_size=args.vocab_size, maxlen=args.maxlen) +train_x = sequence.pad_sequences(train_x, maxlen=overall_maxlen) +test_x = sequence.pad_sequences(test_x, maxlen=overall_maxlen) + +print('Number of training examples: ', len(train_x)) +print('Length of vocab: ', len(vocab)) + +def sentence_batch_generator(data, batch_size): + n_batch = len(data) / batch_size + batch_count = 0 + np.random.shuffle(data) + + while True: + if batch_count == n_batch: + np.random.shuffle(data) + batch_count = 0 + + batch = data[batch_count*batch_size: (batch_count+1)*batch_size] + batch_count += 1 + yield batch + +def negative_batch_generator(data, batch_size, neg_size): + data_len = data.shape[0] + dim = data.shape[1] + + while True: + indices = np.random.choice(data_len, batch_size * neg_size) + samples = data[indices].reshape(batch_size, neg_size, dim) + yield samples + + + +############################################################################################################################### +## Optimizaer algorithm +# + +from optimizers import get_optimizer + +optimizer = get_optimizer(args) + + + +############################################################################################################################### +## Building model + +from model import create_model +import keras.backend as K + +logger.info(' Building model') + +print(W) +print("***") +def max_margin_loss(y_true, y_pred): + return K.mean(y_pred) + +model = create_model(args, overall_maxlen, vocab) +# freeze the word embedding layer +model.get_layer('word_emb').trainable=False +model.compile(optimizer=optimizer, loss=max_margin_loss, metrics=[max_margin_loss]) + + +############################################################################################################################### +## Training +# +from keras.models import load_model +from tqdm import tqdm + +logger.info('--------------------------------------------------------------------------------------------------------------------------') + +vocab_inv = {} +for w, ind in vocab.items(): + vocab_inv[ind] = w + + +sen_gen = sentence_batch_generator(train_x, args.batch_size) +neg_gen = negative_batch_generator(train_x, args.batch_size, args.neg_size) +# batches_per_epoch = len(train_x) / args.batch_size +batches_per_epoch = 1000 + +min_loss = float('inf') +for ii in range(args.epochs): + t0 = time() + loss, max_margin_loss = 0., 0. + + for b in tqdm(range(batches_per_epoch)): + sen_input = next(sen_gen) + neg_input = next(neg_gen) + + batch_loss, batch_max_margin_loss = model.train_on_batch([sen_input, neg_input], np.ones((args.batch_size, 1))) + loss += batch_loss / batches_per_epoch + max_margin_loss += batch_max_margin_loss / batches_per_epoch + + tr_time = time() - t0 + + if loss < min_loss: + min_loss = loss + word_emb = model.get_layer('word_emb').W.get_value() + aspect_emb = model.get_layer('aspect_emb').W.get_value() + word_emb = word_emb / np.linalg.norm(word_emb, axis=-1, keepdims=True) + aspect_emb = aspect_emb / np.linalg.norm(aspect_emb, axis=-1, keepdims=True) + aspect_file = codecs.open(out_dir+'/aspect.log', 'w', 'utf-8') + model.save_weights(out_dir+'/model_param') + + for ind in range(len(aspect_emb)): + desc = aspect_emb[ind] + sims = word_emb.dot(desc.T) + ordered_words = np.argsort(sims)[::-1] + desc_list = [vocab_inv[w] for w in ordered_words[:100]] + print('Aspect %d:' % ind) + print(desc_list) + aspect_file.write('Aspect %d:\n' % ind) + aspect_file.write(' '.join(desc_list) + '\n\n') + + logger.info('Epoch %d, train: %is' % (ii, tr_time)) + logger.info('Total loss: %.4f, max_margin_loss: %.4f, ortho_reg: %.4f' % (loss, max_margin_loss, loss-max_margin_loss)) + + + + + + + + + diff --git a/code/utils.py b/code/utils.py old mode 100755 new mode 100644 index a134e62..33509ac --- a/code/utils.py +++ b/code/utils.py @@ -1,165 +1,165 @@ -import sys -import os, errno -import logging - -#-----------------------------------------------------------------------------------------------------------# - -def set_logger(out_dir=None): - console_format = BColors.OKBLUE + '[%(levelname)s]' + BColors.ENDC + ' (%(name)s) %(message)s' - #datefmt='%Y-%m-%d %Hh-%Mm-%Ss' - logger = logging.getLogger() - logger.setLevel(logging.DEBUG) - console = logging.StreamHandler() - console.setLevel(logging.DEBUG) - console.setFormatter(logging.Formatter(console_format)) - logger.addHandler(console) - if out_dir: - file_format = '[%(levelname)s] (%(name)s) %(message)s' - log_file = logging.FileHandler(out_dir + '/log.txt', mode='w') - log_file.setLevel(logging.DEBUG) - log_file.setFormatter(logging.Formatter(file_format)) - logger.addHandler(log_file) - -#-----------------------------------------------------------------------------------------------------------# - -def mkdir_p(path): - if path == '': - return - try: - os.makedirs(path) - except OSError as exc: # Python >2.5 - if exc.errno == errno.EEXIST and os.path.isdir(path): - pass - else: raise - -def get_root_dir(): - return os.path.dirname(sys.argv[0]) - -def bincounts(array): - num_rows = array.shape[0] - if array.ndim > 1: - num_cols = array.shape[1] - else: - num_cols = 1 - array = array[:, None] - counters = [] - mfe_list = [] - for col in range(num_cols): - counter = {} - for row in range(num_rows): - element = array[row,col] - if element in counter: - counter[element] += 1 - else: - counter[element] = 1 - max_count = 0 - for element in counter: - if counter[element] > max_count: - max_count = counter[element] - mfe = element - counters.append(counter) - mfe_list.append(mfe) - return counters, mfe_list - -# Convert all arguments to strings -def ltos(*args): - outputs = [] - for arg in args: - if type(arg) == list: - out = ' '.join(['%.3f' % e for e in arg]) - if len(arg) == 1: - outputs.append(out) - else: - outputs.append('[' + out + ']') - else: - outputs.append(str(arg)) - return tuple(outputs) - -#-----------------------------------------------------------------------------------------------------------# - -import re - -class BColors: - HEADER = '\033[95m' - OKBLUE = '\033[94m' - OKGREEN = '\033[92m' - WARNING = '\033[93m' - FAIL = '\033[91m' - ENDC = '\033[0m' - BOLD = '\033[1m' - UNDERLINE = '\033[4m' - WHITE = '\033[37m' - YELLOW = '\033[33m' - GREEN = '\033[32m' - BLUE = '\033[34m' - CYAN = '\033[36m' - RED = '\033[31m' - MAGENTA = '\033[35m' - BLACK = '\033[30m' - BHEADER = BOLD + '\033[95m' - BOKBLUE = BOLD + '\033[94m' - BOKGREEN = BOLD + '\033[92m' - BWARNING = BOLD + '\033[93m' - BFAIL = BOLD + '\033[91m' - BUNDERLINE = BOLD + '\033[4m' - BWHITE = BOLD + '\033[37m' - BYELLOW = BOLD + '\033[33m' - BGREEN = BOLD + '\033[32m' - BBLUE = BOLD + '\033[34m' - BCYAN = BOLD + '\033[36m' - BRED = BOLD + '\033[31m' - BMAGENTA = BOLD + '\033[35m' - BBLACK = BOLD + '\033[30m' - - @staticmethod - def cleared(s): - return re.sub("\033\[[0-9][0-9]?m", "", s) - -def red(message): - return BColors.RED + str(message) + BColors.ENDC - -def b_red(message): - return BColors.BRED + str(message) + BColors.ENDC - -def blue(message): - return BColors.BLUE + str(message) + BColors.ENDC - -def b_yellow(message): - return BColors.BYELLOW + str(message) + BColors.ENDC - -def green(message): - return BColors.GREEN + str(message) + BColors.ENDC - -def b_green(message): - return BColors.BGREEN + str(message) + BColors.ENDC - -#-----------------------------------------------------------------------------------------------------------# - -def print_args(args, path=None): - if path: - output_file = open(path, 'w') - logger = logging.getLogger(__name__) - logger.info("Arguments:") - args.command = ' '.join(sys.argv) - items = vars(args) - for key in sorted(items.keys(), key=lambda s: s.lower()): - value = items[key] - if not value: - value = "None" - logger.info(" " + key + ": " + str(items[key])) - if path is not None: - output_file.write(" " + key + ": " + str(items[key]) + "\n") - if path: - output_file.close() - del args.command - -def get_args(args): - items = vars(args) - output_string = '' - for key in sorted(items.keys(), key=lambda s: s.lower()): - value = items[key] - if not value: - value = "None" - output_string += " " + key + ": " + str(items[key] + "\n") - return output_string - +import sys +import os, errno +import logging + +#-----------------------------------------------------------------------------------------------------------# + +def set_logger(out_dir=None): + console_format = BColors.OKBLUE + '[%(levelname)s]' + BColors.ENDC + ' (%(name)s) %(message)s' + #datefmt='%Y-%m-%d %Hh-%Mm-%Ss' + logger = logging.getLogger() + logger.setLevel(logging.DEBUG) + console = logging.StreamHandler() + console.setLevel(logging.DEBUG) + console.setFormatter(logging.Formatter(console_format)) + logger.addHandler(console) + if out_dir: + file_format = '[%(levelname)s] (%(name)s) %(message)s' + log_file = logging.FileHandler(out_dir + '/log.txt', mode='w') + log_file.setLevel(logging.DEBUG) + log_file.setFormatter(logging.Formatter(file_format)) + logger.addHandler(log_file) + +#-----------------------------------------------------------------------------------------------------------# + +def mkdir_p(path): + if path == '': + return + try: + os.makedirs(path) + except OSError as exc: # Python >2.5 + if exc.errno == errno.EEXIST and os.path.isdir(path): + pass + else: raise + +def get_root_dir(): + return os.path.dirname(sys.argv[0]) + +def bincounts(array): + num_rows = array.shape[0] + if array.ndim > 1: + num_cols = array.shape[1] + else: + num_cols = 1 + array = array[:, None] + counters = [] + mfe_list = [] + for col in range(num_cols): + counter = {} + for row in range(num_rows): + element = array[row,col] + if element in counter: + counter[element] += 1 + else: + counter[element] = 1 + max_count = 0 + for element in counter: + if counter[element] > max_count: + max_count = counter[element] + mfe = element + counters.append(counter) + mfe_list.append(mfe) + return counters, mfe_list + +# Convert all arguments to strings +def ltos(*args): + outputs = [] + for arg in args: + if type(arg) == list: + out = ' '.join(['%.3f' % e for e in arg]) + if len(arg) == 1: + outputs.append(out) + else: + outputs.append('[' + out + ']') + else: + outputs.append(str(arg)) + return tuple(outputs) + +#-----------------------------------------------------------------------------------------------------------# + +import re + +class BColors: + HEADER = '\033[95m' + OKBLUE = '\033[94m' + OKGREEN = '\033[92m' + WARNING = '\033[93m' + FAIL = '\033[91m' + ENDC = '\033[0m' + BOLD = '\033[1m' + UNDERLINE = '\033[4m' + WHITE = '\033[37m' + YELLOW = '\033[33m' + GREEN = '\033[32m' + BLUE = '\033[34m' + CYAN = '\033[36m' + RED = '\033[31m' + MAGENTA = '\033[35m' + BLACK = '\033[30m' + BHEADER = BOLD + '\033[95m' + BOKBLUE = BOLD + '\033[94m' + BOKGREEN = BOLD + '\033[92m' + BWARNING = BOLD + '\033[93m' + BFAIL = BOLD + '\033[91m' + BUNDERLINE = BOLD + '\033[4m' + BWHITE = BOLD + '\033[37m' + BYELLOW = BOLD + '\033[33m' + BGREEN = BOLD + '\033[32m' + BBLUE = BOLD + '\033[34m' + BCYAN = BOLD + '\033[36m' + BRED = BOLD + '\033[31m' + BMAGENTA = BOLD + '\033[35m' + BBLACK = BOLD + '\033[30m' + + @staticmethod + def cleared(s): + return re.sub("\033\[[0-9][0-9]?m", "", s) + +def red(message): + return BColors.RED + str(message) + BColors.ENDC + +def b_red(message): + return BColors.BRED + str(message) + BColors.ENDC + +def blue(message): + return BColors.BLUE + str(message) + BColors.ENDC + +def b_yellow(message): + return BColors.BYELLOW + str(message) + BColors.ENDC + +def green(message): + return BColors.GREEN + str(message) + BColors.ENDC + +def b_green(message): + return BColors.BGREEN + str(message) + BColors.ENDC + +#-----------------------------------------------------------------------------------------------------------# + +def print_args(args, path=None): + if path: + output_file = open(path, 'w') + logger = logging.getLogger(__name__) + logger.info("Arguments:") + args.command = ' '.join(sys.argv) + items = vars(args) + for key in sorted(items.keys(), key=lambda s: s.lower()): + value = items[key] + if not value: + value = "None" + logger.info(" " + key + ": " + str(items[key])) + if path is not None: + output_file.write(" " + key + ": " + str(items[key]) + "\n") + if path: + output_file.close() + del args.command + +def get_args(args): + items = vars(args) + output_string = '' + for key in sorted(items.keys(), key=lambda s: s.lower()): + value = items[key] + if not value: + value = "None" + output_string += " " + key + ": " + str(items[key] + "\n") + return output_string + diff --git a/code/w2vEmbReader.py b/code/w2vEmbReader.py index 5c90c7d..b446d60 100644 --- a/code/w2vEmbReader.py +++ b/code/w2vEmbReader.py @@ -1,71 +1,71 @@ -import codecs -import logging -import numpy as np -import gensim -from sklearn.cluster import KMeans -import pickle - - -logging.basicConfig(level=logging.INFO, - format='%(asctime)s %(levelname)s %(message)s') -logger = logging.getLogger(__name__) - -class W2VEmbReader: - - def __init__(self, emb_path, emb_dim=None): - - logger.info('Loading embeddings from: ' + emb_path) - self.embeddings = {} - emb_matrix = [] - - model = gensim.models.Word2Vec.load(emb_path) - self.emb_dim = emb_dim - for word in model.vocab: - self.embeddings[word] = list(model[word]) - emb_matrix.append(list(model[word])) - - if emb_dim != None: - assert self.emb_dim == len(self.embeddings['nice']) - - self.vector_size = len(self.embeddings) - self.emb_matrix = np.asarray(emb_matrix) - - logger.info(' #vectors: %i, #dimensions: %i' % (self.vector_size, self.emb_dim)) - - - def get_emb_given_word(self, word): - try: - return self.embeddings[word] - except KeyError: - return None - - def get_emb_matrix_given_vocab(self, vocab, emb_matrix): - counter = 0. - for word, index in vocab.iteritems(): - try: - emb_matrix[index] = self.embeddings[word] - counter += 1 - except KeyError: - pass - - logger.info('%i/%i word vectors initialized (hit rate: %.2f%%)' % (counter, len(vocab), 100*counter/len(vocab))) - # L2 normalization - norm_emb_matrix = emb_matrix / np.linalg.norm(emb_matrix, axis=-1, keepdims=True) - return norm_emb_matrix - - - def get_aspect_matrix(self, n_clusters): - km = KMeans(n_clusters=n_clusters) - km.fit(self.emb_matrix) - clusters = km.cluster_centers_ - - # L2 normalization - norm_aspect_matrix = clusters / np.linalg.norm(clusters, axis=-1, keepdims=True) - return norm_aspect_matrix.astype(np.float32) - - def get_emb_dim(self): - return self.emb_dim - - - - +import codecs +import logging +import numpy as np +import gensim +from sklearn.cluster import KMeans +import pickle + + +logging.basicConfig(level=logging.INFO, + format='%(asctime)s %(levelname)s %(message)s') +logger = logging.getLogger(__name__) + +class W2VEmbReader: + + def __init__(self, emb_path, emb_dim=None): + + logger.info('Loading embeddings from: ' + emb_path) + self.embeddings = {} + emb_matrix = [] + + model = gensim.models.Word2Vec.load(emb_path) + self.emb_dim = emb_dim + for word in model.vocab: + self.embeddings[word] = list(model[word]) + emb_matrix.append(list(model[word])) + + if emb_dim != None: + assert self.emb_dim == len(self.embeddings['nice']) + + self.vector_size = len(self.embeddings) + self.emb_matrix = np.asarray(emb_matrix) + + logger.info(' #vectors: %i, #dimensions: %i' % (self.vector_size, self.emb_dim)) + + + def get_emb_given_word(self, word): + try: + return self.embeddings[word] + except KeyError: + return None + + def get_emb_matrix_given_vocab(self, vocab, emb_matrix): + counter = 0. + for word, index in vocab.iteritems(): + try: + emb_matrix[index] = self.embeddings[word] + counter += 1 + except KeyError: + pass + + logger.info('%i/%i word vectors initialized (hit rate: %.2f%%)' % (counter, len(vocab), 100*counter/len(vocab))) + # L2 normalization + norm_emb_matrix = emb_matrix / np.linalg.norm(emb_matrix, axis=-1, keepdims=True) + return norm_emb_matrix + + + def get_aspect_matrix(self, n_clusters): + km = KMeans(n_clusters=n_clusters) + km.fit(self.emb_matrix) + clusters = km.cluster_centers_ + + # L2 normalization + norm_aspect_matrix = clusters / np.linalg.norm(clusters, axis=-1, keepdims=True) + return norm_aspect_matrix.astype(np.float32) + + def get_emb_dim(self): + return self.emb_dim + + + + diff --git a/code/word2vec.py b/code/word2vec.py old mode 100755 new mode 100644 index 4b524ff..a083e05 --- a/code/word2vec.py +++ b/code/word2vec.py @@ -1,26 +1,26 @@ -import gensim -import codecs - -class MySentences(object): - def __init__(self, filename): - self.filename = filename - - def __iter__(self): - for line in codecs.open(self.filename, 'r', 'utf-8'): - yield line.split() - - -def main(domain): - source = '../preprocessed_data/%s/train.txt' % (domain) - model_file = '../preprocessed_data/%s/w2v_embedding' % (domain) - sentences = MySentences(source) - model = gensim.models.Word2Vec(sentences, size=200, window=5, min_count=10, workers=4) - model.save(model_file) - - -print 'Pre-training word embeddings ...' -main('restaurant') -main('beer') - - - +import gensim +import codecs + +class MySentences(object): + def __init__(self, filename): + self.filename = filename + + def __iter__(self): + for line in codecs.open(self.filename, 'r', 'utf-8'): + yield line.split() + + +def main(domain): + source = '../preprocessed_data/%s/train.txt' % (domain) + model_file = '../preprocessed_data/%s/w2v_embedding' % (domain) + sentences = MySentences(source) + model = gensim.models.Word2Vec(sentences, size=200, window=5, min_count=10, workers=4) + model.save(model_file) + + +print('Pre-training word embeddings ...') +main('restaurant') +main('beer') + + +