diff --git a/bin/lmplz b/bin/lmplz index a7072aa..28bc143 100755 Binary files a/bin/lmplz and b/bin/lmplz differ diff --git a/scripts/comparison.pdf b/scripts/comparison.pdf index 1865219..4077531 100644 Binary files a/scripts/comparison.pdf and b/scripts/comparison.pdf differ diff --git a/scripts/comparison.pdf b/scripts/comparison_all_lang.pdf similarity index 78% copy from scripts/comparison.pdf copy to scripts/comparison_all_lang.pdf index 1865219..b99516e 100644 Binary files a/scripts/comparison.pdf and b/scripts/comparison_all_lang.pdf differ diff --git a/scripts/draw_accuracy.py b/scripts/draw_accuracy.py index 680b387..e90a9a2 100644 --- a/scripts/draw_accuracy.py +++ b/scripts/draw_accuracy.py @@ -1,145 +1,153 @@ #!/bin/bash/python3 import sys, os from pickle import load from collections import namedtuple, Counter try: import numpy as np import matplotlib.pyplot as plt from matplotlib.ticker import MaxNLocator except ImportError: - raise ImportError('Please install matplotlib') + raise ImportError('Please run `pip3 install matplotlib\' in command line.') def heatmap(path): with open(path, 'rb') as f: data = load(f) mat = process(data) labels = sorted(data) fig, ax = plt.subplots() fig.set_size_inches(100,100) heatmap = ax.matshow(mat, cmap='Blues') fig = plt.gcf() ax.set_frame_on(False) ax.set_yticks(np.arange(len(labels)), minor=False) ax.set_xticks(np.arange(len(labels)), minor=False) ax.set_xlabel('Classification of test files') ax.set_ylabel('Ground truth class of test files') ax.set_xticklabels(labels, minor=False) ax.set_yticklabels(labels, minor=False) ax.xaxis.tick_top() ax.xaxis.set_label_position('top') plt.xticks(rotation=90) ax.grid(False) ''' for i in np.arange(len(mat)): for j in np.arange(len(mat[i])): ax.text(i, j, "%.1f" % (mat[i][j] * 100), color='white') ''' ax = plt.gca() for t in ax.xaxis.get_major_ticks(): t.tick1On = False t.tick2On = False for t in ax.yaxis.get_major_ticks(): t.tick1On = False t.tick2On = False fig.savefig("results.pdf", bbox_inches='tight') def process(data): ''' ''' ldata = sorted(data) length = len(ldata) out = [[0 for x in range(length)] for y in range(length)] for lang in ldata: index_lan = ldata.index(lang) ok = data[lang][0] if data[lang][1] > 1000 : test_size = 1000 else: test_size = data[lang][1] result = [x[1] for x in data[lang][3]] counter = dict(Counter(result)) for res_lan in counter.keys(): index_res = ldata.index(res_lan) out[index_lan][index_res] = counter.get(res_lan, 0) / test_size return out def get_accuracy(data): ldata = sorted(data) out = {} + total_ok = 0 + total_size = 0 for lang in ldata: ok = data[lang][0] if data[lang][1] > 1000: test_size = 1000 else: test_size = data[lang][1] result = [x[1] for x in data[lang][3]] counter = dict(Counter(result)) out[lang] = counter.get(lang, 0) / test_size + + total_ok += counter.get(lang,0) + total_size += test_size + print(total_ok) + print(total_size) + print(total_ok / total_size) return out def compare(results): datas = [] for result in results: with open(result, 'rb') as f: datas.append(load(f)) dicts = [] for data in datas: dicts.append(get_accuracy(data)) all_lang = sorted(list(set().union(dicts[0].keys(),dicts[1].keys())))[::-1] n = len(all_lang) accs = [] for d in dicts: accs.append([d.get(lang, 0) for lang in all_lang]) fig, ax = plt.subplots() fig.set_size_inches(10, 75 * len(results)) ind = np.arange(n) width = 0.75 / len(results) opacity = 0.4 rectss = [] colors = ['b', 'r', 'c', 'm', 'y', 'g'] for idx, result in enumerate(results): rectss.append(ax.barh(ind - (idx - len(results) / 2) * width, accs[idx], width, alpha=opacity, color=colors[idx % len(colors)], label=os.path.basename(result))) ax.set_xlabel('Accuracy / %') ax.set_yticks(ind + width / 2) ax.set_yticklabels(all_lang) vals = ax.get_xticks() ax.set_xticklabels(['{:3.0f}%'.format(x * 100) for x in vals]) ax.xaxis.tick_top() ax.legend() def autolabel(rects): for rect in rects: width = rect.get_width() ax.text(width + 0.01, rect.get_y() + rect.get_height() / 2., '{0:.1f}%'.format(width * 100), ha='left', va='center') for rects in rectss: autolabel(rects) plt.ylim([-1,n+1]) fig.tight_layout() fig.savefig("comparison.pdf", bbox_inches='tight') if __name__ == '__main__': if len(sys.argv) == 2: heatmap(sys.argv[1]) elif len(sys.argv) > 2: compare(sys.argv[1:]) else: print('Please check arguments.') diff --git a/swh/langdetect/cnn.py b/swh/langdetect/cnn.py index 008f01e..46b60db 100644 --- a/swh/langdetect/cnn.py +++ b/swh/langdetect/cnn.py @@ -1,309 +1,350 @@ import os import sys import subprocess import time import random import csv import numpy as np import warnings with warnings.catch_warnings(): warnings.simplefilter("ignore") import tensorflow as tf import json import argparse from ast import literal_eval from pickle import dump from pickle import load from numpy import array from .utils.common import Tokenizer from .utils.common import file_to_string from keras.preprocessing.sequence import pad_sequences from keras.callbacks import EarlyStopping from keras.models import Model from keras.models import Sequential from keras.models import load_model from keras.layers import Input from keras.layers import Dense from keras.layers import Flatten -from keras.layers import Dropout +from keras.layers import Dropout, AlphaDropout from keras.layers import ThresholdedReLU from keras.layers import Activation from keras.layers import Lambda from keras.layers import Embedding -from keras.layers.convolutional import Convolution1D -from keras.layers.convolutional import MaxPooling1D +from keras.layers import Concatenate, GlobalMaxPooling1D +from keras.layers.convolutional import Convolution1D, MaxPooling1D from keras.layers.normalization import BatchNormalization from keras.utils import np_utils from keras.optimizers import SGD from pyspark import SparkContext, SparkConf from elephas.spark_model import SparkModel # pip install flask from elephas import optimizers as elephas_optimizers from elephas.utils.rdd_utils import to_labeled_point csv.field_size_limit(sys.maxsize) -conf = SparkConf().setAppName('Elephas_App').setMaster('local[8]') # Set up on cluster. -sc = SparkContext(conf=conf) +from keras import backend as K +K.set_session(K.tf.Session(config=K.tf.ConfigProto(intra_op_parallelism_threads=1, inter_op_parallelism_threads=1))) def main(): parser = argparse.ArgumentParser(description='Training and test tool of charactor-level ConvNet text categorisation.') subparsers = parser.add_subparsers(dest='sub_command') parser_train = subparsers.add_parser('train', help='Training on the dataset, dataset must be a *.csv file. A model will be created in the same directory.') parser_train.add_argument('-s', '--spark', type=bool, help='Training on cluster.', dest='train_spark') parser_train.add_argument('train_path', metavar='PATH', type=str, help='Path of the training dataset.') parser_train.add_argument('-ms', '--maxsize', metavar='SIZE', dest='train_maxsize', type=int, help='Set maximum input size of ConvNet, default 1024.') parser_train.add_argument('-e', '--epochs', metavar='N', dest='train_epochs', type=int, help='Number of training epochs (iterations), default 50.') parser_test = subparsers.add_parser('test', help='Test on the dataset, dataset must be a directory with *.csv dataset named by corresponding language.') parser_test.add_argument('test_root', metavar='ROOT', type=str, help='Root of the test dataset.') if len(sys.argv[1:]) == 0: parser.print_help() parser.exit() args = parser.parse_args() if args.sub_command == 'train' : maxsize = 1024 - epochs = 50 + epochs = 15 if args.train_maxsize: maxsize = args.train_maxsize if args.train_epochs: epochs = args.train_epochs n = CNN(args.train_path, maxsize=maxsize, epochs=epochs) if args.train_spark: n.train_on_cluster() else: n.train() elif args.sub_command == 'test': n = CNN(args.test_root) n.test() else: parser.parse_args('-h') class CNN: def __init__(self, path, maxsize, epochs): self._path = path # Root of model folder self._root_model = os.path.join(os.path.dirname(path), 'model_cnn') try: os.mkdir(self._root_model) except: pass # Path of result self._path_result = os.path.join(os.path.dirname(path), 'result_cnn') dir_path = os.path.dirname(os.path.abspath(__file__)) - with open(os.path.join(dir_path, 'static_data', 'languages.json'), 'r') as f: + with open(os.path.join(dir_path, 'static_data', 'languages_less.json'), 'r') as f: self._languages = json.load(f) self._path_test_csv = path self._input_size = maxsize self._vocab_size = 256 self._num_of_classes = len(self._languages) - self._batch_size = 128 + self._batch_size = 64 self._epochs = epochs def file_len(self, fname): with open(fname) as f: count = 0 for l in f: count += 1 return count def train(self): model = self._get_model() - earlystop = EarlyStopping(monitor='loss', min_delta=0, patience=5, verbose=0, mode='auto') + earlystop = EarlyStopping(monitor='loss', min_delta=0, patience=3, verbose=0, mode='auto') callbacks = [earlystop] model.fit_generator( self._generator(self._input_size, self._num_of_classes, self._batch_size), steps_per_epoch=self.file_len(self._path) / self._batch_size, epochs=self._epochs, callbacks=callbacks) model.save(os.path.join(self._root_model, 'model.h5')) def train_on_cluster(self): rdd = self._get_train_rdd() model = self._get_model() adagrad = elephas_optimizers.Adagrad() spark_model = SparkModel(sc, model, optimizer=adagrad, frequency='epoch', mode='asynchronous', num_workers=2) spark_model.train(rdd, nb_epoch=self._epochs, batch_size=self._batch_size, verbose=0, categorical=True, nb_classes=self._num_of_classes) model.save(os.path.join(self._root_model, 'model.h5')) def _get_train_rdd(self): print('Prepairing RDD for training...') X_train = np.empty((0, self._input_size)) Y_train = np.empty((0, self._num_of_classes)) with open(self._path, newline='') as csvfile: r = csv.reader(csvfile, delimiter=' ', quotechar='|') for pair in r: label, string = pair label = int(label) print(label, end='\r') string = literal_eval(string) tokens = [x + 1 for x in Tokenizer.tokenize(string, 'letter')] X_train = np.append(X_train, pad_sequences([tokens], maxlen=self._input_size), axis=0) label = array(np_utils.to_categorical([label], self._num_of_classes)) Y_train = np.append(Y_train, label, axis=0) rdd = to_labeled_point(sc, X_train, Y_train, categorical=True) def _generator(self, length, total_class, batch_size=128): counter = 0 while True: with open(self._path, newline='') as csvfile: r = csv.reader(csvfile, delimiter=' ', quotechar='|') for pair in r: if counter == 0: X = np.empty((0, length)) Y = np.empty((0, total_class)) label, string = pair label = int(label) string = literal_eval(string) tokens = [x + 1 for x in Tokenizer.tokenize(string, 'letter')] X = np.append(X, pad_sequences([tokens], maxlen=length), axis=0) label = array(np_utils.to_categorical([label], total_class)) Y = np.append(Y, label, axis=0) counter += 1 if counter == batch_size: counter = 0 yield(X,Y) - def _get_model(self): + def _get_model_zhang(self): input_size = self._input_size alphabet_size = self._vocab_size - embedding_size = 256 + embedding_size = 128 conv_layers = [(256,7,3), (256,7,3), (256,3,-1), (256,3,-1), (256,3,-1), (256,3,3)] threshold = 1e-6 fully_connected_layers = [1024, 1024] dropout_p = 0.2 optimizer = 'adam' loss = 'categorical_crossentropy' num_of_classes = self._num_of_classes # Input layer inputs = Input(shape=(input_size,), name='sent_input', dtype='int64') # Embedding layers x = Embedding(alphabet_size + 1, embedding_size, input_length=input_size)(inputs) # Convolution layers for cl in conv_layers: x = Convolution1D(cl[0], cl[1])(x) x = ThresholdedReLU(threshold)(x) if cl[2] != -1: x = MaxPooling1D(cl[2])(x) x = Flatten()(x) # Fully connected layers for fl in fully_connected_layers: x = Dense(fl)(x) x = ThresholdedReLU(threshold)(x) x = Dropout(dropout_p)(x) # Output layer predictions = Dense(num_of_classes, activation='softmax')(x) # Build and compile model model = Model(inputs=inputs, outputs=predictions) model.compile(optimizer=optimizer, loss=loss, metrics=['accuracy']) print(model.summary()) return model + def _get_model(self): + + input_size = self._input_size + alphabet_size = self._vocab_size + embedding_size = 32 + conv_layers = [(256,10), (256,7), (256,5), (256,3)] + threshold = 1e-6 + fully_connected_layers = [1024, 1024] + dropout_p = 0.1 + optimizer = 'adam' + loss = 'categorical_crossentropy' + num_of_classes = self._num_of_classes + + # Input layer + inputs = Input(shape=(input_size,), name='sent_input', dtype='int64') + # Embedding layers + x = Embedding(alphabet_size + 1, embedding_size, input_length=input_size)(inputs) + convolution_output = [] + # Convolution layers + for num_filters, filter_width in conv_layers: + conv = Convolution1D(filters=num_filters, + kernel_size=filter_width, + activation='tanh', + name='Conv1D_{}_{}'.format(num_filters, filter_width))(x) + pool = GlobalMaxPooling1D(name='MaxPoolingOverTime_{}_{}'.format(num_filters, filter_width))(conv) + convolution_output.append(pool) + x = Concatenate()(convolution_output) + # Fully connected layers + for fl in fully_connected_layers: + x = Dense(fl, activation='selu', kernel_initializer='lecun_normal')(x) + x = Dropout(dropout_p)(x) + # Output layer + predictions = Dense(num_of_classes, activation='softmax')(x) + # Build and compile model + model = Model(inputs=inputs, outputs=predictions) + model.compile(optimizer=optimizer, loss=loss, metrics=['accuracy']) + + print(model.summary()) + + return model + def _max_len(self, texts): return max([len(text) for text in texts]) def test(self): csv.field_size_limit(sys.maxsize) try: r = open(self._path_result, 'rb') test_result = load(r) r.close() except FileNotFoundError: test_result = {} model = self._load_model() for language in [x for x in self._languages if x not in test_result.keys()]: test_result[language] = self.test_class(model, language) with open(self._path_result, 'wb') as f: dump(test_result, f) def _load_model(self): model = load_model(os.path.join(self._root_model, 'model.h5')) return model def _count_size(self, files): size = 0 for f in files: size += os.path.getsize(f) return size def test_class(self, model, language): ok = 0 results = [] count = 0 total_test = self.file_len(os.path.join(self._path_test_csv, language + '.csv')) with open(os.path.join(self._path_test_csv, language + '.csv'), newline='') as csvfile: r = csv.reader(csvfile, delimiter=' ', quotechar='|') for pair in r: label, string = pair label = int(label) string = literal_eval(string) tokens = [x + 1 for x in Tokenizer.tokenize(string, 'letter')] result = self._guess_file_language(model, tokens) count += 1 print('[{0:4d}/{1:4d}] {2}:{3} '.format(count, total_test, result[0][1], result[0][0]),end='\r') results.append(result[0]) if result[0][1] == language: ok += 1 accuracy = ok / total_test print('Tests for {} '.format(language)) print('Total test files : {}'.format(total_test)) print('Correctly classified files : {}'.format(ok)) print('Accuracy : {}%'.format(accuracy * 100)) return (ok, total_test, accuracy, results) def speed_benchmark(self): language = self._languages[10] model = self._load_model() test_set = self._get_test_set(language) total_size = self._count_size(test_set) print('{} kB in total'.format(total_size / 1024)) t_start = time.perf_counter() self.test_class(model, language) t_end = time.perf_counter() print('{} seconds.'.format(t_end - t_start)) print('{} seconds per KiB'.format(((t_end - t_start) / total_size) * 1024)) def _guess_file_language(self, model, tokens): X = pad_sequences([tokens], maxlen=self._input_size) result = list(model.predict(X))[0] result = [(s, self._languages[i]) for i, s in enumerate(result)] return sorted(result, reverse=True) if __name__ == '__main__': main() diff --git a/swh/langdetect/cnn_w.py b/swh/langdetect/cnn_w.py index 7dce467..b622abb 100644 --- a/swh/langdetect/cnn_w.py +++ b/swh/langdetect/cnn_w.py @@ -1,300 +1,300 @@ import os import sys import subprocess import time import random import csv import numpy as np import warnings with warnings.catch_warnings(): warnings.simplefilter("ignore") import tensorflow as tf import json import argparse from ast import literal_eval from pickle import dump from pickle import load from numpy import array from .utils.common import Tokenizer from keras.preprocessing.sequence import pad_sequences from keras.callbacks import EarlyStopping from keras.models import Model from keras.models import Sequential from keras.models import load_model from keras.layers import Input from keras.layers import Dense from keras.layers import Flatten from keras.layers import Merge from keras.layers import Dropout from keras.layers import ThresholdedReLU from keras.layers import Activation from keras.layers import Lambda from keras.layers import Embedding from keras.layers.convolutional import Convolution1D from keras.layers.convolutional import MaxPooling1D from keras.layers.normalization import BatchNormalization from keras.layers import Concatenate from keras.utils import np_utils from keras.optimizers import SGD from collections import Counter csv.field_size_limit(sys.maxsize) from keras import backend as K K.set_session(K.tf.Session(config=K.tf.ConfigProto(intra_op_parallelism_threads=1, inter_op_parallelism_threads=1))) def main(): parser = argparse.ArgumentParser(description='Training and test tool of charactor-level ConvNet text categorisation.') subparsers = parser.add_subparsers(dest='sub_command') parser_train = subparsers.add_parser('train', help='Training on the dataset, dataset must be a *.csv file. A model will be created in the same directory.') parser_train.add_argument('train_path', metavar='PATH', type=str, help='Path of the training dataset.') parser_train.add_argument('-ms', '--maxsize', metavar='SIZE', dest='train_maxsize', type=int, help='Set maximum input size of ConvNet, default 1024.') parser_train.add_argument('-e', '--epochs', metavar='N', dest='train_epochs', type=int, help='Number of training epochs (iterations), default 50.') parser_test = subparsers.add_parser('test', help='Test on the dataset, dataset must be a directory with *.csv dataset named by corresponding language.') parser_test.add_argument('test_root', metavar='ROOT', type=str, help='Root of the test dataset.') if len(sys.argv[1:]) == 0: parser.print_help() parser.exit() args = parser.parse_args() if args.sub_command == "train": if args.train_maxsize: if args.train_epochs: n = CNNword(args.train_path, maxsize=args.train_maxsize, epochs=args.train_epochs) n.train() else: n = CNNword(args.train_path, maxsize=args.train_maxsize) n.train() else: if args.train_epochs: n = CNNword(args.train_path, epochs=args.train_epochs) n.train() else: n = CNNword(args.train_path) n.train() elif args.sub_command == "test": n = CNNword(args.test_root) print(args.test_root) n.test() else: parser.parse_args('-h') class CNNword: def __init__(self, path, maxsize=1024, epochs=30): self._path = path # Root of model folder self._root_model = os.path.join(os.path.dirname(path), 'model_cnn_word') try: os.mkdir(self._root_model) except: pass # Path of result self._path_result = os.path.join(os.path.dirname(path), 'result_cnn_word') dir_path = os.path.dirname(os.path.abspath(__file__)) - with open(os.path.join(dir_path, 'static_data', 'languages.json'), 'r') as f: + with open(os.path.join(dir_path, 'static_data', 'languages_less.json'), 'r') as f: self._languages = json.load(f) self._path_test_csv = path self._path_vocab = os.path.join(self._root_model, 'vocab') self._input_size = maxsize self._vocab_size = 15001 self._num_of_classes = len(self._languages) self._batch_size = 64 self._epochs = epochs if not os.path.isfile(self._path_vocab): self._learn_vocab(self._input_size, self._num_of_classes) with open(self._path_vocab, 'rb') as f: c = load(f) l = c.most_common(15000) print(l) self._indexer = dict((v[0], i + 1) for i, v in enumerate(l)) self._oov_index = len(self._indexer) + 1 def file_len(self, fname): with open(fname) as f: count = 0 for l in f: count += 1 return count def train(self): model = self._get_model() earlystop = EarlyStopping(monitor='loss', min_delta=0, patience=3, verbose=0, mode='auto') callbacks = [earlystop] model.fit_generator( self._generator(self._input_size, self._num_of_classes, self._batch_size), steps_per_epoch=self.file_len(self._path) / self._batch_size, epochs=self._epochs, callbacks=callbacks) model.save(os.path.join(self._root_model, 'model.h5')) def _learn_vocab(self, length, total_class): c = Counter() with open(self._path, newline='') as csvfile: r = csv.reader(csvfile, delimiter=' ', quotechar='|') for pair in r: label, string = pair label = int(label) print(label, end='\r') string = literal_eval(string) tokens = Tokenizer.tokenize(string, 'word') c.update(tokens) with open(self._path_vocab, 'wb') as f: dump(c, f) def _generator(self, length, total_class, batch_size=64): counter = 0 while True: with open(self._path, newline='') as csvfile: r = csv.reader(csvfile, delimiter=' ', quotechar='|') for pair in r: if counter == 0: X = np.empty((0, length)) Y = np.empty((0, total_class)) label, string = pair label = int(label) string = literal_eval(string) tokens = [self._indexer.get(x, self._oov_index) for x in Tokenizer.tokenize(string, 'word')] X = np.append(X, pad_sequences([tokens], maxlen=length), axis=0) label = array(np_utils.to_categorical([label], total_class)) Y = np.append(Y, label, axis=0) counter += 1 if counter == batch_size: counter = 0 yield(X,Y) def _get_model(self): input_size = self._input_size vocab_size = self._vocab_size embedding_size = 128 optimizer = 'adam' loss = 'categorical_crossentropy' num_of_classes = self._num_of_classes embedding_layer = Embedding(vocab_size + 1, embedding_size, input_length=input_size, ) # applying a more complex convolutional approach convs = [] filter_sizes = [3,4,5] sequence_input = Input(shape=(input_size,), dtype='int64') embedded_sequences = embedding_layer(sequence_input) for fsz in filter_sizes: l_conv = Convolution1D(filters=10, kernel_size=fsz, activation='relu')(embedded_sequences) l_pool = MaxPooling1D(3)(l_conv) convs.append(l_pool) l_merge = Concatenate(axis=1)(convs) l_conv1= Convolution1D(128, 3, activation='relu')(l_merge) l_pool1 = MaxPooling1D(5)(l_conv1) l_conv2 = Convolution1D(128, 3, activation='relu')(l_pool1) l_pool2 = MaxPooling1D(5)(l_conv2) l_flat = Flatten()(l_pool2) l_dense = Dense(512, activation='relu')(l_flat) preds = Dense(num_of_classes, activation='softmax')(l_dense) model = Model(sequence_input, preds) model.compile(optimizer=optimizer, loss=loss, metrics=['accuracy']) print(model.summary()) return model def _max_len(self, texts): return max([len(text) for text in texts]) def test(self): csv.field_size_limit(sys.maxsize) try: r = open(self._path_result, 'rb') test_result = load(r) r.close() except FileNotFoundError: test_result = {} model = self._load_model() for language in [x for x in self._languages if x not in test_result.keys()]: test_result[language] = self.test_class(model, language) with open(self._path_result, 'wb') as f: dump(test_result, f) def _load_model(self): model = load_model(os.path.join(self._root_model, 'model.h5')) return model def _count_size(self, files): size = 0 for f in files: size += os.path.getsize(f) return size def test_class(self, model, language): ok = 0 results = [] count = 0 total_test = self.file_len(os.path.join(self._path_test_csv, language + '.csv')) with open(os.path.join(self._path_test_csv, language + '.csv'), newline='') as csvfile: r = csv.reader(csvfile, delimiter=' ', quotechar='|') for pair in r: label, string = pair label = int(label) string = literal_eval(string) tokens = [self._indexer.get(x, self._oov_index) for x in Tokenizer.tokenize(string, 'word')] result = self._guess_file_language(model, tokens) count += 1 print('[{0:4d}/{1:4d}] {2}:{3} '.format(count, total_test, result[0][1], result[0][0]),end='\r') results.append(result[0]) if result[0][1] == language: ok += 1 accuracy = ok / total_test print('Tests for {} '.format(language)) print('Total test files : {}'.format(total_test)) print('Correctly classified files : {}'.format(ok)) print('Accuracy : {}%'.format(accuracy * 100)) return (ok, total_test, accuracy, results) def speed_benchmark(self): language = self._languages[10] model = self._load_model() test_set = self._get_test_set(language) total_size = self._count_size(test_set) print('{} kB in total'.format(total_size / 1024)) t_start = time.perf_counter() self.test_class(model, language) t_end = time.perf_counter() print('{} seconds.'.format(t_end - t_start)) print('{} seconds per KiB'.format(((t_end - t_start) / total_size) * 1024)) def _guess_file_language(self, model, tokens): X = pad_sequences([tokens], maxlen=self._input_size) result = list(model.predict(X))[0] result = [(s, self._languages[i]) for i, s in enumerate(result)] return sorted(result, reverse=True) if __name__ == '__main__': main() diff --git a/swh/langdetect/unsupervised.py b/swh/langdetect/hierarchical.py similarity index 55% rename from swh/langdetect/unsupervised.py rename to swh/langdetect/hierarchical.py index c87b001..684e412 100644 --- a/swh/langdetect/unsupervised.py +++ b/swh/langdetect/hierarchical.py @@ -1,267 +1,238 @@ -""" -Naive Bayesian -""" - import os import sys import operator import nltk import random import time import numpy as np import csv import argparse import json import matplotlib.pyplot as plt import matplotlib as mpl from ast import literal_eval from itertools import islice from pickle import dump, load -from .utils.common import tokenizer, file_to_string, find_file, count_files +from .utils.common import Tokenizer from nltk.util import ngrams from collections import Counter -from sklearn.naive_bayes import MultinomialNB -from sklearn.feature_extraction.text import HashingVectorizer +from sklearn.feature_extraction.text import HashingVectorizer, TfidfVectorizer from sklearn.metrics.pairwise import cosine_similarity from sklearn.externals import joblib from sklearn.cluster import KMeans, MiniBatchKMeans -from sklearn.metrics.pairwise import cosine_similarity -from scipy.cluster.hierarchy import ward, dendrogram +from sklearn.metrics.pairwise import cosine_similarity, cosine_distances, euclidean_distances +from scipy.sparse import vstack +from scipy.sparse import csr_matrix +from scipy.cluster.hierarchy import ward, dendrogram, centroid, complete, average, weighted, median from sklearn.manifold import MDS csv.field_size_limit(sys.maxsize) def main(): parser = argparse.ArgumentParser(description='Training and test tool of multinumial naive bayesian.') subparsers = parser.add_subparsers(dest='sub_command') parser_train = subparsers.add_parser('train', help='Training on the dataset, dataset must be a *.csv file. A model will be created in the same directory.') parser_train.add_argument('train_path', metavar='PATH', type=str, help='Path of the training dataset.') # parser_train.add_argument('-n', '--ngrams', metavar='N', dest='train_maxsize', type=int, help='Set maximum input size of ConvNet, default 5.') parser_test = subparsers.add_parser('test', help='Test on the dataset, dataset must be a directory with *.csv dataset named by corresponding language.') parser_test.add_argument('test_root', metavar='ROOT', type=str, help='Root of the test dataset.') if len(sys.argv[1:]) == 0: parser.print_help() parser.exit() args = parser.parse_args() if args.sub_command == 'train' : n = Unsupervised(args.train_path) n.train() - # n.clustering() - # n.graph() + n.graph_top_20() elif args.sub_command == 'test': n = Unsupervised(args.test_root) n.test() else: parser.parse_args('-h') class Unsupervised: def __init__(self, path): self._path = path # Root of model folder self._root_model = os.path.join(os.path.dirname(path), 'model_unsupervised') try: os.mkdir(self._root_model) except: pass # Path of result self._path_result = os.path.join(os.path.dirname(path), 'result_unsupervised') dir_path = os.path.dirname(os.path.abspath(__file__)) with open(os.path.join(dir_path, 'static_data', 'languages.json'), 'r') as f: self._languages = json.load(f) self._path_test_csv = path self._num_of_classes = len(self._languages) def train(self): - cv = HashingVectorizer(analyzer='char', ngram_range=(1, 4), n_features=2**16, alternate_sign=False) + cv = HashingVectorizer(analyzer='char', ngram_range=(1, 5), n_features=2**24, alternate_sign=False) texts = [] label = 0 string = '' + top_20 = ['Python', 'Java', 'JavaScript', 'PHP', 'C#', 'C', 'C++', + 'R', 'Objective-C', 'Swift', 'Matlab', 'Ruby', 'TypeScript', + 'Visual Basic', 'Scala', 'Kotlin', 'Go', 'Perl', 'Lua', + 'Rust', 'Haskell'] + top_20 = [self._languages.index(x) for x in top_20] + print(top_20) + with open(self._path, newline='') as csvfile: r = csv.reader(csvfile, delimiter=' ', quotechar='|') for pair in r: label_new, string_new = pair + print(label_new, end=' \r') if not int(label_new) == label: if not os.path.isfile(os.path.join(self._root_model, 'counts{}.pkl'.format(label))): - counts = cv.fit_transform(texts) - self.clustering(counts, 1, label) - self.graph(label) + if label in top_20: + counts = cv.fit_transform(texts) + self.clustering(counts, 1, label) + self.graph(label) texts = [] label = int(label_new) - string = literal_eval(string_new) - tokens = tokenizer(string, 'letter') - text = ''.join([chr(token) for token in tokens]) - texts.append(text) - + if label in top_20: + string = literal_eval(string_new) + #tokens = Tokenizer.tokenize(string, 'word') + #text = ' '.join([''.join([chr(x) for x in token]) for token in tokens]) + tokens = Tokenizer.tokenize(string, 'letter') + text = ''.join([chr(token) for token in tokens]) + texts.append(text) with open(os.path.join(self._root_model, 'classifier.cv'), 'wb') as f: joblib.dump(cv, f) def clustering(self, counts, num_clusters, label): - km = KMeans(n_clusters=num_clusters) - km.fit(counts) + # km = KMeans(n_clusters=num_clusters) + # km.fit(counts) with open(os.path.join(self._root_model, 'counts{}.pkl'.format(label)), 'wb') as f: joblib.dump(counts, f) - with open(os.path.join(self._root_model, 'cluster{}.pkl'.format(label)), 'wb') as f: - joblib.dump(km, f) + #with open(os.path.join(self._root_model, 'cluster{}.pkl'.format(label)), 'wb') as f: + # joblib.dump(km, f) + + def graph_top_20(self): + + top_20 = ['Python', 'Java', 'JavaScript', 'PHP', 'C#', 'C', 'C++', + 'R', 'Objective-C', 'Swift', 'Matlab', 'Ruby', 'TypeScript', + 'Visual Basic', 'Scala', 'Kotlin', 'Go', 'Perl', 'Lua', + 'Rust', 'Haskell'] + top_20 = [self._languages.index(x) for x in top_20] + counts = csr_matrix((0, 2 ** 24)) + for label in top_20: + with open(os.path.join(self._root_model, 'counts{}.pkl'.format(label)), 'rb') as f: + counts = vstack((counts, joblib.load(f))) + print(counts.shape) + + if not os.path.isfile(os.path.join(self._root_model, 'linkage_matrix')): + + dist = euclidean_distances(counts) + linkage_matrix = ward(dist) + + with open(os.path.join(self._root_model, 'linkage_matrix'), 'wb') as f: + joblib.dump(linkage_matrix, f) + else: + with open(os.path.join(self._root_model, 'linkage_matrix'), 'rb') as f: + linkage_matrix = joblib.load(f) + print(linkage_matrix) + + fig, ax = plt.subplots(figsize=(15, 150)) + titles = [self._languages[top_20[x // 500]] for x in list(range(0,counts.shape[0]))] + ax = dendrogram(linkage_matrix, orientation="right", labels=titles) + + plt.tick_params(axis= 'x', + which='both', + bottom=False, + top=False, + labelbottom=False) + + plt.tight_layout() + plt.savefig(os.path.join(self._root_model, 'top_20_cluster.pdf')) def graph(self, label): with open(os.path.join(self._root_model, 'counts{}.pkl'.format(label)), 'rb') as f: counts = joblib.load(f) - dist = 1 - cosine_similarity(counts) - linkage_matrix = ward(dist) + dist = euclidean_distances(counts) + linkage_matrix = ward(dist) fig, ax = plt.subplots(figsize=(15, 40)) titles = list(range(1,counts.shape[0]+1)) ax = dendrogram(linkage_matrix, orientation="right", labels=titles) plt.tick_params(axis= 'x', which='both', - bottom='off', - top='off', - labelbottom='off') + bottom=False, + top=False, + labelbottom=False) plt.tight_layout() plt.savefig(os.path.join(self._root_model, '{}_cluster.pdf'.format(self._languages[label]))) - def test(self): - try: - r = open(self._path_result, 'rb') - test_result = load(r) - r.close() - except FileNotFoundError: - test_result = {} - - with open(os.path.join(self._root_model, 'classifier.clf'), 'rb') as f: - clf = joblib.load(f) - with open(os.path.join(self._root_model, 'classifier.hv'), 'rb') as f: - cv = joblib.load(f) - - for language in [x for x in self._languages if x not in test_result.keys()]: - test_result[language] = self.test_class((clf, cv), language) - with open(self._path_result, 'wb') as f: - dump(test_result, f) - def speed_benchmark(self): language = [x for x in os.listdir(self._root_training_set) if not x.startswith('.')][10] models = self._load_models() test_set = self._get_test_set(language) total_size = self._count_size(test_set) print('{} kB in total'.format(total_size / 1024)) t_start = time.perf_counter() self.test_class(models, language) t_end = time.perf_counter() print('{} seconds.'.format(t_end - t_start)) print('{} seconds per kB'.format(((t_end - t_start) / total_size) * 1024)) - - - def _get_test_set(self, language): - root_training_language = os.path.join(self._root_training_set, language) - root_language = os.path.join(self._root_language_dataset, language) - total = count_files(root_language) - training_set = [int(os.path.splitext(x)[0]) for x in os.listdir(root_training_language) if not x.startswith('.')] - it = (find_file(root_language, x) for x in range(1, total + 1) if x not in training_set and os.path.getsize(find_file(root_language, x)) <= 1048576) - test_set = list(islice(it, 1000)) - if len(test_set) == 0: - it = (find_file(root_language, x) for x in range(1, total + 1) if x not in training_set) - test_set = list(islice(it, 1000)) - return test_set def _count_size(self, files): size = 0 for f in files: size += os.path.getsize(f) return size - def test_class(self, clf, language): - ok = 0 - results = [] - count = 0 - total_test = self.file_len(os.path.join(self._path_test_csv, language + '.csv')) - with open(os.path.join(self._path_test_csv, language + '.csv'), newline='') as csvfile: - r = csv.reader(csvfile, delimiter=' ', quotechar='|') - for pair in r: - label, string = pair - label = int(label) - string = literal_eval(string) - result = self._guess_file_language(clf, string) - count += 1 - print('[{0:4d}/{1:4d}] {2}:{3} '.format(count, total_test, result[0][1], result[0][0]),end='\r') - results.append(result[0]) - if result[0][1] == language: - ok += 1 - - accuracy = ok / total_test - print('Tests for {} '.format(language)) - print('Total test files : {}'.format(total_test)) - print('Correctly classified files : {}'.format(ok)) - print('Accuracy : {}%'.format(accuracy * 100)) - return (ok, total_test, accuracy, results) - - def test_single(self, filename): - self._guess_file_language(clf, filename) - def file_len(self, fname): with open(fname) as f: count = 0 for l in f: count += 1 return count - def _guess_file_language(self, cc, string): - clf = cc[0] - cv = cc[1] - tokens = tokenizer(string, 'letter') - text = ''.join([chr(token) for token in tokens]) - counts = cv.fit_transform([text]) - tf = TfidfTransformer().fit(counts) - normalised = tf.transform(counts) - - result = clf.predict_log_proba(normalised) - - result = [(val, self._languages[idx]) for idx, val in enumerate(result[0])] - - return sorted(result, reverse=True) - def _distance(self, model_profile, test_profile): distance = 0 maximum = len(test_profile) for test_ngram in test_profile.keys(): test_rank = test_profile.get(test_ngram) model_rank = model_profile.get(test_ngram, maximum) d = abs(test_rank - model_rank) distance += d return distance ''' def _prob(model, trigrams): print('Checking {} model ...'.format(model)) with open(model, 'rb') as f: kneser_ney = load(f) result = 1 for trigram in trigrams: prob = kneser_ney.prob(trigram) result = result * prob return result ''' if __name__ == '__main__': main() diff --git a/swh/langdetect/naivebayesian.py b/swh/langdetect/naivebayesian.py index 5fe1e9c..d1691e2 100644 --- a/swh/langdetect/naivebayesian.py +++ b/swh/langdetect/naivebayesian.py @@ -1,241 +1,240 @@ """ Naive Bayesian """ import os import sys import operator import nltk import random import time import numpy as np import csv import argparse import json from ast import literal_eval from itertools import islice from pickle import dump, load -from .utils.common import tokenizer, file_to_string, find_file, count_files +from .utils.common import Tokenizer, file_to_string, find_file, count_files from nltk.util import ngrams from collections import Counter from sklearn.naive_bayes import MultinomialNB from sklearn.feature_extraction.text import HashingVectorizer, TfidfTransformer from sklearn.externals import joblib csv.field_size_limit(sys.maxsize) def main(): parser = argparse.ArgumentParser(description='Training and test tool of multinumial naive bayesian.') subparsers = parser.add_subparsers(dest='sub_command') parser_train = subparsers.add_parser('train', help='Training on the dataset, dataset must be a *.csv file. A model will be created in the same directory.') parser_train.add_argument('train_path', metavar='PATH', type=str, help='Path of the training dataset.') # parser_train.add_argument('-n', '--ngrams', metavar='N', dest='train_maxsize', type=int, help='Set maximum input size of ConvNet, default 5.') parser_test = subparsers.add_parser('test', help='Test on the dataset, dataset must be a directory with *.csv dataset named by corresponding language.') parser_test.add_argument('test_root', metavar='ROOT', type=str, help='Root of the test dataset.') if len(sys.argv[1:]) == 0: parser.print_help() parser.exit() args = parser.parse_args() if args.sub_command == 'train' : n = NaiveBayesian(args.train_path) n.train() elif args.sub_command == 'test': n = NaiveBayesian(args.test_root) n.test() else: parser.parse_args('-h') class NaiveBayesian: def __init__(self, path): self._path = path # Root of model folder self._root_model = os.path.join(os.path.dirname(path), 'model_bayesian') try: os.mkdir(self._root_model) except: pass # Path of result self._path_result = os.path.join(os.path.dirname(path), 'result_bayesian') dir_path = os.path.dirname(os.path.abspath(__file__)) - with open(os.path.join(dir_path, 'static_data', 'languages.json'), 'r') as f: + with open(os.path.join(dir_path, 'static_data', 'languages_less.json'), 'r') as f: self._languages = json.load(f) self._path_test_csv = path self._num_of_classes = len(self._languages) def train(self): ''' train () generates and stores counted n-grams in '_root_model' folder ''' ''' Calculate frequencies of generated n-grams then store them into a sorted list of (ngram, count) ''' clf = MultinomialNB(alpha=0.001) cv = HashingVectorizer(analyzer='char', ngram_range=(1, 4), n_features=2**16, alternate_sign=False) indices = list(range(len(self._languages))) with open(self._path, newline='') as csvfile: r = csv.reader(csvfile, delimiter=' ', quotechar='|') for pair in r: label, string = pair label = int(label) print(label, end='\r') string = literal_eval(string) - print(string) - tokens = tokenizer(string, 'letter') + tokens = Tokenizer.tokenize(string, 'letter') text = ''.join([chr(token) for token in tokens]) counts = cv.fit_transform([text]) tf = TfidfTransformer().fit(counts) normalised = tf.transform(counts) clf.partial_fit(normalised, np.array([label]), indices) with open(os.path.join(self._root_model, 'classifier.clf'), 'wb') as f: joblib.dump(clf, f) with open(os.path.join(self._root_model, 'classifier.hv'), 'wb') as f: joblib.dump(cv, f) def test(self): try: r = open(self._path_result, 'rb') test_result = load(r) r.close() except FileNotFoundError: test_result = {} with open(os.path.join(self._root_model, 'classifier.clf'), 'rb') as f: clf = joblib.load(f) with open(os.path.join(self._root_model, 'classifier.hv'), 'rb') as f: cv = joblib.load(f) for language in [x for x in self._languages if x not in test_result.keys()]: test_result[language] = self.test_class((clf, cv), language) with open(self._path_result, 'wb') as f: dump(test_result, f) def speed_benchmark(self): language = [x for x in os.listdir(self._root_training_set) if not x.startswith('.')][10] models = self._load_models() test_set = self._get_test_set(language) total_size = self._count_size(test_set) print('{} kB in total'.format(total_size / 1024)) t_start = time.perf_counter() self.test_class(models, language) t_end = time.perf_counter() print('{} seconds.'.format(t_end - t_start)) print('{} seconds per kB'.format(((t_end - t_start) / total_size) * 1024)) def _get_test_set(self, language): root_training_language = os.path.join(self._root_training_set, language) root_language = os.path.join(self._root_language_dataset, language) total = count_files(root_language) training_set = [int(os.path.splitext(x)[0]) for x in os.listdir(root_training_language) if not x.startswith('.')] it = (find_file(root_language, x) for x in range(1, total + 1) if x not in training_set and os.path.getsize(find_file(root_language, x)) <= 1048576) test_set = list(islice(it, 1000)) if len(test_set) == 0: it = (find_file(root_language, x) for x in range(1, total + 1) if x not in training_set) test_set = list(islice(it, 1000)) return test_set def _count_size(self, files): size = 0 for f in files: size += os.path.getsize(f) return size def test_class(self, clf, language): ok = 0 results = [] count = 0 total_test = self.file_len(os.path.join(self._path_test_csv, language + '.csv')) with open(os.path.join(self._path_test_csv, language + '.csv'), newline='') as csvfile: r = csv.reader(csvfile, delimiter=' ', quotechar='|') for pair in r: label, string = pair label = int(label) string = literal_eval(string) result = self._guess_file_language(clf, string) count += 1 print('[{0:4d}/{1:4d}] {2}:{3} '.format(count, total_test, result[0][1], result[0][0]),end='\r') results.append(result[0]) if result[0][1] == language: ok += 1 accuracy = ok / total_test print('Tests for {} '.format(language)) print('Total test files : {}'.format(total_test)) print('Correctly classified files : {}'.format(ok)) print('Accuracy : {}%'.format(accuracy * 100)) return (ok, total_test, accuracy, results) def test_single(self, filename): self._guess_file_language(clf, filename) def file_len(self, fname): with open(fname) as f: count = 0 for l in f: count += 1 return count def _guess_file_language(self, cc, string): clf = cc[0] cv = cc[1] - tokens = tokenizer(string, 'letter') + tokens = Tokenizer.tokenize(string, 'letter') text = ''.join([chr(token) for token in tokens]) counts = cv.fit_transform([text]) tf = TfidfTransformer().fit(counts) normalised = tf.transform(counts) result = clf.predict_log_proba(normalised) result = [(val, self._languages[idx]) for idx, val in enumerate(result[0])] return sorted(result, reverse=True) def _distance(self, model_profile, test_profile): distance = 0 maximum = len(test_profile) for test_ngram in test_profile.keys(): test_rank = test_profile.get(test_ngram) model_rank = model_profile.get(test_ngram, maximum) d = abs(test_rank - model_rank) distance += d return distance ''' def _prob(model, trigrams): print('Checking {} model ...'.format(model)) with open(model, 'rb') as f: kneser_ney = load(f) result = 1 for trigram in trigrams: prob = kneser_ney.prob(trigram) result = result * prob return result ''' if __name__ == '__main__': main() diff --git a/swh/langdetect/ngramdist.py b/swh/langdetect/ngramdist.py index 3dc4e9b..004fdd8 100644 --- a/swh/langdetect/ngramdist.py +++ b/swh/langdetect/ngramdist.py @@ -1,234 +1,235 @@ import os import sys import time import random import csv import json import argparse import nltk import operator from ast import literal_eval from itertools import islice from pickle import dump, load from nltk.util import ngrams -from .utils.common import tokenizer, file_to_string, find_file, count_files +from .utils.common import Tokenizer, file_to_string, find_file, count_files csv.field_size_limit(sys.maxsize) def main(): parser = argparse.ArgumentParser(description='Training and test tool of frequency distance of n-grams.') subparsers = parser.add_subparsers(dest='sub_command') parser_train = subparsers.add_parser('train', help='Training on the dataset, dataset must be a *.csv file. A model will be created in the same directory.') parser_train.add_argument('train_path', metavar='PATH', type=str, help='Path of the training dataset.') # parser_train.add_argument('-n', '--ngrams', metavar='N', dest='train_maxsize', type=int, help='Set maximum input size of ConvNet, default 5.') parser_test = subparsers.add_parser('test', help='Test on the dataset, dataset must be a directory with *.csv dataset named by corresponding language.') parser_test.add_argument('test_root', metavar='ROOT', type=str, help='Root of the test dataset.') if len(sys.argv[1:]) == 0: parser.print_help() parser.exit() args = parser.parse_args() if args.sub_command == 'train' : n = NGramDist(args.train_path) n.train() elif args.sub_command == 'test': n = NGramDist(args.test_root) n.test() else: parser.parse_args('-h') class NGramDist: def __init__(self, path): self._path = path # Root of model folder self._root_model = os.path.join(os.path.dirname(path), 'model_ngram_dist') try: os.mkdir(self._root_model) except: pass # Path of result self._path_result = os.path.join(os.path.dirname(path), 'result_ngram_dist') dir_path = os.path.dirname(os.path.abspath(__file__)) - with open(os.path.join(dir_path, 'static_data', 'languages.json'), 'r') as f: + with open(os.path.join(dir_path, 'static_data', 'languages_less.json'), 'r') as f: self._languages = json.load(f) self._path_test_csv = path self._num_of_classes = len(self._languages) def file_len(self, fname): with open(fname) as f: count = 0 for l in f: count += 1 return count def train(self): statistics = {} with open(self._path, newline='') as csvfile: r = csv.reader(csvfile, delimiter=' ', quotechar='|') for pair in r: label, string = pair label = int(label) language = self._languages[label] print(language, end='\r') statistics_lang = statistics.get(language, {}) string = literal_eval(string) - tokens = tokenizer(string, 'letter') + tokens = Tokenizer.tokenize(string, 'letter') generated_ngrams = self._generate_ngrams([chr(token) for token in tokens], 3) self._count_ngrams(statistics_lang, generated_ngrams) + statistics[language] = statistics_lang for language in self._languages: with open(os.path.join(self._root_model, language), 'wb') as f: - dump(self._sort_by_value(statistics[language], f)) + dump(self._sort_by_value(statistics[language]), f) def _generate_ngrams(self, tokens, n): generated_ngrams = [] for i in range(1, n+1): igrams = ngrams(tokens, i, pad_left=True, pad_right=True, left_pad_symbol = '$BOF$', right_pad_symbol = '$EOF$') for igram in igrams: generated_ngrams.append(''.join(igram)) return generated_ngrams def _count_ngrams(self, statistics, ngrams): for ngram in ngrams: statistics[ngram] = statistics.get(ngram, 0) + 1 def test(self): try: r = open(self._path_result, 'rb') test_result = load(r) r.close() except FileNotFoundError: test_result = {} model = self._load_models() for language in [x for x in self._languages if x not in test_result.keys()]: test_result[language] = self.test_class(model, language) with open(self._path_result, 'wb') as f: dump(test_result, f) def _load_models(self): models = {} for model in [model for model in os.listdir(self._root_model) if not model.startswith('.')]: root_model = os.path.join(self._root_model, model) with open(root_model, 'rb') as sorted_file: models[model] = self._list_to_dict(load(sorted_file)) return models def _list_to_dict(self, model): model_ngrams = [x[0] for x in model] model_dict = {} index = 0 for ngram in model_ngrams: index += 1 model_dict[ngram] = index return model_dict def _count_size(self, files): size = 0 for f in files: size += os.path.getsize(f) return size def test_class(self, model, language): ok = 0 results = [] count = 0 total_test = self.file_len(os.path.join(self._path_test_csv, language + '.csv')) with open(os.path.join(self._path_test_csv, language + '.csv'), newline='') as csvfile: r = csv.reader(csvfile, delimiter=' ', quotechar='|') for pair in r: label, string = pair label = int(label) string = literal_eval(string) result = self._guess_file_language(model, string) count += 1 print('[{0:4d}/{1:4d}] {2}:{3} '.format(count, total_test, result[0][1], result[0][0]),end='\r') results.append(result[0]) if result[0][1] == language: ok += 1 accuracy = ok / total_test print('Tests for {} '.format(language)) print('Total test files : {}'.format(total_test)) print('Correctly classified files : {}'.format(ok)) print('Accuracy : {}%'.format(accuracy * 100)) return (ok, total_test, accuracy, results) def speed_benchmark(self): language = self._languages[10] model = self._load_model() test_set = self._get_test_set(language) total_size = self._count_size(test_set) print('{} kB in total'.format(total_size / 1024)) t_start = time.perf_counter() self.test_class(model, language) t_end = time.perf_counter() print('{} seconds.'.format(t_end - t_start)) print('{} seconds per KiB'.format(((t_end - t_start) / total_size) * 1024)) def _guess_file_language(self, models, string): - tokens = tokenizer(string, 'letter') + tokens = Tokenizer.tokenize(string, 'letter') generated_ngrams = self._generate_ngrams([chr(token) for token in tokens], 3) statistics = {} self._count_ngrams(statistics, generated_ngrams) test_profile = self._list_to_dict(self._sort_by_value(statistics)) result = [] for model in models.keys(): root_model = os.path.join(self._root_model, model) model_profile = models[model] distance = self._distance(model_profile, test_profile) result.append((distance, model)) return sorted(result) def _sort_by_value(self, statistics): statistics_sorted = sorted(statistics.items(), key = operator.itemgetter(1), reverse = True)[:500] return statistics_sorted def _distance(self, model_profile, test_profile): distance = 0 maximum = len(test_profile) for test_ngram in test_profile.keys(): test_rank = test_profile.get(test_ngram) model_rank = model_profile.get(test_ngram, maximum) d = abs(test_rank - model_rank) distance += d return distance if __name__ == '__main__': main() diff --git a/swh/langdetect/ngramprob.py b/swh/langdetect/ngramprob.py index ec9e372..ff2b0ee 100644 --- a/swh/langdetect/ngramprob.py +++ b/swh/langdetect/ngramprob.py @@ -1,170 +1,191 @@ -import os, sys, subprocess, time +import os, sys, subprocess, time, csv, argparse, json import kenlm +from ast import literal_eval from itertools import islice from pickle import dump, load -from .utils.common import tokenizer, file_to_string, find_file, count_files, remove_comment +from .utils.common import Tokenizer, file_to_string, find_file, count_files, remove_comment -class NGramProb: +csv.field_size_limit(sys.maxsize) - def __init__(self, root): - # Root of dataset - self._root = root +def main(): + parser = argparse.ArgumentParser(description='Training and test tool of n-grams model.') - # Root of training set - self._root_training_set = os.path.join(self._root, '..', 'training_set') + subparsers = parser.add_subparsers(dest='sub_command') - # Root of model folder - self._root_model = os.path.join(self._root, '..', 'model_ngram_prob') + parser_train = subparsers.add_parser('train', help='Training on the dataset, dataset must be a *.csv file. A model will be created in the same directory.') + parser_train.add_argument('train_path', metavar='PATH', type=str, help='Path of the training dataset.') + # parser_train.add_argument('-n', '--ngrams', metavar='N', dest='train_maxsize', type=int, help='Set maximum input size of ConvNet, default 5.') + parser_test = subparsers.add_parser('test', help='Test on the dataset, dataset must be a directory with *.csv dataset named by corresponding language.') + parser_test.add_argument('test_root', metavar='ROOT', type=str, help='Root of the test dataset.') + + if len(sys.argv[1:]) == 0: + parser.print_help() + parser.exit() + args = parser.parse_args() + + if args.sub_command == 'train' : + n = NGramProb(args.train_path) + n.train() + elif args.sub_command == 'test': + n = NGramProb(args.test_root) + n.test() + else: + parser.parse_args('-h') - # Root of arranged dataset - self._root_language_dataset = os.path.join(self._root, '..', 'code_by_language') + +class NGramProb: + + def __init__(self, path): + + self._path = path + + # Root of model folder + self._root_model = os.path.join(os.path.dirname(path), 'model_ngram_prob') + try: + os.mkdir(self._root_model) + except: + pass # Path of result - self._path_result = os.path.join(self._root, '..', 'result_prob') + self._path_result = os.path.join(os.path.dirname(path), 'result_ngram_prob') - def train(self): + dir_path = os.path.dirname(os.path.abspath(__file__)) + with open(os.path.join(dir_path, 'static_data', 'languages_less.json'), 'r') as f: + self._languages = json.load(f) - for language in [x for x in os.listdir(self._root_training_set) if not x.startswith('.')]: - root_training_set_language = os.path.join(self._root_training_set, language) + self._path_test_csv = path + + self._num_of_classes = len(self._languages) + + def file_len(self, fname): + with open(fname) as f: + count = 0 + for l in f: + count += 1 + return count + + def train(self): + command = [os.path.join(os.path.dirname(os.path.abspath(__file__)), + '..' , '..', 'bin', 'lmplz'), + '-o', '3', '-T', '/tmp', '--discount_fallback'] + + with open(self._path, newline='') as csvfile: + r = csv.reader(csvfile, delimiter=' ', quotechar='|') + label = 0 + language = self._languages[label] texts = [] - root_stat_language = os.path.join(self._root_model, language) - if os.path.isfile(root_stat_language): - continue - - for f in [x for x in os.listdir(root_training_set_language) if not x.startswith('.')]: - print(f) - filename = os.path.join(root_training_set_language, f) - text = file_to_string(filename) - text = remove_comment(text, language) - tokens = tokenizer(text, 'letter') + for pair in r: + label_new, _ = pair + if label != int(label_new): + with open(os.path.join(self._root_model, language), 'wb') as f: + train_text = ' '.join(texts) + proc = subprocess.Popen(command, stdin=subprocess.PIPE, stdout=f) + proc.communicate(train_text.encode()) + texts = [] + label, string = pair + label = int(label) + language = self._languages[label] + print(language, end='\r') + + text = literal_eval(string) + tokens = Tokenizer.tokenize(text, 'letter') + texts.append(' '.join(chr(token) for token in tokens)) - #tokens = tokenizer(text, 'word')[-1024:] - #tokens = b' '.join(tokens) - #texts.append((''.join(chr(token) for token in list(tokens)))) - train_text = ' '.join(texts) - command = ['../../bin/lmplz', '-o', '3', '-T', '/tmp', '--discount_fallback'] - - with open(root_stat_language, 'wb') as f: + + with open(os.path.join(self._root_model, language), 'wb') as f: + train_text = ' '.join(texts) proc = subprocess.Popen(command, stdin=subprocess.PIPE, stdout=f) proc.communicate(train_text.encode()) - if os.path.getsize(root_stat_language) == 0: - os.remove(root_stat_language) - - # st = os.stat(root_stat_language) - # os.chmod(root_stat_language, st.st_mode | stat.S_IEXEC) def test(self): try: r = open(self._path_result, 'rb') test_result = load(r) r.close() except FileNotFoundError: test_result = {} models = self._load_models() - for language in [x for x in os.listdir(self._root_model) if not x.startswith('.') and x not in test_result.keys()]: + for language in [x for x in self._languages if x not in test_result.keys()]: test_result[language] = self.test_class(models, language) with open(self._path_result, 'wb') as f: dump(test_result, f) def _load_models(self): models = {} for model in [model for model in os.listdir(self._root_model) if not model.startswith('.')]: root_model = os.path.join(self._root_model, model) models[model] = kenlm.LanguageModel(root_model) return models - - def _get_test_set(self, language): - root_training_language = os.path.join(self._root_training_set, language) - root_language = os.path.join(self._root_language_dataset, language) - total = count_files(root_language) - training_set = [int(os.path.splitext(x)[0]) for x in os.listdir(root_training_language) if not x.startswith('.')] - it = (find_file(root_language, x) for x in range(1, total + 1) if x not in training_set and os.path.getsize(find_file(root_language, x)) <= 1048576) - test_set = list(islice(it, 1000)) - if len(test_set) == 0: - it = (find_file(root_language, x) for x in range(1, total + 1) if x not in training_set) - test_set = list(islice(it, 1000)) - return test_set def _count_size(self, files): size = 0 for f in files: size += os.path.getsize(f) return size - def test_class(self, models, language): - test_set = self._get_test_set(language) - + def test_class(self, model, language): ok = 0 results = [] count = 0 - length = len(test_set) - for test in test_set: - result = self._guess_file_language(models, test) - count += 1 - print('[{0:4d}/{1:4d}] {2}:{3} '.format(count, length, result[0][1], result[0][0]),end='\r') - results.append(result[0]) - if result[0][1] == language: - ok += 1 - - total_test = len(test_set) - accuracy = ok / len(test_set) + total_test = self.file_len(os.path.join(self._path_test_csv, language + '.csv')) + + with open(os.path.join(self._path_test_csv, language + '.csv'), newline='') as csvfile: + r = csv.reader(csvfile, delimiter=' ', quotechar='|') + for pair in r: + label, string = pair + label = int(label) + string = literal_eval(string) + result = self._guess_file_language(model, string) + count += 1 + print('[{0:4d}/{1:4d}] {2}:{3} '.format(count, total_test, result[0][1], result[0][0]),end='\r') + results.append(result[0]) + if result[0][1] == language: + ok += 1 + + accuracy = ok / total_test print('Tests for {} '.format(language)) print('Total test files : {}'.format(total_test)) print('Correctly classified files : {}'.format(ok)) print('Accuracy : {}%'.format(accuracy * 100)) - return (ok, len(test_set), accuracy, results) + return (ok, total_test, accuracy, results) def speed_benchmark(self): - language = [x for x in os.listdir(self._root_model) if not x.startswith('.')][10] - models = self._load_models() + language = self._languages[10] + model = self._load_model() test_set = self._get_test_set(language) total_size = self._count_size(test_set) print('{} kB in total'.format(total_size / 1024)) t_start = time.perf_counter() - self.test_class(models, language) + self.test_class(model, language) t_end = time.perf_counter() print('{} seconds.'.format(t_end - t_start)) - print('{} seconds per kB'.format(((t_end - t_start) / total_size) * 1024)) + print('{} seconds per KiB'.format(((t_end - t_start) / total_size) * 1024)) - def _guess_file_language(self, models, filename): - tokens = tokenizer(file_to_string(filename), 'letter') - tokens = tokens[-1024:] + def _guess_file_language(self, models, string): + tokens = Tokenizer.tokenize(string, 'letter') text = ' '.join(chr(token) for token in tokens) #text = file_to_string(filename) #tokens = tokenizer(text, 'word') #tokens = b' '.join(tokens) #text = ''.join(chr(token) for token in list(tokens)) result = [] for model_key in models.keys(): root_model = os.path.join(self._root_model, model_key) model = models[model_key] score = model.score(text) result.append((score, model_key)) return sorted(result, reverse=True) if __name__ == '__main__': - if len(sys.argv) == 3 and sys.argv[1] == '--train': - n = NGramProb(sys.argv[2]) - n.train() - elif len(sys.argv) == 3 and sys.argv[1] == '--test': - n = NGramProb(sys.argv[2]) - n.test() - elif len(sys.argv) == 3 and sys.argv[1] == '--benchmark': - n = NGramProb(sys.argv[2]) - n.speed_benchmark() - elif len(sys.argv) == 4 and sys.argv[1] == '--test': - n = NGramProb(sys.argv[2]) - n.test_class(n.load_models(), sys.argv[3]) - else: - print('Wrong arguments, please check your input.') + main() diff --git a/swh/langdetect/static_data/languages_less.json b/swh/langdetect/static_data/languages_less.json new file mode 100644 index 0000000..695c179 --- /dev/null +++ b/swh/langdetect/static_data/languages_less.json @@ -0,0 +1,2 @@ +["ActionScript", "Ada", "AMPL", "AngelScript", "AspectJ", "Assembly", "ATS", "AutoHotkey", "AutoIt", "Batchfile", "BitBake", "C", "C#", "C++", "Ceylon", "Clean", "Clojure", "CMake", "CoffeeScript", "ColdFusion", "Common Lisp", "Coq", "CSS", "D", "Dart", "DM", "Dylan", "E", "Eiffel", "Elixir", "Erlang", "F#", "Factor", "Forth", "Fortran", "Game Maker Language", "GLSL", "Go", "Groovy", "Haskell", "Haxe", "HTML", "IDL", "Isabelle", "Java", "JavaScript", "Kotlin", "LilyPond", "Limbo", "LLVM", "Lua", "M", "Makefile", "Mathematica", "Matlab", "Max", "Modelica", "Modula-2", "NCL", "nesC", "NewLisp", "Nim", "Objective-C", "Objective-J", "OCaml", "OpenEdge ABL", "P4", "Pan", "Pascal", "Perl", "Perl 6", "PHP", "PLpgSQL", "PowerBuilder", "Processing", "Prolog", "Python", "QML", "R", "Racket", "RAML", "Red", "Roff", "Ruby", "Rust", "Scala", "Scheme", "Scilab", "Shell", "Smali", "Smalltalk", "Smarty", "SMT", "SourcePawn", "SQF", "Standard ML", "Swift", "SystemVerilog", "Tcl", "TeX", "TypeScript", "UnrealScript", "Vala", "Verilog", "VHDL", "Vim script", "Visual Basic", "xBase", "XSLT"] + diff --git a/swh/langdetect/t-SNE.py b/swh/langdetect/t-SNE.py new file mode 100644 index 0000000..c03423e --- /dev/null +++ b/swh/langdetect/t-SNE.py @@ -0,0 +1,226 @@ +import os +import sys +import operator +import nltk +import random +import time +import numpy as np +import csv +import argparse +import json + +import matplotlib.pyplot as plt +import matplotlib as mpl +import sklearn.metrics as metrics + +from ast import literal_eval +from itertools import islice +from pickle import dump, load +from .utils.common import Tokenizer +from nltk.util import ngrams +from collections import Counter +from sklearn.feature_extraction.text import HashingVectorizer, TfidfVectorizer +from sklearn.externals import joblib +from sklearn.metrics.pairwise import euclidean_distances +from sklearn.cluster import DBSCAN +from sklearn.decomposition import TruncatedSVD +from scipy.sparse import vstack +from scipy.sparse import csr_matrix + +csv.field_size_limit(sys.maxsize) + +def main(): + parser = argparse.ArgumentParser(description='Training and test tool of multinumial naive bayesian.') + + subparsers = parser.add_subparsers(dest='sub_command') + + parser_train = subparsers.add_parser('train', help='Training on the dataset, dataset must be a *.csv file. A model will be created in the same directory.') + parser_train.add_argument('train_path', metavar='PATH', type=str, help='Path of the training dataset.') + # parser_train.add_argument('-n', '--ngrams', metavar='N', dest='train_maxsize', type=int, help='Set maximum input size of ConvNet, default 5.') + parser_test = subparsers.add_parser('test', help='Test on the dataset, dataset must be a directory with *.csv dataset named by corresponding language.') + parser_test.add_argument('test_root', metavar='ROOT', type=str, help='Root of the test dataset.') + + if len(sys.argv[1:]) == 0: + parser.print_help() + parser.exit() + args = parser.parse_args() + + if args.sub_command == 'train' : + n = Unsupervised(args.train_path) + # n.train() + n.graph_top_20() + elif args.sub_command == 'test': + n = Unsupervised(args.test_root) + n.test() + else: + parser.parse_args('-h') + +class Unsupervised: + + def __init__(self, path): + + self._path = path + + # Root of model folder + self._root_model = os.path.join(os.path.dirname(path), 'model_dbscan') + try: + os.mkdir(self._root_model) + except: + pass + + # Path of result + self._path_result = os.path.join(os.path.dirname(path), 'result_dbscan') + + dir_path = os.path.dirname(os.path.abspath(__file__)) + with open(os.path.join(dir_path, 'static_data', 'languages.json'), 'r') as f: + self._languages = json.load(f) + + self._path_test_csv = path + + self._num_of_classes = len(self._languages) + + def train(self): + cv = HashingVectorizer(analyzer='char', ngram_range=(1, 5), n_features=2**20, alternate_sign=False) + texts = [] + label = 0 + string = '' + + top_20 = ['Python', 'Java', 'JavaScript', 'PHP', 'C#', 'C', 'C++', + 'R', 'Objective-C', 'Swift', 'Matlab', 'Ruby', 'TypeScript', + 'Visual Basic', 'Scala', 'Kotlin', 'Go', 'Perl', 'Lua', + 'Rust', 'Haskell'] + top_20 = [self._languages.index(x) for x in top_20] + print(top_20) + + with open(self._path, newline='') as csvfile: + r = csv.reader(csvfile, delimiter=' ', quotechar='|') + for pair in r: + label_new, string_new = pair + print(label_new, end=' \r') + if not int(label_new) == label: + if not os.path.isfile(os.path.join(self._root_model, 'counts{}.pkl'.format(label))): + if label in top_20: + counts = cv.fit_transform(texts) + self.clustering(counts, 1, label) + texts = [] + label = int(label_new) + if label in top_20: + string = literal_eval(string_new) + #tokens = Tokenizer.tokenize(string, 'word') + #text = ' '.join([''.join([chr(x) for x in token]) for token in tokens]) + tokens = Tokenizer.tokenize(string, 'letter') + text = ''.join([chr(token) for token in tokens]) + texts.append(text) + with open(os.path.join(self._root_model, 'classifier.cv'), 'wb') as f: + joblib.dump(cv, f) + + + def clustering(self, counts, num_clusters, label): + # km = KMeans(n_clusters=num_clusters) + # km.fit(counts) + + with open(os.path.join(self._root_model, 'counts{}.pkl'.format(label)), 'wb') as f: + joblib.dump(counts, f) + #with open(os.path.join(self._root_model, 'cluster{}.pkl'.format(label)), 'wb') as f: + # joblib.dump(km, f) + + def graph_top_20(self): + + top_20 = ['Python', 'Java', 'JavaScript', 'PHP', 'C#', 'C', 'C++', + 'R', 'Objective-C', 'Swift', 'Matlab', 'Ruby', 'TypeScript', + 'Visual Basic', 'Scala', 'Kotlin', 'Go', 'Perl', 'Lua', + 'Rust', 'Haskell'] + top_20 = [self._languages.index(x) for x in top_20] + counts = csr_matrix((0, 2 ** 20)) + for label in top_20: + with open(os.path.join(self._root_model, 'counts{}.pkl'.format(label)), 'rb') as f: + counts = vstack((counts, joblib.load(f))) + print(counts.shape) + if not os.path.isfile(os.path.join(self._root_model, 'dist')): + dist = euclidean_distances(counts) + with open(os.path.join(self._root_model, 'dist'), 'wb') as f: + joblib.dump(dist, f) + else: + with open(os.path.join(self._root_model, 'dist'), 'rb') as f: + dist = joblib.load(f) + + + model = DBSCAN(eps=0.3, min_samples=10) + print('dbscan fit start') + db = model.fit(counts) + print('dbscan fit done') + + labels = db.labels_ + + # Number of clusters in labels, ignoring noise if present. + n_clusters_ = len(set(labels)) - (1 if -1 in labels else 0) + + print('Estimated number of clusters: %d' % n_clusters_) + + + #fig, ax = plt.subplots(figsize=(15, 550)) + #titles = [self._languages[top_20[x // 500]] for x in list(range(0,counts.shape[0]))] + #ax = dendrogram(linkage_matrix, orientation="right", labels=titles) + + #plt.tick_params(axis= 'x', + # which='both', + # bottom=False, + # top=False, + # labelbottom=False) + + #plt.tight_layout() + #plt.savefig(os.path.join(self._root_model, 'top_20_dbscan.pdf')) + + def speed_benchmark(self): + language = [x for x in os.listdir(self._root_training_set) if not x.startswith('.')][10] + models = self._load_models() + + test_set = self._get_test_set(language) + total_size = self._count_size(test_set) + print('{} kB in total'.format(total_size / 1024)) + + t_start = time.perf_counter() + self.test_class(models, language) + t_end = time.perf_counter() + + print('{} seconds.'.format(t_end - t_start)) + print('{} seconds per kB'.format(((t_end - t_start) / total_size) * 1024)) + + def _count_size(self, files): + size = 0 + for f in files: + size += os.path.getsize(f) + return size + + def file_len(self, fname): + with open(fname) as f: + count = 0 + for l in f: + count += 1 + return count + + def _distance(self, model_profile, test_profile): + distance = 0 + maximum = len(test_profile) + + for test_ngram in test_profile.keys(): + test_rank = test_profile.get(test_ngram) + model_rank = model_profile.get(test_ngram, maximum) + d = abs(test_rank - model_rank) + distance += d + + return distance + ''' + def _prob(model, trigrams): + print('Checking {} model ...'.format(model)) + with open(model, 'rb') as f: + kneser_ney = load(f) + result = 1 + for trigram in trigrams: + prob = kneser_ney.prob(trigram) + result = result * prob + return result + ''' + +if __name__ == '__main__': + main() diff --git a/swh/langdetect/utils/training.py b/swh/langdetect/utils/training.py index 8bdbed7..09c90c9 100644 --- a/swh/langdetect/utils/training.py +++ b/swh/langdetect/utils/training.py @@ -1,110 +1,115 @@ import os import random import csv +import json from .common import count_files, find_file, file_to_string from itertools import islice from shutil import copyfile class Dataset: def __init__(self, root): self.root_code = os.path.join(root, '..', 'code_by_language') self.root_training = os.path.join(root, '..', 'training_set') self.root_training_csv = os.path.join(root, '..', 'training_set_csv') self.root_test = os.path.join(root, '..', 'test_set') self.root_test_csv = os.path.join(root, '..', 'test_set_csv') try: os.mkdir(self.root_training) except FileExistsError: pass try: os.mkdir(self.root_training_csv) except FileExistsError: pass try: os.mkdir(self.root_test) except FileExistsError: pass try: os.mkdir(self.root_test_csv) except FileExistsError: pass + dir_path = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + print(dir_path) + with open(os.path.join(dir_path, 'static_data', 'languages_less.json'), 'r') as f: + self._languages = json.load(f) - def build_training_set(self, languages): - for language in languages: + def build_training_set(self): + for language in self._languages: # limit defines the size of training set # upper defines the maximum size root_code_language = os.path.join(self.root_code, language) root_training_language = os.path.join(self.root_training, language) total = count_files(root_code_language) try: os.mkdir(root_training_language) except FileExistsError: pass upper = 1000 if total >= upper: limit = upper // 2 else: limit = total // 2 indices = random.sample(range(1, total + 1), limit) files = map(lambda x : find_file(root_code_language, x), indices) for src in files: basename = os.path.basename(src) des = os.path.join(root_training_language, basename) os.symlink(src, des) - def build_test_set(self, languages, extension=True): - for language in languages: + def build_test_set(self, extension=True): + for language in self._languages: root_language = os.path.join(self.root_code, language) root_test_language = os.path.join(self.root_test, language) try: os.mkdir(root_test_language) except FileExistsError: pass files = self.get_test_set(language) for src in files: if extension: des = os.path.join(root_test_language, os.path.basename(src)) else: des = os.path.join(root_test_language, os.path.splitext(os.path.basename(src))[0]) copyfile(src, des) - def train_files_with_label(self, languages): + def train_files_with_label(self): with open(os.path.join(self.root_training_csv, 'training_set.csv'), 'w', newline='') as csvfile: setwriter = csv.writer(csvfile, delimiter=' ', quotechar='|', quoting=csv.QUOTE_MINIMAL) - for language in languages: + for language in self._languages: print(language) root_training_language = os.path.join(self.root_training, language) - index_lang = languages.index(language) + index_lang = self._languages.index(language) for f in [x for x in os.listdir(root_training_language) if not x.startswith('.')]: filename = os.path.join(root_training_language, f) tokens = file_to_string(filename) # 10240 setwriter.writerow([index_lang, tokens]) def get_test_set(self, language): root_training_language = os.path.join(self.root_training, language) root_language = os.path.join(self.root_code, language) total = count_files(root_language) training_set = [int(os.path.splitext(x)[0]) for x in os.listdir(root_training_language) if not x.startswith('.')] it = (find_file(root_language, x) for x in range(1, total + 1) if x not in training_set and os.path.getsize(find_file(root_language, x)) <= 1048576) test_set = list(islice(it, 1000)) if len(test_set) == 0: it = (find_file(root_language, x) for x in range(1, total + 1) if x not in training_set) test_set = list(islice(it, 1000)) return test_set - def test_files_with_label(self, languages): - for language in languages: + def test_files_with_label(self): + for language in self._languages: root_test_language = os.path.join(self.root_test, language) - index_lang = languages.index(language) + index_lang = self._languages.index(language) with open(os.path.join(self.root_test_csv, language + '.csv'), 'w', newline='') as csvfile: setwriter = csv.writer(csvfile, delimiter=' ', quotechar='|', quoting=csv.QUOTE_MINIMAL) for f in [x for x in os.listdir(root_test_language) if not x.startswith('.')]: filename = os.path.join(root_test_language, f) tokens = file_to_string(filename) setwriter.writerow([index_lang, tokens])