From f91a4d40ed541cb516a3a72f70cb1f5fb280b2eb Mon Sep 17 00:00:00 2001 From: ofir Date: Mon, 2 Nov 2015 23:37:40 +0200 Subject: [PATCH 01/15] added private proposal api. this will be used for future auto-tagging --- apis/resources/__init__.py | 5 +++-- laws/api.py | 31 ++++++++++++++++++++++++++++++- 2 files changed, 33 insertions(+), 3 deletions(-) diff --git a/apis/resources/__init__.py b/apis/resources/__init__.py index 218349b8..5174f0a5 100644 --- a/apis/resources/__init__.py +++ b/apis/resources/__init__.py @@ -6,7 +6,7 @@ from mks.api import MemberResource, PartyResource, MemberBillsResource, MemberAgendasResource from video.api import VideoResource from links.api import LinkResource -from laws.api import BillResource, LawResource, VoteResource, VoteActionResource +from laws.api import BillResource, LawResource, VoteResource, VoteActionResource, ProvateProposalResource from agendas.api import AgendaResource, AgendaTodoResource from committees.api import CommitteeResource, CommitteeMeetingResource, ProtocolPartResource from auxiliary.api import PostResource, TagResource @@ -27,6 +27,7 @@ v2_api.register(VoteResource()) v2_api.register(VoteActionResource()) v2_api.register(LawResource()) +v2_api.register(ProvateProposalResource()) v2_api.register(AgendaResource()) v2_api.register(AgendaTodoResource()) v2_api.register(CommitteeResource()) @@ -39,4 +40,4 @@ v2_api.register(PersonResource()) v2_api.register(LobbyistsChangeResource()) v2_api.register(LobbyistResource()) -v2_api.register(LobbyistCorporationResource()) \ No newline at end of file +v2_api.register(LobbyistCorporationResource()) diff --git a/laws/api.py b/laws/api.py index db9b96a7..fb3127f8 100755 --- a/laws/api.py +++ b/laws/api.py @@ -5,7 +5,6 @@ from django.core.urlresolvers import reverse from tastypie.constants import ALL import tastypie.fields as fields - from agendas.templatetags.agendas_tags import agendas_for from apis.resources.base import BaseResource @@ -27,6 +26,36 @@ class LawResource(BaseResource): class Meta(BaseResource.Meta): queryset = Law.objects.all() allowed_methods = ['get'] +class TagResource(BaseResource):pass +class ProvateProposalResource(BaseResource): + class Meta(BaseResource.Meta): + queryset = PrivateProposal.objects.all() + allowed_methods = ['get'] + + filtering = dict(from_date=ALL, + to_date=ALL) + + + tags = fields.ToManyField(TagResource, + 'tags', + + null=True, + full=False) + + def dehydrate_tags(self, bundle): + return [tag.name for tag in bundle.obj.bill.tags] + + def build_filters(self, filters={}): + orm_filters = super(ProvateProposalResource, self).build_filters(filters) + if 'from_date' in filters: + orm_filters["date__gte"] = filters['from_date'] + if 'to_date' in filters: + # the to_date needs to be incremented by a day since when humans say to_date=2014-07-30 they + # actually mean midnight between 30 to 31. python on the other hand interperts this as midnight between + # 29 and 30 + to_date = datetime.strptime(filters["to_date"], "%Y-%M-%d")+timedelta(days=1) + orm_filters["date__lte"] = to_date.strftime("%Y-%M-%d") + return orm_filters class VoteActionResource(BaseResource): class Meta(BaseResource.Meta): From 8c2109d9f1e3c6cd99e7595309c7428571cb59bd Mon Sep 17 00:00:00 2001 From: ofir Date: Mon, 9 Nov 2015 23:00:59 +0200 Subject: [PATCH 02/15] added parsing scripts for pps --- laws/make_histograms.py | 33 +++++++++++++++++++++++++++++++++ laws/parse_htmls.py | 30 ++++++++++++++++++++++++++++++ laws/read_pp.py | 22 ++++++++++++++++++++++ 3 files changed, 85 insertions(+) create mode 100644 laws/make_histograms.py create mode 100644 laws/parse_htmls.py create mode 100644 laws/read_pp.py diff --git a/laws/make_histograms.py b/laws/make_histograms.py new file mode 100644 index 00000000..d6c9ce48 --- /dev/null +++ b/laws/make_histograms.py @@ -0,0 +1,33 @@ +import cPickle as pickle +from collections import defaultdict +import re + +split_re = re.compile(r'\s+', re.U) + +REMOVE = ['"', "'", '/', '\\', ',', '.', '!', '@', '#', '$', '%', '^', '&', '*', '(', ')', '-', '=', '_', '+', '\n', '[', ']', '{', '}', '|', '<', '>', '?', '~', ":", ';'] +REMOVE.extend([str(i) for i in xrange(10)]) +def make_histogram(t): + dd = defaultdict(int) + for w in t.split(): + dd[reduce(lambda o, s: o.replace(s, ''), REMOVE, w)] += 1 + return dict(dd) + + +pp_with_tags = pickle.load(open('pp_with_tags.pkl', "rb")) + +keywords = set() +data = [] +for i, pp in enumerate(pp_with_tags): + pp_text, pp_tag, pp_id = pp['text'], pp['tags'], pp['id'] + try: + histogram = make_histogram(pp_text) + except: + import traceback; traceback.print_exc() + print "at pp", i + import pdb; pdb.set_trace() + data.append({'histogram' : histogram, 'tags' : pp_tag, 'id' : pp_id}) + keywords |= set(histogram.keys()) # this will add the keys + + + +pickle.dump((keywords, data), open('histograms_with_tags.pkl', 'wb')) diff --git a/laws/parse_htmls.py b/laws/parse_htmls.py new file mode 100644 index 00000000..06cdd68b --- /dev/null +++ b/laws/parse_htmls.py @@ -0,0 +1,30 @@ +from BeautifulSoup import BeautifulSoup + +import cPickle as pickle + +pp_without_tags = [] +pp_with_tags = [] +for i, pp in enumerate(pickle.load(open('pps.pkl', 'rb'))): + content_html = pp['content_html'] + parsed_html = BeautifulSoup(content_html) + + current_tag = parsed_html.find('p', attrs={'class':'explanation-header'}) + if current_tag is None: + print "no text for pp", pp['id'] + continue + current_tag = current_tag.findNext() + text = '' + while current_tag is not None and '---' not in current_tag.text: + + text += current_tag.text + current_tag = current_tag.findNext() + + tags = pp['content_html'] + if len(tags) == 0: + pp_without_tags.append({'text' : text, 'id' : pp['id']}) + else: + pp_with_tags.append({'text' : text, 'id' : pp['id'], 'tags' : tags}) + +pickle.dump(pp_without_tags, open("pp_without_tags.pkl", "wb")) +pickle.dump(pp_with_tags, open("pp_with_tags.pkl", "wb")) + diff --git a/laws/read_pp.py b/laws/read_pp.py new file mode 100644 index 00000000..b2432b39 --- /dev/null +++ b/laws/read_pp.py @@ -0,0 +1,22 @@ +import urllib +import json +import sys +import cPickle as pickle +base_url = "http://127.0.0.1:8000" +next_url = "/api/v2/provateproposal/" +pps = [] + +while next_url != None: + uo = urllib.urlopen(base_url + next_url.replace("\\/", "/")) + try: + + json_obj = json.load(uo) # .read(), {'null' : None}) # just for eval to work + print json_obj['meta']['offset'], "/", json_obj['meta']['total_count']; sys.stdout.flush() + pps.extend(json_obj['objects']) + next_url = json_obj['meta']['next'] + print next_url + finally: + uo.close() + +with open("pps.pkl", "wb") as po: + pickle.dump(pps, po) From a216d1ef17d667992d6b859934eae2cab0053786 Mon Sep 17 00:00:00 2001 From: ofir Date: Mon, 16 Nov 2015 22:51:30 +0200 Subject: [PATCH 03/15] added scikit-learn to learn multilabel data for the pps --- laws/parse_htmls.py | 3 +- laws/tags_autolearn_play.py | 124 ++++++++++++++++++++++++++++++++++++ 2 files changed, 126 insertions(+), 1 deletion(-) create mode 100644 laws/tags_autolearn_play.py diff --git a/laws/parse_htmls.py b/laws/parse_htmls.py index 06cdd68b..ba83a263 100644 --- a/laws/parse_htmls.py +++ b/laws/parse_htmls.py @@ -19,7 +19,8 @@ text += current_tag.text current_tag = current_tag.findNext() - tags = pp['content_html'] + tags = pp['tags'] + if len(tags) == 0: pp_without_tags.append({'text' : text, 'id' : pp['id']}) else: diff --git a/laws/tags_autolearn_play.py b/laws/tags_autolearn_play.py new file mode 100644 index 00000000..7237b975 --- /dev/null +++ b/laws/tags_autolearn_play.py @@ -0,0 +1,124 @@ +# print(__doc__) +# +# import numpy as np +# import matplotlib.pyplot as plt +# +# from sklearn.datasets import make_multilabel_classification +# from sklearn.multiclass import OneVsRestClassifier +# from sklearn.svm import SVC +# from sklearn.preprocessing import LabelBinarizer +# from sklearn.decomposition import PCA +# from sklearn.cross_decomposition import CCA + + +def plot_hyperplane(clf, min_x, max_x, linestyle, label): + # get the separating hyperplane + w = clf.coef_[0] + a = -w[0] / w[1] + xx = np.linspace(min_x - 5, max_x + 5) # make sure the line is long enough + yy = a * xx - (clf.intercept_[0]) / w[1] + plt.plot(xx, yy, linestyle, label=label) + + +def plot_subfigure(X, Y, subplot, title, transform): + if transform == "pca": + X = PCA(n_components=2).fit_transform(X) + elif transform == "cca": + X = CCA(n_components=2).fit(X, Y).transform(X) + else: + raise ValueError + + min_x = np.min(X[:, 0]) + max_x = np.max(X[:, 0]) + + min_y = np.min(X[:, 1]) + max_y = np.max(X[:, 1]) + + classif = OneVsRestClassifier(SVC(kernel='linear')) + classif.fit(X, Y) + + plt.subplot(2, 2, subplot) + plt.title(title) + + zero_class = np.where(Y[:, 0]) + one_class = np.where(Y[:, 1]) + plt.scatter(X[:, 0], X[:, 1], s=40, c='gray') + plt.scatter(X[zero_class, 0], X[zero_class, 1], s=160, edgecolors='b', + facecolors='none', linewidths=2, label='Class 1') + plt.scatter(X[one_class, 0], X[one_class, 1], s=80, edgecolors='orange', + facecolors='none', linewidths=2, label='Class 2') + + plot_hyperplane(classif.estimators_[0], min_x, max_x, 'k--', + 'Boundary\nfor class 1') + plot_hyperplane(classif.estimators_[1], min_x, max_x, 'k-.', + 'Boundary\nfor class 2') + plt.xticks(()) + plt.yticks(()) + + plt.xlim(min_x - .5 * max_x, max_x + .5 * max_x) + plt.ylim(min_y - .5 * max_y, max_y + .5 * max_y) + if subplot == 2: + plt.xlabel('First principal component') + plt.ylabel('Second principal component') + plt.legend(loc="upper left") + + + + +import numpy +import cPickle as pickle +import sys +from sklearn import svm +print >>sys.stderr, 'loading data'; sys.stderr.flush() +keywords, data = pickle.load(open('histograms_with_tags.pkl', 'rb')) + +tags = numpy.array([a['tags'] for a in data]) +ids = [a['id'] for a in data] +print >>sys.stderr, 'preparing examples'; sys.stderr.flush() +keyword_dict = {} +for i,w in enumerate(keywords): + keyword_dict[w] = i + + +learn_data = [] +for a in data: + histogram = [0,] * len(keywords) + for k,v in a['histogram'].iteritems(): + histogram[keyword_dict[k]] = v + learn_data.append(numpy.array(histogram)) +learn_data = numpy.array(learn_data) + +from sklearn.preprocessing import LabelBinarizer +print >>sys.stderr, 'vectorizing labels'; sys.stderr.flush() + +tags = tags[:100] +learn_data = learn_data[:100] + + +labels = LabelBinarizer().fit_transform(tags) + + +#plt.figure(figsize=(8, 6)) +#plot_subfigure(learn_data, labels, 1, "With unlabeled samples + CCA", "cca") +#plot_subfigure(learn_data, labels, 2, "With unlabeled samples + PCA", "pca") +#plt.subplots_adjust(.04, .02, .97, .94, .09, .2) +#plt.show() + + +from sklearn.multiclass import OneVsRestClassifier + +from sklearn.svm import LinearSVC + + +#print >>sys.stderr, 'learning data'; sys.stderr.flush() +#classifier = OneVsRestClassifier(svm.SVC(kernel='linear')) +#trained_classifier = classifier.fit(learn_data[:-10], labels[:-10]) +#print >>sys.stderr, 'classifying'; sys.stderr.flush() +#predicted_labels = trained_classifier.predict(learn_data[-10:]) +#print zip(predicted_labels, labels[-10:]) + +from sklearn import cross_validation +from sklearn import metrics +classifier = OneVsRestClassifier(svm.SVC(kernel='linear', C=1)) +scores = cross_validation.cross_val_score(classifier, learn_data, labels, cv=5) #, scoring='f1_weighted') +print "Accuracy: %0.2f (+/- %0.2f)" % (scores.mean(), scores.std() * 2) From fee92d3d47be4aff94c856c41af4c68b57e9e873 Mon Sep 17 00:00:00 2001 From: ofir Date: Mon, 16 Nov 2015 22:54:18 +0200 Subject: [PATCH 04/15] auto param cv --- laws/tags_autolearn_play.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/laws/tags_autolearn_play.py b/laws/tags_autolearn_play.py index 7237b975..229fde5b 100644 --- a/laws/tags_autolearn_play.py +++ b/laws/tags_autolearn_play.py @@ -120,5 +120,5 @@ def plot_subfigure(X, Y, subplot, title, transform): from sklearn import cross_validation from sklearn import metrics classifier = OneVsRestClassifier(svm.SVC(kernel='linear', C=1)) -scores = cross_validation.cross_val_score(classifier, learn_data, labels, cv=5) #, scoring='f1_weighted') +scores = cross_validation.cross_val_score(classifier, learn_data, labels) #, scoring='f1_weighted') print "Accuracy: %0.2f (+/- %0.2f)" % (scores.mean(), scores.std() * 2) From 72cad984947e57268e627348a74cd089337f7d07 Mon Sep 17 00:00:00 2001 From: ofir Date: Mon, 23 Nov 2015 22:54:37 +0200 Subject: [PATCH 05/15] middle changes for multilabel cv --- laws/tags_autolearn_play.py | 27 +++++++++++++++++++++++++-- 1 file changed, 25 insertions(+), 2 deletions(-) diff --git a/laws/tags_autolearn_play.py b/laws/tags_autolearn_play.py index 229fde5b..c8a5ec3f 100644 --- a/laws/tags_autolearn_play.py +++ b/laws/tags_autolearn_play.py @@ -1,3 +1,4 @@ +import numpy as np # print(__doc__) # # import numpy as np @@ -94,8 +95,8 @@ def plot_subfigure(X, Y, subplot, title, transform): tags = tags[:100] learn_data = learn_data[:100] - -labels = LabelBinarizer().fit_transform(tags) +lb = LabelBinarizer() +labels = lb.fit_transform(tags) #plt.figure(figsize=(8, 6)) @@ -106,6 +107,13 @@ def plot_subfigure(X, Y, subplot, title, transform): from sklearn.multiclass import OneVsRestClassifier +def _OneVsRestClassifier_multilabel_score(self, X, y): + if self.multilabel_: + from sklearn.metrics import accuracy_score + return np.array(accuracy_score(yy, XX) for yy, XX in zip(y.transpose(), self.predict(X).transpose())).mean() + else: + return super(OneVsRestClassifier, self).score(X, y) +OneVsRestClassifier.score = _OneVsRestClassifier_multilabel_score from sklearn.svm import LinearSVC @@ -117,7 +125,22 @@ def plot_subfigure(X, Y, subplot, title, transform): #predicted_labels = trained_classifier.predict(learn_data[-10:]) #print zip(predicted_labels, labels[-10:]) + from sklearn import cross_validation + +def _StratifiedKFold_multilabel_iter_test_indices(self): + n_folds = self.n_folds + if len(self.y.shape) > 1: + y1dim = [int(reduce(lambda o,n: o + str(n), yy, '0'),2) for yy in self.y] + else: + y1dim = self.y + idx = np.argsort(y1dim) + for i in range(n_folds): + yield idx[i::n_folds] +cross_validation.StratifiedKFold._iter_test_indices = _StratifiedKFold_multilabel_iter_test_indices + + + from sklearn import metrics classifier = OneVsRestClassifier(svm.SVC(kernel='linear', C=1)) scores = cross_validation.cross_val_score(classifier, learn_data, labels) #, scoring='f1_weighted') From c452002a6b25e335f7b74e0eed14e059b55feeac Mon Sep 17 00:00:00 2001 From: ofir Date: Mon, 7 Dec 2015 21:41:45 +0200 Subject: [PATCH 06/15] cross validation gives 0.99 accuracy. what does it means? --- laws/tags_autolearn_play.py | 71 +++++++++++++++++++++---------------- 1 file changed, 41 insertions(+), 30 deletions(-) diff --git a/laws/tags_autolearn_play.py b/laws/tags_autolearn_play.py index c8a5ec3f..2e0332b3 100644 --- a/laws/tags_autolearn_play.py +++ b/laws/tags_autolearn_play.py @@ -92,11 +92,14 @@ def plot_subfigure(X, Y, subplot, title, transform): from sklearn.preprocessing import LabelBinarizer print >>sys.stderr, 'vectorizing labels'; sys.stderr.flush() +lb = LabelBinarizer() +lb.fit(tags) + + tags = tags[:100] learn_data = learn_data[:100] -lb = LabelBinarizer() -labels = lb.fit_transform(tags) +labels = lb.transform(tags) #plt.figure(figsize=(8, 6)) @@ -110,7 +113,8 @@ def plot_subfigure(X, Y, subplot, title, transform): def _OneVsRestClassifier_multilabel_score(self, X, y): if self.multilabel_: from sklearn.metrics import accuracy_score - return np.array(accuracy_score(yy, XX) for yy, XX in zip(y.transpose(), self.predict(X).transpose())).mean() + + return np.array([accuracy_score(yy, XX) for yy, XX in zip(y.transpose(), self.predict(X).transpose())]).mean() else: return super(OneVsRestClassifier, self).score(X, y) OneVsRestClassifier.score = _OneVsRestClassifier_multilabel_score @@ -118,30 +122,37 @@ def _OneVsRestClassifier_multilabel_score(self, X, y): from sklearn.svm import LinearSVC -#print >>sys.stderr, 'learning data'; sys.stderr.flush() -#classifier = OneVsRestClassifier(svm.SVC(kernel='linear')) -#trained_classifier = classifier.fit(learn_data[:-10], labels[:-10]) -#print >>sys.stderr, 'classifying'; sys.stderr.flush() -#predicted_labels = trained_classifier.predict(learn_data[-10:]) -#print zip(predicted_labels, labels[-10:]) - - -from sklearn import cross_validation - -def _StratifiedKFold_multilabel_iter_test_indices(self): - n_folds = self.n_folds - if len(self.y.shape) > 1: - y1dim = [int(reduce(lambda o,n: o + str(n), yy, '0'),2) for yy in self.y] - else: - y1dim = self.y - idx = np.argsort(y1dim) - for i in range(n_folds): - yield idx[i::n_folds] -cross_validation.StratifiedKFold._iter_test_indices = _StratifiedKFold_multilabel_iter_test_indices - - - -from sklearn import metrics -classifier = OneVsRestClassifier(svm.SVC(kernel='linear', C=1)) -scores = cross_validation.cross_val_score(classifier, learn_data, labels) #, scoring='f1_weighted') -print "Accuracy: %0.2f (+/- %0.2f)" % (scores.mean(), scores.std() * 2) +print >>sys.stderr, 'learning data'; sys.stderr.flush() +classifier = OneVsRestClassifier(svm.SVC(kernel='linear')) +trained_classifier = classifier.fit(learn_data[:-10], labels[:-10]) +print >>sys.stderr, 'classifying'; sys.stderr.flush() +predicted_labels = trained_classifier.predict(learn_data[-10:]) +predicted_tags = lb.inverse_transform(predicted_labels) + +from pprint import pprint +for _predicted_tags, _tags in zip(predicted_tags, tags[-10:]): + for _predicted_tag in _predicted_tags: + print _predicted_tag[::-1], ",", + print "-", + for _tag in _tags: + print _tag[::-1], ",", + print "" +import pdb; pdb.set_trace() + +#from sklearn import cross_validation + +#def _StratifiedKFold_multilabel_iter_test_indices(self): +# n_folds = self.n_folds +# if len(self.y.shape) > 1: +# y1dim = [int(reduce(lambda o,n: o + str(n), yy, '0'),2) for yy in self.y] +# else: +# y1dim = self.y +# idx = np.argsort(y1dim) +# for i in range(n_folds): +# yield idx[i::n_folds] +#cross_validation.StratifiedKFold._iter_test_indices = _StratifiedKFold_multilabel_iter_test_indices + +#from sklearn import metrics +#classifier = OneVsRestClassifier(svm.SVC(kernel='linear', C=1)) +#scores = cross_validation.cross_val_score(classifier, learn_data, labels) #, scoring='f1_weighted') +#print "Accuracy: %0.2f (+/- %0.2f)" % (scores.mean(), scores.std() * 2) From 29e6ff9aed5f28236adc5884173286dd8e3f7f76 Mon Sep 17 00:00:00 2001 From: ofir Date: Mon, 7 Dec 2015 22:45:10 +0200 Subject: [PATCH 07/15] should use prec and recall for scoring? --- laws/tags_autolearn_play.py | 93 ++++++++++++++++++++----------------- 1 file changed, 51 insertions(+), 42 deletions(-) diff --git a/laws/tags_autolearn_play.py b/laws/tags_autolearn_play.py index 2e0332b3..83a70d05 100644 --- a/laws/tags_autolearn_play.py +++ b/laws/tags_autolearn_play.py @@ -95,9 +95,10 @@ def plot_subfigure(X, Y, subplot, title, transform): lb = LabelBinarizer() lb.fit(tags) +TRIM_SAMPLES = 100 -tags = tags[:100] -learn_data = learn_data[:100] +tags = tags[:TRIM_SAMPLES] +learn_data = learn_data[:TRIM_SAMPLES] labels = lb.transform(tags) @@ -111,48 +112,56 @@ def plot_subfigure(X, Y, subplot, title, transform): from sklearn.multiclass import OneVsRestClassifier def _OneVsRestClassifier_multilabel_score(self, X, y): - if self.multilabel_: - from sklearn.metrics import accuracy_score - - return np.array([accuracy_score(yy, XX) for yy, XX in zip(y.transpose(), self.predict(X).transpose())]).mean() - else: - return super(OneVsRestClassifier, self).score(X, y) + from sklearn.metrics import accuracy_score + # should use sklearn.metrics.precision_recall_fscore_support ??? + yp = self.predict(X) + import pdb; pdb.set_trace() + a = accuracy_score(y, yp) + return a + + return super(OneVsRestClassifier, self).score(X, y) +# if self.multilabel_: +# from sklearn.metrics import accuracy_score +# +# return np.array([accuracy_score(yy, XX) for yy, XX in zip(y.transpose(), self.predict(X).transpose())]).mean() +# else: +# return super(OneVsRestClassifier, self).score(X, y) OneVsRestClassifier.score = _OneVsRestClassifier_multilabel_score from sklearn.svm import LinearSVC -print >>sys.stderr, 'learning data'; sys.stderr.flush() -classifier = OneVsRestClassifier(svm.SVC(kernel='linear')) -trained_classifier = classifier.fit(learn_data[:-10], labels[:-10]) -print >>sys.stderr, 'classifying'; sys.stderr.flush() -predicted_labels = trained_classifier.predict(learn_data[-10:]) -predicted_tags = lb.inverse_transform(predicted_labels) - -from pprint import pprint -for _predicted_tags, _tags in zip(predicted_tags, tags[-10:]): - for _predicted_tag in _predicted_tags: - print _predicted_tag[::-1], ",", - print "-", - for _tag in _tags: - print _tag[::-1], ",", - print "" -import pdb; pdb.set_trace() - -#from sklearn import cross_validation - -#def _StratifiedKFold_multilabel_iter_test_indices(self): -# n_folds = self.n_folds -# if len(self.y.shape) > 1: -# y1dim = [int(reduce(lambda o,n: o + str(n), yy, '0'),2) for yy in self.y] -# else: -# y1dim = self.y -# idx = np.argsort(y1dim) -# for i in range(n_folds): -# yield idx[i::n_folds] -#cross_validation.StratifiedKFold._iter_test_indices = _StratifiedKFold_multilabel_iter_test_indices - -#from sklearn import metrics -#classifier = OneVsRestClassifier(svm.SVC(kernel='linear', C=1)) -#scores = cross_validation.cross_val_score(classifier, learn_data, labels) #, scoring='f1_weighted') -#print "Accuracy: %0.2f (+/- %0.2f)" % (scores.mean(), scores.std() * 2) +#print >>sys.stderr, 'learning data'; sys.stderr.flush() +#classifier = OneVsRestClassifier(svm.SVC(kernel='linear')) +#trained_classifier = classifier.fit(learn_data[:-10], labels[:-10]) +#print >>sys.stderr, 'classifying'; sys.stderr.flush() +#predicted_labels = trained_classifier.predict(learn_data[-10:]) +#predicted_tags = lb.inverse_transform(predicted_labels) +# +#from pprint import pprint +#for _predicted_tags, _tags in zip(predicted_tags, tags[-10:]): +# for _predicted_tag in _predicted_tags: +# print _predicted_tag[::-1], ",", +# print "-", +# for _tag in _tags: +# print _tag[::-1], ",", +# print "" +#import pdb; pdb.set_trace() + +from sklearn import cross_validation + +def _StratifiedKFold_multilabel_iter_test_indices(self): + n_folds = self.n_folds + if len(self.y.shape) > 1: + y1dim = [int(reduce(lambda o,n: o + str(n), yy, '0'),2) for yy in self.y] + else: + y1dim = self.y + idx = np.argsort(y1dim) + for i in range(n_folds): + yield idx[i::n_folds] +cross_validation.StratifiedKFold._iter_test_indices = _StratifiedKFold_multilabel_iter_test_indices + +from sklearn import metrics +classifier = OneVsRestClassifier(svm.SVC(kernel='linear', C=1)) +scores = cross_validation.cross_val_score(classifier, learn_data, labels) #, scoring='f1_weighted') +print "Accuracy: %0.2f (+/- %0.2f)" % (scores.mean(), scores.std() * 2) From 7259f5282292e7a1a78ba6a345279da9a41a2b1e Mon Sep 17 00:00:00 2001 From: ofir Date: Sat, 12 Dec 2015 17:28:36 -0800 Subject: [PATCH 08/15] better preprocessing, splitting on special characters, instead of removing them. should be better preprocessing.\nadded html build to see tag-keyword relations.\nadded rst file for autotagging. --- docs/devel/source/autotagging.rst | 65 ++++++++++++++ laws/build_important_keyword_html.py | 124 +++++++++++++++++++++++++++ laws/make_histograms.py | 13 ++- laws/play_with_trained_classifier.py | 20 +++++ laws/tags_autolearn_play.py | 91 +++++++++----------- 5 files changed, 261 insertions(+), 52 deletions(-) create mode 100644 docs/devel/source/autotagging.rst create mode 100644 laws/build_important_keyword_html.py create mode 100644 laws/play_with_trained_classifier.py diff --git a/docs/devel/source/autotagging.rst b/docs/devel/source/autotagging.rst new file mode 100644 index 00000000..80f15c38 --- /dev/null +++ b/docs/devel/source/autotagging.rst @@ -0,0 +1,65 @@ +.. autotagging: + +======= +Intro +======= + +Auto tagging is used to create auto tags out of the explanition info of a private proposal. +It uses machine learning techniques. + +====== +Flow +====== + +* Reading - Read the data from the server using API. +* Parsing - Parse the downloaded HTML files of the PP. +* Building - Make histograms out of the parsed HTML. +* Train/Test the information. + +Reading +========= + +Run: + +.. code-block:: sh + + python read_pp.py + +Parsing +========= + +Run: + +.. code-block:: sh + + python parse_htmls.py + +Building +========== + +Run: + +.. code-block:: sh + + python make_histograms.py + +Training/Testing +================== + +Auto tagging is in testing phase, so no auto-tagging script exist. +To execute the test, run: + +.. code-block:: sh + + python tags_autolearn_play.py + +Building HTML +=============== + +This is used to understand to find errors in the tagging process. Just use your logic here. +To execute the test, run: + +.. code-block:: sh + + python build_important_keyword_html.py + diff --git a/laws/build_important_keyword_html.py b/laws/build_important_keyword_html.py new file mode 100644 index 00000000..3f28f87b --- /dev/null +++ b/laws/build_important_keyword_html.py @@ -0,0 +1,124 @@ +#!/usr/bin/python +# -*- coding: UTF-8 -*- +import numpy +from sklearn.externals import joblib +import cPickle as pickle +from collections import defaultdict +important_keywords = pickle.load(open('meaningful_keyword_for_tag.pkl', 'rb')) +lb = pickle.load(open("classifier_data/label_binarizer.pkl", "rb")) +keywords, data = pickle.load(open('histograms_with_tags.pkl', 'rb')) +trained_classifier = joblib.load('classifier_data/linear_svc_classifier.jlb') +keywords = numpy.array(sorted(list(keywords))) +import os +import codecs +try: + os.makedirs("important_keywords/tags") +except OSError, e: + pass +try: + os.makedirs("important_keywords/keywords") +except OSError, e: + pass + +doc_in_class = defaultdict(list) + +print "creating tag dicts" +for a in data: + for tag in a['tags']: + doc_in_class[tag].append(a) + + +REMOVE = [u'"', u"'", u'/', u'\\', u',', u'.', u'!', u'@', u'#', u'$', u'%', u'^', u'&', u'*', u'(', u')', u'-', u'=', u'_', u'+', u'\n', u'[', u']', u'{', u'}', u'|', u'<', u'>', u'?', u'~', u":", u';', u'–' , u'”', u'`'] +REMOVE.extend([unicode(i) for i in xrange(10)]) + +print "creating keyword dict" +keyword_dict = {} +for i,w in enumerate(keywords): + keyword_dict[w] = i + + +print "splitting docs" +pp_with_tags = pickle.load(open('pp_with_tags.pkl', "rb")) +pp_dict = {} +for pp in pp_with_tags: + pp_text, pp_tag, pp_id = pp['text'], pp['tags'], pp['id'] + pp_dict[pp_id] = reduce(lambda o, s: o.replace(s, u' '), REMOVE, pp_text).split() # [reduce(lambda o, s: o.replace(s, u''), REMOVE, w) for w in pp_text.split()] + +def output_tag(class_file, w, estimator_max, estimator_min, weight, hyperlink = False): + orig_weight = weight + pre = post = '' + if weight > 0 and estimator_max != 0: + weight /= estimator_max + if weight > 0.05 and hyperlink: + pre = u"" + post = u"" + print >>class_file, pre + u"" % (int(255 * weight),) + w + u"" + post + " " + elif weight < 0 and estimator_min != 0: + weight /= estimator_min + if weight > 0.05 and hyperlink: + pre = u"" + post = u"" + print >>class_file, pre + u"" % (int(255 * weight),) + w + u"" + post + " " + else: + print >>class_file, u"" + w + u" " + +def output_word(class_file, w, estimator_max, estimator_min, estimator, keyword_dict, hyperlink = False): + weight = estimator.coef_[0, keyword_dict[w]] + pre = post = '' + if weight > 0 and estimator_max != 0: + weight /= estimator_max + if weight > 0.05 and hyperlink: + pre = u"" + post = u"" + print >>class_file, pre + u"" % (int(255 * weight),) + w + u"" + post + " " + elif weight < 0 and estimator_min != 0: + weight /= estimator_min + if weight > 0.05 and hyperlink: + pre = u"" + post = u"" + print >>class_file, pre + u"" % (int(255 * weight),) + w + u"" + post + " " + else: + print >>class_file, u"" + w + u" " + +print "creating files" +for i, (class_name, important_keyword_for_class, estimator) in enumerate(zip(lb.classes_, important_keywords, trained_classifier.estimators_)): + print "tag", str(i), "/", str(len(lb.classes_)) + class_file = codecs.open("important_keywords/tags/" + class_name.replace("/", ' ') + ".html", "wt", encoding="utf-8") + + print >>class_file, u"Keyword Explanation for Class " + class_name + u"" + print >>class_file, u"

" + class_name + u"


" + print >>class_file, u"

Most influencing words

" + estimator_min, estimator_max = estimator.coef_[0].min(), estimator.coef_[0].max() + for keyword in important_keyword_for_class: + output_word(class_file, keyword, estimator_max, estimator_min, estimator, keyword_dict) + print >>class_file, u"(", str(estimator.coef_[0, keyword_dict[keyword]]), u")
" + print >>class_file, u"" + for doc in doc_in_class[class_name]: + + print >>class_file, u"" + print >>class_file, u"
" + unicode(doc['id']) + u"" + + for w in pp_dict[doc['id']]: + output_word(class_file, w, estimator_max, estimator_min, estimator, keyword_dict, hyperlink = True) + print >>class_file, u"
" + class_file.close() + +coef_abs = numpy.abs(trained_classifier.coef_) +sorted_keywords = coef_abs.argsort(axis = 0) +min_weight, max_weight = trained_classifier.coef_.min(axis = 1), trained_classifier.coef_.max(axis = 1) +for keyword_count, i in enumerate(coef_abs.sum(axis = 0).argsort()[::-1]): + keyword = keywords[i] + print "keyword", str(keyword_count), "/", str(len(keywords)), "(", str(i), ")" + keyword_file = codecs.open("important_keywords/keywords/" + keyword.replace("/", ' ') + ".html", "wt", encoding="utf-8") + print >>keyword_file, u"Tag Explanation for Keyword " + keyword + u"" + print >>keyword_file, u"

" + keyword + u"


" + print >>keyword_file, u"

Most influenced tags

" + weights = trained_classifier.coef_[:,i] + class_keywords = sorted_keywords[::-1,i] + + for class_name, weight, max_weight_, min_weight_ in zip(lb.classes_[class_keywords], weights[class_keywords], max_weight[class_keywords], min_weight[class_keywords]): + if weight != 0: + output_tag(keyword_file, class_name, max_weight_, min_weight_, weight, hyperlink = True) + print >>keyword_file, u"
" + print >>keyword_file, u"" + keyword_file.close() diff --git a/laws/make_histograms.py b/laws/make_histograms.py index d6c9ce48..e1cdf31e 100644 --- a/laws/make_histograms.py +++ b/laws/make_histograms.py @@ -1,15 +1,22 @@ +#!/usr/bin/python +# -*- coding: UTF-8 -*- import cPickle as pickle from collections import defaultdict import re split_re = re.compile(r'\s+', re.U) -REMOVE = ['"', "'", '/', '\\', ',', '.', '!', '@', '#', '$', '%', '^', '&', '*', '(', ')', '-', '=', '_', '+', '\n', '[', ']', '{', '}', '|', '<', '>', '?', '~', ":", ';'] +#REMOVE = ['"', "'", '/', '\\', ',', '.', '!', '@', '#', '$', '%', '^', '&', '*', '(', ')', '-', '=', '_', '+', '\n', '[', ']', '{', '}', '|', '<', '>', '?', '~', ":", ';'] +REMOVE = [u'"', u"'", u'/', u'\\', u',', u'.', u'!', u'@', u'#', u'$', u'%', u'^', u'&', u'*', u'(', u')', u'-', u'=', u'_', u'+', u'\n', u'[', u']', u'{', u'}', u'|', u'<', u'>', u'?', u'~', u":", u';', u'–' , u'”', u'`'] +REMOVE.extend([unicode(i) for i in xrange(10)]) + REMOVE.extend([str(i) for i in xrange(10)]) def make_histogram(t): dd = defaultdict(int) - for w in t.split(): - dd[reduce(lambda o, s: o.replace(s, ''), REMOVE, w)] += 1 +# for w in t.split(): +# dd[reduce(lambda o, s: o.replace(s, ' '), REMOVE, w)] += 1 + for w in reduce(lambda o, s: o.replace(s, u' '), REMOVE, t).split(): + dd[w] += 1 return dict(dd) diff --git a/laws/play_with_trained_classifier.py b/laws/play_with_trained_classifier.py new file mode 100644 index 00000000..a7e6733d --- /dev/null +++ b/laws/play_with_trained_classifier.py @@ -0,0 +1,20 @@ +import numpy +from sklearn.externals import joblib +import cPickle as pickle + +ATTR_AMOUNT = 10 + +trained_classifier = joblib.load('classifier_data/linear_svc_classifier.jlb') +lb = pickle.load(open("classifier_data/label_binarizer.pkl", "rb")) +keywords, data = pickle.load(open('histograms_with_tags.pkl', 'rb')) +keywords = numpy.array(sorted(list(keywords))) + + +#numpy.abs(trained_classifier.coef_).argsort()[:,:10] +important_keywords = keywords[numpy.abs(trained_classifier.coef_).argsort()[:,-ATTR_AMOUNT:]] +pickle.dump(important_keywords, open('meaningful_keyword_for_tag.pkl', 'wb')) + +#for estimator in trained_classifier.estimators_: +# keywords[numpy.abs(estimator.coef_).argsort()[0][:ATTR_AMOUNT]] + +import IPython; IPython.embed() diff --git a/laws/tags_autolearn_play.py b/laws/tags_autolearn_play.py index 83a70d05..0669929c 100644 --- a/laws/tags_autolearn_play.py +++ b/laws/tags_autolearn_play.py @@ -69,9 +69,10 @@ def plot_subfigure(X, Y, subplot, title, transform): import numpy import cPickle as pickle import sys -from sklearn import svm + print >>sys.stderr, 'loading data'; sys.stderr.flush() keywords, data = pickle.load(open('histograms_with_tags.pkl', 'rb')) +keywords = sorted(list(keywords)) tags = numpy.array([a['tags'] for a in data]) ids = [a['id'] for a in data] @@ -80,28 +81,32 @@ def plot_subfigure(X, Y, subplot, title, transform): for i,w in enumerate(keywords): keyword_dict[w] = i - -learn_data = [] -for a in data: - histogram = [0,] * len(keywords) +learn_data = numpy.zeros((len(data), len(keywords))) +for i,a in enumerate(data): + histogram = learn_data[i] for k,v in a['histogram'].iteritems(): histogram[keyword_dict[k]] = v - learn_data.append(numpy.array(histogram)) -learn_data = numpy.array(learn_data) -from sklearn.preprocessing import LabelBinarizer +from sklearn import svm +#from sklearn.preprocessing import LabelBinarizer +from sklearn.preprocessing import MultiLabelBinarizer print >>sys.stderr, 'vectorizing labels'; sys.stderr.flush() -lb = LabelBinarizer() -lb.fit(tags) +#lb = LabelBinarizer() +lb = MultiLabelBinarizer() -TRIM_SAMPLES = 100 +TRIM_SAMPLES = len(tags) #/ 10 tags = tags[:TRIM_SAMPLES] learn_data = learn_data[:TRIM_SAMPLES] +lb.fit(tags) labels = lb.transform(tags) +print "using\t",TRIM_SAMPLES,"samples" +print "\t",len(keywords),"keywords" +print "\t",len(lb.classes_),"tags" + #plt.figure(figsize=(8, 6)) #plot_subfigure(learn_data, labels, 1, "With unlabeled samples + CCA", "cca") @@ -109,59 +114,47 @@ def plot_subfigure(X, Y, subplot, title, transform): #plt.subplots_adjust(.04, .02, .97, .94, .09, .2) #plt.show() - from sklearn.multiclass import OneVsRestClassifier -def _OneVsRestClassifier_multilabel_score(self, X, y): - from sklearn.metrics import accuracy_score - # should use sklearn.metrics.precision_recall_fscore_support ??? - yp = self.predict(X) - import pdb; pdb.set_trace() - a = accuracy_score(y, yp) - return a - - return super(OneVsRestClassifier, self).score(X, y) -# if self.multilabel_: -# from sklearn.metrics import accuracy_score -# -# return np.array([accuracy_score(yy, XX) for yy, XX in zip(y.transpose(), self.predict(X).transpose())]).mean() -# else: -# return super(OneVsRestClassifier, self).score(X, y) -OneVsRestClassifier.score = _OneVsRestClassifier_multilabel_score - -from sklearn.svm import LinearSVC +#single_svc = svm.SVC(kernel='polynomial', C=4) + +#single_svc = svm.SVC(kernel='linear', cache_size = 2048) + +single_svc = svm.LinearSVC() + +#single_svc = svm.SVC(kernel='rbf', cache_size = 2048) +classifier = OneVsRestClassifier(single_svc) + +trained_classifier = classifier.fit(learn_data, labels) +from sklearn.externals import joblib +import cPickle as pickle +import os +try: + os.makedirs('classifier_data') +except OSError, e: + pass +joblib.dump(trained_classifier, 'classifier_data/linear_svc_classifier.jlb') +pickle.dump(lb, open("classifier_data/label_binarizer.pkl", "wb")) #print >>sys.stderr, 'learning data'; sys.stderr.flush() -#classifier = OneVsRestClassifier(svm.SVC(kernel='linear')) #trained_classifier = classifier.fit(learn_data[:-10], labels[:-10]) #print >>sys.stderr, 'classifying'; sys.stderr.flush() #predicted_labels = trained_classifier.predict(learn_data[-10:]) #predicted_tags = lb.inverse_transform(predicted_labels) # #from pprint import pprint -#for _predicted_tags, _tags in zip(predicted_tags, tags[-10:]): +#for _predicted_tags, _tags, _d in zip(predicted_tags, tags[-10:], data[-10:]): +# print _d['id'],'--', # for _predicted_tag in _predicted_tags: # print _predicted_tag[::-1], ",", # print "-", # for _tag in _tags: # print _tag[::-1], ",", # print "" -#import pdb; pdb.set_trace() -from sklearn import cross_validation +import IPython; IPython.embed() -def _StratifiedKFold_multilabel_iter_test_indices(self): - n_folds = self.n_folds - if len(self.y.shape) > 1: - y1dim = [int(reduce(lambda o,n: o + str(n), yy, '0'),2) for yy in self.y] - else: - y1dim = self.y - idx = np.argsort(y1dim) - for i in range(n_folds): - yield idx[i::n_folds] -cross_validation.StratifiedKFold._iter_test_indices = _StratifiedKFold_multilabel_iter_test_indices - -from sklearn import metrics -classifier = OneVsRestClassifier(svm.SVC(kernel='linear', C=1)) -scores = cross_validation.cross_val_score(classifier, learn_data, labels) #, scoring='f1_weighted') -print "Accuracy: %0.2f (+/- %0.2f)" % (scores.mean(), scores.std() * 2) +#from sklearn import cross_validation +#from sklearn import metrics +#scores = cross_validation.cross_val_score(classifier, learn_data, labels, scoring='f1_weighted') +#print "Accuracy: %0.2f (+/- %0.2f)" % (scores.mean(), scores.std() * 2) From ddac7b5f9761d8138a46a706446226936bede919 Mon Sep 17 00:00:00 2001 From: ofir Date: Mon, 14 Dec 2015 22:28:19 +0200 Subject: [PATCH 09/15] removing unneeded data in preprocessing (previous posted pp). added POS tagging before feature extraction to histograms --- laws/make_histograms.py | 22 ++++++++++++-- laws/parse_htmls.py | 59 +++++++++++++++++++++++++++++++++++++ laws/tags_autolearn_play.py | 12 +++++--- 3 files changed, 87 insertions(+), 6 deletions(-) diff --git a/laws/make_histograms.py b/laws/make_histograms.py index e1cdf31e..cf734f61 100644 --- a/laws/make_histograms.py +++ b/laws/make_histograms.py @@ -9,16 +9,32 @@ #REMOVE = ['"', "'", '/', '\\', ',', '.', '!', '@', '#', '$', '%', '^', '&', '*', '(', ')', '-', '=', '_', '+', '\n', '[', ']', '{', '}', '|', '<', '>', '?', '~', ":", ';'] REMOVE = [u'"', u"'", u'/', u'\\', u',', u'.', u'!', u'@', u'#', u'$', u'%', u'^', u'&', u'*', u'(', u')', u'-', u'=', u'_', u'+', u'\n', u'[', u']', u'{', u'}', u'|', u'<', u'>', u'?', u'~', u":", u';', u'–' , u'”', u'`'] REMOVE.extend([unicode(i) for i in xrange(10)]) - REMOVE.extend([str(i) for i in xrange(10)]) + + +ACCEPTED_POS = ['NN', 'JJ', 'VB'] + def make_histogram(t): dd = defaultdict(int) # for w in t.split(): # dd[reduce(lambda o, s: o.replace(s, ' '), REMOVE, w)] += 1 - for w in reduce(lambda o, s: o.replace(s, u' '), REMOVE, t).split(): +# for w in reduce(lambda o, s: o.replace(s, u' '), REMOVE, t).split(): + for w in [ppp.split('\t')[1].decode('utf-8') for ppp in t.splitlines() if ppp.count('\t') > 5 and (ppp.split('\t')[3] in ACCEPTED_POS or ppp.split('\t')[4] in ACCEPTED_POS)]: dd[w] += 1 return dict(dd) +# needed phrases: NN, JJ, VB +# Use this: +# http://www.cs.bgu.ac.il/~yoavg/depparse/gparse +# or this: +# http://www.cs.bgu.ac.il/~yoavg/constparse/gparse +# or this: +# http://www.cs.bgu.ac.il/~nlpproj/demo/ +# from here: +# http://www.cs.bgu.ac.il/~nlpproj/ +# seems like: +# http://www.cs.bgu.ac.il/~yoavg/software/hebparsers/hebdepparser/hebdepparser.tgz +# is working good enough pp_with_tags = pickle.load(open('pp_with_tags.pkl', "rb")) @@ -26,6 +42,8 @@ def make_histogram(t): data = [] for i, pp in enumerate(pp_with_tags): pp_text, pp_tag, pp_id = pp['text'], pp['tags'], pp['id'] + + try: histogram = make_histogram(pp_text) except: diff --git a/laws/parse_htmls.py b/laws/parse_htmls.py index ba83a263..5e616f98 100644 --- a/laws/parse_htmls.py +++ b/laws/parse_htmls.py @@ -1,9 +1,60 @@ +#!/usr/bin/python +# -*- coding: UTF-8 -*- + from BeautifulSoup import BeautifulSoup import cPickle as pickle +similar_texts = [ + u"הצעת חוק דומה בעיקרה הונחה", + u"הצעת חוק זהה הונחה", + u"הצעות חוק דומות בעיקרן הונחו", +] pp_without_tags = [] pp_with_tags = [] + + +import subprocess +import sys + +def parse_lang_tree(text): +# p = subprocess.Popen(["python", "/home/o/otadmor/Downloads/hebdepparser/parse.py"], stdin=subprocess.PIPE, stdout=subprocess.PIPE) + p = subprocess.Popen("python /home/o/otadmor/Downloads/hebdepparser/code/utils/sentences.py | python /home/o/otadmor/Downloads/hebdepparser/parse.py", shell=True, stdin=subprocess.PIPE, stdout=subprocess.PIPE) + + p.stdin.write(text.encode('utf-8')) + + p.stdin.flush() + p.stdin.close() + a = p.stdout.read() + res = '' + while a != '': + res += a + a = p.stdout.read() + p.stdout.close() + p.wait() + return res + +REMOVE = [u'"', u"'", u'/', u'\\', u',', u'.', u'!', u'@', u'#', u'$', u'%', u'^', u'&', u'*', u'(', u')', u'-', u'=', u'_', u'+', u'\n', u'[', u']', u'{', u'}', u'|', u'<', u'>', u'?', u'~', u":", u';', u'–' , u'”', u'`'] +REMOVE.extend([unicode(i) for i in xrange(10)]) +REMOVE.extend([str(i) for i in xrange(10)]) + + +ACCEPTED_POS = ['NN', 'JJ', 'VB'] + +# for w in reduce(lambda o, s: o.replace(s, u' '), REMOVE, t).split(): + + +import sys +sys.path.append("/home/o/otadmor/Downloads/hebdepparser/") +#sys.path.append("/home/o/otadmor/Downloads/hebdepparser/code/utils/") +from parse import parse_sent +from cStringIO import StringIO +def parse_lang_tree(text): + text = '\n'.join([l.strip() for l in reduce(lambda o, s: o.replace(s, s + u'\n'), ['.', '!', '?'], text).splitlines()]) + out = StringIO() + parse_sent(text, out) + return out.getvalue() + for i, pp in enumerate(pickle.load(open('pps.pkl', 'rb'))): content_html = pp['content_html'] parsed_html = BeautifulSoup(content_html) @@ -21,11 +72,19 @@ tags = pp['tags'] + for similar_text in similar_texts: + similar_loc = text.find(similar_text) + if similar_loc != -1: + text = text[:similar_loc] + + text = parse_lang_tree(text) + if len(tags) == 0: pp_without_tags.append({'text' : text, 'id' : pp['id']}) else: pp_with_tags.append({'text' : text, 'id' : pp['id'], 'tags' : tags}) + pickle.dump(pp_without_tags, open("pp_without_tags.pkl", "wb")) pickle.dump(pp_with_tags, open("pp_with_tags.pkl", "wb")) diff --git a/laws/tags_autolearn_play.py b/laws/tags_autolearn_play.py index 0669929c..42d73595 100644 --- a/laws/tags_autolearn_play.py +++ b/laws/tags_autolearn_play.py @@ -88,12 +88,16 @@ def plot_subfigure(X, Y, subplot, title, transform): histogram[keyword_dict[k]] = v from sklearn import svm -#from sklearn.preprocessing import LabelBinarizer -from sklearn.preprocessing import MultiLabelBinarizer +# + print >>sys.stderr, 'vectorizing labels'; sys.stderr.flush() -#lb = LabelBinarizer() -lb = MultiLabelBinarizer() +try: + from sklearn.preprocessing import MultiLabelBinarizer + lb = MultiLabelBinarizer() +except ImportError, e: + from sklearn.preprocessing import LabelBinarizer + lb = LabelBinarizer() TRIM_SAMPLES = len(tags) #/ 10 From 7140873dde437697a61d62298d3db8c8859a79da Mon Sep 17 00:00:00 2001 From: ofir Date: Mon, 21 Dec 2015 07:28:52 -0800 Subject: [PATCH 10/15] many changes - 1)parse pos tags is working 2) nicer html builder 3) tested with different classifiers --- laws/parse_htmls.py | 90 ------------- laws/play_with_trained_classifier.py | 20 --- .../build_important_keyword_html.py | 80 ++++++++++-- laws/{ => tags}/make_histograms.py | 30 +++-- laws/tags/parse_htmls.py | 77 +++++++++++ laws/tags/parse_pos_tag.py | 68 ++++++++++ laws/{ => tags}/read_pp.py | 0 laws/tags/results.txt | 19 +++ laws/{ => tags}/tags_autolearn_play.py | 122 +++++++++++++++--- 9 files changed, 353 insertions(+), 153 deletions(-) delete mode 100644 laws/parse_htmls.py delete mode 100644 laws/play_with_trained_classifier.py rename laws/{ => tags}/build_important_keyword_html.py (63%) rename laws/{ => tags}/make_histograms.py (68%) create mode 100644 laws/tags/parse_htmls.py create mode 100644 laws/tags/parse_pos_tag.py rename laws/{ => tags}/read_pp.py (100%) create mode 100644 laws/tags/results.txt rename laws/{ => tags}/tags_autolearn_play.py (56%) diff --git a/laws/parse_htmls.py b/laws/parse_htmls.py deleted file mode 100644 index 5e616f98..00000000 --- a/laws/parse_htmls.py +++ /dev/null @@ -1,90 +0,0 @@ -#!/usr/bin/python -# -*- coding: UTF-8 -*- - -from BeautifulSoup import BeautifulSoup - -import cPickle as pickle - -similar_texts = [ - u"הצעת חוק דומה בעיקרה הונחה", - u"הצעת חוק זהה הונחה", - u"הצעות חוק דומות בעיקרן הונחו", -] -pp_without_tags = [] -pp_with_tags = [] - - -import subprocess -import sys - -def parse_lang_tree(text): -# p = subprocess.Popen(["python", "/home/o/otadmor/Downloads/hebdepparser/parse.py"], stdin=subprocess.PIPE, stdout=subprocess.PIPE) - p = subprocess.Popen("python /home/o/otadmor/Downloads/hebdepparser/code/utils/sentences.py | python /home/o/otadmor/Downloads/hebdepparser/parse.py", shell=True, stdin=subprocess.PIPE, stdout=subprocess.PIPE) - - p.stdin.write(text.encode('utf-8')) - - p.stdin.flush() - p.stdin.close() - a = p.stdout.read() - res = '' - while a != '': - res += a - a = p.stdout.read() - p.stdout.close() - p.wait() - return res - -REMOVE = [u'"', u"'", u'/', u'\\', u',', u'.', u'!', u'@', u'#', u'$', u'%', u'^', u'&', u'*', u'(', u')', u'-', u'=', u'_', u'+', u'\n', u'[', u']', u'{', u'}', u'|', u'<', u'>', u'?', u'~', u":", u';', u'–' , u'”', u'`'] -REMOVE.extend([unicode(i) for i in xrange(10)]) -REMOVE.extend([str(i) for i in xrange(10)]) - - -ACCEPTED_POS = ['NN', 'JJ', 'VB'] - -# for w in reduce(lambda o, s: o.replace(s, u' '), REMOVE, t).split(): - - -import sys -sys.path.append("/home/o/otadmor/Downloads/hebdepparser/") -#sys.path.append("/home/o/otadmor/Downloads/hebdepparser/code/utils/") -from parse import parse_sent -from cStringIO import StringIO -def parse_lang_tree(text): - text = '\n'.join([l.strip() for l in reduce(lambda o, s: o.replace(s, s + u'\n'), ['.', '!', '?'], text).splitlines()]) - out = StringIO() - parse_sent(text, out) - return out.getvalue() - -for i, pp in enumerate(pickle.load(open('pps.pkl', 'rb'))): - content_html = pp['content_html'] - parsed_html = BeautifulSoup(content_html) - - current_tag = parsed_html.find('p', attrs={'class':'explanation-header'}) - if current_tag is None: - print "no text for pp", pp['id'] - continue - current_tag = current_tag.findNext() - text = '' - while current_tag is not None and '---' not in current_tag.text: - - text += current_tag.text - current_tag = current_tag.findNext() - - tags = pp['tags'] - - for similar_text in similar_texts: - similar_loc = text.find(similar_text) - if similar_loc != -1: - text = text[:similar_loc] - - text = parse_lang_tree(text) - - if len(tags) == 0: - pp_without_tags.append({'text' : text, 'id' : pp['id']}) - else: - pp_with_tags.append({'text' : text, 'id' : pp['id'], 'tags' : tags}) - - -pickle.dump(pp_without_tags, open("pp_without_tags.pkl", "wb")) -pickle.dump(pp_with_tags, open("pp_with_tags.pkl", "wb")) - diff --git a/laws/play_with_trained_classifier.py b/laws/play_with_trained_classifier.py deleted file mode 100644 index a7e6733d..00000000 --- a/laws/play_with_trained_classifier.py +++ /dev/null @@ -1,20 +0,0 @@ -import numpy -from sklearn.externals import joblib -import cPickle as pickle - -ATTR_AMOUNT = 10 - -trained_classifier = joblib.load('classifier_data/linear_svc_classifier.jlb') -lb = pickle.load(open("classifier_data/label_binarizer.pkl", "rb")) -keywords, data = pickle.load(open('histograms_with_tags.pkl', 'rb')) -keywords = numpy.array(sorted(list(keywords))) - - -#numpy.abs(trained_classifier.coef_).argsort()[:,:10] -important_keywords = keywords[numpy.abs(trained_classifier.coef_).argsort()[:,-ATTR_AMOUNT:]] -pickle.dump(important_keywords, open('meaningful_keyword_for_tag.pkl', 'wb')) - -#for estimator in trained_classifier.estimators_: -# keywords[numpy.abs(estimator.coef_).argsort()[0][:ATTR_AMOUNT]] - -import IPython; IPython.embed() diff --git a/laws/build_important_keyword_html.py b/laws/tags/build_important_keyword_html.py similarity index 63% rename from laws/build_important_keyword_html.py rename to laws/tags/build_important_keyword_html.py index 3f28f87b..8428d8e3 100644 --- a/laws/build_important_keyword_html.py +++ b/laws/tags/build_important_keyword_html.py @@ -1,12 +1,14 @@ #!/usr/bin/python # -*- coding: UTF-8 -*- import numpy +import re +from functools import partial from sklearn.externals import joblib import cPickle as pickle from collections import defaultdict -important_keywords = pickle.load(open('meaningful_keyword_for_tag.pkl', 'rb')) lb = pickle.load(open("classifier_data/label_binarizer.pkl", "rb")) keywords, data = pickle.load(open('histograms_with_tags.pkl', 'rb')) +pp_with_tags = pickle.load(open('pp_with_tags.pkl', "rb")) trained_classifier = joblib.load('classifier_data/linear_svc_classifier.jlb') keywords = numpy.array(sorted(list(keywords))) import os @@ -23,13 +25,11 @@ doc_in_class = defaultdict(list) print "creating tag dicts" -for a in data: +for a in pp_with_tags: for tag in a['tags']: doc_in_class[tag].append(a) -REMOVE = [u'"', u"'", u'/', u'\\', u',', u'.', u'!', u'@', u'#', u'$', u'%', u'^', u'&', u'*', u'(', u')', u'-', u'=', u'_', u'+', u'\n', u'[', u']', u'{', u'}', u'|', u'<', u'>', u'?', u'~', u":", u';', u'–' , u'”', u'`'] -REMOVE.extend([unicode(i) for i in xrange(10)]) print "creating keyword dict" keyword_dict = {} @@ -41,8 +41,7 @@ pp_with_tags = pickle.load(open('pp_with_tags.pkl', "rb")) pp_dict = {} for pp in pp_with_tags: - pp_text, pp_tag, pp_id = pp['text'], pp['tags'], pp['id'] - pp_dict[pp_id] = reduce(lambda o, s: o.replace(s, u' '), REMOVE, pp_text).split() # [reduce(lambda o, s: o.replace(s, u''), REMOVE, w) for w in pp_text.split()] + pp_dict[pp['id']] = pp['text'] def output_tag(class_file, w, estimator_max, estimator_min, weight, hyperlink = False): orig_weight = weight @@ -80,6 +79,62 @@ def output_word(class_file, w, estimator_max, estimator_min, estimator, keyword_ else: print >>class_file, u"" + w + u" " + +def format_word(w, estimator_max, estimator_min, estimator, keyword_dict, hyperlink = False): + weight = estimator.coef_[0, keyword_dict[w]] + pre = post = '' + if weight > 0 and estimator_max != 0: + weight /= estimator_max + if hyperlink: + pre = u"" + post = u"" + v = u"" % (int(255 * weight),) + w + u"" + elif weight < 0 and estimator_min != 0: + weight /= estimator_min + if hyperlink: + pre = u"" + post = u"" + v = u"" % (int(255 * weight),) + w + u"" + else: + v = u"" + w + u"" + return u"%s%s%s" % (pre, v, post,) + + +ATTR_AMOUNT = 10 +important_keywords = keywords[numpy.abs(trained_classifier.coef_).argsort()[:,-ATTR_AMOUNT:]] + +MAX_RE_GROUP = 100 +def create_re(estimator_max, estimator_min, estimator, keyword_dict): + #return ("|".join(p) for p in zip(*( + # ( + # u"(?P\\b)%s(?P\\b)" % (i, re.escape(k), i, ), + # u"\\g%s\\g" % (i, re.escape(format_word(k, estimator_max, estimator_min, estimator, keyword_dict, hyperlink = True)), i, ), + # ) + # for i, k + # in enumerate(keywords[numpy.where(estimator.coef_[0] != 0)]) + # ) + #)) + + return { + k : format_word(k, estimator_max, estimator_min, estimator, keyword_dict, hyperlink = True) + for k + in keywords[numpy.where(estimator.coef_[0] != 0)] + + } +ALLOWED_PRELETTERS = 3 +MIN_WORD_LEN = 2 +def replace_match(replacements, match): + whole_match_str = match_str = match.group(0) + if match_str in replacements: + return replacements[match_str] + + for i in xrange(1, min(ALLOWED_PRELETTERS + 1, len(match_str) - MIN_WORD_LEN + 1)): + pre, pre_match_str = match_str[:i], match_str[i:] + if pre_match_str in replacements: + return pre + replacements[pre_match_str] + + return u"[%s]" % (match.group(0),) + print "creating files" for i, (class_name, important_keyword_for_class, estimator) in enumerate(zip(lb.classes_, important_keywords, trained_classifier.estimators_)): print "tag", str(i), "/", str(len(lb.classes_)) @@ -92,13 +147,20 @@ def output_word(class_file, w, estimator_max, estimator_min, estimator, keyword_ for keyword in important_keyword_for_class: output_word(class_file, keyword, estimator_max, estimator_min, estimator, keyword_dict) print >>class_file, u"(", str(estimator.coef_[0, keyword_dict[keyword]]), u")
" + +# tag_pattern, tag_replacement_pattern = create_re(estimator_max, estimator_min, estimator, keyword_dict) +# tag_re = re.compile(tag_pattern, re.MULTILINE | re.UNICODE) + replacements = create_re(estimator_max, estimator_min, estimator, keyword_dict) + replacements_re = re.compile('|'.join(('|'.join((u'\\b%s%s\\b' % (u'\\w' * i, p,) for p in replacements.iterkeys())) for i in xrange(ALLOWED_PRELETTERS + 1))), re.UNICODE) print >>class_file, u"" for doc in doc_in_class[class_name]: print >>class_file, u"" print >>class_file, u"
" + unicode(doc['id']) + u"" - - for w in pp_dict[doc['id']]: - output_word(class_file, w, estimator_max, estimator_min, estimator, keyword_dict, hyperlink = True) + print >>class_file, replacements_re.sub(partial(replace_match, replacements), pp_dict[doc['id']]) + + #import pdb; pdb.set_trace() +# print >>class_file, reduce(lambda s,(o,r): re.sub(o,r,s, flags=re.UNICODE), replacements, pp_dict[doc['id']]) +# print >>class_file, tag_re.sub(tag_replacement_pattern, pp_dict[doc['id']]) print >>class_file, u"
" class_file.close() diff --git a/laws/make_histograms.py b/laws/tags/make_histograms.py similarity index 68% rename from laws/make_histograms.py rename to laws/tags/make_histograms.py index cf734f61..c0dc347a 100644 --- a/laws/make_histograms.py +++ b/laws/tags/make_histograms.py @@ -2,28 +2,30 @@ # -*- coding: UTF-8 -*- import cPickle as pickle from collections import defaultdict -import re - -split_re = re.compile(r'\s+', re.U) #REMOVE = ['"', "'", '/', '\\', ',', '.', '!', '@', '#', '$', '%', '^', '&', '*', '(', ')', '-', '=', '_', '+', '\n', '[', ']', '{', '}', '|', '<', '>', '?', '~', ":", ';'] REMOVE = [u'"', u"'", u'/', u'\\', u',', u'.', u'!', u'@', u'#', u'$', u'%', u'^', u'&', u'*', u'(', u')', u'-', u'=', u'_', u'+', u'\n', u'[', u']', u'{', u'}', u'|', u'<', u'>', u'?', u'~', u":", u';', u'–' , u'”', u'`'] REMOVE.extend([unicode(i) for i in xrange(10)]) REMOVE.extend([str(i) for i in xrange(10)]) - -ACCEPTED_POS = ['NN', 'JJ', 'VB'] - def make_histogram(t): dd = defaultdict(int) # for w in t.split(): # dd[reduce(lambda o, s: o.replace(s, ' '), REMOVE, w)] += 1 -# for w in reduce(lambda o, s: o.replace(s, u' '), REMOVE, t).split(): - for w in [ppp.split('\t')[1].decode('utf-8') for ppp in t.splitlines() if ppp.count('\t') > 5 and (ppp.split('\t')[3] in ACCEPTED_POS or ppp.split('\t')[4] in ACCEPTED_POS)]: + for w in reduce(lambda o, s: o.replace(s, u' '), REMOVE, t).split(): dd[w] += 1 return dict(dd) + +from collections import defaultdict +ACCEPTED_POS = ['NN', 'JJ', 'VB', 'NNT', 'BN', 'NNP', 'TTL', ] +def make_pos_histogram(t): + dd = defaultdict(int) + for w in [ppp.split('\t')[1] for ppp in t.splitlines() if ppp.count('\t') > 5 and (ppp.split('\t')[3] in ACCEPTED_POS or ppp.split('\t')[4] in ACCEPTED_POS)]: + dd[w] += 1 + return dict(dd) + -# needed phrases: NN, JJ, VB +# needed phrases: NN, JJ, VB, NNT, BN # Use this: # http://www.cs.bgu.ac.il/~yoavg/depparse/gparse # or this: @@ -36,16 +38,17 @@ def make_histogram(t): # http://www.cs.bgu.ac.il/~yoavg/software/hebparsers/hebdepparser/hebdepparser.tgz # is working good enough -pp_with_tags = pickle.load(open('pp_with_tags.pkl', "rb")) +pp_with_tags = pickle.load(open('pp_pos_with_tags.pkl', "rb")) keywords = set() data = [] for i, pp in enumerate(pp_with_tags): - pp_text, pp_tag, pp_id = pp['text'], pp['tags'], pp['id'] - + pp_pos, pp_tag, pp_id = pp['pos'], pp['tags'], pp['id'] + #print str(i), "/", str(len(pp_with_tags)) try: - histogram = make_histogram(pp_text) + histogram = make_pos_histogram(pp_pos) +# histogram = make_histogram(pp_text) except: import traceback; traceback.print_exc() print "at pp", i @@ -56,3 +59,4 @@ def make_histogram(t): pickle.dump((keywords, data), open('histograms_with_tags.pkl', 'wb')) + diff --git a/laws/tags/parse_htmls.py b/laws/tags/parse_htmls.py new file mode 100644 index 00000000..1872c95e --- /dev/null +++ b/laws/tags/parse_htmls.py @@ -0,0 +1,77 @@ +#!/usr/bin/python +# -*- coding: UTF-8 -*- + +from BeautifulSoup import BeautifulSoup + +import cPickle as pickle + +similar_texts = [ + u"הצעת חוק דומה בעיקרה הונחה", + u"הצעת חוק זהה הונחה", + u"הצעות חוק דומות בעיקרן הונחו", + u"הצעות חוק זהות הונחו", + u"הצעות חוק זהות הונחו", + u"הצעת חוק זהות הונחו", + u"הצעת חוק זהות הונחו", + u"הצעת חוק זהה הוגשה", + u"הצעת חוק זהה הונח", + u"הצעת חוק יסוד זהה הונחה", + u"הצעת חוק זהה הוגשה", + u"הצעות חוק זהות הונחו", + u"הצעת חוק זהות הונחו", + u"הצעות חוק זהות הונחו", + u"הצעות חוק זהות הונחו", + u"הצעות חוק זהות הונחו", + u"סעיפי הצעת חוק זו מבוססים", + u"הצעת החוק נכתבה בסיוע", + u"הצעת חוק דומות הונחו", + u"הצעת חוק דומה בעיקרה זהה הוגשה", + u"הצעות חוק זהות הונחו", + u"הצעות חוק זהות הונחו", + u"הצעות חוק דומות הונחו", + u"הצעת חוק-יסוד זהה הונחה", + u"הצעות חוק-יסוד זהות הונחו", + u"הצעות חוק זהות הונחו", + u"הצעת חוק דומה בעיקרה זהה הוגשה", + u"הצעות חוק זהות הונחה", + u"הצעת חוק דומות בעיקרן הונחו", + u"הצעת חוק דומות בעיקרן הונחו", + +] +pp_without_tags = [] +pp_with_tags = [] + + +for i, pp in enumerate(pickle.load(open('pps.pkl', 'rb'))): + content_html = pp['content_html'] + parsed_html = BeautifulSoup(content_html) + + current_tag = parsed_html.find('p', attrs={'class':'explanation-header'}) + if current_tag is None: + print "no text for pp", pp['id'] + continue + current_tag = current_tag.findNext() + text = '' + while current_tag is not None and '---' not in current_tag.text: + + text += current_tag.text + current_tag = current_tag.findNext() + + tags = pp['tags'] + + for similar_text in similar_texts: + similar_loc = text.find(similar_text) + if similar_loc != -1: + text = text[:similar_loc] + + + + if len(tags) == 0: + pp_without_tags.append({'text' : text, 'id' : pp['id']}) + else: + pp_with_tags.append({'text' : text, 'id' : pp['id'], 'tags' : tags}) + + +pickle.dump(pp_without_tags, open("pp_without_tags.pkl", "wb")) +pickle.dump(pp_with_tags, open("pp_with_tags.pkl", "wb")) + diff --git a/laws/tags/parse_pos_tag.py b/laws/tags/parse_pos_tag.py new file mode 100644 index 00000000..807670a3 --- /dev/null +++ b/laws/tags/parse_pos_tag.py @@ -0,0 +1,68 @@ +#!/usr/bin/python +# -*- coding: UTF-8 -*- +import cPickle as pickle + +#hebdepparser_path = "/home/o/otadmor/Downloads/hebdepparser" +hebdepparser_path = "/home/someone/hebdepparser" +import subprocess + +def parse_lang_tree_popen(text): +# p = subprocess.Popen(["python", "/home/o/otadmor/Downloads/hebdepparser/parse.py"], stdin=subprocess.PIPE, stdout=subprocess.PIPE) + p = subprocess.Popen("python " + hebdepparser_path + "/code/utils/sentences.py | python " + hebdepparser_path + "/parse.py", shell=True, stdin=subprocess.PIPE, stdout=subprocess.PIPE) + + p.stdin.write(text.encode('utf-8')) + + p.stdin.flush() + p.stdin.close() + a = p.stdout.read() + res = '' + while a != '': + res += a + a = p.stdout.read() + p.stdout.close() + p.wait() + return res.decode('utf-8') + + + + +# for w in reduce(lambda o, s: o.replace(s, u' '), REMOVE, t).split(): + + +import sys +sys.path.append(hebdepparser_path) +from parse import parse_sent +from cStringIO import StringIO +def split_sentences(text): + for l in reduce(lambda o, s: o.replace(s, s + u'\n'), ['.', '!', '?'], text).splitlines(): + l = l.strip() + if not l.endswith("."): + l += "." + yield l + +import codecs +def parse_lang_tree(text): + out = StringIO() + for l in split_sentences(text): + parse_sent(l, out) + return out.getvalue().decode('utf-8') + +pp_with_tags = pickle.load(open('pp_with_tags.pkl', "rb")) + +data = [] +for i, pp in enumerate(pp_with_tags): + pp_text, pp_tag, pp_id = pp['text'], pp['tags'], pp['id'] + + print str(i), "/", str(len(pp_with_tags)) + try: + parsed_pos = parse_lang_tree(pp_text) + except: + import traceback; traceback.print_exc() + print "at pp", i + import pdb; pdb.set_trace() + data.append({'pos' : parsed_pos, 'tags' : pp_tag, 'id' : pp_id}) + + + +pickle.dump(data, open('pp_pos_with_tags.pkl', 'wb')) + diff --git a/laws/read_pp.py b/laws/tags/read_pp.py similarity index 100% rename from laws/read_pp.py rename to laws/tags/read_pp.py diff --git a/laws/tags/results.txt b/laws/tags/results.txt new file mode 100644 index 00000000..27fa9e51 --- /dev/null +++ b/laws/tags/results.txt @@ -0,0 +1,19 @@ +features classifier score +NN_JJ_VB_NNT_BN_NNP_TTL OneVsRest_LinearSVC 0.65 +- 0.13 +NN_JJ_VB_NNT_BN OneVsRest_LinearSVC 0.64 +- 0.11 +NN_JJ_VB_NNT_BN_NNP_TTL OneVsRest_GausianNB 0.51 +- 0.12 +NN_JJ_VB_NNT_BN_NNP_TTL OneVsRest_MultinomialNB 0.44 +- 0.40 +NN_JJ_VB_NNT_BN_NNP_TTL OneVsRest_BernoulliNB 0.08 +- 0.00 +NN_JJ_VB_NNT_BN_NNP_TTL KNeighborsClassifier 0.35 +- 0.05 +NN_JJ_VB_NNT_BN_NNP_TTL OneVsRest_TfidfTransformer_LinearSVC 0.64 +- 0.10 +NN_JJ_VB_NNT_BN OneVsRest_TfidfTransformer_LinearSVC 0.64 +- 0.10 +NN_JJ_VB_NNT_BN_NNP_TTL TfidfTransformer_KNeighborsClassifier 0.47 +- 0.03 very fast + +NN_JJ_VB_NNT_BN_NNP_TTL TfidfTransformer_GausianNB +NN_JJ_VB_NNT_BN_NNP_TTL TfidfTransformer_MultinomialNB +NN_JJ_VB_NNT_BN_NNP_TTL TfidfTransformer_BernoulliNB + + +NN_JJ_VB_NNT_BN_NNP_TTL OneVsRest_TfidfTransformer_GausianNB 0.00 +- 0.00 +NN_JJ_VB_NNT_BN_NNP_TTL OneVsRest_TfidfTransformer_MultinomialNB 0.00 +- 0.00 +NN_JJ_VB_NNT_BN_NNP_TTL OneVsRest_TfidfTransformer_BernoulliNB 0.08 +- 0.00 diff --git a/laws/tags_autolearn_play.py b/laws/tags/tags_autolearn_play.py similarity index 56% rename from laws/tags_autolearn_play.py rename to laws/tags/tags_autolearn_play.py index 42d73595..e5b9bcf0 100644 --- a/laws/tags_autolearn_play.py +++ b/laws/tags/tags_autolearn_play.py @@ -87,7 +87,6 @@ def plot_subfigure(X, Y, subplot, title, transform): for k,v in a['histogram'].iteritems(): histogram[keyword_dict[k]] = v -from sklearn import svm # print >>sys.stderr, 'vectorizing labels'; sys.stderr.flush() @@ -110,6 +109,11 @@ def plot_subfigure(X, Y, subplot, title, transform): print "using\t",TRIM_SAMPLES,"samples" print "\t",len(keywords),"keywords" print "\t",len(lb.classes_),"tags" +metadata = learn_data.sum(axis=1) + +print "\t",metadata.mean(), "avg words in document" +print "\t",metadata.max(), "biggest document" +print "\t",metadata.min(), "smallest document" #plt.figure(figsize=(8, 6)) @@ -118,28 +122,107 @@ def plot_subfigure(X, Y, subplot, title, transform): #plt.subplots_adjust(.04, .02, .97, .94, .09, .2) #plt.show() -from sklearn.multiclass import OneVsRestClassifier +from sklearn.multiclass import OneVsRestClassifier, OneVsOneClassifier + +#from sklearn.utils.validation import check_consistent_length +from sklearn.multiclass import _fit_ovo_binary, check_consistent_length, np, Parallel, delayed +class OneVsOneClassifierMultiLabel(OneVsOneClassifier): + + def fit(self, X, y): + y = np.asarray(y) + check_consistent_length(X, y) + + self.classes_ = np.arange(y.shape[1]) + 1 + n_classes = self.classes_.shape[0] + self.estimators_ = Parallel(n_jobs=self.n_jobs)( + delayed(_fit_ovo_binary)( + self.estimator, X, self.classes_[i] * y[:,i] + self.classes_[j] * y[:,j], self.classes_[i], self.classes_[j]) + for i in range(n_classes) for j in range(i + 1, n_classes)) + + return self + + def predict(self, X): + Y = self.decision_function(X) + + return self.classes_[Y.argmax(axis=1)] + + def decision_function(self, X): + check_is_fitted(self, 'estimators_') + import pdb; pdb.set_trace() -#single_svc = svm.SVC(kernel='polynomial', C=4) + n_samples = X.shape[0] + n_classes = self.classes_.shape[0] + votes = np.zeros((n_samples, n_classes, n_classes)) -#single_svc = svm.SVC(kernel='linear', cache_size = 2048) + k = 0 + for i in range(n_classes): + for j in range(i + 1, n_classes): + pred = self.estimators_[k].predict(X) + votes[:, i, j] += pred + k += 1 + return votes -single_svc = svm.LinearSVC() +from sklearn.pipeline import make_pipeline + +from sklearn import preprocessing + +from sklearn.feature_extraction.text import TfidfTransformer + + +from sklearn import svm +from sklearn import naive_bayes +from sklearn import neighbors +from sklearn import linear_model +#single_classifier = svm.SVC(kernel='polynomial', C=4) + +#single_classifier = svm.SVC(kernel='linear', cache_size = 2048) + +#single_classifier = svm.LinearSVC() +#single_classifier = naive_bayes.GaussianNB() +single_classifier = naive_bayes.MultinomialNB() +#single_classifier = naive_bayes.BernoulliNB() #single_svc = svm.SVC(kernel='rbf', cache_size = 2048) -classifier = OneVsRestClassifier(single_svc) +#single_classifier = make_pipeline(TfidfTransformer(), single_classifier) + +#classifier = OneVsRestClassifier(make_pipeline(preprocessing.StandardScaler(),single_classifier)) + +#classifier = OneVsRestClassifier(single_classifier) + +classifier = OneVsOneClassifierMultiLabel(single_classifier) +#classifier = neighbors.KNeighborsClassifier() +#classifier = make_pipeline(TfidfTransformer(), neighbors.KNeighborsClassifier()) +#classifier = linear_model.Ridge() + + + +print "running", classifier +TEST = True + +if not TEST: + + trained_classifier = classifier.fit(learn_data, labels) + from sklearn.externals import joblib + import cPickle as pickle + import os + try: + os.makedirs('classifier_data') + except OSError, e: + pass + joblib.dump(trained_classifier, 'classifier_data/linear_svc_classifier.jlb') + pickle.dump(lb, open("classifier_data/label_binarizer.pkl", "wb")) +else: + + + from sklearn import cross_validation + from sklearn import metrics + scores = cross_validation.cross_val_score(classifier, learn_data, labels, scoring='f1_weighted') + print "Accuracy: %0.2f (+/- %0.2f)" % (scores.mean(), scores.std() * 2) + + + -trained_classifier = classifier.fit(learn_data, labels) -from sklearn.externals import joblib -import cPickle as pickle -import os -try: - os.makedirs('classifier_data') -except OSError, e: - pass -joblib.dump(trained_classifier, 'classifier_data/linear_svc_classifier.jlb') -pickle.dump(lb, open("classifier_data/label_binarizer.pkl", "wb")) #print >>sys.stderr, 'learning data'; sys.stderr.flush() #trained_classifier = classifier.fit(learn_data[:-10], labels[:-10]) #print >>sys.stderr, 'classifying'; sys.stderr.flush() @@ -156,9 +239,6 @@ def plot_subfigure(X, Y, subplot, title, transform): # print _tag[::-1], ",", # print "" -import IPython; IPython.embed() -#from sklearn import cross_validation -#from sklearn import metrics -#scores = cross_validation.cross_val_score(classifier, learn_data, labels, scoring='f1_weighted') -#print "Accuracy: %0.2f (+/- %0.2f)" % (scores.mean(), scores.std() * 2) + +import IPython; IPython.embed() From 83b0f447fe7077045b406516e291aaaee0e84049 Mon Sep 17 00:00:00 2001 From: ofir Date: Mon, 28 Dec 2015 23:02:11 +0200 Subject: [PATCH 11/15] added per-class f1 scoring --- laws/tags/parse_pos_tag.py | 4 +- laws/tags/tags_autolearn_play.py | 73 +++++++++++++++++++++++++++++--- 2 files changed, 69 insertions(+), 8 deletions(-) diff --git a/laws/tags/parse_pos_tag.py b/laws/tags/parse_pos_tag.py index 807670a3..85bd1105 100644 --- a/laws/tags/parse_pos_tag.py +++ b/laws/tags/parse_pos_tag.py @@ -2,8 +2,8 @@ # -*- coding: UTF-8 -*- import cPickle as pickle -#hebdepparser_path = "/home/o/otadmor/Downloads/hebdepparser" -hebdepparser_path = "/home/someone/hebdepparser" +hebdepparser_path = "/home/o/otadmor/Downloads/hebdepparser" +#hebdepparser_path = "/home/someone/hebdepparser" import subprocess def parse_lang_tree_popen(text): diff --git a/laws/tags/tags_autolearn_play.py b/laws/tags/tags_autolearn_play.py index e5b9bcf0..ed426744 100644 --- a/laws/tags/tags_autolearn_play.py +++ b/laws/tags/tags_autolearn_play.py @@ -179,7 +179,7 @@ def decision_function(self, X): #single_classifier = svm.LinearSVC() #single_classifier = naive_bayes.GaussianNB() -single_classifier = naive_bayes.MultinomialNB() +#single_classifier = naive_bayes.MultinomialNB() #single_classifier = naive_bayes.BernoulliNB() #single_svc = svm.SVC(kernel='rbf', cache_size = 2048) @@ -190,13 +190,12 @@ def decision_function(self, X): #classifier = OneVsRestClassifier(single_classifier) -classifier = OneVsOneClassifierMultiLabel(single_classifier) -#classifier = neighbors.KNeighborsClassifier() +#classifier = OneVsOneClassifierMultiLabel(single_classifier) +classifier = neighbors.KNeighborsClassifier() #classifier = make_pipeline(TfidfTransformer(), neighbors.KNeighborsClassifier()) #classifier = linear_model.Ridge() - print "running", classifier TEST = True @@ -215,9 +214,71 @@ def decision_function(self, X): else: - from sklearn import cross_validation from sklearn import metrics - scores = cross_validation.cross_val_score(classifier, learn_data, labels, scoring='f1_weighted') + from functools import partial + f1_scorer_no_average = metrics.make_scorer(partial(metrics.f1_score, average=None)) + + def multilabel_score(estimator, X_test, y_test, scorer): + """Compute the score of an estimator on a given test set.""" + if y_test is None: + score = scorer(estimator, X_test) + else: + score = scorer(estimator, X_test, y_test) +# if not isinstance(score, numbers.Number): +# raise ValueError("scoring must return a number, got %s (%s) instead." +# % (str(score), type(score))) + return score + from sklearn import cross_validation + cross_validation._score = multilabel_score + + def default_nan_prf_divide(numerator, denominator, metric, modifier, average, warn_for): + """Performs division and handles divide-by-zero. + + On zero-division, sets the corresponding result elements to zero + and raises a warning. + + The metric, modifier and average arguments are used only for determining + an appropriate warning. + """ + result = numerator / denominator + mask = denominator == 0.0 + if not np.any(mask): + return result + + # remove infs + result[mask] = np.nan + + # build appropriate warning + # E.g. "Precision and F-score are ill-defined and being set to 0.0 in + # labels with no predicted samples" + axis0 = 'sample' + axis1 = 'label' + if average == 'samples': + axis0, axis1 = axis1, axis0 + + if metric in warn_for and 'f-score' in warn_for: + msg_start = '{0} and F-score are'.format(metric.title()) + elif metric in warn_for: + msg_start = '{0} is'.format(metric.title()) + elif 'f-score' in warn_for: + msg_start = 'F-score is' + else: + return result + + msg = ('{0} ill-defined and being set to np.nan {{0}} ' + 'no {1} {2}s.'.format(msg_start, modifier, axis0)) + if len(mask) == 1: + msg = msg.format('due to') + else: + msg = msg.format('in {0}s with'.format(axis1)) + warnings.warn(msg, UndefinedMetricWarning, stacklevel=2) + return result + from sklearn.metrics import classification + classification._prf_divide = default_nan_prf_divide + + from sklearn import cross_validation + scores = cross_validation.cross_val_score(classifier, learn_data, labels, scoring=f1_scorer_no_average)# 'f1_weighted') + import IPython; IPython.embed() print "Accuracy: %0.2f (+/- %0.2f)" % (scores.mean(), scores.std() * 2) From 68d3130c23d24f099d3f42b5f1fee504ac94ca9b Mon Sep 17 00:00:00 2001 From: ofir Date: Mon, 28 Dec 2015 14:07:19 -0800 Subject: [PATCH 12/15] commit before merge - added more learners --- laws/tags/results.txt | 7 +++- laws/tags/tags_autolearn_play.py | 59 ++++++++++++++++++++++++++++++-- 2 files changed, 63 insertions(+), 3 deletions(-) diff --git a/laws/tags/results.txt b/laws/tags/results.txt index 27fa9e51..e53fee37 100644 --- a/laws/tags/results.txt +++ b/laws/tags/results.txt @@ -1,6 +1,11 @@ features classifier score NN_JJ_VB_NNT_BN_NNP_TTL OneVsRest_LinearSVC 0.65 +- 0.13 -NN_JJ_VB_NNT_BN OneVsRest_LinearSVC 0.64 +- 0.11 +NN_JJ_VB_NNT_BN_NNP_TTL OneVsRest_LinearSVC 0.65 +- 0.13 +NN_JJ_VB_NNT_BN OneVsRest_InfoGainFilter_TfidfTransformer_LinearSVC 0.58 +- 0.04 +NN_JJ_VB_NNT_BN InfoGainFilter_OneVsRest_TfidfTransformer_LinearSVC 0.64 +- 0.08 +NN_JJ_VB_NNT_BN InfoGainFilter_OneVsRest_StandardScaler_LinearSVC 0.50 +- 0.13 +NN_JJ_VB_NNT_BN InfoGainFilter_StandardScaler_MLPClassifier_10_levels_100_each 0.48 +- 0.08 +NN_JJ_VB_NNT_BN StandardScaler_MLPClassifier_10_levels_100_each 0.52 +- 0.11 NN_JJ_VB_NNT_BN_NNP_TTL OneVsRest_GausianNB 0.51 +- 0.12 NN_JJ_VB_NNT_BN_NNP_TTL OneVsRest_MultinomialNB 0.44 +- 0.40 NN_JJ_VB_NNT_BN_NNP_TTL OneVsRest_BernoulliNB 0.08 +- 0.00 diff --git a/laws/tags/tags_autolearn_play.py b/laws/tags/tags_autolearn_play.py index e5b9bcf0..eb8f50b0 100644 --- a/laws/tags/tags_autolearn_play.py +++ b/laws/tags/tags_autolearn_play.py @@ -162,35 +162,90 @@ def decision_function(self, X): k += 1 return votes +class KoppelSameAuthorClassifier(object): + def __init__(self): + super(KoppelSameAuthorClassifier, self).__init__() + self.class_data = None + self.orig_data = None + self.orig_class = None + + + def fit(self, X, y): + self.class_data = np.mat(y).transpose() * np.mat(X) + self.orig_data = X + self.orig_class = y + + def predict(self, X): + impostors_amount = 5 + top_similar = 3 + for x in X: + for i, y in enumerate(self.class_data): + impostors = self.orig_data[np.array(self.orig_class[:,i] == 0).flatten()] + # pick random top 5 impostors + np.random.shuffle(impostors) + impostors = impostors[:impostors_amount] + + distances = self.cals_distances(impostors, x, y) + sorted_distances = distances.argsort() + + # top 3 similar + same_author = (same_author[:top_similar] == impostors_amount).any() + + + def cals_distances(impostors, x, y): + # impostors[:, np.newaxis] - y # only if all y are sent (y is two dimensional matrix) + y_distance = np.power(impostors - y, 2).sum(axis = -1) + x_distance = np.power(impostors - x, 2).sum(axis = -1) + imposters_two_way_distance = np.sqrt(y_distance) * np.sqrt(x_distance) + + two_way_distance = np.power(x - y, 2).sum(axis = -1) + + + return np.append(imposters_two_way_distance,two_way_distance) + + from sklearn.pipeline import make_pipeline from sklearn import preprocessing from sklearn.feature_extraction.text import TfidfTransformer +from sklearn.ensemble import ExtraTreesClassifier +from sklearn.feature_selection import SelectFromModel from sklearn import svm from sklearn import naive_bayes from sklearn import neighbors from sklearn import linear_model +from sklearn import neural_network #single_classifier = svm.SVC(kernel='polynomial', C=4) #single_classifier = svm.SVC(kernel='linear', cache_size = 2048) +single_classifier = neural_network.MLPClassifier(hidden_layer_sizes=10 * (100,)) + #single_classifier = svm.LinearSVC() + #single_classifier = naive_bayes.GaussianNB() -single_classifier = naive_bayes.MultinomialNB() +#single_classifier = naive_bayes.MultinomialNB() #single_classifier = naive_bayes.BernoulliNB() #single_svc = svm.SVC(kernel='rbf', cache_size = 2048) #single_classifier = make_pipeline(TfidfTransformer(), single_classifier) +single_classifier = make_pipeline(preprocessing.StandardScaler(), single_classifier) +#single_classifier = make_pipeline(SelectFromModel(ExtraTreesClassifier(), prefit = False), TfidfTransformer(), single_classifier) #classifier = OneVsRestClassifier(make_pipeline(preprocessing.StandardScaler(),single_classifier)) #classifier = OneVsRestClassifier(single_classifier) +#classifier = make_pipeline(SelectFromModel(ExtraTreesClassifier(), prefit = False), OneVsRestClassifier(single_classifier)) + +#classifier = make_pipeline(SelectFromModel(ExtraTreesClassifier(), prefit = False), single_classifier) + +classifier = single_classifier -classifier = OneVsOneClassifierMultiLabel(single_classifier) +#classifier = OneVsOneClassifierMultiLabel(single_classifier) #classifier = neighbors.KNeighborsClassifier() #classifier = make_pipeline(TfidfTransformer(), neighbors.KNeighborsClassifier()) #classifier = linear_model.Ridge() From 5bde87c46de0059b4f7fe55dd7f3c1bd974d21e8 Mon Sep 17 00:00:00 2001 From: ofir Date: Fri, 1 Jan 2016 11:57:14 -0800 Subject: [PATCH 13/15] changed f1 score calculation to better infer there is a bad class;changed crossvalidation to be different when 1vall multilabel - each label is tested individually before splitting the data --- laws/tags/build_important_keyword_html.py | 36 ++++++------ laws/tags/results.txt | 1 + laws/tags/tags_autolearn_play.py | 69 ++++++++++++++++++----- 3 files changed, 75 insertions(+), 31 deletions(-) diff --git a/laws/tags/build_important_keyword_html.py b/laws/tags/build_important_keyword_html.py index 8428d8e3..36d87b4c 100644 --- a/laws/tags/build_important_keyword_html.py +++ b/laws/tags/build_important_keyword_html.py @@ -1,6 +1,7 @@ #!/usr/bin/python # -*- coding: UTF-8 -*- import numpy +import numpy as np import re from functools import partial from sklearn.externals import joblib @@ -11,6 +12,9 @@ pp_with_tags = pickle.load(open('pp_with_tags.pkl', "rb")) trained_classifier = joblib.load('classifier_data/linear_svc_classifier.jlb') keywords = numpy.array(sorted(list(keywords))) + +coefs = np.vstack([e.steps[-1][1].coef_ for e in trained_classifier.estimators_]) + import os import codecs try: @@ -61,38 +65,38 @@ def output_tag(class_file, w, estimator_max, estimator_min, weight, hyperlink = else: print >>class_file, u"" + w + u" " -def output_word(class_file, w, estimator_max, estimator_min, estimator, keyword_dict, hyperlink = False): - weight = estimator.coef_[0, keyword_dict[w]] +def output_word(class_file, w, estimator_max, estimator_min, coefs, keyword_dict, hyperlink = False): + weight = coefs[keyword_dict[w]] pre = post = '' if weight > 0 and estimator_max != 0: weight /= estimator_max if weight > 0.05 and hyperlink: - pre = u"" + pre = u"" post = u"" print >>class_file, pre + u"" % (int(255 * weight),) + w + u"" + post + " " elif weight < 0 and estimator_min != 0: weight /= estimator_min if weight > 0.05 and hyperlink: - pre = u"" + pre = u"" post = u"" print >>class_file, pre + u"" % (int(255 * weight),) + w + u"" + post + " " else: print >>class_file, u"" + w + u" " -def format_word(w, estimator_max, estimator_min, estimator, keyword_dict, hyperlink = False): - weight = estimator.coef_[0, keyword_dict[w]] +def format_word(w, estimator_max, estimator_min, coefs, keyword_dict, hyperlink = False): + orig_Weight = weight = coefs[keyword_dict[w]] pre = post = '' if weight > 0 and estimator_max != 0: weight /= estimator_max if hyperlink: - pre = u"" + pre = u"" post = u"" v = u"" % (int(255 * weight),) + w + u"" elif weight < 0 and estimator_min != 0: weight /= estimator_min if hyperlink: - pre = u"" + pre = u"" post = u"" v = u"" % (int(255 * weight),) + w + u"" else: @@ -101,10 +105,10 @@ def format_word(w, estimator_max, estimator_min, estimator, keyword_dict, hyperl ATTR_AMOUNT = 10 -important_keywords = keywords[numpy.abs(trained_classifier.coef_).argsort()[:,-ATTR_AMOUNT:]] +important_keywords = keywords[numpy.abs(coefs).argsort()[:,-ATTR_AMOUNT:]] MAX_RE_GROUP = 100 -def create_re(estimator_max, estimator_min, estimator, keyword_dict): +def create_re(estimator_max, estimator_min, coefs, keyword_dict): #return ("|".join(p) for p in zip(*( # ( # u"(?P\\b)%s(?P\\b)" % (i, re.escape(k), i, ), @@ -116,9 +120,9 @@ def create_re(estimator_max, estimator_min, estimator, keyword_dict): #)) return { - k : format_word(k, estimator_max, estimator_min, estimator, keyword_dict, hyperlink = True) + k : format_word(k, estimator_max, estimator_min, coefs, keyword_dict, hyperlink = True) for k - in keywords[numpy.where(estimator.coef_[0] != 0)] + in keywords[numpy.where(coefs != 0)] } ALLOWED_PRELETTERS = 3 @@ -143,14 +147,14 @@ def replace_match(replacements, match): print >>class_file, u"Keyword Explanation for Class " + class_name + u"" print >>class_file, u"

" + class_name + u"


" print >>class_file, u"

Most influencing words

" - estimator_min, estimator_max = estimator.coef_[0].min(), estimator.coef_[0].max() + estimator_min, estimator_max = coefs[i,:].min(), coefs[i,:].max() for keyword in important_keyword_for_class: - output_word(class_file, keyword, estimator_max, estimator_min, estimator, keyword_dict) - print >>class_file, u"(", str(estimator.coef_[0, keyword_dict[keyword]]), u")
" + output_word(class_file, keyword, estimator_max, estimator_min, coefs[i], keyword_dict) + print >>class_file, u"(", str(coefs[i,keyword_dict[keyword]]), u")
" # tag_pattern, tag_replacement_pattern = create_re(estimator_max, estimator_min, estimator, keyword_dict) # tag_re = re.compile(tag_pattern, re.MULTILINE | re.UNICODE) - replacements = create_re(estimator_max, estimator_min, estimator, keyword_dict) + replacements = create_re(estimator_max, estimator_min, coefs[i], keyword_dict) replacements_re = re.compile('|'.join(('|'.join((u'\\b%s%s\\b' % (u'\\w' * i, p,) for p in replacements.iterkeys())) for i in xrange(ALLOWED_PRELETTERS + 1))), re.UNICODE) print >>class_file, u"" for doc in doc_in_class[class_name]: diff --git a/laws/tags/results.txt b/laws/tags/results.txt index e53fee37..844ff58d 100644 --- a/laws/tags/results.txt +++ b/laws/tags/results.txt @@ -10,6 +10,7 @@ NN_JJ_VB_NNT_BN_NNP_TTL OneVsRest_GausianNB 0.51 +- 0.12 NN_JJ_VB_NNT_BN_NNP_TTL OneVsRest_MultinomialNB 0.44 +- 0.40 NN_JJ_VB_NNT_BN_NNP_TTL OneVsRest_BernoulliNB 0.08 +- 0.00 NN_JJ_VB_NNT_BN_NNP_TTL KNeighborsClassifier 0.35 +- 0.05 +NN_JJ_VB_NNT_BN_NNP_TTL OneVsRest_TfidfTransformer_LinearSVC_10folds 0.74 +- 0.34 NN_JJ_VB_NNT_BN_NNP_TTL OneVsRest_TfidfTransformer_LinearSVC 0.64 +- 0.10 NN_JJ_VB_NNT_BN OneVsRest_TfidfTransformer_LinearSVC 0.64 +- 0.10 NN_JJ_VB_NNT_BN_NNP_TTL TfidfTransformer_KNeighborsClassifier 0.47 +- 0.03 very fast diff --git a/laws/tags/tags_autolearn_play.py b/laws/tags/tags_autolearn_play.py index 8b4e52ba..056c3bb5 100644 --- a/laws/tags/tags_autolearn_play.py +++ b/laws/tags/tags_autolearn_play.py @@ -222,9 +222,9 @@ def cals_distances(impostors, x, y): #single_classifier = svm.SVC(kernel='linear', cache_size = 2048) -single_classifier = neural_network.MLPClassifier(hidden_layer_sizes=10 * (100,)) +#single_classifier = neural_network.MLPClassifier(hidden_layer_sizes=10 * (100,)) -#single_classifier = svm.LinearSVC() +single_classifier = svm.LinearSVC() #single_classifier = naive_bayes.GaussianNB() #single_classifier = naive_bayes.MultinomialNB() @@ -232,33 +232,34 @@ def cals_distances(impostors, x, y): #single_svc = svm.SVC(kernel='rbf', cache_size = 2048) -#single_classifier = make_pipeline(TfidfTransformer(), single_classifier) -single_classifier = make_pipeline(preprocessing.StandardScaler(), single_classifier) +single_classifier = make_pipeline(TfidfTransformer(), single_classifier) +#single_classifier = make_pipeline(preprocessing.StandardScaler(), single_classifier) #single_classifier = make_pipeline(SelectFromModel(ExtraTreesClassifier(), prefit = False), TfidfTransformer(), single_classifier) #classifier = OneVsRestClassifier(make_pipeline(preprocessing.StandardScaler(),single_classifier)) -#classifier = OneVsRestClassifier(single_classifier) +classifier = OneVsRestClassifier(single_classifier) #classifier = make_pipeline(SelectFromModel(ExtraTreesClassifier(), prefit = False), OneVsRestClassifier(single_classifier)) #classifier = make_pipeline(SelectFromModel(ExtraTreesClassifier(), prefit = False), single_classifier) -classifier = single_classifier +#classifier = single_classifier #classifier = OneVsOneClassifierMultiLabel(single_classifier) -classifier = neighbors.KNeighborsClassifier() +#classifier = neighbors.KNeighborsClassifier() #classifier = make_pipeline(TfidfTransformer(), neighbors.KNeighborsClassifier()) #classifier = linear_model.Ridge() print "running", classifier -TEST = True +TEST = False if not TEST: trained_classifier = classifier.fit(learn_data, labels) + from sklearn.externals import joblib import cPickle as pickle import os @@ -281,13 +282,17 @@ def multilabel_score(estimator, X_test, y_test, scorer): score = scorer(estimator, X_test) else: score = scorer(estimator, X_test, y_test) -# if not isinstance(score, numbers.Number): -# raise ValueError("scoring must return a number, got %s (%s) instead." -# % (str(score), type(score))) - return score + if getattr(scorer, 'keywords', {}).get('average', None) is None: + return list(score) + elif not isinstance(score, numbers.Number): + raise ValueError("scoring must return a number, got %s (%s) instead." + % (str(score), type(score))) + return list(score) from sklearn import cross_validation cross_validation._score = multilabel_score + import warnings + from sklearn.metrics.base import UndefinedMetricWarning def default_nan_prf_divide(numerator, denominator, metric, modifier, average, warn_for): """Performs division and handles divide-by-zero. @@ -297,13 +302,16 @@ def default_nan_prf_divide(numerator, denominator, metric, modifier, average, wa The metric, modifier and average arguments are used only for determining an appropriate warning. """ - result = numerator / denominator + result = numerator.astype(np.float_) / denominator.astype(np.float_) mask = denominator == 0.0 if not np.any(mask): return result # remove infs - result[mask] = np.nan + if average is None: + result[mask] = np.nan + else: + result[mask] = 0.0 # build appropriate warning # E.g. "Precision and F-score are ill-defined and being set to 0.0 in @@ -330,11 +338,42 @@ def default_nan_prf_divide(numerator, denominator, metric, modifier, average, wa msg = msg.format('in {0}s with'.format(axis1)) warnings.warn(msg, UndefinedMetricWarning, stacklevel=2) return result + from sklearn.metrics import classification + _precision_recall_fscore_support = classification.precision_recall_fscore_support + def precision_recall_fscore_support(*args, **kargs): + precision, recall, f_score, true_sum = _precision_recall_fscore_support(*args, **kargs) + if kargs.get('average', None) is None: + f_score[np.isnan(precision) * np.isnan(recall)] = 1.0 # both true_sum and pred_sum are 0 + return precision, recall, f_score, true_sum + classification.precision_recall_fscore_support = precision_recall_fscore_support classification._prf_divide = default_nan_prf_divide from sklearn import cross_validation - scores = cross_validation.cross_val_score(classifier, learn_data, labels, scoring=f1_scorer_no_average)# 'f1_weighted') + from sklearn.utils.multiclass import type_of_target + from copy import deepcopy + _cross_val_score = cross_validation.cross_val_score + def cross_val_score(estimator, X, y=None, *args, **kargs): + y_type = type_of_target(y) + positive_example_amount = y.sum(axis=0) + error = "" + if (positive_example_amount < kargs['cv']).any(): + error = str((positive_example_amount < kargs['cv']).sum()) + " : too little examples for " + str(np.where(positive_example_amount < kargs['cv'])) + str(positive_example_amount[np.where(positive_example_amount < kargs['cv'])]) + if (positive_example_amount > y.shape[0] - kargs['cv']).any(): + error += str((positive_example_amount > y.shape[0] - kargs['cv']).sum()) + " : too many examples for " + str(np.where(positive_example_amount > y.shape[0] - kargs['cv'])) + str(positive_example_amount[np.where(positive_example_amount > y.shape[0] - kargs['cv'])]) + if error: + raise Exception(error) + if y_type.startswith('multilabel') and isinstance(estimator, OneVsRestClassifier): + res = [] + for yy in y.transpose(): + res.append(_cross_val_score(deepcopy(estimator.estimator), X, yy, *args, **kargs)) + import pdb; pdb.set_trace() + else: + res = _cross_val_score(estimator, X, y, *args, **kargs) + return np.array(list(res)) + cross_validation.cross_val_score = cross_val_score + + scores = cross_validation.cross_val_score(classifier, learn_data, labels, scoring=f1_scorer_no_average, cv=10)# 'f1_weighted') import IPython; IPython.embed() print "Accuracy: %0.2f (+/- %0.2f)" % (scores.mean(), scores.std() * 2) From 5aac86f7b5b6376f7e30edee2acbcc919818dda9 Mon Sep 17 00:00:00 2001 From: ofir Date: Sun, 3 Jan 2016 15:38:19 -0800 Subject: [PATCH 14/15] added tag renaming for merging tags --- laws/tags/build_important_keyword_html.py | 6 +- laws/tags/parse_htmls.py | 1 + laws/tags/tag_rename.py | 116 ++++++++++++++++++++++ laws/tags/tags_autolearn_play.py | 12 +-- 4 files changed, 126 insertions(+), 9 deletions(-) create mode 100644 laws/tags/tag_rename.py diff --git a/laws/tags/build_important_keyword_html.py b/laws/tags/build_important_keyword_html.py index 36d87b4c..08dfd2be 100644 --- a/laws/tags/build_important_keyword_html.py +++ b/laws/tags/build_important_keyword_html.py @@ -169,9 +169,9 @@ def replace_match(replacements, match): print >>class_file, u"
" class_file.close() -coef_abs = numpy.abs(trained_classifier.coef_) +coef_abs = numpy.abs(coefs) sorted_keywords = coef_abs.argsort(axis = 0) -min_weight, max_weight = trained_classifier.coef_.min(axis = 1), trained_classifier.coef_.max(axis = 1) +min_weight, max_weight = coefs.min(axis = 1), coefs.max(axis = 1) for keyword_count, i in enumerate(coef_abs.sum(axis = 0).argsort()[::-1]): keyword = keywords[i] print "keyword", str(keyword_count), "/", str(len(keywords)), "(", str(i), ")" @@ -179,7 +179,7 @@ def replace_match(replacements, match): print >>keyword_file, u"Tag Explanation for Keyword " + keyword + u"" print >>keyword_file, u"

" + keyword + u"


" print >>keyword_file, u"

Most influenced tags

" - weights = trained_classifier.coef_[:,i] + weights = coefs[:,i] class_keywords = sorted_keywords[::-1,i] for class_name, weight, max_weight_, min_weight_ in zip(lb.classes_[class_keywords], weights[class_keywords], max_weight[class_keywords], min_weight[class_keywords]): diff --git a/laws/tags/parse_htmls.py b/laws/tags/parse_htmls.py index 1872c95e..1b464d0a 100644 --- a/laws/tags/parse_htmls.py +++ b/laws/tags/parse_htmls.py @@ -36,6 +36,7 @@ u"הצעות חוק זהות הונחה", u"הצעת חוק דומות בעיקרן הונחו", u"הצעת חוק דומות בעיקרן הונחו", + u"הצעת דומה בעיקרה הונחה", ] pp_without_tags = [] diff --git a/laws/tags/tag_rename.py b/laws/tags/tag_rename.py new file mode 100644 index 00000000..d2e38d40 --- /dev/null +++ b/laws/tags/tag_rename.py @@ -0,0 +1,116 @@ + +class_map = { +"אומנה" : "אימוץ", +"אונס" : "אלימות מינית", +"הטרדה מינית" : "אלימות מינית", +"אחים שכולים" : "קצבאות", +"אשראי" : "בנקאות", +#"בחירות לנשיאות", +"בחירות לכנסת" : "בחירות", +"בית דין לעבודה" : "ביטוח לאומי", +# "בית הדין לענייני משפחה", +# "בני זוג", +# "בנייה ירוקה", +# "בריאות הפה", +"בתי דין רבנים" : "בית דין רבני", +"בתי דין רבניים" : "בית דין רבני", +# "גז ונפט", +"גיור", +"דיור מוגן", +"דיני חוזים", +"הורים" : "הורות", +"הזכות להליך הוגן", +"הימורים", +"הכרזת מלחמה", +"המגזר הבדואי", +"הסברה", +"הסדר מדיני", +"הסתה לגזענות", +"הפליה", +"השמה חוץ ביתית", +"השעייה", +"השתלת אברים", +"התאגדות", +"התרת נישואים", +"ועדת הכלכלה", +"זיהום אוויר", +"זיכרון השואה", +"זכויות הנכים", +"זכויות חיילים", +"זכויות יוצרים", +"חוק אוויר נקי" : "זיהום אוויר", +"חוק גיל הפרישה", +"חוק האזרחים הותיקים", +"חוק הביטוח הלאומי" : "ביטוח לאומי", +"חוק הספורט", +"חוק הרשות השניה", +"חוק התקשורת", +"חוק חג המצות", +"חוק חינוך ממלכתי", +"חוק טל", +"חוק לימוד חובה", +"חינוך בלתי-פורמלי", +"חללי צה\"ל", +"חניה", +"חסכון", +"חריגות בניה", +"טכנולוגיה", +"טלוויזיה", +"יבוא ויצוא", +"ימי זכרון", +"ישיבות", +"יתמות", +"לסביות" : "להט\"ב", +"לשון הרע", +"מבחן הכנסה", +"מהגרים בלתי חוקיים", +"מינהל מקרקעי ישראל", +"מנהל מקרקעי ישראל", +"משטר נשיאותי", +"משרד התחבורה, התשתיות הלאומיות והבטיחות בדרכים", +"משרד התקשורת", +"נגב", +"נהיגה בשכרות", +"נהיגה תחת השפעת סמים", +"נוכחים-נפקדים", +"ניירות ערך", +"נכי צה\"ל", +"נפגעי פעולות איבה", +"נשק", +"סחר נשים", +"סיוע לעסקים", +"סכסוך עבודה", +"סלולר", +"עובדי ציבור", +"עיתונות", +"פונדקאות", +"פינוי", +"פלסטינאים", +"פקודת מס הכנסה", +"פרסומות", +"ציבור דתי", +"צפון", +"קהילה אתיופית", +"קצבאות ילדים", +"קרן קיימת לישראל", +"ראש הממשלה", +"שב\"כ", +"שביתה", +"שוק חופשי", +"שירות המדינה", +"שירותי קהילה", +"שכר והוצאות שרים וחברי כנסת", +"שעון קיץ", +"שעת חירום", +"תושבי חוץ", +"תיווך מקרקעין", +"תכנון ובנייה" : "תכנון ובניה", +"תלונות הציבור", +"תמלוגים", +"תנאי תעסוקה", +"תנועות נוער", +"תעודת זהות", +"תעופה", +"תקנות שעת חירום", +"תרומות", +} diff --git a/laws/tags/tags_autolearn_play.py b/laws/tags/tags_autolearn_play.py index 056c3bb5..c0996b22 100644 --- a/laws/tags/tags_autolearn_play.py +++ b/laws/tags/tags_autolearn_play.py @@ -254,7 +254,7 @@ def cals_distances(impostors, x, y): print "running", classifier -TEST = False +TEST = True if not TEST: @@ -353,7 +353,7 @@ def precision_recall_fscore_support(*args, **kargs): from sklearn.utils.multiclass import type_of_target from copy import deepcopy _cross_val_score = cross_validation.cross_val_score - def cross_val_score(estimator, X, y=None, *args, **kargs): + def cross_val_score_one_vs_all_per_class(estimator, X, y=None, *args, **kargs): y_type = type_of_target(y) positive_example_amount = y.sum(axis=0) error = "" @@ -361,8 +361,8 @@ def cross_val_score(estimator, X, y=None, *args, **kargs): error = str((positive_example_amount < kargs['cv']).sum()) + " : too little examples for " + str(np.where(positive_example_amount < kargs['cv'])) + str(positive_example_amount[np.where(positive_example_amount < kargs['cv'])]) if (positive_example_amount > y.shape[0] - kargs['cv']).any(): error += str((positive_example_amount > y.shape[0] - kargs['cv']).sum()) + " : too many examples for " + str(np.where(positive_example_amount > y.shape[0] - kargs['cv'])) + str(positive_example_amount[np.where(positive_example_amount > y.shape[0] - kargs['cv'])]) - if error: - raise Exception(error) +# if error: +# raise Exception(error) if y_type.startswith('multilabel') and isinstance(estimator, OneVsRestClassifier): res = [] for yy in y.transpose(): @@ -371,9 +371,9 @@ def cross_val_score(estimator, X, y=None, *args, **kargs): else: res = _cross_val_score(estimator, X, y, *args, **kargs) return np.array(list(res)) - cross_validation.cross_val_score = cross_val_score +# cross_validation.cross_val_score = cross_val_score_one_vs_all_per_class - scores = cross_validation.cross_val_score(classifier, learn_data, labels, scoring=f1_scorer_no_average, cv=10)# 'f1_weighted') + scores = cross_validation.cross_val_score(classifier, learn_data, labels, scoring=f1_scorer_no_average, cv=3)# 'f1_weighted') import IPython; IPython.embed() print "Accuracy: %0.2f (+/- %0.2f)" % (scores.mean(), scores.std() * 2) From 0ca89ba1024a750dfdf11d5cd52b0c7f17b0cbf4 Mon Sep 17 00:00:00 2001 From: ofir Date: Sun, 17 Jan 2016 16:02:37 -0800 Subject: [PATCH 15/15] added tag renaming - doesnt seems like its changes very much --- laws/tags/parse_htmls.py | 68 +++++---- laws/tags/parse_pos_tag.py | 4 +- laws/tags/results.txt | 41 ++++-- laws/tags/tag_rename.py | 242 ++++++++++++++++--------------- laws/tags/tags_autolearn_play.py | 10 +- 5 files changed, 206 insertions(+), 159 deletions(-) diff --git a/laws/tags/parse_htmls.py b/laws/tags/parse_htmls.py index 1b464d0a..00f37160 100644 --- a/laws/tags/parse_htmls.py +++ b/laws/tags/parse_htmls.py @@ -5,6 +5,8 @@ import cPickle as pickle +from tag_rename import change_docs_tags, class_map + similar_texts = [ u"הצעת חוק דומה בעיקרה הונחה", u"הצעת חוק זהה הונחה", @@ -37,6 +39,11 @@ u"הצעת חוק דומות בעיקרן הונחו", u"הצעת חוק דומות בעיקרן הונחו", u"הצעת דומה בעיקרה הונחה", + u"הצעות חוק דומות בעיקרון הונחו", + u"הצעות חוק זהה הונחה", + u"הצעת חוק דומות בעיקרן הונחן", + u"הצעת חוק זו הוכנה בסיוע", + u"הצעת חוק דומות בעיקרן הונחו", ] pp_without_tags = [] @@ -44,35 +51,38 @@ for i, pp in enumerate(pickle.load(open('pps.pkl', 'rb'))): - content_html = pp['content_html'] - parsed_html = BeautifulSoup(content_html) - - current_tag = parsed_html.find('p', attrs={'class':'explanation-header'}) - if current_tag is None: - print "no text for pp", pp['id'] - continue - current_tag = current_tag.findNext() - text = '' - while current_tag is not None and '---' not in current_tag.text: - - text += current_tag.text - current_tag = current_tag.findNext() + content_html = pp['content_html'] + parsed_html = BeautifulSoup(content_html) + + current_tag = parsed_html.find('p', attrs={'class':'explanation-header'}) + if current_tag is None: + print "no text for pp", pp['id'] + continue + current_tag = current_tag.findNext() + text = '' + while current_tag is not None and '---' not in current_tag.text: + + text += current_tag.text + current_tag = current_tag.findNext() + + tags = [class_map.get(t, t) for t in pp['tags']] - tags = pp['tags'] - - for similar_text in similar_texts: - similar_loc = text.find(similar_text) - if similar_loc != -1: - text = text[:similar_loc] - - - - if len(tags) == 0: - pp_without_tags.append({'text' : text, 'id' : pp['id']}) - else: - pp_with_tags.append({'text' : text, 'id' : pp['id'], 'tags' : tags}) - - + if pp['id'] in change_docs_tags: + tags = [change_docs_tags[pp['id']]] + + for similar_text in similar_texts: + similar_loc = text.find(similar_text) + if similar_loc != -1: + text = text[:similar_loc] + + + + if len(tags) == 0: + pp_without_tags.append({'text' : text, 'id' : pp['id']}) + else: + pp_with_tags.append({'text' : text, 'id' : pp['id'], 'tags' : tags}) + + pickle.dump(pp_without_tags, open("pp_without_tags.pkl", "wb")) pickle.dump(pp_with_tags, open("pp_with_tags.pkl", "wb")) - + diff --git a/laws/tags/parse_pos_tag.py b/laws/tags/parse_pos_tag.py index 85bd1105..807670a3 100644 --- a/laws/tags/parse_pos_tag.py +++ b/laws/tags/parse_pos_tag.py @@ -2,8 +2,8 @@ # -*- coding: UTF-8 -*- import cPickle as pickle -hebdepparser_path = "/home/o/otadmor/Downloads/hebdepparser" -#hebdepparser_path = "/home/someone/hebdepparser" +#hebdepparser_path = "/home/o/otadmor/Downloads/hebdepparser" +hebdepparser_path = "/home/someone/hebdepparser" import subprocess def parse_lang_tree_popen(text): diff --git a/laws/tags/results.txt b/laws/tags/results.txt index 844ff58d..3f6e498d 100644 --- a/laws/tags/results.txt +++ b/laws/tags/results.txt @@ -1,19 +1,20 @@ +original tags features classifier score -NN_JJ_VB_NNT_BN_NNP_TTL OneVsRest_LinearSVC 0.65 +- 0.13 -NN_JJ_VB_NNT_BN_NNP_TTL OneVsRest_LinearSVC 0.65 +- 0.13 -NN_JJ_VB_NNT_BN OneVsRest_InfoGainFilter_TfidfTransformer_LinearSVC 0.58 +- 0.04 -NN_JJ_VB_NNT_BN InfoGainFilter_OneVsRest_TfidfTransformer_LinearSVC 0.64 +- 0.08 -NN_JJ_VB_NNT_BN InfoGainFilter_OneVsRest_StandardScaler_LinearSVC 0.50 +- 0.13 -NN_JJ_VB_NNT_BN InfoGainFilter_StandardScaler_MLPClassifier_10_levels_100_each 0.48 +- 0.08 -NN_JJ_VB_NNT_BN StandardScaler_MLPClassifier_10_levels_100_each 0.52 +- 0.11 -NN_JJ_VB_NNT_BN_NNP_TTL OneVsRest_GausianNB 0.51 +- 0.12 -NN_JJ_VB_NNT_BN_NNP_TTL OneVsRest_MultinomialNB 0.44 +- 0.40 +NN_JJ_VB_NNT_BN_NNP_TTL OneVsRest_LinearSVC 0.65 +- 0.06 +NN_JJ_VB_NNT_BN_NNP_TTL OneVsRest_LinearSVC 0.65 +- 0.06 +NN_JJ_VB_NNT_BN OneVsRest_InfoGainFilter_TfidfTransformer_LinearSVC 0.58 +- 0.02 +NN_JJ_VB_NNT_BN InfoGainFilter_OneVsRest_TfidfTransformer_LinearSVC 0.64 +- 0.04 +NN_JJ_VB_NNT_BN InfoGainFilter_OneVsRest_StandardScaler_LinearSVC 0.50 +- 0.06 +NN_JJ_VB_NNT_BN InfoGainFilter_StandardScaler_MLPClassifier_10_levels_100_each 0.48 +- 0.04 +NN_JJ_VB_NNT_BN StandardScaler_MLPClassifier_10_levels_100_each 0.52 +- 0.05 +NN_JJ_VB_NNT_BN_NNP_TTL OneVsRest_GausianNB 0.51 +- 0.06 +NN_JJ_VB_NNT_BN_NNP_TTL OneVsRest_MultinomialNB 0.44 +- 0.20 NN_JJ_VB_NNT_BN_NNP_TTL OneVsRest_BernoulliNB 0.08 +- 0.00 -NN_JJ_VB_NNT_BN_NNP_TTL KNeighborsClassifier 0.35 +- 0.05 +NN_JJ_VB_NNT_BN_NNP_TTL KNeighborsClassifier 0.35 +- 0.02 NN_JJ_VB_NNT_BN_NNP_TTL OneVsRest_TfidfTransformer_LinearSVC_10folds 0.74 +- 0.34 -NN_JJ_VB_NNT_BN_NNP_TTL OneVsRest_TfidfTransformer_LinearSVC 0.64 +- 0.10 -NN_JJ_VB_NNT_BN OneVsRest_TfidfTransformer_LinearSVC 0.64 +- 0.10 -NN_JJ_VB_NNT_BN_NNP_TTL TfidfTransformer_KNeighborsClassifier 0.47 +- 0.03 very fast +NN_JJ_VB_NNT_BN_NNP_TTL OneVsRest_TfidfTransformer_LinearSVC 0.64 +- 0.05 +NN_JJ_VB_NNT_BN OneVsRest_TfidfTransformer_LinearSVC 0.64 +- 0.05 +NN_JJ_VB_NNT_BN_NNP_TTL TfidfTransformer_KNeighborsClassifier 0.47 +- 0.01 very fast NN_JJ_VB_NNT_BN_NNP_TTL TfidfTransformer_GausianNB NN_JJ_VB_NNT_BN_NNP_TTL TfidfTransformer_MultinomialNB @@ -23,3 +24,17 @@ NN_JJ_VB_NNT_BN_NNP_TTL TfidfTransformer_BernoulliNB NN_JJ_VB_NNT_BN_NNP_TTL OneVsRest_TfidfTransformer_GausianNB 0.00 +- 0.00 NN_JJ_VB_NNT_BN_NNP_TTL OneVsRest_TfidfTransformer_MultinomialNB 0.00 +- 0.00 NN_JJ_VB_NNT_BN_NNP_TTL OneVsRest_TfidfTransformer_BernoulliNB 0.08 +- 0.00 + + + + + + +reduced tags 300 +features classifier score +NN_JJ_VB_NNT_BN_NNP_TTL StandardScaler_OneVsRest_LinearSVC_3folds 0.03 +- 0.04 +NN_JJ_VB_NNT_BN_NNP_TTL OneVsRest_LinearSVC_3folds 0.59 +- 0.31 +NN_JJ_VB_NNT_BN_NNP_TTL OneVsRest_LinearSVC_10folds 0.73 +- 0.32 + + + diff --git a/laws/tags/tag_rename.py b/laws/tags/tag_rename.py index d2e38d40..87c901c5 100644 --- a/laws/tags/tag_rename.py +++ b/laws/tags/tag_rename.py @@ -1,116 +1,132 @@ +#!/usr/bin/python +# -*- coding: UTF-8 -*- class_map = { -"אומנה" : "אימוץ", -"אונס" : "אלימות מינית", -"הטרדה מינית" : "אלימות מינית", -"אחים שכולים" : "קצבאות", -"אשראי" : "בנקאות", -#"בחירות לנשיאות", -"בחירות לכנסת" : "בחירות", -"בית דין לעבודה" : "ביטוח לאומי", -# "בית הדין לענייני משפחה", -# "בני זוג", -# "בנייה ירוקה", -# "בריאות הפה", -"בתי דין רבנים" : "בית דין רבני", -"בתי דין רבניים" : "בית דין רבני", -# "גז ונפט", -"גיור", -"דיור מוגן", -"דיני חוזים", -"הורים" : "הורות", -"הזכות להליך הוגן", -"הימורים", -"הכרזת מלחמה", -"המגזר הבדואי", -"הסברה", -"הסדר מדיני", -"הסתה לגזענות", -"הפליה", -"השמה חוץ ביתית", -"השעייה", -"השתלת אברים", -"התאגדות", -"התרת נישואים", -"ועדת הכלכלה", -"זיהום אוויר", -"זיכרון השואה", -"זכויות הנכים", -"זכויות חיילים", -"זכויות יוצרים", -"חוק אוויר נקי" : "זיהום אוויר", -"חוק גיל הפרישה", -"חוק האזרחים הותיקים", -"חוק הביטוח הלאומי" : "ביטוח לאומי", -"חוק הספורט", -"חוק הרשות השניה", -"חוק התקשורת", -"חוק חג המצות", -"חוק חינוך ממלכתי", -"חוק טל", -"חוק לימוד חובה", -"חינוך בלתי-פורמלי", -"חללי צה\"ל", -"חניה", -"חסכון", -"חריגות בניה", -"טכנולוגיה", -"טלוויזיה", -"יבוא ויצוא", -"ימי זכרון", -"ישיבות", -"יתמות", -"לסביות" : "להט\"ב", -"לשון הרע", -"מבחן הכנסה", -"מהגרים בלתי חוקיים", -"מינהל מקרקעי ישראל", -"מנהל מקרקעי ישראל", -"משטר נשיאותי", -"משרד התחבורה, התשתיות הלאומיות והבטיחות בדרכים", -"משרד התקשורת", -"נגב", -"נהיגה בשכרות", -"נהיגה תחת השפעת סמים", -"נוכחים-נפקדים", -"ניירות ערך", -"נכי צה\"ל", -"נפגעי פעולות איבה", -"נשק", -"סחר נשים", -"סיוע לעסקים", -"סכסוך עבודה", -"סלולר", -"עובדי ציבור", -"עיתונות", -"פונדקאות", -"פינוי", -"פלסטינאים", -"פקודת מס הכנסה", -"פרסומות", -"ציבור דתי", -"צפון", -"קהילה אתיופית", -"קצבאות ילדים", -"קרן קיימת לישראל", -"ראש הממשלה", -"שב\"כ", -"שביתה", -"שוק חופשי", -"שירות המדינה", -"שירותי קהילה", -"שכר והוצאות שרים וחברי כנסת", -"שעון קיץ", -"שעת חירום", -"תושבי חוץ", -"תיווך מקרקעין", -"תכנון ובנייה" : "תכנון ובניה", -"תלונות הציבור", -"תמלוגים", -"תנאי תעסוקה", -"תנועות נוער", -"תעודת זהות", -"תעופה", -"תקנות שעת חירום", -"תרומות", + u"אומנה" : u"אימוץ", + u"אונס" : u"אלימות מינית", + u"הטרדה מינית" : u"אלימות מינית", + u"אחים שכולים" : u"קצבאות", + u"אשראי" : u"בנקאות", + u"אפליה" : u"שוויון", + #u"בחירות לנשיאות", + u"בחירות לכנסת" : u"בחירות", + u"בית דין לעבודה" : u"ביטוח לאומי", + # u"בית הדין לענייני משפחה", + # u"בני זוג", + # u"בנייה ירוקה", + # u"בריאות הפה", + u"בטחון" : u"ביטחון", + u"בתי דין רבנים" : u"בית דין רבני", + u"בתי דין רבניים" : u"בית דין רבני", + # u"גז ונפט", + u"גיור" : u"דת", + u"דת ומדינה" : u"דת", + # uחופש דת, יהדות, נישואים אזרחיים, נישואין וגירושין, דת ומדינה, גיור, דת + u"דיור מוגן" : u"אזרחים ותיקים", + u"דיור ציבורי" : u"דיור בר השגה", + # u"דיני חוזים", + # uשוק חופשי, ריכוזיות, תחרותיות במשק + u"הורים" : u"הורות", + # u"הזכות להליך הוגן", + # u"הימורים", + # u"הכרזת מלחמה", + # u"המגזר הבדואי", + u"הסברה" : u"חינוך", + # u"הסדר מדיני", + u"הסתה לגזענות" : u"גזענות", + u"הפליה" : u"שוויון", + + # u"השמה חוץ ביתית" : "קטינים ונוער", + u"השעייה" : u"שחיתות", + # u"השתלת אברים", + #u"התאגדות", + u"התרת נישואים" : u"בית דין רבני", + u"ועדת הכלכלה" : u"כלכלה", + u"זיהום אוויר" : u"איכות הסביבה", + u"פסולת" : u"איכות הסביבה", + u"זיהום" : u"איכות הסביבה", + u"חוק אוויר נקי" : u"איכות הסביבה", + # u"זיכרון השואה", + u"זכויות הנכים" : u"ניצולי שואה", + u"זכויות חיילים" : u"מילואים", + # u"זכויות יוצרים", + u"חוק אוויר נקי" : u"איכות הסביבה", + u"חוק גיל הפרישה" : u"פנסיה", + u"חוק האזרחים הותיקים" : u"אזרחים ותיקים", + u"חוק הביטוח הלאומי" : u"ביטוח לאומי", + u"חוק הספורט" : u"ספורט", + u"חוק הרשות השניה" : u"טלוויזיה", + u"חוק התקשורת" : u"תקשורת", + u"חוק חג המצות" : u"חגים ומועדים", + u"חוק חינוך ממלכתי" : u"חינוך", + u"חוק טל" : u"ישיבות הסדר", + u"חוק לימוד חובה" : u"חינוך יסודי ועל יסודי", + # u"חינוך בלתי-פורמלי", + u"חללי צה\"ל" : u"צבא", + u"חניה" : u"תחבורה", + # u"חסכון", + u"חריגות בניה" : u"תכנון ובניה", + # u"טכנולוגיה", + # u"טלוויזיה", + # u"יבוא ויצוא", + u"יהודה ושומרון" : u"יהודה ושומרון", + u"ימי זכרון" : u"חגים ומועדים", + # u"ישיבות", + # u"יתמות", + u"לסביות" : u"להט\"ב", + u"לשון הרע" : u"חופש ביטוי", + u"מבחן הכנסה" : u"נישואין וגירושין", + u"מהגרים בלתי חוקיים" : u"מהגרי עבודה", + u"מינהל מקרקעי ישראל" : u"מנהל מקרקעי ישראל", + # u"משטר נשיאותי", + u"משרד התחבורה, התשתיות הלאומיות והבטיחות בדרכים" : u"תחבורה", + u"משרד התקשורת" : u"תקשורת", + # u"נגב" : "פריפריה", + u"נהיגה בשכרות" : u"בטיחות בדרכים", + u"נהיגה תחת השפעת סמים" : u"בטיחות בדרכים", + u"נוכחים-נפקדים" : u"הגירה וגבולות", + # u"ניירות ערך", + # u"נכי צה\"ל", + # u"נפגעי פעולות איבה", + # u"נשק", + # u"סחר נשים", + # u"סיוע לעסקים", + # u"סכסוך עבודה", + # u"סלולר", + u"עובדי ציבור" : u"השירות הציבורי", + # u"עיתונות", + # u"פונדקאות", + u"פינוי" : u"התנחלויות", + u"פלסטינאים" : u"הסדר מדיני", + u"פקודת מס הכנסה" : u"מס הכנסה", + u"פרסומות" : u"פרסום ושיווק", + # u"ציבור דתי", + # u"צפון" : "פריפריה", + # u"קהילה אתיופית", + # u"קצבאות ילדים", + # u"קרן קיימת לישראל", + # u"ראש הממשלה", + # u"שב\"כ", + u"שביתה" : u"סכסוך עבודה", + u"אי שוויון/שוויון" : u"שוויון", + # u"שוק חופשי", + u"שירות המדינה" : u"עובדי ציבור", + # u"שירותי קהילה", + u"שכר והוצאות שרים וחברי כנסת" : u"מנהל תקין", + # u"שעון קיץ", + u"שעת חירום" : u"תקנות שעת חירום", + u"תושבי חוץ" : u"דיור", + # u"תיווך מקרקעין", + u"תכנון ובנייה" : u"תכנון ובניה", + # u"תלונות הציבור", + # u"תמלוגים", + u"תנאי תעסוקה" : u"זכויות עובדים", + # u"תנועות נוער" : "קטינים ונוער", + # u"תעודת זהות" : "משרד הפנים", + # u"תעופה", + # u"תרומות", +} + +change_docs_tags = { + 11992 : u'תחבורה', } diff --git a/laws/tags/tags_autolearn_play.py b/laws/tags/tags_autolearn_play.py index c0996b22..3977efde 100644 --- a/laws/tags/tags_autolearn_play.py +++ b/laws/tags/tags_autolearn_play.py @@ -211,7 +211,7 @@ def cals_distances(impostors, x, y): from sklearn.feature_extraction.text import TfidfTransformer from sklearn.ensemble import ExtraTreesClassifier -from sklearn.feature_selection import SelectFromModel +#from sklearn.feature_selection import SelectFromModel from sklearn import svm from sklearn import naive_bayes @@ -239,6 +239,11 @@ def cals_distances(impostors, x, y): #classifier = OneVsRestClassifier(make_pipeline(preprocessing.StandardScaler(),single_classifier)) classifier = OneVsRestClassifier(single_classifier) + +# learn_data = preprocessing.scale(learn_data) + +#classifier = make_pipeline(preprocessing.StandardScaler(), classifier) + #classifier = make_pipeline(SelectFromModel(ExtraTreesClassifier(), prefit = False), OneVsRestClassifier(single_classifier)) #classifier = make_pipeline(SelectFromModel(ExtraTreesClassifier(), prefit = False), single_classifier) @@ -373,7 +378,8 @@ def cross_val_score_one_vs_all_per_class(estimator, X, y=None, *args, **kargs): return np.array(list(res)) # cross_validation.cross_val_score = cross_val_score_one_vs_all_per_class - scores = cross_validation.cross_val_score(classifier, learn_data, labels, scoring=f1_scorer_no_average, cv=3)# 'f1_weighted') + scores = cross_validation.cross_val_score(classifier, learn_data, labels, scoring=f1_scorer_no_average, cv=10)# 'f1_weighted') + scores = np.array(list(scores)) import IPython; IPython.embed() print "Accuracy: %0.2f (+/- %0.2f)" % (scores.mean(), scores.std() * 2)