diff --git a/apis/resources/__init__.py b/apis/resources/__init__.py index 218349b8..5174f0a5 100644 --- a/apis/resources/__init__.py +++ b/apis/resources/__init__.py @@ -6,7 +6,7 @@ from mks.api import MemberResource, PartyResource, MemberBillsResource, MemberAgendasResource from video.api import VideoResource from links.api import LinkResource -from laws.api import BillResource, LawResource, VoteResource, VoteActionResource +from laws.api import BillResource, LawResource, VoteResource, VoteActionResource, ProvateProposalResource from agendas.api import AgendaResource, AgendaTodoResource from committees.api import CommitteeResource, CommitteeMeetingResource, ProtocolPartResource from auxiliary.api import PostResource, TagResource @@ -27,6 +27,7 @@ v2_api.register(VoteResource()) v2_api.register(VoteActionResource()) v2_api.register(LawResource()) +v2_api.register(ProvateProposalResource()) v2_api.register(AgendaResource()) v2_api.register(AgendaTodoResource()) v2_api.register(CommitteeResource()) @@ -39,4 +40,4 @@ v2_api.register(PersonResource()) v2_api.register(LobbyistsChangeResource()) v2_api.register(LobbyistResource()) -v2_api.register(LobbyistCorporationResource()) \ No newline at end of file +v2_api.register(LobbyistCorporationResource()) diff --git a/docs/devel/source/autotagging.rst b/docs/devel/source/autotagging.rst new file mode 100644 index 00000000..80f15c38 --- /dev/null +++ b/docs/devel/source/autotagging.rst @@ -0,0 +1,65 @@ +.. autotagging: + +======= +Intro +======= + +Auto tagging is used to create auto tags out of the explanition info of a private proposal. +It uses machine learning techniques. + +====== +Flow +====== + +* Reading - Read the data from the server using API. +* Parsing - Parse the downloaded HTML files of the PP. +* Building - Make histograms out of the parsed HTML. +* Train/Test the information. + +Reading +========= + +Run: + +.. code-block:: sh + + python read_pp.py + +Parsing +========= + +Run: + +.. code-block:: sh + + python parse_htmls.py + +Building +========== + +Run: + +.. code-block:: sh + + python make_histograms.py + +Training/Testing +================== + +Auto tagging is in testing phase, so no auto-tagging script exist. +To execute the test, run: + +.. code-block:: sh + + python tags_autolearn_play.py + +Building HTML +=============== + +This is used to understand to find errors in the tagging process. Just use your logic here. +To execute the test, run: + +.. code-block:: sh + + python build_important_keyword_html.py + diff --git a/laws/api.py b/laws/api.py index db9b96a7..fb3127f8 100755 --- a/laws/api.py +++ b/laws/api.py @@ -5,7 +5,6 @@ from django.core.urlresolvers import reverse from tastypie.constants import ALL import tastypie.fields as fields - from agendas.templatetags.agendas_tags import agendas_for from apis.resources.base import BaseResource @@ -27,6 +26,36 @@ class LawResource(BaseResource): class Meta(BaseResource.Meta): queryset = Law.objects.all() allowed_methods = ['get'] +class TagResource(BaseResource):pass +class ProvateProposalResource(BaseResource): + class Meta(BaseResource.Meta): + queryset = PrivateProposal.objects.all() + allowed_methods = ['get'] + + filtering = dict(from_date=ALL, + to_date=ALL) + + + tags = fields.ToManyField(TagResource, + 'tags', + + null=True, + full=False) + + def dehydrate_tags(self, bundle): + return [tag.name for tag in bundle.obj.bill.tags] + + def build_filters(self, filters={}): + orm_filters = super(ProvateProposalResource, self).build_filters(filters) + if 'from_date' in filters: + orm_filters["date__gte"] = filters['from_date'] + if 'to_date' in filters: + # the to_date needs to be incremented by a day since when humans say to_date=2014-07-30 they + # actually mean midnight between 30 to 31. python on the other hand interperts this as midnight between + # 29 and 30 + to_date = datetime.strptime(filters["to_date"], "%Y-%M-%d")+timedelta(days=1) + orm_filters["date__lte"] = to_date.strftime("%Y-%M-%d") + return orm_filters class VoteActionResource(BaseResource): class Meta(BaseResource.Meta): diff --git a/laws/tags/build_important_keyword_html.py b/laws/tags/build_important_keyword_html.py new file mode 100644 index 00000000..08dfd2be --- /dev/null +++ b/laws/tags/build_important_keyword_html.py @@ -0,0 +1,190 @@ +#!/usr/bin/python +# -*- coding: UTF-8 -*- +import numpy +import numpy as np +import re +from functools import partial +from sklearn.externals import joblib +import cPickle as pickle +from collections import defaultdict +lb = pickle.load(open("classifier_data/label_binarizer.pkl", "rb")) +keywords, data = pickle.load(open('histograms_with_tags.pkl', 'rb')) +pp_with_tags = pickle.load(open('pp_with_tags.pkl', "rb")) +trained_classifier = joblib.load('classifier_data/linear_svc_classifier.jlb') +keywords = numpy.array(sorted(list(keywords))) + +coefs = np.vstack([e.steps[-1][1].coef_ for e in trained_classifier.estimators_]) + +import os +import codecs +try: + os.makedirs("important_keywords/tags") +except OSError, e: + pass +try: + os.makedirs("important_keywords/keywords") +except OSError, e: + pass + +doc_in_class = defaultdict(list) + +print "creating tag dicts" +for a in pp_with_tags: + for tag in a['tags']: + doc_in_class[tag].append(a) + + + +print "creating keyword dict" +keyword_dict = {} +for i,w in enumerate(keywords): + keyword_dict[w] = i + + +print "splitting docs" +pp_with_tags = pickle.load(open('pp_with_tags.pkl', "rb")) +pp_dict = {} +for pp in pp_with_tags: + pp_dict[pp['id']] = pp['text'] + +def output_tag(class_file, w, estimator_max, estimator_min, weight, hyperlink = False): + orig_weight = weight + pre = post = '' + if weight > 0 and estimator_max != 0: + weight /= estimator_max + if weight > 0.05 and hyperlink: + pre = u"" + post = u"" + print >>class_file, pre + u"" % (int(255 * weight),) + w + u"" + post + " " + elif weight < 0 and estimator_min != 0: + weight /= estimator_min + if weight > 0.05 and hyperlink: + pre = u"" + post = u"" + print >>class_file, pre + u"" % (int(255 * weight),) + w + u"" + post + " " + else: + print >>class_file, u"" + w + u" " + +def output_word(class_file, w, estimator_max, estimator_min, coefs, keyword_dict, hyperlink = False): + weight = coefs[keyword_dict[w]] + pre = post = '' + if weight > 0 and estimator_max != 0: + weight /= estimator_max + if weight > 0.05 and hyperlink: + pre = u"" + post = u"" + print >>class_file, pre + u"" % (int(255 * weight),) + w + u"" + post + " " + elif weight < 0 and estimator_min != 0: + weight /= estimator_min + if weight > 0.05 and hyperlink: + pre = u"" + post = u"" + print >>class_file, pre + u"" % (int(255 * weight),) + w + u"" + post + " " + else: + print >>class_file, u"" + w + u" " + + +def format_word(w, estimator_max, estimator_min, coefs, keyword_dict, hyperlink = False): + orig_Weight = weight = coefs[keyword_dict[w]] + pre = post = '' + if weight > 0 and estimator_max != 0: + weight /= estimator_max + if hyperlink: + pre = u"" + post = u"" + v = u"" % (int(255 * weight),) + w + u"" + elif weight < 0 and estimator_min != 0: + weight /= estimator_min + if hyperlink: + pre = u"" + post = u"" + v = u"" % (int(255 * weight),) + w + u"" + else: + v = u"" + w + u"" + return u"%s%s%s" % (pre, v, post,) + + +ATTR_AMOUNT = 10 +important_keywords = keywords[numpy.abs(coefs).argsort()[:,-ATTR_AMOUNT:]] + +MAX_RE_GROUP = 100 +def create_re(estimator_max, estimator_min, coefs, keyword_dict): + #return ("|".join(p) for p in zip(*( + # ( + # u"(?P\\b)%s(?P\\b)" % (i, re.escape(k), i, ), + # u"\\g%s\\g" % (i, re.escape(format_word(k, estimator_max, estimator_min, estimator, keyword_dict, hyperlink = True)), i, ), + # ) + # for i, k + # in enumerate(keywords[numpy.where(estimator.coef_[0] != 0)]) + # ) + #)) + + return { + k : format_word(k, estimator_max, estimator_min, coefs, keyword_dict, hyperlink = True) + for k + in keywords[numpy.where(coefs != 0)] + + } +ALLOWED_PRELETTERS = 3 +MIN_WORD_LEN = 2 +def replace_match(replacements, match): + whole_match_str = match_str = match.group(0) + if match_str in replacements: + return replacements[match_str] + + for i in xrange(1, min(ALLOWED_PRELETTERS + 1, len(match_str) - MIN_WORD_LEN + 1)): + pre, pre_match_str = match_str[:i], match_str[i:] + if pre_match_str in replacements: + return pre + replacements[pre_match_str] + + return u"[%s]" % (match.group(0),) + +print "creating files" +for i, (class_name, important_keyword_for_class, estimator) in enumerate(zip(lb.classes_, important_keywords, trained_classifier.estimators_)): + print "tag", str(i), "/", str(len(lb.classes_)) + class_file = codecs.open("important_keywords/tags/" + class_name.replace("/", ' ') + ".html", "wt", encoding="utf-8") + + print >>class_file, u"Keyword Explanation for Class " + class_name + u"" + print >>class_file, u"

" + class_name + u"


" + print >>class_file, u"

Most influencing words

" + estimator_min, estimator_max = coefs[i,:].min(), coefs[i,:].max() + for keyword in important_keyword_for_class: + output_word(class_file, keyword, estimator_max, estimator_min, coefs[i], keyword_dict) + print >>class_file, u"(", str(coefs[i,keyword_dict[keyword]]), u")
" + +# tag_pattern, tag_replacement_pattern = create_re(estimator_max, estimator_min, estimator, keyword_dict) +# tag_re = re.compile(tag_pattern, re.MULTILINE | re.UNICODE) + replacements = create_re(estimator_max, estimator_min, coefs[i], keyword_dict) + replacements_re = re.compile('|'.join(('|'.join((u'\\b%s%s\\b' % (u'\\w' * i, p,) for p in replacements.iterkeys())) for i in xrange(ALLOWED_PRELETTERS + 1))), re.UNICODE) + print >>class_file, u"" + for doc in doc_in_class[class_name]: + + print >>class_file, u"" + print >>class_file, u"
" + unicode(doc['id']) + u"" + print >>class_file, replacements_re.sub(partial(replace_match, replacements), pp_dict[doc['id']]) + + #import pdb; pdb.set_trace() +# print >>class_file, reduce(lambda s,(o,r): re.sub(o,r,s, flags=re.UNICODE), replacements, pp_dict[doc['id']]) +# print >>class_file, tag_re.sub(tag_replacement_pattern, pp_dict[doc['id']]) + print >>class_file, u"
" + class_file.close() + +coef_abs = numpy.abs(coefs) +sorted_keywords = coef_abs.argsort(axis = 0) +min_weight, max_weight = coefs.min(axis = 1), coefs.max(axis = 1) +for keyword_count, i in enumerate(coef_abs.sum(axis = 0).argsort()[::-1]): + keyword = keywords[i] + print "keyword", str(keyword_count), "/", str(len(keywords)), "(", str(i), ")" + keyword_file = codecs.open("important_keywords/keywords/" + keyword.replace("/", ' ') + ".html", "wt", encoding="utf-8") + print >>keyword_file, u"Tag Explanation for Keyword " + keyword + u"" + print >>keyword_file, u"

" + keyword + u"


" + print >>keyword_file, u"

Most influenced tags

" + weights = coefs[:,i] + class_keywords = sorted_keywords[::-1,i] + + for class_name, weight, max_weight_, min_weight_ in zip(lb.classes_[class_keywords], weights[class_keywords], max_weight[class_keywords], min_weight[class_keywords]): + if weight != 0: + output_tag(keyword_file, class_name, max_weight_, min_weight_, weight, hyperlink = True) + print >>keyword_file, u"
" + print >>keyword_file, u"" + keyword_file.close() diff --git a/laws/tags/make_histograms.py b/laws/tags/make_histograms.py new file mode 100644 index 00000000..c0dc347a --- /dev/null +++ b/laws/tags/make_histograms.py @@ -0,0 +1,62 @@ +#!/usr/bin/python +# -*- coding: UTF-8 -*- +import cPickle as pickle +from collections import defaultdict + +#REMOVE = ['"', "'", '/', '\\', ',', '.', '!', '@', '#', '$', '%', '^', '&', '*', '(', ')', '-', '=', '_', '+', '\n', '[', ']', '{', '}', '|', '<', '>', '?', '~', ":", ';'] +REMOVE = [u'"', u"'", u'/', u'\\', u',', u'.', u'!', u'@', u'#', u'$', u'%', u'^', u'&', u'*', u'(', u')', u'-', u'=', u'_', u'+', u'\n', u'[', u']', u'{', u'}', u'|', u'<', u'>', u'?', u'~', u":", u';', u'–' , u'”', u'`'] +REMOVE.extend([unicode(i) for i in xrange(10)]) +REMOVE.extend([str(i) for i in xrange(10)]) + +def make_histogram(t): + dd = defaultdict(int) +# for w in t.split(): +# dd[reduce(lambda o, s: o.replace(s, ' '), REMOVE, w)] += 1 + for w in reduce(lambda o, s: o.replace(s, u' '), REMOVE, t).split(): + dd[w] += 1 + return dict(dd) + +from collections import defaultdict +ACCEPTED_POS = ['NN', 'JJ', 'VB', 'NNT', 'BN', 'NNP', 'TTL', ] +def make_pos_histogram(t): + dd = defaultdict(int) + for w in [ppp.split('\t')[1] for ppp in t.splitlines() if ppp.count('\t') > 5 and (ppp.split('\t')[3] in ACCEPTED_POS or ppp.split('\t')[4] in ACCEPTED_POS)]: + dd[w] += 1 + return dict(dd) + + +# needed phrases: NN, JJ, VB, NNT, BN +# Use this: +# http://www.cs.bgu.ac.il/~yoavg/depparse/gparse +# or this: +# http://www.cs.bgu.ac.il/~yoavg/constparse/gparse +# or this: +# http://www.cs.bgu.ac.il/~nlpproj/demo/ +# from here: +# http://www.cs.bgu.ac.il/~nlpproj/ +# seems like: +# http://www.cs.bgu.ac.il/~yoavg/software/hebparsers/hebdepparser/hebdepparser.tgz +# is working good enough + +pp_with_tags = pickle.load(open('pp_pos_with_tags.pkl', "rb")) + +keywords = set() +data = [] +for i, pp in enumerate(pp_with_tags): + pp_pos, pp_tag, pp_id = pp['pos'], pp['tags'], pp['id'] + + #print str(i), "/", str(len(pp_with_tags)) + try: + histogram = make_pos_histogram(pp_pos) +# histogram = make_histogram(pp_text) + except: + import traceback; traceback.print_exc() + print "at pp", i + import pdb; pdb.set_trace() + data.append({'histogram' : histogram, 'tags' : pp_tag, 'id' : pp_id}) + keywords |= set(histogram.keys()) # this will add the keys + + + +pickle.dump((keywords, data), open('histograms_with_tags.pkl', 'wb')) + diff --git a/laws/tags/parse_htmls.py b/laws/tags/parse_htmls.py new file mode 100644 index 00000000..00f37160 --- /dev/null +++ b/laws/tags/parse_htmls.py @@ -0,0 +1,88 @@ +#!/usr/bin/python +# -*- coding: UTF-8 -*- + +from BeautifulSoup import BeautifulSoup + +import cPickle as pickle + +from tag_rename import change_docs_tags, class_map + +similar_texts = [ + u"הצעת חוק דומה בעיקרה הונחה", + u"הצעת חוק זהה הונחה", + u"הצעות חוק דומות בעיקרן הונחו", + u"הצעות חוק זהות הונחו", + u"הצעות חוק זהות הונחו", + u"הצעת חוק זהות הונחו", + u"הצעת חוק זהות הונחו", + u"הצעת חוק זהה הוגשה", + u"הצעת חוק זהה הונח", + u"הצעת חוק יסוד זהה הונחה", + u"הצעת חוק זהה הוגשה", + u"הצעות חוק זהות הונחו", + u"הצעת חוק זהות הונחו", + u"הצעות חוק זהות הונחו", + u"הצעות חוק זהות הונחו", + u"הצעות חוק זהות הונחו", + u"סעיפי הצעת חוק זו מבוססים", + u"הצעת החוק נכתבה בסיוע", + u"הצעת חוק דומות הונחו", + u"הצעת חוק דומה בעיקרה זהה הוגשה", + u"הצעות חוק זהות הונחו", + u"הצעות חוק זהות הונחו", + u"הצעות חוק דומות הונחו", + u"הצעת חוק-יסוד זהה הונחה", + u"הצעות חוק-יסוד זהות הונחו", + u"הצעות חוק זהות הונחו", + u"הצעת חוק דומה בעיקרה זהה הוגשה", + u"הצעות חוק זהות הונחה", + u"הצעת חוק דומות בעיקרן הונחו", + u"הצעת חוק דומות בעיקרן הונחו", + u"הצעת דומה בעיקרה הונחה", + u"הצעות חוק דומות בעיקרון הונחו", + u"הצעות חוק זהה הונחה", + u"הצעת חוק דומות בעיקרן הונחן", + u"הצעת חוק זו הוכנה בסיוע", + u"הצעת חוק דומות בעיקרן הונחו", + +] +pp_without_tags = [] +pp_with_tags = [] + + +for i, pp in enumerate(pickle.load(open('pps.pkl', 'rb'))): + content_html = pp['content_html'] + parsed_html = BeautifulSoup(content_html) + + current_tag = parsed_html.find('p', attrs={'class':'explanation-header'}) + if current_tag is None: + print "no text for pp", pp['id'] + continue + current_tag = current_tag.findNext() + text = '' + while current_tag is not None and '---' not in current_tag.text: + + text += current_tag.text + current_tag = current_tag.findNext() + + tags = [class_map.get(t, t) for t in pp['tags']] + + if pp['id'] in change_docs_tags: + tags = [change_docs_tags[pp['id']]] + + for similar_text in similar_texts: + similar_loc = text.find(similar_text) + if similar_loc != -1: + text = text[:similar_loc] + + + + if len(tags) == 0: + pp_without_tags.append({'text' : text, 'id' : pp['id']}) + else: + pp_with_tags.append({'text' : text, 'id' : pp['id'], 'tags' : tags}) + + +pickle.dump(pp_without_tags, open("pp_without_tags.pkl", "wb")) +pickle.dump(pp_with_tags, open("pp_with_tags.pkl", "wb")) + diff --git a/laws/tags/parse_pos_tag.py b/laws/tags/parse_pos_tag.py new file mode 100644 index 00000000..807670a3 --- /dev/null +++ b/laws/tags/parse_pos_tag.py @@ -0,0 +1,68 @@ +#!/usr/bin/python +# -*- coding: UTF-8 -*- +import cPickle as pickle + +#hebdepparser_path = "/home/o/otadmor/Downloads/hebdepparser" +hebdepparser_path = "/home/someone/hebdepparser" +import subprocess + +def parse_lang_tree_popen(text): +# p = subprocess.Popen(["python", "/home/o/otadmor/Downloads/hebdepparser/parse.py"], stdin=subprocess.PIPE, stdout=subprocess.PIPE) + p = subprocess.Popen("python " + hebdepparser_path + "/code/utils/sentences.py | python " + hebdepparser_path + "/parse.py", shell=True, stdin=subprocess.PIPE, stdout=subprocess.PIPE) + + p.stdin.write(text.encode('utf-8')) + + p.stdin.flush() + p.stdin.close() + a = p.stdout.read() + res = '' + while a != '': + res += a + a = p.stdout.read() + p.stdout.close() + p.wait() + return res.decode('utf-8') + + + + +# for w in reduce(lambda o, s: o.replace(s, u' '), REMOVE, t).split(): + + +import sys +sys.path.append(hebdepparser_path) +from parse import parse_sent +from cStringIO import StringIO +def split_sentences(text): + for l in reduce(lambda o, s: o.replace(s, s + u'\n'), ['.', '!', '?'], text).splitlines(): + l = l.strip() + if not l.endswith("."): + l += "." + yield l + +import codecs +def parse_lang_tree(text): + out = StringIO() + for l in split_sentences(text): + parse_sent(l, out) + return out.getvalue().decode('utf-8') + +pp_with_tags = pickle.load(open('pp_with_tags.pkl', "rb")) + +data = [] +for i, pp in enumerate(pp_with_tags): + pp_text, pp_tag, pp_id = pp['text'], pp['tags'], pp['id'] + + print str(i), "/", str(len(pp_with_tags)) + try: + parsed_pos = parse_lang_tree(pp_text) + except: + import traceback; traceback.print_exc() + print "at pp", i + import pdb; pdb.set_trace() + data.append({'pos' : parsed_pos, 'tags' : pp_tag, 'id' : pp_id}) + + + +pickle.dump(data, open('pp_pos_with_tags.pkl', 'wb')) + diff --git a/laws/tags/read_pp.py b/laws/tags/read_pp.py new file mode 100644 index 00000000..b2432b39 --- /dev/null +++ b/laws/tags/read_pp.py @@ -0,0 +1,22 @@ +import urllib +import json +import sys +import cPickle as pickle +base_url = "http://127.0.0.1:8000" +next_url = "/api/v2/provateproposal/" +pps = [] + +while next_url != None: + uo = urllib.urlopen(base_url + next_url.replace("\\/", "/")) + try: + + json_obj = json.load(uo) # .read(), {'null' : None}) # just for eval to work + print json_obj['meta']['offset'], "/", json_obj['meta']['total_count']; sys.stdout.flush() + pps.extend(json_obj['objects']) + next_url = json_obj['meta']['next'] + print next_url + finally: + uo.close() + +with open("pps.pkl", "wb") as po: + pickle.dump(pps, po) diff --git a/laws/tags/results.txt b/laws/tags/results.txt new file mode 100644 index 00000000..3f6e498d --- /dev/null +++ b/laws/tags/results.txt @@ -0,0 +1,40 @@ +original tags +features classifier score +NN_JJ_VB_NNT_BN_NNP_TTL OneVsRest_LinearSVC 0.65 +- 0.06 +NN_JJ_VB_NNT_BN_NNP_TTL OneVsRest_LinearSVC 0.65 +- 0.06 +NN_JJ_VB_NNT_BN OneVsRest_InfoGainFilter_TfidfTransformer_LinearSVC 0.58 +- 0.02 +NN_JJ_VB_NNT_BN InfoGainFilter_OneVsRest_TfidfTransformer_LinearSVC 0.64 +- 0.04 +NN_JJ_VB_NNT_BN InfoGainFilter_OneVsRest_StandardScaler_LinearSVC 0.50 +- 0.06 +NN_JJ_VB_NNT_BN InfoGainFilter_StandardScaler_MLPClassifier_10_levels_100_each 0.48 +- 0.04 +NN_JJ_VB_NNT_BN StandardScaler_MLPClassifier_10_levels_100_each 0.52 +- 0.05 +NN_JJ_VB_NNT_BN_NNP_TTL OneVsRest_GausianNB 0.51 +- 0.06 +NN_JJ_VB_NNT_BN_NNP_TTL OneVsRest_MultinomialNB 0.44 +- 0.20 +NN_JJ_VB_NNT_BN_NNP_TTL OneVsRest_BernoulliNB 0.08 +- 0.00 +NN_JJ_VB_NNT_BN_NNP_TTL KNeighborsClassifier 0.35 +- 0.02 +NN_JJ_VB_NNT_BN_NNP_TTL OneVsRest_TfidfTransformer_LinearSVC_10folds 0.74 +- 0.34 +NN_JJ_VB_NNT_BN_NNP_TTL OneVsRest_TfidfTransformer_LinearSVC 0.64 +- 0.05 +NN_JJ_VB_NNT_BN OneVsRest_TfidfTransformer_LinearSVC 0.64 +- 0.05 +NN_JJ_VB_NNT_BN_NNP_TTL TfidfTransformer_KNeighborsClassifier 0.47 +- 0.01 very fast + +NN_JJ_VB_NNT_BN_NNP_TTL TfidfTransformer_GausianNB +NN_JJ_VB_NNT_BN_NNP_TTL TfidfTransformer_MultinomialNB +NN_JJ_VB_NNT_BN_NNP_TTL TfidfTransformer_BernoulliNB + + +NN_JJ_VB_NNT_BN_NNP_TTL OneVsRest_TfidfTransformer_GausianNB 0.00 +- 0.00 +NN_JJ_VB_NNT_BN_NNP_TTL OneVsRest_TfidfTransformer_MultinomialNB 0.00 +- 0.00 +NN_JJ_VB_NNT_BN_NNP_TTL OneVsRest_TfidfTransformer_BernoulliNB 0.08 +- 0.00 + + + + + + +reduced tags 300 +features classifier score +NN_JJ_VB_NNT_BN_NNP_TTL StandardScaler_OneVsRest_LinearSVC_3folds 0.03 +- 0.04 +NN_JJ_VB_NNT_BN_NNP_TTL OneVsRest_LinearSVC_3folds 0.59 +- 0.31 +NN_JJ_VB_NNT_BN_NNP_TTL OneVsRest_LinearSVC_10folds 0.73 +- 0.32 + + + diff --git a/laws/tags/tag_rename.py b/laws/tags/tag_rename.py new file mode 100644 index 00000000..87c901c5 --- /dev/null +++ b/laws/tags/tag_rename.py @@ -0,0 +1,132 @@ +#!/usr/bin/python +# -*- coding: UTF-8 -*- + +class_map = { + u"אומנה" : u"אימוץ", + u"אונס" : u"אלימות מינית", + u"הטרדה מינית" : u"אלימות מינית", + u"אחים שכולים" : u"קצבאות", + u"אשראי" : u"בנקאות", + u"אפליה" : u"שוויון", + #u"בחירות לנשיאות", + u"בחירות לכנסת" : u"בחירות", + u"בית דין לעבודה" : u"ביטוח לאומי", + # u"בית הדין לענייני משפחה", + # u"בני זוג", + # u"בנייה ירוקה", + # u"בריאות הפה", + u"בטחון" : u"ביטחון", + u"בתי דין רבנים" : u"בית דין רבני", + u"בתי דין רבניים" : u"בית דין רבני", + # u"גז ונפט", + u"גיור" : u"דת", + u"דת ומדינה" : u"דת", + # uחופש דת, יהדות, נישואים אזרחיים, נישואין וגירושין, דת ומדינה, גיור, דת + u"דיור מוגן" : u"אזרחים ותיקים", + u"דיור ציבורי" : u"דיור בר השגה", + # u"דיני חוזים", + # uשוק חופשי, ריכוזיות, תחרותיות במשק + u"הורים" : u"הורות", + # u"הזכות להליך הוגן", + # u"הימורים", + # u"הכרזת מלחמה", + # u"המגזר הבדואי", + u"הסברה" : u"חינוך", + # u"הסדר מדיני", + u"הסתה לגזענות" : u"גזענות", + u"הפליה" : u"שוויון", + + # u"השמה חוץ ביתית" : "קטינים ונוער", + u"השעייה" : u"שחיתות", + # u"השתלת אברים", + #u"התאגדות", + u"התרת נישואים" : u"בית דין רבני", + u"ועדת הכלכלה" : u"כלכלה", + u"זיהום אוויר" : u"איכות הסביבה", + u"פסולת" : u"איכות הסביבה", + u"זיהום" : u"איכות הסביבה", + u"חוק אוויר נקי" : u"איכות הסביבה", + # u"זיכרון השואה", + u"זכויות הנכים" : u"ניצולי שואה", + u"זכויות חיילים" : u"מילואים", + # u"זכויות יוצרים", + u"חוק אוויר נקי" : u"איכות הסביבה", + u"חוק גיל הפרישה" : u"פנסיה", + u"חוק האזרחים הותיקים" : u"אזרחים ותיקים", + u"חוק הביטוח הלאומי" : u"ביטוח לאומי", + u"חוק הספורט" : u"ספורט", + u"חוק הרשות השניה" : u"טלוויזיה", + u"חוק התקשורת" : u"תקשורת", + u"חוק חג המצות" : u"חגים ומועדים", + u"חוק חינוך ממלכתי" : u"חינוך", + u"חוק טל" : u"ישיבות הסדר", + u"חוק לימוד חובה" : u"חינוך יסודי ועל יסודי", + # u"חינוך בלתי-פורמלי", + u"חללי צה\"ל" : u"צבא", + u"חניה" : u"תחבורה", + # u"חסכון", + u"חריגות בניה" : u"תכנון ובניה", + # u"טכנולוגיה", + # u"טלוויזיה", + # u"יבוא ויצוא", + u"יהודה ושומרון" : u"יהודה ושומרון", + u"ימי זכרון" : u"חגים ומועדים", + # u"ישיבות", + # u"יתמות", + u"לסביות" : u"להט\"ב", + u"לשון הרע" : u"חופש ביטוי", + u"מבחן הכנסה" : u"נישואין וגירושין", + u"מהגרים בלתי חוקיים" : u"מהגרי עבודה", + u"מינהל מקרקעי ישראל" : u"מנהל מקרקעי ישראל", + # u"משטר נשיאותי", + u"משרד התחבורה, התשתיות הלאומיות והבטיחות בדרכים" : u"תחבורה", + u"משרד התקשורת" : u"תקשורת", + # u"נגב" : "פריפריה", + u"נהיגה בשכרות" : u"בטיחות בדרכים", + u"נהיגה תחת השפעת סמים" : u"בטיחות בדרכים", + u"נוכחים-נפקדים" : u"הגירה וגבולות", + # u"ניירות ערך", + # u"נכי צה\"ל", + # u"נפגעי פעולות איבה", + # u"נשק", + # u"סחר נשים", + # u"סיוע לעסקים", + # u"סכסוך עבודה", + # u"סלולר", + u"עובדי ציבור" : u"השירות הציבורי", + # u"עיתונות", + # u"פונדקאות", + u"פינוי" : u"התנחלויות", + u"פלסטינאים" : u"הסדר מדיני", + u"פקודת מס הכנסה" : u"מס הכנסה", + u"פרסומות" : u"פרסום ושיווק", + # u"ציבור דתי", + # u"צפון" : "פריפריה", + # u"קהילה אתיופית", + # u"קצבאות ילדים", + # u"קרן קיימת לישראל", + # u"ראש הממשלה", + # u"שב\"כ", + u"שביתה" : u"סכסוך עבודה", + u"אי שוויון/שוויון" : u"שוויון", + # u"שוק חופשי", + u"שירות המדינה" : u"עובדי ציבור", + # u"שירותי קהילה", + u"שכר והוצאות שרים וחברי כנסת" : u"מנהל תקין", + # u"שעון קיץ", + u"שעת חירום" : u"תקנות שעת חירום", + u"תושבי חוץ" : u"דיור", + # u"תיווך מקרקעין", + u"תכנון ובנייה" : u"תכנון ובניה", + # u"תלונות הציבור", + # u"תמלוגים", + u"תנאי תעסוקה" : u"זכויות עובדים", + # u"תנועות נוער" : "קטינים ונוער", + # u"תעודת זהות" : "משרד הפנים", + # u"תעופה", + # u"תרומות", +} + +change_docs_tags = { + 11992 : u'תחבורה', +} diff --git a/laws/tags/tags_autolearn_play.py b/laws/tags/tags_autolearn_play.py new file mode 100644 index 00000000..3977efde --- /dev/null +++ b/laws/tags/tags_autolearn_play.py @@ -0,0 +1,407 @@ +import numpy as np +# print(__doc__) +# +# import numpy as np +# import matplotlib.pyplot as plt +# +# from sklearn.datasets import make_multilabel_classification +# from sklearn.multiclass import OneVsRestClassifier +# from sklearn.svm import SVC +# from sklearn.preprocessing import LabelBinarizer +# from sklearn.decomposition import PCA +# from sklearn.cross_decomposition import CCA + + +def plot_hyperplane(clf, min_x, max_x, linestyle, label): + # get the separating hyperplane + w = clf.coef_[0] + a = -w[0] / w[1] + xx = np.linspace(min_x - 5, max_x + 5) # make sure the line is long enough + yy = a * xx - (clf.intercept_[0]) / w[1] + plt.plot(xx, yy, linestyle, label=label) + + +def plot_subfigure(X, Y, subplot, title, transform): + if transform == "pca": + X = PCA(n_components=2).fit_transform(X) + elif transform == "cca": + X = CCA(n_components=2).fit(X, Y).transform(X) + else: + raise ValueError + + min_x = np.min(X[:, 0]) + max_x = np.max(X[:, 0]) + + min_y = np.min(X[:, 1]) + max_y = np.max(X[:, 1]) + + classif = OneVsRestClassifier(SVC(kernel='linear')) + classif.fit(X, Y) + + plt.subplot(2, 2, subplot) + plt.title(title) + + zero_class = np.where(Y[:, 0]) + one_class = np.where(Y[:, 1]) + plt.scatter(X[:, 0], X[:, 1], s=40, c='gray') + plt.scatter(X[zero_class, 0], X[zero_class, 1], s=160, edgecolors='b', + facecolors='none', linewidths=2, label='Class 1') + plt.scatter(X[one_class, 0], X[one_class, 1], s=80, edgecolors='orange', + facecolors='none', linewidths=2, label='Class 2') + + plot_hyperplane(classif.estimators_[0], min_x, max_x, 'k--', + 'Boundary\nfor class 1') + plot_hyperplane(classif.estimators_[1], min_x, max_x, 'k-.', + 'Boundary\nfor class 2') + plt.xticks(()) + plt.yticks(()) + + plt.xlim(min_x - .5 * max_x, max_x + .5 * max_x) + plt.ylim(min_y - .5 * max_y, max_y + .5 * max_y) + if subplot == 2: + plt.xlabel('First principal component') + plt.ylabel('Second principal component') + plt.legend(loc="upper left") + + + + +import numpy +import cPickle as pickle +import sys + +print >>sys.stderr, 'loading data'; sys.stderr.flush() +keywords, data = pickle.load(open('histograms_with_tags.pkl', 'rb')) +keywords = sorted(list(keywords)) + +tags = numpy.array([a['tags'] for a in data]) +ids = [a['id'] for a in data] +print >>sys.stderr, 'preparing examples'; sys.stderr.flush() +keyword_dict = {} +for i,w in enumerate(keywords): + keyword_dict[w] = i + +learn_data = numpy.zeros((len(data), len(keywords))) +for i,a in enumerate(data): + histogram = learn_data[i] + for k,v in a['histogram'].iteritems(): + histogram[keyword_dict[k]] = v + +# + +print >>sys.stderr, 'vectorizing labels'; sys.stderr.flush() + +try: + from sklearn.preprocessing import MultiLabelBinarizer + lb = MultiLabelBinarizer() +except ImportError, e: + from sklearn.preprocessing import LabelBinarizer + lb = LabelBinarizer() + + +TRIM_SAMPLES = len(tags) #/ 10 +tags = tags[:TRIM_SAMPLES] +learn_data = learn_data[:TRIM_SAMPLES] + +lb.fit(tags) +labels = lb.transform(tags) + +print "using\t",TRIM_SAMPLES,"samples" +print "\t",len(keywords),"keywords" +print "\t",len(lb.classes_),"tags" +metadata = learn_data.sum(axis=1) + +print "\t",metadata.mean(), "avg words in document" +print "\t",metadata.max(), "biggest document" +print "\t",metadata.min(), "smallest document" + + +#plt.figure(figsize=(8, 6)) +#plot_subfigure(learn_data, labels, 1, "With unlabeled samples + CCA", "cca") +#plot_subfigure(learn_data, labels, 2, "With unlabeled samples + PCA", "pca") +#plt.subplots_adjust(.04, .02, .97, .94, .09, .2) +#plt.show() + +from sklearn.multiclass import OneVsRestClassifier, OneVsOneClassifier + +#from sklearn.utils.validation import check_consistent_length +from sklearn.multiclass import _fit_ovo_binary, check_consistent_length, np, Parallel, delayed +class OneVsOneClassifierMultiLabel(OneVsOneClassifier): + + def fit(self, X, y): + y = np.asarray(y) + check_consistent_length(X, y) + + self.classes_ = np.arange(y.shape[1]) + 1 + n_classes = self.classes_.shape[0] + self.estimators_ = Parallel(n_jobs=self.n_jobs)( + delayed(_fit_ovo_binary)( + self.estimator, X, self.classes_[i] * y[:,i] + self.classes_[j] * y[:,j], self.classes_[i], self.classes_[j]) + for i in range(n_classes) for j in range(i + 1, n_classes)) + + return self + + def predict(self, X): + Y = self.decision_function(X) + + return self.classes_[Y.argmax(axis=1)] + + def decision_function(self, X): + check_is_fitted(self, 'estimators_') + import pdb; pdb.set_trace() + + n_samples = X.shape[0] + n_classes = self.classes_.shape[0] + votes = np.zeros((n_samples, n_classes, n_classes)) + + k = 0 + for i in range(n_classes): + for j in range(i + 1, n_classes): + pred = self.estimators_[k].predict(X) + votes[:, i, j] += pred + k += 1 + return votes + +class KoppelSameAuthorClassifier(object): + def __init__(self): + super(KoppelSameAuthorClassifier, self).__init__() + self.class_data = None + self.orig_data = None + self.orig_class = None + + + def fit(self, X, y): + self.class_data = np.mat(y).transpose() * np.mat(X) + self.orig_data = X + self.orig_class = y + + def predict(self, X): + impostors_amount = 5 + top_similar = 3 + for x in X: + for i, y in enumerate(self.class_data): + impostors = self.orig_data[np.array(self.orig_class[:,i] == 0).flatten()] + # pick random top 5 impostors + np.random.shuffle(impostors) + impostors = impostors[:impostors_amount] + + distances = self.cals_distances(impostors, x, y) + sorted_distances = distances.argsort() + + # top 3 similar + same_author = (same_author[:top_similar] == impostors_amount).any() + + + def cals_distances(impostors, x, y): + # impostors[:, np.newaxis] - y # only if all y are sent (y is two dimensional matrix) + y_distance = np.power(impostors - y, 2).sum(axis = -1) + x_distance = np.power(impostors - x, 2).sum(axis = -1) + imposters_two_way_distance = np.sqrt(y_distance) * np.sqrt(x_distance) + + two_way_distance = np.power(x - y, 2).sum(axis = -1) + + + return np.append(imposters_two_way_distance,two_way_distance) + + +from sklearn.pipeline import make_pipeline + +from sklearn import preprocessing + +from sklearn.feature_extraction.text import TfidfTransformer + +from sklearn.ensemble import ExtraTreesClassifier +#from sklearn.feature_selection import SelectFromModel + +from sklearn import svm +from sklearn import naive_bayes +from sklearn import neighbors +from sklearn import linear_model +from sklearn import neural_network +#single_classifier = svm.SVC(kernel='polynomial', C=4) + +#single_classifier = svm.SVC(kernel='linear', cache_size = 2048) + +#single_classifier = neural_network.MLPClassifier(hidden_layer_sizes=10 * (100,)) + +single_classifier = svm.LinearSVC() + +#single_classifier = naive_bayes.GaussianNB() +#single_classifier = naive_bayes.MultinomialNB() +#single_classifier = naive_bayes.BernoulliNB() + +#single_svc = svm.SVC(kernel='rbf', cache_size = 2048) + +single_classifier = make_pipeline(TfidfTransformer(), single_classifier) +#single_classifier = make_pipeline(preprocessing.StandardScaler(), single_classifier) +#single_classifier = make_pipeline(SelectFromModel(ExtraTreesClassifier(), prefit = False), TfidfTransformer(), single_classifier) + +#classifier = OneVsRestClassifier(make_pipeline(preprocessing.StandardScaler(),single_classifier)) + +classifier = OneVsRestClassifier(single_classifier) + +# learn_data = preprocessing.scale(learn_data) + +#classifier = make_pipeline(preprocessing.StandardScaler(), classifier) + +#classifier = make_pipeline(SelectFromModel(ExtraTreesClassifier(), prefit = False), OneVsRestClassifier(single_classifier)) + +#classifier = make_pipeline(SelectFromModel(ExtraTreesClassifier(), prefit = False), single_classifier) + +#classifier = single_classifier + +#classifier = OneVsOneClassifierMultiLabel(single_classifier) + +#classifier = neighbors.KNeighborsClassifier() + +#classifier = make_pipeline(TfidfTransformer(), neighbors.KNeighborsClassifier()) +#classifier = linear_model.Ridge() + + +print "running", classifier +TEST = True + +if not TEST: + + trained_classifier = classifier.fit(learn_data, labels) + + from sklearn.externals import joblib + import cPickle as pickle + import os + try: + os.makedirs('classifier_data') + except OSError, e: + pass + joblib.dump(trained_classifier, 'classifier_data/linear_svc_classifier.jlb') + pickle.dump(lb, open("classifier_data/label_binarizer.pkl", "wb")) +else: + + + from sklearn import metrics + from functools import partial + f1_scorer_no_average = metrics.make_scorer(partial(metrics.f1_score, average=None)) + + def multilabel_score(estimator, X_test, y_test, scorer): + """Compute the score of an estimator on a given test set.""" + if y_test is None: + score = scorer(estimator, X_test) + else: + score = scorer(estimator, X_test, y_test) + if getattr(scorer, 'keywords', {}).get('average', None) is None: + return list(score) + elif not isinstance(score, numbers.Number): + raise ValueError("scoring must return a number, got %s (%s) instead." + % (str(score), type(score))) + return list(score) + from sklearn import cross_validation + cross_validation._score = multilabel_score + + import warnings + from sklearn.metrics.base import UndefinedMetricWarning + def default_nan_prf_divide(numerator, denominator, metric, modifier, average, warn_for): + """Performs division and handles divide-by-zero. + + On zero-division, sets the corresponding result elements to zero + and raises a warning. + + The metric, modifier and average arguments are used only for determining + an appropriate warning. + """ + result = numerator.astype(np.float_) / denominator.astype(np.float_) + mask = denominator == 0.0 + if not np.any(mask): + return result + + # remove infs + if average is None: + result[mask] = np.nan + else: + result[mask] = 0.0 + + # build appropriate warning + # E.g. "Precision and F-score are ill-defined and being set to 0.0 in + # labels with no predicted samples" + axis0 = 'sample' + axis1 = 'label' + if average == 'samples': + axis0, axis1 = axis1, axis0 + + if metric in warn_for and 'f-score' in warn_for: + msg_start = '{0} and F-score are'.format(metric.title()) + elif metric in warn_for: + msg_start = '{0} is'.format(metric.title()) + elif 'f-score' in warn_for: + msg_start = 'F-score is' + else: + return result + + msg = ('{0} ill-defined and being set to np.nan {{0}} ' + 'no {1} {2}s.'.format(msg_start, modifier, axis0)) + if len(mask) == 1: + msg = msg.format('due to') + else: + msg = msg.format('in {0}s with'.format(axis1)) + warnings.warn(msg, UndefinedMetricWarning, stacklevel=2) + return result + + from sklearn.metrics import classification + _precision_recall_fscore_support = classification.precision_recall_fscore_support + def precision_recall_fscore_support(*args, **kargs): + precision, recall, f_score, true_sum = _precision_recall_fscore_support(*args, **kargs) + if kargs.get('average', None) is None: + f_score[np.isnan(precision) * np.isnan(recall)] = 1.0 # both true_sum and pred_sum are 0 + return precision, recall, f_score, true_sum + classification.precision_recall_fscore_support = precision_recall_fscore_support + classification._prf_divide = default_nan_prf_divide + + from sklearn import cross_validation + from sklearn.utils.multiclass import type_of_target + from copy import deepcopy + _cross_val_score = cross_validation.cross_val_score + def cross_val_score_one_vs_all_per_class(estimator, X, y=None, *args, **kargs): + y_type = type_of_target(y) + positive_example_amount = y.sum(axis=0) + error = "" + if (positive_example_amount < kargs['cv']).any(): + error = str((positive_example_amount < kargs['cv']).sum()) + " : too little examples for " + str(np.where(positive_example_amount < kargs['cv'])) + str(positive_example_amount[np.where(positive_example_amount < kargs['cv'])]) + if (positive_example_amount > y.shape[0] - kargs['cv']).any(): + error += str((positive_example_amount > y.shape[0] - kargs['cv']).sum()) + " : too many examples for " + str(np.where(positive_example_amount > y.shape[0] - kargs['cv'])) + str(positive_example_amount[np.where(positive_example_amount > y.shape[0] - kargs['cv'])]) +# if error: +# raise Exception(error) + if y_type.startswith('multilabel') and isinstance(estimator, OneVsRestClassifier): + res = [] + for yy in y.transpose(): + res.append(_cross_val_score(deepcopy(estimator.estimator), X, yy, *args, **kargs)) + import pdb; pdb.set_trace() + else: + res = _cross_val_score(estimator, X, y, *args, **kargs) + return np.array(list(res)) +# cross_validation.cross_val_score = cross_val_score_one_vs_all_per_class + + scores = cross_validation.cross_val_score(classifier, learn_data, labels, scoring=f1_scorer_no_average, cv=10)# 'f1_weighted') + scores = np.array(list(scores)) + import IPython; IPython.embed() + print "Accuracy: %0.2f (+/- %0.2f)" % (scores.mean(), scores.std() * 2) + + + + +#print >>sys.stderr, 'learning data'; sys.stderr.flush() +#trained_classifier = classifier.fit(learn_data[:-10], labels[:-10]) +#print >>sys.stderr, 'classifying'; sys.stderr.flush() +#predicted_labels = trained_classifier.predict(learn_data[-10:]) +#predicted_tags = lb.inverse_transform(predicted_labels) +# +#from pprint import pprint +#for _predicted_tags, _tags, _d in zip(predicted_tags, tags[-10:], data[-10:]): +# print _d['id'],'--', +# for _predicted_tag in _predicted_tags: +# print _predicted_tag[::-1], ",", +# print "-", +# for _tag in _tags: +# print _tag[::-1], ",", +# print "" + + + +import IPython; IPython.embed()