From b38525ba8018ccdd30718a962fe4962445c4de7d Mon Sep 17 00:00:00 2001 From: Raviv Barzilay Date: Mon, 12 Jun 2017 22:19:08 +0300 Subject: [PATCH 1/4] wip kns_person scraper and model --- knesset_data_django/__init__.py | 1 + .../scrapers/root_datapackage_scraper.py | 11 +++- knesset_data_django/kns_persons/__init__.py | 0 knesset_data_django/kns_persons/models.py | 45 +++++++++++++++ .../kns_persons/scrapers/__init__.py | 0 .../kns_persons/scrapers/person.py | 56 +++++++++++++++++++ knesset_data_django/persons/models.py | 2 +- 7 files changed, 111 insertions(+), 4 deletions(-) create mode 100644 knesset_data_django/kns_persons/__init__.py create mode 100644 knesset_data_django/kns_persons/models.py create mode 100644 knesset_data_django/kns_persons/scrapers/__init__.py create mode 100644 knesset_data_django/kns_persons/scrapers/person.py diff --git a/knesset_data_django/__init__.py b/knesset_data_django/__init__.py index 2b5bad0..a7bae1c 100644 --- a/knesset_data_django/__init__.py +++ b/knesset_data_django/__init__.py @@ -1,4 +1,5 @@ KNESSET_DATA_DJANGO_APPS = ( 'knesset_data_django.committees', 'knesset_data_django.common', + 'knesset_data_django.kns_persons', ) diff --git a/knesset_data_django/common/scrapers/root_datapackage_scraper.py b/knesset_data_django/common/scrapers/root_datapackage_scraper.py index b112b0c..2df81c6 100644 --- a/knesset_data_django/common/scrapers/root_datapackage_scraper.py +++ b/knesset_data_django/common/scrapers/root_datapackage_scraper.py @@ -178,9 +178,14 @@ def log_scrape_return_value(self, scrape_classes_return_value): scraper_instance, scrape_return_values = scrape_class_return_value if scraper_instance: i = 0 - for scrape_return_value in scrape_return_values: - scraper_instance.log_return_value(*scrape_return_value) - i += 1 + try: + for scrape_return_value in scrape_return_values: + scraper_instance.log_return_value(*scrape_return_value) + i += 1 + except Exception as e: + message = "unexpected exception parsing return value for {}".format(scraper_class) + self.logger.exception(message) + raise Exception(message) self.logger.info("processed {} items for scraper {}".format(i, scraper_class.__name__)) else: self.logger.debug("skipping scraper {}".format(scraper_class.__name__)) diff --git a/knesset_data_django/kns_persons/__init__.py b/knesset_data_django/kns_persons/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/knesset_data_django/kns_persons/models.py b/knesset_data_django/kns_persons/models.py new file mode 100644 index 0000000..b996a5c --- /dev/null +++ b/knesset_data_django/kns_persons/models.py @@ -0,0 +1,45 @@ +from django.db import models + + +class KnessetPerson(models.Model): + source_id = models.IntegerField() + last_name = models.CharField(max_length=64) + first_name = models.CharField(max_length=64) + gender_id = models.IntegerField() + gender_description = models.CharField(max_length=64) + email = models.EmailField(blank=True, null=True) + is_current = models.BooleanField() + last_update = models.DateTimeField() + + +class KnessetPosition(models.Model): + source_id = models.IntegerField() + description = models.CharField(max_length=256) + gender_id = models.IntegerField() + gender_description = models.CharField(max_length=64) + last_update = models.DateTimeField() + + +class KnessetPersonToPosition(models.Model): + source_id = models.IntegerField() + person_id = models.IntegerField() + position_id = models.IntegerField() + knesset_num = models.IntegerField() + ministry_id = models.IntegerField() + ministry_name = models.CharField(max_length=256) + duty_description = models.CharField(max_length=256) + faction_id = models.IntegerField() + faction_name = models.CharField(max_length=64) + gov_num = models.IntegerField() + committee_id = models.IntegerField() + committee_name = models.CharField(max_length=64) + start_update = models.DateTimeField() + finish_update = models.DateTimeField() + is_current = models.BooleanField() + last_update = models.DateTimeField() + + +class KnessetSiteCode(models.Model): + source_id = models.IntegerField() + kns_id = models.IntegerField() + site_id = models.IntegerField() diff --git a/knesset_data_django/kns_persons/scrapers/__init__.py b/knesset_data_django/kns_persons/scrapers/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/knesset_data_django/kns_persons/scrapers/person.py b/knesset_data_django/kns_persons/scrapers/person.py new file mode 100644 index 0000000..9f210a1 --- /dev/null +++ b/knesset_data_django/kns_persons/scrapers/person.py @@ -0,0 +1,56 @@ +from ...common.scrapers.base_datapackage_scraper import BaseDatapackageScraper +from knesset_data_django.common.exceptions import TooManyObjectsException +from ..models import KnessetPerson + + +class PersonScraper(BaseDatapackageScraper): + DATAPACKAGE_RESOURCE_NAME = "person" + + def _handle_datapackage_item(self, person_data): + """ + updates or create a committee object based on dataservice_person + :param person_data: dataservice person object + :return: tuple(person, created) the updated or created person model object and True/False if it was created + """ + person_knesset_id = person_data["id"] + person_model_data = { + "last_name": person_data["last_name"], + "first_name": person_data["first_name"], + "gender_id": person_data["gender_id"], + "gender_description": person_data["gender_description"], + "email": person_data["email"], + "is_current": person_data["is_current"], + "last_update": person_data["last_update"] + } + person_qs = KnessetPerson.objects.filter(source_id=person_knesset_id) + person_qs_count = person_qs.count() + if person_qs_count == 1: + person = person_qs.first() + needs_update = False + for attr, scraped_value in person_model_data.iteritems(): + db_value = getattr(person, attr) + if db_value != scraped_value: + needs_update = True + break + if needs_update: + [setattr(person, k, v) for k, v in person_model_data.iteritems()] + created, updated, message = False, True, "detected a change in one of the fields, updating person" + else: + created, updated, message = False, False, "existing person in DB, no change" + elif person_qs_count == 0: + person = KnessetPerson(knesset_id=person_knesset_id, **person_model_data) + created, updated, message = True, False, "created person" + else: + raise TooManyObjectsException("committee_knesset_id={}, matching db ids: {}".format(person_knesset_id, + [c.id for c in + person_qs])) + if updated or created: + person.save() + return person, created, updated, message + + def log_return_value(self, person, created, updated, message): + prefix = u"person {} - {} {}".format(person.id, person.first_name, person.last_name) + if created or updated: + self.logger.info(u"{}: {}".format(prefix, message)) + else: + self.logger.debug(u'{}: {}'.format(prefix, message)) diff --git a/knesset_data_django/persons/models.py b/knesset_data_django/persons/models.py index df339e2..bc64b57 100644 --- a/knesset_data_django/persons/models.py +++ b/knesset_data_django/persons/models.py @@ -1,2 +1,2 @@ # currently uses Open Knesset models. TODO: move to a separate data access layer -from persons.models import Person, PersonAlias \ No newline at end of file +from persons.models import Person, PersonAlias From 9f05ee76df7f15e850dee3d2e132d7ec88861881 Mon Sep 17 00:00:00 2001 From: Ori Hoch Date: Sun, 18 Jun 2017 19:18:13 +0300 Subject: [PATCH 2/4] add person scrapers --- .../scrapers/root_datapackage_scraper.py | 3 + .../scrapers/simple_datapackage_scraper.py | 64 +++++++++ .../kns_persons/migrations/0001_initial.py | 133 ++++++++++++++++++ .../{scrapers => migrations}/__init__.py | 0 knesset_data_django/kns_persons/models.py | 20 +-- knesset_data_django/kns_persons/scrapers.py | 89 ++++++++++++ .../kns_persons/scrapers/person.py | 56 -------- 7 files changed, 299 insertions(+), 66 deletions(-) create mode 100644 knesset_data_django/common/scrapers/simple_datapackage_scraper.py create mode 100644 knesset_data_django/kns_persons/migrations/0001_initial.py rename knesset_data_django/kns_persons/{scrapers => migrations}/__init__.py (100%) create mode 100644 knesset_data_django/kns_persons/scrapers.py delete mode 100644 knesset_data_django/kns_persons/scrapers/person.py diff --git a/knesset_data_django/common/scrapers/root_datapackage_scraper.py b/knesset_data_django/common/scrapers/root_datapackage_scraper.py index 2df81c6..c5de674 100644 --- a/knesset_data_django/common/scrapers/root_datapackage_scraper.py +++ b/knesset_data_django/common/scrapers/root_datapackage_scraper.py @@ -17,6 +17,9 @@ class RootDatapackageScraper(BaseScraper): "knesset_data_django.committees.scrapers.committees.CommitteesScraper", "knesset_data_django.committees.scrapers.committee_meetings.CommitteeMeetingsScraper", "knesset_data_django.committees.scrapers.committee_meeting_protocols.CommitteeMeetingProtocolsScraper", + "knesset_data_django.kns_persons.scrapers.PersonScraper", + "knesset_data_django.kns_persons.scrapers.PersonPositionScraper", + "knesset_data_django.kns_persons.scrapers.PersonToPositionScraper", ) def __init__(self, **kwargs): diff --git a/knesset_data_django/common/scrapers/simple_datapackage_scraper.py b/knesset_data_django/common/scrapers/simple_datapackage_scraper.py new file mode 100644 index 0000000..6f2ada7 --- /dev/null +++ b/knesset_data_django/common/scrapers/simple_datapackage_scraper.py @@ -0,0 +1,64 @@ +from .base_datapackage_scraper import BaseDatapackageScraper +from knesset_data_django.common.exceptions import TooManyObjectsException + + + +class SimpleDatapackageScraper(BaseDatapackageScraper): + + # extending classes should define this with the name of the source resource from the datapackage + # DATAPACKAGE_RESOURCE_NAME = "" + + def _get_source_id(self, item_data): + return item_data["id"] + + def _get_model_data(self, item_data): + raise NotImplementedError() + + def _get_model_class(self): + raise NotImplementedError() + + def _get_source_id_attr(self): + return "source_id" + + def _get_object_prefix(self, object): + return str(object) + + def _handle_datapackage_item(self, item_data): + """ + updates or create an object based on the data from the datapackage + :param item_data: source data, conforms to the datapackage schema + :return: tuple(object, created, updated, message) the created/updated db object, whether it was created or updated and a message + """ + source_id = self._get_source_id(item_data) + model_data = self._get_model_data(item_data) + qs = self._get_model_class().objects.filter(**{self._get_source_id_attr(): source_id}) + qs_count = qs.count() + if qs_count == 1: + object = qs.first() + needs_update = False + for attr, scraped_value in model_data.iteritems(): + db_value = getattr(object, attr) + if db_value != scraped_value: + self.logger.debug("change in field {} db_value {} scraped_value {}".format(attr, db_value, scraped_value)) + needs_update = True + break + if needs_update: + [setattr(object, k, v) for k, v in model_data.iteritems()] + created, updated, message = False, True, "detected a change in one of the fields, updating object" + else: + created, updated, message = False, False, "existing object in DB, no change" + elif qs_count == 0: + object = self._get_model_class()(**dict(model_data, **{self._get_source_id_attr(): source_id})) + created, updated, message = True, False, "created object" + else: + raise TooManyObjectsException("source_id={0}, matching db ids: {1}".format(source_id, [o.id for o in qs])) + if updated or created: + object.save() + return object, created, updated, message + + def log_return_value(self, object, created, updated, message): + prefix = self._get_object_prefix(object) + if created or updated: + self.logger.info(u"{}: {}".format(prefix, message)) + else: + self.logger.debug(u'{}: {}'.format(prefix, message)) diff --git a/knesset_data_django/kns_persons/migrations/0001_initial.py b/knesset_data_django/kns_persons/migrations/0001_initial.py new file mode 100644 index 0000000..08d2bb7 --- /dev/null +++ b/knesset_data_django/kns_persons/migrations/0001_initial.py @@ -0,0 +1,133 @@ +# -*- coding: utf-8 -*- +from south.utils import datetime_utils as datetime +from south.db import db +from south.v2 import SchemaMigration +from django.db import models + + +class Migration(SchemaMigration): + + def forwards(self, orm): + # Adding model 'KnessetPerson' + db.create_table(u'kns_persons_knessetperson', ( + (u'id', self.gf('django.db.models.fields.AutoField')(primary_key=True)), + ('source_id', self.gf('django.db.models.fields.IntegerField')()), + ('last_name', self.gf('django.db.models.fields.CharField')(max_length=64)), + ('first_name', self.gf('django.db.models.fields.CharField')(max_length=64)), + ('gender_id', self.gf('django.db.models.fields.IntegerField')()), + ('gender_description', self.gf('django.db.models.fields.CharField')(max_length=64)), + ('email', self.gf('django.db.models.fields.EmailField')(max_length=75, null=True, blank=True)), + ('is_current', self.gf('django.db.models.fields.CharField')(max_length=64, null=True, blank=True)), + ('last_update', self.gf('django.db.models.fields.DateTimeField')()), + )) + db.send_create_signal(u'kns_persons', ['KnessetPerson']) + + # Adding model 'KnessetPosition' + db.create_table(u'kns_persons_knessetposition', ( + (u'id', self.gf('django.db.models.fields.AutoField')(primary_key=True)), + ('source_id', self.gf('django.db.models.fields.IntegerField')()), + ('description', self.gf('django.db.models.fields.CharField')(max_length=256)), + ('gender_id', self.gf('django.db.models.fields.IntegerField')()), + ('gender_description', self.gf('django.db.models.fields.CharField')(max_length=64)), + ('last_update', self.gf('django.db.models.fields.DateTimeField')()), + )) + db.send_create_signal(u'kns_persons', ['KnessetPosition']) + + # Adding model 'KnessetPersonToPosition' + db.create_table(u'kns_persons_knessetpersontoposition', ( + (u'id', self.gf('django.db.models.fields.AutoField')(primary_key=True)), + ('source_id', self.gf('django.db.models.fields.IntegerField')()), + ('person_id', self.gf('django.db.models.fields.IntegerField')()), + ('position_id', self.gf('django.db.models.fields.IntegerField')()), + ('knesset_num', self.gf('django.db.models.fields.IntegerField')(null=True, blank=True)), + ('ministry_id', self.gf('django.db.models.fields.IntegerField')(null=True, blank=True)), + ('ministry_name', self.gf('django.db.models.fields.CharField')(max_length=256)), + ('duty_description', self.gf('django.db.models.fields.CharField')(max_length=256)), + ('faction_id', self.gf('django.db.models.fields.IntegerField')(null=True, blank=True)), + ('faction_name', self.gf('django.db.models.fields.CharField')(max_length=64)), + ('gov_num', self.gf('django.db.models.fields.IntegerField')(null=True, blank=True)), + ('committee_id', self.gf('django.db.models.fields.IntegerField')(null=True, blank=True)), + ('committee_name', self.gf('django.db.models.fields.CharField')(max_length=64)), + ('start_update', self.gf('django.db.models.fields.DateTimeField')(null=True, blank=True)), + ('finish_update', self.gf('django.db.models.fields.DateTimeField')(null=True, blank=True)), + ('is_current', self.gf('django.db.models.fields.CharField')(max_length=64, null=True, blank=True)), + ('last_update', self.gf('django.db.models.fields.DateTimeField')(null=True, blank=True)), + )) + db.send_create_signal(u'kns_persons', ['KnessetPersonToPosition']) + + # Adding model 'KnessetSiteCode' + db.create_table(u'kns_persons_knessetsitecode', ( + (u'id', self.gf('django.db.models.fields.AutoField')(primary_key=True)), + ('source_id', self.gf('django.db.models.fields.IntegerField')()), + ('kns_id', self.gf('django.db.models.fields.IntegerField')()), + ('site_id', self.gf('django.db.models.fields.IntegerField')()), + )) + db.send_create_signal(u'kns_persons', ['KnessetSiteCode']) + + + def backwards(self, orm): + # Deleting model 'KnessetPerson' + db.delete_table(u'kns_persons_knessetperson') + + # Deleting model 'KnessetPosition' + db.delete_table(u'kns_persons_knessetposition') + + # Deleting model 'KnessetPersonToPosition' + db.delete_table(u'kns_persons_knessetpersontoposition') + + # Deleting model 'KnessetSiteCode' + db.delete_table(u'kns_persons_knessetsitecode') + + + models = { + u'kns_persons.knessetperson': { + 'Meta': {'object_name': 'KnessetPerson'}, + 'email': ('django.db.models.fields.EmailField', [], {'max_length': '75', 'null': 'True', 'blank': 'True'}), + 'first_name': ('django.db.models.fields.CharField', [], {'max_length': '64'}), + 'gender_description': ('django.db.models.fields.CharField', [], {'max_length': '64'}), + 'gender_id': ('django.db.models.fields.IntegerField', [], {}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'is_current': ('django.db.models.fields.CharField', [], {'max_length': '64', 'null': 'True', 'blank': 'True'}), + 'last_name': ('django.db.models.fields.CharField', [], {'max_length': '64'}), + 'last_update': ('django.db.models.fields.DateTimeField', [], {}), + 'source_id': ('django.db.models.fields.IntegerField', [], {}) + }, + u'kns_persons.knessetpersontoposition': { + 'Meta': {'object_name': 'KnessetPersonToPosition'}, + 'committee_id': ('django.db.models.fields.IntegerField', [], {'null': 'True', 'blank': 'True'}), + 'committee_name': ('django.db.models.fields.CharField', [], {'max_length': '64'}), + 'duty_description': ('django.db.models.fields.CharField', [], {'max_length': '256'}), + 'faction_id': ('django.db.models.fields.IntegerField', [], {'null': 'True', 'blank': 'True'}), + 'faction_name': ('django.db.models.fields.CharField', [], {'max_length': '64'}), + 'finish_update': ('django.db.models.fields.DateTimeField', [], {'null': 'True', 'blank': 'True'}), + 'gov_num': ('django.db.models.fields.IntegerField', [], {'null': 'True', 'blank': 'True'}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'is_current': ('django.db.models.fields.CharField', [], {'max_length': '64', 'null': 'True', 'blank': 'True'}), + 'knesset_num': ('django.db.models.fields.IntegerField', [], {'null': 'True', 'blank': 'True'}), + 'last_update': ('django.db.models.fields.DateTimeField', [], {'null': 'True', 'blank': 'True'}), + 'ministry_id': ('django.db.models.fields.IntegerField', [], {'null': 'True', 'blank': 'True'}), + 'ministry_name': ('django.db.models.fields.CharField', [], {'max_length': '256'}), + 'person_id': ('django.db.models.fields.IntegerField', [], {}), + 'position_id': ('django.db.models.fields.IntegerField', [], {}), + 'source_id': ('django.db.models.fields.IntegerField', [], {}), + 'start_update': ('django.db.models.fields.DateTimeField', [], {'null': 'True', 'blank': 'True'}) + }, + u'kns_persons.knessetposition': { + 'Meta': {'object_name': 'KnessetPosition'}, + 'description': ('django.db.models.fields.CharField', [], {'max_length': '256'}), + 'gender_description': ('django.db.models.fields.CharField', [], {'max_length': '64'}), + 'gender_id': ('django.db.models.fields.IntegerField', [], {}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'last_update': ('django.db.models.fields.DateTimeField', [], {}), + 'source_id': ('django.db.models.fields.IntegerField', [], {}) + }, + u'kns_persons.knessetsitecode': { + 'Meta': {'object_name': 'KnessetSiteCode'}, + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'kns_id': ('django.db.models.fields.IntegerField', [], {}), + 'site_id': ('django.db.models.fields.IntegerField', [], {}), + 'source_id': ('django.db.models.fields.IntegerField', [], {}) + } + } + + complete_apps = ['kns_persons'] \ No newline at end of file diff --git a/knesset_data_django/kns_persons/scrapers/__init__.py b/knesset_data_django/kns_persons/migrations/__init__.py similarity index 100% rename from knesset_data_django/kns_persons/scrapers/__init__.py rename to knesset_data_django/kns_persons/migrations/__init__.py diff --git a/knesset_data_django/kns_persons/models.py b/knesset_data_django/kns_persons/models.py index b996a5c..de1d678 100644 --- a/knesset_data_django/kns_persons/models.py +++ b/knesset_data_django/kns_persons/models.py @@ -8,7 +8,7 @@ class KnessetPerson(models.Model): gender_id = models.IntegerField() gender_description = models.CharField(max_length=64) email = models.EmailField(blank=True, null=True) - is_current = models.BooleanField() + is_current = models.CharField(blank=True, null=True, max_length=64) last_update = models.DateTimeField() @@ -24,19 +24,19 @@ class KnessetPersonToPosition(models.Model): source_id = models.IntegerField() person_id = models.IntegerField() position_id = models.IntegerField() - knesset_num = models.IntegerField() - ministry_id = models.IntegerField() + knesset_num = models.IntegerField(blank=True, null=True) + ministry_id = models.IntegerField(blank=True, null=True) ministry_name = models.CharField(max_length=256) duty_description = models.CharField(max_length=256) - faction_id = models.IntegerField() + faction_id = models.IntegerField(blank=True, null=True) faction_name = models.CharField(max_length=64) - gov_num = models.IntegerField() - committee_id = models.IntegerField() + gov_num = models.IntegerField(blank=True, null=True) + committee_id = models.IntegerField(blank=True, null=True) committee_name = models.CharField(max_length=64) - start_update = models.DateTimeField() - finish_update = models.DateTimeField() - is_current = models.BooleanField() - last_update = models.DateTimeField() + start_update = models.DateTimeField(blank=True, null=True) + finish_update = models.DateTimeField(blank=True, null=True) + is_current = models.CharField(blank=True, null=True, max_length=64) + last_update = models.DateTimeField(blank=True, null=True) class KnessetSiteCode(models.Model): diff --git a/knesset_data_django/kns_persons/scrapers.py b/knesset_data_django/kns_persons/scrapers.py new file mode 100644 index 0000000..1482486 --- /dev/null +++ b/knesset_data_django/kns_persons/scrapers.py @@ -0,0 +1,89 @@ +from ..common.scrapers.simple_datapackage_scraper import SimpleDatapackageScraper +from .models import KnessetPerson, KnessetPersonToPosition, KnessetPosition + + +class PersonScraper(SimpleDatapackageScraper): + + DATAPACKAGE_RESOURCE_NAME = "persons-person" + + def _get_source_id(self, item_data): + return item_data["id"] + + def _get_model_data(self, item_data): + return { + "last_name": item_data["last_name"], + "first_name": item_data["first_name"], + "gender_id": item_data["gender_id"], + "gender_description": item_data["gender_description"], + "email": item_data["email"], + "is_current": item_data["is_current"], + "last_update": item_data["last_update"] + } + + def _get_model_class(self): + return KnessetPerson + + def _get_source_id_attr(self): + return "source_id" + + def _get_object_prefix(self, object): + return u"person {} - {} {}".format(object.id, object.first_name, object.last_name) + +class PersonPositionScraper(SimpleDatapackageScraper): + + DATAPACKAGE_RESOURCE_NAME = "persons-position" + + def _get_source_id(self, item_data): + return item_data["id"] + + def _get_model_data(self, item_data): + return { + "description": item_data["description"], + "gender_id": item_data["gender_id"], + "gender_description": item_data["gender_description"], + "last_update": item_data["last_update"] + } + + def _get_model_class(self): + return KnessetPosition + + def _get_source_id_attr(self): + return "source_id" + + def _get_object_prefix(self, object): + return u"person position {} - {}".format(object.id, object.description) + +class PersonToPositionScraper(SimpleDatapackageScraper): + + DATAPACKAGE_RESOURCE_NAME = "persons-persons-to-positions" + + def _get_source_id(self, item_data): + return item_data["id"] + + def _get_model_data(self, item_data): + return { + "person_id": item_data["person_id"], + "position_id": item_data["position_id"], + "knesset_num": item_data["knnesset_num"], + "ministry_id": item_data["ministry_id"], + "ministry_name": item_data["ministry_name"], + "duty_description": item_data["duty_description"], + "faction_id": item_data["faction_id"], + "faction_name": item_data["faction_name"], + "gov_num": item_data["gov_num"], + "committee_id": item_data["committee_id"], + "committee_name": item_data["committee_name"], + "start_update": item_data["start_update"], + "finish_update": item_data["finish_update"], + "is_current": item_data["is_current"], + "last_update": item_data["last_update"], + } + + def _get_model_class(self): + return KnessetPersonToPosition + + def _get_source_id_attr(self): + return "source_id" + + def _get_object_prefix(self, object): + return u"person to position {} - {} {}".format(object.id, object.person_id, object.position_id) diff --git a/knesset_data_django/kns_persons/scrapers/person.py b/knesset_data_django/kns_persons/scrapers/person.py deleted file mode 100644 index 9f210a1..0000000 --- a/knesset_data_django/kns_persons/scrapers/person.py +++ /dev/null @@ -1,56 +0,0 @@ -from ...common.scrapers.base_datapackage_scraper import BaseDatapackageScraper -from knesset_data_django.common.exceptions import TooManyObjectsException -from ..models import KnessetPerson - - -class PersonScraper(BaseDatapackageScraper): - DATAPACKAGE_RESOURCE_NAME = "person" - - def _handle_datapackage_item(self, person_data): - """ - updates or create a committee object based on dataservice_person - :param person_data: dataservice person object - :return: tuple(person, created) the updated or created person model object and True/False if it was created - """ - person_knesset_id = person_data["id"] - person_model_data = { - "last_name": person_data["last_name"], - "first_name": person_data["first_name"], - "gender_id": person_data["gender_id"], - "gender_description": person_data["gender_description"], - "email": person_data["email"], - "is_current": person_data["is_current"], - "last_update": person_data["last_update"] - } - person_qs = KnessetPerson.objects.filter(source_id=person_knesset_id) - person_qs_count = person_qs.count() - if person_qs_count == 1: - person = person_qs.first() - needs_update = False - for attr, scraped_value in person_model_data.iteritems(): - db_value = getattr(person, attr) - if db_value != scraped_value: - needs_update = True - break - if needs_update: - [setattr(person, k, v) for k, v in person_model_data.iteritems()] - created, updated, message = False, True, "detected a change in one of the fields, updating person" - else: - created, updated, message = False, False, "existing person in DB, no change" - elif person_qs_count == 0: - person = KnessetPerson(knesset_id=person_knesset_id, **person_model_data) - created, updated, message = True, False, "created person" - else: - raise TooManyObjectsException("committee_knesset_id={}, matching db ids: {}".format(person_knesset_id, - [c.id for c in - person_qs])) - if updated or created: - person.save() - return person, created, updated, message - - def log_return_value(self, person, created, updated, message): - prefix = u"person {} - {} {}".format(person.id, person.first_name, person.last_name) - if created or updated: - self.logger.info(u"{}: {}".format(prefix, message)) - else: - self.logger.debug(u'{}: {}'.format(prefix, message)) From acce310cb565347fb5ca5703aa7a5cf948d1536e Mon Sep 17 00:00:00 2001 From: Ori Hoch Date: Sun, 18 Jun 2017 19:29:41 +0300 Subject: [PATCH 3/4] fix tests --- .../common/tests/test_datapackage.py | 32 +++++++++++-------- 1 file changed, 19 insertions(+), 13 deletions(-) diff --git a/knesset_data_django/common/tests/test_datapackage.py b/knesset_data_django/common/tests/test_datapackage.py index dd2beb0..74b2740 100644 --- a/knesset_data_django/common/tests/test_datapackage.py +++ b/knesset_data_django/common/tests/test_datapackage.py @@ -33,6 +33,12 @@ class MockRootDatapackageScraper(RootDatapackageScraper): + SCRAPERS = ( + "knesset_data_django.mks.scrapers.members.MembersScraper", + "knesset_data_django.committees.scrapers.committees.CommitteesScraper", + "knesset_data_django.committees.scrapers.committee_meetings.CommitteeMeetingsScraper", + ) + def _scrape_instance_value(self, from_datapackage, fetch_kwargs, child_scraper_instance, return_value): if isinstance(child_scraper_instance, MembersScraper) and return_value[0] and return_value[1]: # add some fake data to allow committee meetings to detect it in attending members @@ -91,7 +97,7 @@ def when_datapackage_was_scraped_with_future_enabled(self): self.assertIn("processed 946 items for scraper MembersScraper", info_messages) self.assertIn("processed 100 items for scraper CommitteesScraper", info_messages) self.assertIn("processed 34 items for scraper CommitteeMeetingsScraper", info_messages) - self.assertIn("processed 6 items for scraper CommitteeMeetingProtocolsScraper", info_messages) + # self.assertIn("processed 6 items for scraper CommitteeMeetingProtocolsScraper", info_messages) # for debugging (add -s to see output when tests pass) print("\n".join([msg for msg in [u"{}: {}".format(r.levelname, r.getMessage()) for r in logger.all_records] if "2014012" in msg or "896" in msg])) @@ -116,22 +122,22 @@ def assert_scraped_data(self): {"knesset_id": 2, "name_eng": "Finance Committee"}) # committee meetings - committee_meeting = self.assert_model(CommitteeMeeting.objects.get(committee=committee, knesset_id="2014012"), - {"date_string": "13/02/2017", - "date": datetime.date(2017, 2, 13), - "topics.strip": u"הוראות ליישום משטר כושר פירעון כלכלי של חברת ביטוח מבוסס Solvency II בהתאם לסעיף 35(ג) לחוק הפיקוח על שירותים פיננסיים (ביטוח), התשמ”א - 1981", - "datetime": datetime.datetime(2017, 2, 13, 13, 30), # 2017-02-13T13:30:00 - "knesset_id": 2014012, - "src_url": "http://fs.knesset.gov.il//20/Committees/20_ptv_368875.doc"}) + # committee_meeting = self.assert_model(CommitteeMeeting.objects.get(committee=committee, knesset_id="2014012"), + # {"date_string": "13/02/2017", + # "date": datetime.date(2017, 2, 13), + # "topics.strip": u"הוראות ליישום משטר כושר פירעון כלכלי של חברת ביטוח מבוסס Solvency II בהתאם לסעיף 35(ג) לחוק הפיקוח על שירותים פיננסיים (ביטוח), התשמ”א - 1981", + # "datetime": datetime.datetime(2017, 2, 13, 13, 30), # 2017-02-13T13:30:00 + # "knesset_id": 2014012, + # "src_url": "http://fs.knesset.gov.il//20/Committees/20_ptv_368875.doc"}) # committee meeting protocol text and parts - self.assertTrue("Solvency II" in committee_meeting.protocol_text) - self.assertTrue(u"פרוטוקול מס' 647" in committee_meeting.parts.get(order=1).body) - self.assertTrue(u"סדר-היום" == committee_meeting.parts.get(order=2).header) + # self.assertTrue("Solvency II" in committee_meeting.protocol_text) + # self.assertTrue(u"פרוטוקול מס' 647" in committee_meeting.parts.get(order=1).body) + # self.assertTrue(u"סדר-היום" == committee_meeting.parts.get(order=2).header) # committee meeting attending members - self.assertEqual({mk.id: mk.name for mk in committee_meeting.mks_attended.all()}, - {mkid : self.TEST_MKS[mkid] for mkid in [35, 862, 896, 939, 943, 951]}) + # self.assertEqual({mk.id: mk.name for mk in committee_meeting.mks_attended.all()}, + # {mkid : self.TEST_MKS[mkid] for mkid in [35, 862, 896, 939, 943, 951]}) def test(self): self.given_clean_db() From 5a3fe8919e0807b51f7ca5f4ca159edf97da388b Mon Sep 17 00:00:00 2001 From: Ori Hoch Date: Sun, 18 Jun 2017 19:36:17 +0300 Subject: [PATCH 4/4] fixin' travis --- .travis.yml | 4 +-- .../common/tests/test_datapackage.py | 27 ++++++++++--------- 2 files changed, 16 insertions(+), 15 deletions(-) diff --git a/.travis.yml b/.travis.yml index 7d9f736..2de3d00 100644 --- a/.travis.yml +++ b/.travis.yml @@ -9,8 +9,8 @@ addons: install: - pip install --upgrade pip - pip install -r requirements.txt - - pip install git+https://github.com/hasadna/knesset-data-python.git@v1.7.2#egg=knesset-data - - pip install git+https://github.com/hasadna/knesset-data-datapackage.git@v1.1.0#egg=knesset-datapackage + - pip install git+https://github.com/hasadna/knesset-data-python.git@v1.8.1#egg=knesset-data + - pip install git+https://github.com/hasadna/knesset-data-datapackage.git@v1.3.0#egg=knesset-datapackage - pip install git+https://github.com/hasadna/Open-Knesset.git@e28339da7ca92df96fc79b89351286e2715fcff0#egg=Open-Knesset - pip install . script: diff --git a/knesset_data_django/common/tests/test_datapackage.py b/knesset_data_django/common/tests/test_datapackage.py index 74b2740..41bf508 100644 --- a/knesset_data_django/common/tests/test_datapackage.py +++ b/knesset_data_django/common/tests/test_datapackage.py @@ -37,6 +37,7 @@ class MockRootDatapackageScraper(RootDatapackageScraper): "knesset_data_django.mks.scrapers.members.MembersScraper", "knesset_data_django.committees.scrapers.committees.CommitteesScraper", "knesset_data_django.committees.scrapers.committee_meetings.CommitteeMeetingsScraper", + "knesset_data_django.committees.scrapers.committee_meeting_protocols.CommitteeMeetingProtocolsScraper", ) def _scrape_instance_value(self, from_datapackage, fetch_kwargs, child_scraper_instance, return_value): @@ -97,7 +98,7 @@ def when_datapackage_was_scraped_with_future_enabled(self): self.assertIn("processed 946 items for scraper MembersScraper", info_messages) self.assertIn("processed 100 items for scraper CommitteesScraper", info_messages) self.assertIn("processed 34 items for scraper CommitteeMeetingsScraper", info_messages) - # self.assertIn("processed 6 items for scraper CommitteeMeetingProtocolsScraper", info_messages) + self.assertIn("processed 6 items for scraper CommitteeMeetingProtocolsScraper", info_messages) # for debugging (add -s to see output when tests pass) print("\n".join([msg for msg in [u"{}: {}".format(r.levelname, r.getMessage()) for r in logger.all_records] if "2014012" in msg or "896" in msg])) @@ -122,22 +123,22 @@ def assert_scraped_data(self): {"knesset_id": 2, "name_eng": "Finance Committee"}) # committee meetings - # committee_meeting = self.assert_model(CommitteeMeeting.objects.get(committee=committee, knesset_id="2014012"), - # {"date_string": "13/02/2017", - # "date": datetime.date(2017, 2, 13), - # "topics.strip": u"הוראות ליישום משטר כושר פירעון כלכלי של חברת ביטוח מבוסס Solvency II בהתאם לסעיף 35(ג) לחוק הפיקוח על שירותים פיננסיים (ביטוח), התשמ”א - 1981", - # "datetime": datetime.datetime(2017, 2, 13, 13, 30), # 2017-02-13T13:30:00 - # "knesset_id": 2014012, - # "src_url": "http://fs.knesset.gov.il//20/Committees/20_ptv_368875.doc"}) + committee_meeting = self.assert_model(CommitteeMeeting.objects.get(committee=committee, knesset_id="2014012"), + {"date_string": "13/02/2017", + "date": datetime.date(2017, 2, 13), + "topics.strip": u"הוראות ליישום משטר כושר פירעון כלכלי של חברת ביטוח מבוסס Solvency II בהתאם לסעיף 35(ג) לחוק הפיקוח על שירותים פיננסיים (ביטוח), התשמ”א - 1981", + "datetime": datetime.datetime(2017, 2, 13, 13, 30), # 2017-02-13T13:30:00 + "knesset_id": 2014012, + "src_url": "http://fs.knesset.gov.il//20/Committees/20_ptv_368875.doc"}) # committee meeting protocol text and parts - # self.assertTrue("Solvency II" in committee_meeting.protocol_text) - # self.assertTrue(u"פרוטוקול מס' 647" in committee_meeting.parts.get(order=1).body) - # self.assertTrue(u"סדר-היום" == committee_meeting.parts.get(order=2).header) + self.assertTrue("Solvency II" in committee_meeting.protocol_text) + self.assertTrue(u"פרוטוקול מס' 647" in committee_meeting.parts.get(order=1).body) + self.assertTrue(u"סדר-היום" == committee_meeting.parts.get(order=2).header) # committee meeting attending members - # self.assertEqual({mk.id: mk.name for mk in committee_meeting.mks_attended.all()}, - # {mkid : self.TEST_MKS[mkid] for mkid in [35, 862, 896, 939, 943, 951]}) + self.assertEqual({mk.id: mk.name for mk in committee_meeting.mks_attended.all()}, + {mkid : self.TEST_MKS[mkid] for mkid in [35, 862, 896, 939, 943, 951]}) def test(self): self.given_clean_db()