diff --git a/.travis.yml b/.travis.yml index 7d9f736..2de3d00 100644 --- a/.travis.yml +++ b/.travis.yml @@ -9,8 +9,8 @@ addons: install: - pip install --upgrade pip - pip install -r requirements.txt - - pip install git+https://github.com/hasadna/knesset-data-python.git@v1.7.2#egg=knesset-data - - pip install git+https://github.com/hasadna/knesset-data-datapackage.git@v1.1.0#egg=knesset-datapackage + - pip install git+https://github.com/hasadna/knesset-data-python.git@v1.8.1#egg=knesset-data + - pip install git+https://github.com/hasadna/knesset-data-datapackage.git@v1.3.0#egg=knesset-datapackage - pip install git+https://github.com/hasadna/Open-Knesset.git@e28339da7ca92df96fc79b89351286e2715fcff0#egg=Open-Knesset - pip install . script: diff --git a/knesset_data_django/__init__.py b/knesset_data_django/__init__.py index 2b5bad0..a7bae1c 100644 --- a/knesset_data_django/__init__.py +++ b/knesset_data_django/__init__.py @@ -1,4 +1,5 @@ KNESSET_DATA_DJANGO_APPS = ( 'knesset_data_django.committees', 'knesset_data_django.common', + 'knesset_data_django.kns_persons', ) diff --git a/knesset_data_django/common/scrapers/root_datapackage_scraper.py b/knesset_data_django/common/scrapers/root_datapackage_scraper.py index b112b0c..c5de674 100644 --- a/knesset_data_django/common/scrapers/root_datapackage_scraper.py +++ b/knesset_data_django/common/scrapers/root_datapackage_scraper.py @@ -17,6 +17,9 @@ class RootDatapackageScraper(BaseScraper): "knesset_data_django.committees.scrapers.committees.CommitteesScraper", "knesset_data_django.committees.scrapers.committee_meetings.CommitteeMeetingsScraper", "knesset_data_django.committees.scrapers.committee_meeting_protocols.CommitteeMeetingProtocolsScraper", + "knesset_data_django.kns_persons.scrapers.PersonScraper", + "knesset_data_django.kns_persons.scrapers.PersonPositionScraper", + "knesset_data_django.kns_persons.scrapers.PersonToPositionScraper", ) def __init__(self, **kwargs): @@ -178,9 +181,14 @@ def log_scrape_return_value(self, scrape_classes_return_value): scraper_instance, scrape_return_values = scrape_class_return_value if scraper_instance: i = 0 - for scrape_return_value in scrape_return_values: - scraper_instance.log_return_value(*scrape_return_value) - i += 1 + try: + for scrape_return_value in scrape_return_values: + scraper_instance.log_return_value(*scrape_return_value) + i += 1 + except Exception as e: + message = "unexpected exception parsing return value for {}".format(scraper_class) + self.logger.exception(message) + raise Exception(message) self.logger.info("processed {} items for scraper {}".format(i, scraper_class.__name__)) else: self.logger.debug("skipping scraper {}".format(scraper_class.__name__)) diff --git a/knesset_data_django/common/scrapers/simple_datapackage_scraper.py b/knesset_data_django/common/scrapers/simple_datapackage_scraper.py new file mode 100644 index 0000000..6f2ada7 --- /dev/null +++ b/knesset_data_django/common/scrapers/simple_datapackage_scraper.py @@ -0,0 +1,64 @@ +from .base_datapackage_scraper import BaseDatapackageScraper +from knesset_data_django.common.exceptions import TooManyObjectsException + + + +class SimpleDatapackageScraper(BaseDatapackageScraper): + + # extending classes should define this with the name of the source resource from the datapackage + # DATAPACKAGE_RESOURCE_NAME = "" + + def _get_source_id(self, item_data): + return item_data["id"] + + def _get_model_data(self, item_data): + raise NotImplementedError() + + def _get_model_class(self): + raise NotImplementedError() + + def _get_source_id_attr(self): + return "source_id" + + def _get_object_prefix(self, object): + return str(object) + + def _handle_datapackage_item(self, item_data): + """ + updates or create an object based on the data from the datapackage + :param item_data: source data, conforms to the datapackage schema + :return: tuple(object, created, updated, message) the created/updated db object, whether it was created or updated and a message + """ + source_id = self._get_source_id(item_data) + model_data = self._get_model_data(item_data) + qs = self._get_model_class().objects.filter(**{self._get_source_id_attr(): source_id}) + qs_count = qs.count() + if qs_count == 1: + object = qs.first() + needs_update = False + for attr, scraped_value in model_data.iteritems(): + db_value = getattr(object, attr) + if db_value != scraped_value: + self.logger.debug("change in field {} db_value {} scraped_value {}".format(attr, db_value, scraped_value)) + needs_update = True + break + if needs_update: + [setattr(object, k, v) for k, v in model_data.iteritems()] + created, updated, message = False, True, "detected a change in one of the fields, updating object" + else: + created, updated, message = False, False, "existing object in DB, no change" + elif qs_count == 0: + object = self._get_model_class()(**dict(model_data, **{self._get_source_id_attr(): source_id})) + created, updated, message = True, False, "created object" + else: + raise TooManyObjectsException("source_id={0}, matching db ids: {1}".format(source_id, [o.id for o in qs])) + if updated or created: + object.save() + return object, created, updated, message + + def log_return_value(self, object, created, updated, message): + prefix = self._get_object_prefix(object) + if created or updated: + self.logger.info(u"{}: {}".format(prefix, message)) + else: + self.logger.debug(u'{}: {}'.format(prefix, message)) diff --git a/knesset_data_django/common/tests/test_datapackage.py b/knesset_data_django/common/tests/test_datapackage.py index dd2beb0..41bf508 100644 --- a/knesset_data_django/common/tests/test_datapackage.py +++ b/knesset_data_django/common/tests/test_datapackage.py @@ -33,6 +33,13 @@ class MockRootDatapackageScraper(RootDatapackageScraper): + SCRAPERS = ( + "knesset_data_django.mks.scrapers.members.MembersScraper", + "knesset_data_django.committees.scrapers.committees.CommitteesScraper", + "knesset_data_django.committees.scrapers.committee_meetings.CommitteeMeetingsScraper", + "knesset_data_django.committees.scrapers.committee_meeting_protocols.CommitteeMeetingProtocolsScraper", + ) + def _scrape_instance_value(self, from_datapackage, fetch_kwargs, child_scraper_instance, return_value): if isinstance(child_scraper_instance, MembersScraper) and return_value[0] and return_value[1]: # add some fake data to allow committee meetings to detect it in attending members diff --git a/knesset_data_django/kns_persons/__init__.py b/knesset_data_django/kns_persons/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/knesset_data_django/kns_persons/migrations/0001_initial.py b/knesset_data_django/kns_persons/migrations/0001_initial.py new file mode 100644 index 0000000..08d2bb7 --- /dev/null +++ b/knesset_data_django/kns_persons/migrations/0001_initial.py @@ -0,0 +1,133 @@ +# -*- coding: utf-8 -*- +from south.utils import datetime_utils as datetime +from south.db import db +from south.v2 import SchemaMigration +from django.db import models + + +class Migration(SchemaMigration): + + def forwards(self, orm): + # Adding model 'KnessetPerson' + db.create_table(u'kns_persons_knessetperson', ( + (u'id', self.gf('django.db.models.fields.AutoField')(primary_key=True)), + ('source_id', self.gf('django.db.models.fields.IntegerField')()), + ('last_name', self.gf('django.db.models.fields.CharField')(max_length=64)), + ('first_name', self.gf('django.db.models.fields.CharField')(max_length=64)), + ('gender_id', self.gf('django.db.models.fields.IntegerField')()), + ('gender_description', self.gf('django.db.models.fields.CharField')(max_length=64)), + ('email', self.gf('django.db.models.fields.EmailField')(max_length=75, null=True, blank=True)), + ('is_current', self.gf('django.db.models.fields.CharField')(max_length=64, null=True, blank=True)), + ('last_update', self.gf('django.db.models.fields.DateTimeField')()), + )) + db.send_create_signal(u'kns_persons', ['KnessetPerson']) + + # Adding model 'KnessetPosition' + db.create_table(u'kns_persons_knessetposition', ( + (u'id', self.gf('django.db.models.fields.AutoField')(primary_key=True)), + ('source_id', self.gf('django.db.models.fields.IntegerField')()), + ('description', self.gf('django.db.models.fields.CharField')(max_length=256)), + ('gender_id', self.gf('django.db.models.fields.IntegerField')()), + ('gender_description', self.gf('django.db.models.fields.CharField')(max_length=64)), + ('last_update', self.gf('django.db.models.fields.DateTimeField')()), + )) + db.send_create_signal(u'kns_persons', ['KnessetPosition']) + + # Adding model 'KnessetPersonToPosition' + db.create_table(u'kns_persons_knessetpersontoposition', ( + (u'id', self.gf('django.db.models.fields.AutoField')(primary_key=True)), + ('source_id', self.gf('django.db.models.fields.IntegerField')()), + ('person_id', self.gf('django.db.models.fields.IntegerField')()), + ('position_id', self.gf('django.db.models.fields.IntegerField')()), + ('knesset_num', self.gf('django.db.models.fields.IntegerField')(null=True, blank=True)), + ('ministry_id', self.gf('django.db.models.fields.IntegerField')(null=True, blank=True)), + ('ministry_name', self.gf('django.db.models.fields.CharField')(max_length=256)), + ('duty_description', self.gf('django.db.models.fields.CharField')(max_length=256)), + ('faction_id', self.gf('django.db.models.fields.IntegerField')(null=True, blank=True)), + ('faction_name', self.gf('django.db.models.fields.CharField')(max_length=64)), + ('gov_num', self.gf('django.db.models.fields.IntegerField')(null=True, blank=True)), + ('committee_id', self.gf('django.db.models.fields.IntegerField')(null=True, blank=True)), + ('committee_name', self.gf('django.db.models.fields.CharField')(max_length=64)), + ('start_update', self.gf('django.db.models.fields.DateTimeField')(null=True, blank=True)), + ('finish_update', self.gf('django.db.models.fields.DateTimeField')(null=True, blank=True)), + ('is_current', self.gf('django.db.models.fields.CharField')(max_length=64, null=True, blank=True)), + ('last_update', self.gf('django.db.models.fields.DateTimeField')(null=True, blank=True)), + )) + db.send_create_signal(u'kns_persons', ['KnessetPersonToPosition']) + + # Adding model 'KnessetSiteCode' + db.create_table(u'kns_persons_knessetsitecode', ( + (u'id', self.gf('django.db.models.fields.AutoField')(primary_key=True)), + ('source_id', self.gf('django.db.models.fields.IntegerField')()), + ('kns_id', self.gf('django.db.models.fields.IntegerField')()), + ('site_id', self.gf('django.db.models.fields.IntegerField')()), + )) + db.send_create_signal(u'kns_persons', ['KnessetSiteCode']) + + + def backwards(self, orm): + # Deleting model 'KnessetPerson' + db.delete_table(u'kns_persons_knessetperson') + + # Deleting model 'KnessetPosition' + db.delete_table(u'kns_persons_knessetposition') + + # Deleting model 'KnessetPersonToPosition' + db.delete_table(u'kns_persons_knessetpersontoposition') + + # Deleting model 'KnessetSiteCode' + db.delete_table(u'kns_persons_knessetsitecode') + + + models = { + u'kns_persons.knessetperson': { + 'Meta': {'object_name': 'KnessetPerson'}, + 'email': ('django.db.models.fields.EmailField', [], {'max_length': '75', 'null': 'True', 'blank': 'True'}), + 'first_name': ('django.db.models.fields.CharField', [], {'max_length': '64'}), + 'gender_description': ('django.db.models.fields.CharField', [], {'max_length': '64'}), + 'gender_id': ('django.db.models.fields.IntegerField', [], {}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'is_current': ('django.db.models.fields.CharField', [], {'max_length': '64', 'null': 'True', 'blank': 'True'}), + 'last_name': ('django.db.models.fields.CharField', [], {'max_length': '64'}), + 'last_update': ('django.db.models.fields.DateTimeField', [], {}), + 'source_id': ('django.db.models.fields.IntegerField', [], {}) + }, + u'kns_persons.knessetpersontoposition': { + 'Meta': {'object_name': 'KnessetPersonToPosition'}, + 'committee_id': ('django.db.models.fields.IntegerField', [], {'null': 'True', 'blank': 'True'}), + 'committee_name': ('django.db.models.fields.CharField', [], {'max_length': '64'}), + 'duty_description': ('django.db.models.fields.CharField', [], {'max_length': '256'}), + 'faction_id': ('django.db.models.fields.IntegerField', [], {'null': 'True', 'blank': 'True'}), + 'faction_name': ('django.db.models.fields.CharField', [], {'max_length': '64'}), + 'finish_update': ('django.db.models.fields.DateTimeField', [], {'null': 'True', 'blank': 'True'}), + 'gov_num': ('django.db.models.fields.IntegerField', [], {'null': 'True', 'blank': 'True'}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'is_current': ('django.db.models.fields.CharField', [], {'max_length': '64', 'null': 'True', 'blank': 'True'}), + 'knesset_num': ('django.db.models.fields.IntegerField', [], {'null': 'True', 'blank': 'True'}), + 'last_update': ('django.db.models.fields.DateTimeField', [], {'null': 'True', 'blank': 'True'}), + 'ministry_id': ('django.db.models.fields.IntegerField', [], {'null': 'True', 'blank': 'True'}), + 'ministry_name': ('django.db.models.fields.CharField', [], {'max_length': '256'}), + 'person_id': ('django.db.models.fields.IntegerField', [], {}), + 'position_id': ('django.db.models.fields.IntegerField', [], {}), + 'source_id': ('django.db.models.fields.IntegerField', [], {}), + 'start_update': ('django.db.models.fields.DateTimeField', [], {'null': 'True', 'blank': 'True'}) + }, + u'kns_persons.knessetposition': { + 'Meta': {'object_name': 'KnessetPosition'}, + 'description': ('django.db.models.fields.CharField', [], {'max_length': '256'}), + 'gender_description': ('django.db.models.fields.CharField', [], {'max_length': '64'}), + 'gender_id': ('django.db.models.fields.IntegerField', [], {}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'last_update': ('django.db.models.fields.DateTimeField', [], {}), + 'source_id': ('django.db.models.fields.IntegerField', [], {}) + }, + u'kns_persons.knessetsitecode': { + 'Meta': {'object_name': 'KnessetSiteCode'}, + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'kns_id': ('django.db.models.fields.IntegerField', [], {}), + 'site_id': ('django.db.models.fields.IntegerField', [], {}), + 'source_id': ('django.db.models.fields.IntegerField', [], {}) + } + } + + complete_apps = ['kns_persons'] \ No newline at end of file diff --git a/knesset_data_django/kns_persons/migrations/__init__.py b/knesset_data_django/kns_persons/migrations/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/knesset_data_django/kns_persons/models.py b/knesset_data_django/kns_persons/models.py new file mode 100644 index 0000000..de1d678 --- /dev/null +++ b/knesset_data_django/kns_persons/models.py @@ -0,0 +1,45 @@ +from django.db import models + + +class KnessetPerson(models.Model): + source_id = models.IntegerField() + last_name = models.CharField(max_length=64) + first_name = models.CharField(max_length=64) + gender_id = models.IntegerField() + gender_description = models.CharField(max_length=64) + email = models.EmailField(blank=True, null=True) + is_current = models.CharField(blank=True, null=True, max_length=64) + last_update = models.DateTimeField() + + +class KnessetPosition(models.Model): + source_id = models.IntegerField() + description = models.CharField(max_length=256) + gender_id = models.IntegerField() + gender_description = models.CharField(max_length=64) + last_update = models.DateTimeField() + + +class KnessetPersonToPosition(models.Model): + source_id = models.IntegerField() + person_id = models.IntegerField() + position_id = models.IntegerField() + knesset_num = models.IntegerField(blank=True, null=True) + ministry_id = models.IntegerField(blank=True, null=True) + ministry_name = models.CharField(max_length=256) + duty_description = models.CharField(max_length=256) + faction_id = models.IntegerField(blank=True, null=True) + faction_name = models.CharField(max_length=64) + gov_num = models.IntegerField(blank=True, null=True) + committee_id = models.IntegerField(blank=True, null=True) + committee_name = models.CharField(max_length=64) + start_update = models.DateTimeField(blank=True, null=True) + finish_update = models.DateTimeField(blank=True, null=True) + is_current = models.CharField(blank=True, null=True, max_length=64) + last_update = models.DateTimeField(blank=True, null=True) + + +class KnessetSiteCode(models.Model): + source_id = models.IntegerField() + kns_id = models.IntegerField() + site_id = models.IntegerField() diff --git a/knesset_data_django/kns_persons/scrapers.py b/knesset_data_django/kns_persons/scrapers.py new file mode 100644 index 0000000..1482486 --- /dev/null +++ b/knesset_data_django/kns_persons/scrapers.py @@ -0,0 +1,89 @@ +from ..common.scrapers.simple_datapackage_scraper import SimpleDatapackageScraper +from .models import KnessetPerson, KnessetPersonToPosition, KnessetPosition + + +class PersonScraper(SimpleDatapackageScraper): + + DATAPACKAGE_RESOURCE_NAME = "persons-person" + + def _get_source_id(self, item_data): + return item_data["id"] + + def _get_model_data(self, item_data): + return { + "last_name": item_data["last_name"], + "first_name": item_data["first_name"], + "gender_id": item_data["gender_id"], + "gender_description": item_data["gender_description"], + "email": item_data["email"], + "is_current": item_data["is_current"], + "last_update": item_data["last_update"] + } + + def _get_model_class(self): + return KnessetPerson + + def _get_source_id_attr(self): + return "source_id" + + def _get_object_prefix(self, object): + return u"person {} - {} {}".format(object.id, object.first_name, object.last_name) + +class PersonPositionScraper(SimpleDatapackageScraper): + + DATAPACKAGE_RESOURCE_NAME = "persons-position" + + def _get_source_id(self, item_data): + return item_data["id"] + + def _get_model_data(self, item_data): + return { + "description": item_data["description"], + "gender_id": item_data["gender_id"], + "gender_description": item_data["gender_description"], + "last_update": item_data["last_update"] + } + + def _get_model_class(self): + return KnessetPosition + + def _get_source_id_attr(self): + return "source_id" + + def _get_object_prefix(self, object): + return u"person position {} - {}".format(object.id, object.description) + +class PersonToPositionScraper(SimpleDatapackageScraper): + + DATAPACKAGE_RESOURCE_NAME = "persons-persons-to-positions" + + def _get_source_id(self, item_data): + return item_data["id"] + + def _get_model_data(self, item_data): + return { + "person_id": item_data["person_id"], + "position_id": item_data["position_id"], + "knesset_num": item_data["knnesset_num"], + "ministry_id": item_data["ministry_id"], + "ministry_name": item_data["ministry_name"], + "duty_description": item_data["duty_description"], + "faction_id": item_data["faction_id"], + "faction_name": item_data["faction_name"], + "gov_num": item_data["gov_num"], + "committee_id": item_data["committee_id"], + "committee_name": item_data["committee_name"], + "start_update": item_data["start_update"], + "finish_update": item_data["finish_update"], + "is_current": item_data["is_current"], + "last_update": item_data["last_update"], + } + + def _get_model_class(self): + return KnessetPersonToPosition + + def _get_source_id_attr(self): + return "source_id" + + def _get_object_prefix(self, object): + return u"person to position {} - {} {}".format(object.id, object.person_id, object.position_id) diff --git a/knesset_data_django/persons/models.py b/knesset_data_django/persons/models.py index df339e2..bc64b57 100644 --- a/knesset_data_django/persons/models.py +++ b/knesset_data_django/persons/models.py @@ -1,2 +1,2 @@ # currently uses Open Knesset models. TODO: move to a separate data access layer -from persons.models import Person, PersonAlias \ No newline at end of file +from persons.models import Person, PersonAlias