[WIP] added persons scraping from datapackage (from new API)

.travis.yml

@@ -9,8 +9,8 @@ addons:
 install:
   - pip install --upgrade pip
   - pip install -r requirements.txt
-  - pip install git+https://github.com/hasadna/knesset-data-python.git@v1.7.2#egg=knesset-data
-  - pip install git+https://github.com/hasadna/knesset-data-datapackage.git@v1.1.0#egg=knesset-datapackage
+  - pip install git+https://github.com/hasadna/knesset-data-python.git@v1.8.1#egg=knesset-data
+  - pip install git+https://github.com/hasadna/knesset-data-datapackage.git@v1.3.0#egg=knesset-datapackage
   - pip install git+https://github.com/hasadna/Open-Knesset.git@e28339da7ca92df96fc79b89351286e2715fcff0#egg=Open-Knesset
   - pip install .
 script:

knesset_data_django/__init__.py

@@ -1,4 +1,5 @@
 KNESSET_DATA_DJANGO_APPS = (
     'knesset_data_django.committees',
     'knesset_data_django.common',
+    'knesset_data_django.kns_persons',
 )

knesset_data_django/common/scrapers/root_datapackage_scraper.py

@@ -17,6 +17,9 @@ class RootDatapackageScraper(BaseScraper):
         "knesset_data_django.committees.scrapers.committees.CommitteesScraper",
         "knesset_data_django.committees.scrapers.committee_meetings.CommitteeMeetingsScraper",
         "knesset_data_django.committees.scrapers.committee_meeting_protocols.CommitteeMeetingProtocolsScraper",
+        "knesset_data_django.kns_persons.scrapers.PersonScraper",
+        "knesset_data_django.kns_persons.scrapers.PersonPositionScraper",
+        "knesset_data_django.kns_persons.scrapers.PersonToPositionScraper",
     )
 
     def __init__(self, **kwargs):
@@ -178,9 +181,14 @@ def log_scrape_return_value(self, scrape_classes_return_value):
                 scraper_instance, scrape_return_values = scrape_class_return_value
                 if scraper_instance:
                     i = 0
-                    for scrape_return_value in scrape_return_values:
-                        scraper_instance.log_return_value(*scrape_return_value)
-                        i += 1
+                    try:
+                        for scrape_return_value in scrape_return_values:
+                            scraper_instance.log_return_value(*scrape_return_value)
+                            i += 1
+                    except Exception as e:
+                        message = "unexpected exception parsing return value for {}".format(scraper_class)
+                        self.logger.exception(message)
+                        raise Exception(message)
                     self.logger.info("processed {} items for scraper {}".format(i, scraper_class.__name__))
                 else:
                     self.logger.debug("skipping scraper {}".format(scraper_class.__name__))

knesset_data_django/common/scrapers/simple_datapackage_scraper.py

@@ -0,0 +1,64 @@
+from .base_datapackage_scraper import BaseDatapackageScraper
+from knesset_data_django.common.exceptions import TooManyObjectsException
+
+
+
+class SimpleDatapackageScraper(BaseDatapackageScraper):
+
+    # extending classes should define this with the name of the source resource from the datapackage
+    # DATAPACKAGE_RESOURCE_NAME = ""
+
+    def _get_source_id(self, item_data):
+        return item_data["id"]
+
+    def _get_model_data(self, item_data):
+        raise NotImplementedError()
+
+    def _get_model_class(self):
+        raise NotImplementedError()
+
+    def _get_source_id_attr(self):
+        return "source_id"
+
+    def _get_object_prefix(self, object):
+        return str(object)
+
+    def _handle_datapackage_item(self, item_data):
+        """
+        updates or create an object based on the data from the datapackage
+        :param item_data: source data, conforms to the datapackage schema
+        :return: tuple(object, created, updated, message) the created/updated db object, whether it was created or updated and a message
+                """
+        source_id = self._get_source_id(item_data)
+        model_data = self._get_model_data(item_data)
+        qs = self._get_model_class().objects.filter(**{self._get_source_id_attr(): source_id})
+        qs_count = qs.count()
+        if qs_count == 1:
+            object = qs.first()
+            needs_update = False
+            for attr, scraped_value in model_data.iteritems():
+                db_value = getattr(object, attr)
+                if db_value != scraped_value:
+                    self.logger.debug("change in field {} db_value {} scraped_value {}".format(attr, db_value, scraped_value))
+                    needs_update = True
+                    break
+            if needs_update:
+                [setattr(object, k, v) for k, v in model_data.iteritems()]
+                created, updated, message = False, True, "detected a change in one of the fields, updating object"
+            else:
+                created, updated, message = False, False, "existing object in DB, no change"
+        elif qs_count == 0:
+            object = self._get_model_class()(**dict(model_data, **{self._get_source_id_attr(): source_id}))
+            created, updated, message = True, False, "created object"
+        else:
+            raise TooManyObjectsException("source_id={0}, matching db ids: {1}".format(source_id, [o.id for o in qs]))
+        if updated or created:
+            object.save()
+        return object, created, updated, message
+
+    def log_return_value(self, object, created, updated, message):
+        prefix = self._get_object_prefix(object)
+        if created or updated:
+            self.logger.info(u"{}: {}".format(prefix, message))
+        else:
+            self.logger.debug(u'{}: {}'.format(prefix, message))

knesset_data_django/common/tests/test_datapackage.py

@@ -33,6 +33,13 @@
 
 class MockRootDatapackageScraper(RootDatapackageScraper):
 
+    SCRAPERS = (
+        "knesset_data_django.mks.scrapers.members.MembersScraper",
+        "knesset_data_django.committees.scrapers.committees.CommitteesScraper",
+        "knesset_data_django.committees.scrapers.committee_meetings.CommitteeMeetingsScraper",
+        "knesset_data_django.committees.scrapers.committee_meeting_protocols.CommitteeMeetingProtocolsScraper",
+    )
+
     def _scrape_instance_value(self, from_datapackage, fetch_kwargs, child_scraper_instance, return_value):
         if isinstance(child_scraper_instance, MembersScraper) and return_value[0] and return_value[1]:
             # add some fake data to allow committee meetings to detect it in attending members

knesset_data_django/kns_persons/migrations/0001_initial.py

@@ -0,0 +1,133 @@
+# -*- coding: utf-8 -*-
+from south.utils import datetime_utils as datetime
+from south.db import db
+from south.v2 import SchemaMigration
+from django.db import models
+
+
+class Migration(SchemaMigration):
+
+    def forwards(self, orm):
+        # Adding model 'KnessetPerson'
+        db.create_table(u'kns_persons_knessetperson', (
+            (u'id', self.gf('django.db.models.fields.AutoField')(primary_key=True)),
+            ('source_id', self.gf('django.db.models.fields.IntegerField')()),
+            ('last_name', self.gf('django.db.models.fields.CharField')(max_length=64)),
+            ('first_name', self.gf('django.db.models.fields.CharField')(max_length=64)),
+            ('gender_id', self.gf('django.db.models.fields.IntegerField')()),
+            ('gender_description', self.gf('django.db.models.fields.CharField')(max_length=64)),
+            ('email', self.gf('django.db.models.fields.EmailField')(max_length=75, null=True, blank=True)),
+            ('is_current', self.gf('django.db.models.fields.CharField')(max_length=64, null=True, blank=True)),
+            ('last_update', self.gf('django.db.models.fields.DateTimeField')()),
+        ))
+        db.send_create_signal(u'kns_persons', ['KnessetPerson'])
+
+        # Adding model 'KnessetPosition'
+        db.create_table(u'kns_persons_knessetposition', (
+            (u'id', self.gf('django.db.models.fields.AutoField')(primary_key=True)),
+            ('source_id', self.gf('django.db.models.fields.IntegerField')()),
+            ('description', self.gf('django.db.models.fields.CharField')(max_length=256)),
+            ('gender_id', self.gf('django.db.models.fields.IntegerField')()),
+            ('gender_description', self.gf('django.db.models.fields.CharField')(max_length=64)),
+            ('last_update', self.gf('django.db.models.fields.DateTimeField')()),
+        ))
+        db.send_create_signal(u'kns_persons', ['KnessetPosition'])
+
+        # Adding model 'KnessetPersonToPosition'
+        db.create_table(u'kns_persons_knessetpersontoposition', (
+            (u'id', self.gf('django.db.models.fields.AutoField')(primary_key=True)),
+            ('source_id', self.gf('django.db.models.fields.IntegerField')()),
+            ('person_id', self.gf('django.db.models.fields.IntegerField')()),
+            ('position_id', self.gf('django.db.models.fields.IntegerField')()),
+            ('knesset_num', self.gf('django.db.models.fields.IntegerField')(null=True, blank=True)),
+            ('ministry_id', self.gf('django.db.models.fields.IntegerField')(null=True, blank=True)),
+            ('ministry_name', self.gf('django.db.models.fields.CharField')(max_length=256)),
+            ('duty_description', self.gf('django.db.models.fields.CharField')(max_length=256)),
+            ('faction_id', self.gf('django.db.models.fields.IntegerField')(null=True, blank=True)),
+            ('faction_name', self.gf('django.db.models.fields.CharField')(max_length=64)),
+            ('gov_num', self.gf('django.db.models.fields.IntegerField')(null=True, blank=True)),
+            ('committee_id', self.gf('django.db.models.fields.IntegerField')(null=True, blank=True)),
+            ('committee_name', self.gf('django.db.models.fields.CharField')(max_length=64)),
+            ('start_update', self.gf('django.db.models.fields.DateTimeField')(null=True, blank=True)),
+            ('finish_update', self.gf('django.db.models.fields.DateTimeField')(null=True, blank=True)),
+            ('is_current', self.gf('django.db.models.fields.CharField')(max_length=64, null=True, blank=True)),
+            ('last_update', self.gf('django.db.models.fields.DateTimeField')(null=True, blank=True)),
+        ))
+        db.send_create_signal(u'kns_persons', ['KnessetPersonToPosition'])
+
+        # Adding model 'KnessetSiteCode'
+        db.create_table(u'kns_persons_knessetsitecode', (
+            (u'id', self.gf('django.db.models.fields.AutoField')(primary_key=True)),
+            ('source_id', self.gf('django.db.models.fields.IntegerField')()),
+            ('kns_id', self.gf('django.db.models.fields.IntegerField')()),
+            ('site_id', self.gf('django.db.models.fields.IntegerField')()),
+        ))
+        db.send_create_signal(u'kns_persons', ['KnessetSiteCode'])
+
+
+    def backwards(self, orm):
+        # Deleting model 'KnessetPerson'
+        db.delete_table(u'kns_persons_knessetperson')
+
+        # Deleting model 'KnessetPosition'
+        db.delete_table(u'kns_persons_knessetposition')
+
+        # Deleting model 'KnessetPersonToPosition'
+        db.delete_table(u'kns_persons_knessetpersontoposition')
+
+        # Deleting model 'KnessetSiteCode'
+        db.delete_table(u'kns_persons_knessetsitecode')
+
+
+    models = {
+        u'kns_persons.knessetperson': {
+            'Meta': {'object_name': 'KnessetPerson'},
+            'email': ('django.db.models.fields.EmailField', [], {'max_length': '75', 'null': 'True', 'blank': 'True'}),
+            'first_name': ('django.db.models.fields.CharField', [], {'max_length': '64'}),
+            'gender_description': ('django.db.models.fields.CharField', [], {'max_length': '64'}),
+            'gender_id': ('django.db.models.fields.IntegerField', [], {}),
+            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'is_current': ('django.db.models.fields.CharField', [], {'max_length': '64', 'null': 'True', 'blank': 'True'}),
+            'last_name': ('django.db.models.fields.CharField', [], {'max_length': '64'}),
+            'last_update': ('django.db.models.fields.DateTimeField', [], {}),
+            'source_id': ('django.db.models.fields.IntegerField', [], {})
+        },
+        u'kns_persons.knessetpersontoposition': {
+            'Meta': {'object_name': 'KnessetPersonToPosition'},
+            'committee_id': ('django.db.models.fields.IntegerField', [], {'null': 'True', 'blank': 'True'}),
+            'committee_name': ('django.db.models.fields.CharField', [], {'max_length': '64'}),
+            'duty_description': ('django.db.models.fields.CharField', [], {'max_length': '256'}),
+            'faction_id': ('django.db.models.fields.IntegerField', [], {'null': 'True', 'blank': 'True'}),
+            'faction_name': ('django.db.models.fields.CharField', [], {'max_length': '64'}),
+            'finish_update': ('django.db.models.fields.DateTimeField', [], {'null': 'True', 'blank': 'True'}),
+            'gov_num': ('django.db.models.fields.IntegerField', [], {'null': 'True', 'blank': 'True'}),
+            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'is_current': ('django.db.models.fields.CharField', [], {'max_length': '64', 'null': 'True', 'blank': 'True'}),
+            'knesset_num': ('django.db.models.fields.IntegerField', [], {'null': 'True', 'blank': 'True'}),
+            'last_update': ('django.db.models.fields.DateTimeField', [], {'null': 'True', 'blank': 'True'}),
+            'ministry_id': ('django.db.models.fields.IntegerField', [], {'null': 'True', 'blank': 'True'}),
+            'ministry_name': ('django.db.models.fields.CharField', [], {'max_length': '256'}),
+            'person_id': ('django.db.models.fields.IntegerField', [], {}),
+            'position_id': ('django.db.models.fields.IntegerField', [], {}),
+            'source_id': ('django.db.models.fields.IntegerField', [], {}),
+            'start_update': ('django.db.models.fields.DateTimeField', [], {'null': 'True', 'blank': 'True'})
+        },
+        u'kns_persons.knessetposition': {
+            'Meta': {'object_name': 'KnessetPosition'},
+            'description': ('django.db.models.fields.CharField', [], {'max_length': '256'}),
+            'gender_description': ('django.db.models.fields.CharField', [], {'max_length': '64'}),
+            'gender_id': ('django.db.models.fields.IntegerField', [], {}),
+            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'last_update': ('django.db.models.fields.DateTimeField', [], {}),
+            'source_id': ('django.db.models.fields.IntegerField', [], {})
+        },
+        u'kns_persons.knessetsitecode': {
+            'Meta': {'object_name': 'KnessetSiteCode'},
+            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'kns_id': ('django.db.models.fields.IntegerField', [], {}),
+            'site_id': ('django.db.models.fields.IntegerField', [], {}),
+            'source_id': ('django.db.models.fields.IntegerField', [], {})
+        }
+    }
+
+    complete_apps = ['kns_persons']

knesset_data_django/kns_persons/models.py

@@ -0,0 +1,45 @@
+from django.db import models
+
+
+class KnessetPerson(models.Model):
+    source_id = models.IntegerField()
+    last_name = models.CharField(max_length=64)
+    first_name = models.CharField(max_length=64)
+    gender_id = models.IntegerField()
+    gender_description = models.CharField(max_length=64)
+    email = models.EmailField(blank=True, null=True)
+    is_current = models.CharField(blank=True, null=True, max_length=64)
+    last_update = models.DateTimeField()
+
+
+class KnessetPosition(models.Model):
+    source_id = models.IntegerField()
+    description = models.CharField(max_length=256)
+    gender_id = models.IntegerField()
+    gender_description = models.CharField(max_length=64)
+    last_update = models.DateTimeField()
+
+
+class KnessetPersonToPosition(models.Model):
+    source_id = models.IntegerField()
+    person_id = models.IntegerField()
+    position_id = models.IntegerField()
+    knesset_num = models.IntegerField(blank=True, null=True)
+    ministry_id = models.IntegerField(blank=True, null=True)
+    ministry_name = models.CharField(max_length=256)
+    duty_description = models.CharField(max_length=256)
+    faction_id = models.IntegerField(blank=True, null=True)
+    faction_name = models.CharField(max_length=64)
+    gov_num = models.IntegerField(blank=True, null=True)
+    committee_id = models.IntegerField(blank=True, null=True)
+    committee_name = models.CharField(max_length=64)
+    start_update = models.DateTimeField(blank=True, null=True)
+    finish_update = models.DateTimeField(blank=True, null=True)
+    is_current = models.CharField(blank=True, null=True, max_length=64)
+    last_update = models.DateTimeField(blank=True, null=True)
+
+
+class KnessetSiteCode(models.Model):
+    source_id = models.IntegerField()
+    kns_id = models.IntegerField()
+    site_id = models.IntegerField()

knesset_data_django/kns_persons/scrapers.py

@@ -0,0 +1,89 @@
+from ..common.scrapers.simple_datapackage_scraper import SimpleDatapackageScraper
+from .models import KnessetPerson, KnessetPersonToPosition, KnessetPosition
+
+
+class PersonScraper(SimpleDatapackageScraper):
+
+    DATAPACKAGE_RESOURCE_NAME = "persons-person"
+
+    def _get_source_id(self, item_data):
+        return item_data["id"]
+
+    def _get_model_data(self, item_data):
+        return {
+            "last_name": item_data["last_name"],
+            "first_name": item_data["first_name"],
+            "gender_id": item_data["gender_id"],
+            "gender_description": item_data["gender_description"],
+            "email": item_data["email"],
+            "is_current": item_data["is_current"],
+            "last_update": item_data["last_update"]
+        }
+
+    def _get_model_class(self):
+        return KnessetPerson
+
+    def _get_source_id_attr(self):
+        return "source_id"
+
+    def _get_object_prefix(self, object):
+        return u"person {} - {} {}".format(object.id, object.first_name, object.last_name)
+
+class PersonPositionScraper(SimpleDatapackageScraper):
+
+    DATAPACKAGE_RESOURCE_NAME = "persons-position"
+
+    def _get_source_id(self, item_data):
+        return item_data["id"]
+
+    def _get_model_data(self, item_data):
+        return {
+            "description": item_data["description"],
+            "gender_id": item_data["gender_id"],
+            "gender_description": item_data["gender_description"],
+            "last_update": item_data["last_update"]
+        }
+
+    def _get_model_class(self):
+        return KnessetPosition
+
+    def _get_source_id_attr(self):
+        return "source_id"
+
+    def _get_object_prefix(self, object):
+        return u"person position {} - {}".format(object.id, object.description)
+
+class PersonToPositionScraper(SimpleDatapackageScraper):
+
+    DATAPACKAGE_RESOURCE_NAME = "persons-persons-to-positions"
+
+    def _get_source_id(self, item_data):
+        return item_data["id"]
+
+    def _get_model_data(self, item_data):
+        return {
+            "person_id": item_data["person_id"],
+            "position_id": item_data["position_id"],
+            "knesset_num": item_data["knnesset_num"],
+            "ministry_id": item_data["ministry_id"],
+            "ministry_name": item_data["ministry_name"],
+            "duty_description": item_data["duty_description"],
+            "faction_id": item_data["faction_id"],
+            "faction_name": item_data["faction_name"],
+            "gov_num": item_data["gov_num"],
+            "committee_id": item_data["committee_id"],
+            "committee_name": item_data["committee_name"],
+            "start_update": item_data["start_update"],
+            "finish_update": item_data["finish_update"],
+            "is_current": item_data["is_current"],
+            "last_update": item_data["last_update"],
+        }
+
+    def _get_model_class(self):
+        return KnessetPersonToPosition
+
+    def _get_source_id_attr(self):
+        return "source_id"
+
+    def _get_object_prefix(self, object):
+        return u"person to position {} - {} {}".format(object.id, object.person_id, object.position_id)

knesset_data_django/persons/models.py

@@ -1,2 +1,2 @@
 # currently uses Open Knesset models. TODO: move to a separate data access layer
-from persons.models import Person, PersonAlias
+from persons.models import Person, PersonAlias