[wip] kns_person scraper and model

knesset_data_django/__init__.py

@@ -1,4 +1,5 @@
 KNESSET_DATA_DJANGO_APPS = (
     'knesset_data_django.committees',
     'knesset_data_django.common',
+    'knesset_data_django.kns_persons',
 )

knesset_data_django/common/scrapers/root_datapackage_scraper.py

@@ -178,9 +178,14 @@ def log_scrape_return_value(self, scrape_classes_return_value):
                 scraper_instance, scrape_return_values = scrape_class_return_value
                 if scraper_instance:
                     i = 0
-                    for scrape_return_value in scrape_return_values:
-                        scraper_instance.log_return_value(*scrape_return_value)
-                        i += 1
+                    try:
+                        for scrape_return_value in scrape_return_values:
+                            scraper_instance.log_return_value(*scrape_return_value)
+                            i += 1
+                    except Exception as e:
+                        message = "unexpected exception parsing return value for {}".format(scraper_class)
+                        self.logger.exception(message)
+                        raise Exception(message)
                     self.logger.info("processed {} items for scraper {}".format(i, scraper_class.__name__))
                 else:
                     self.logger.debug("skipping scraper {}".format(scraper_class.__name__))

knesset_data_django/kns_persons/models.py

@@ -0,0 +1,45 @@
+from django.db import models
+
+
+class KnessetPerson(models.Model):
+    source_id = models.IntegerField()
+    last_name = models.CharField(max_length=64)
+    first_name = models.CharField(max_length=64)
+    gender_id = models.IntegerField()
+    gender_description = models.CharField(max_length=64)
+    email = models.EmailField(blank=True, null=True)
+    is_current = models.BooleanField()
+    last_update = models.DateTimeField()
+
+
+class KnessetPosition(models.Model):
+    source_id = models.IntegerField()
+    description = models.CharField(max_length=256)
+    gender_id = models.IntegerField()
+    gender_description = models.CharField(max_length=64)
+    last_update = models.DateTimeField()
+
+
+class KnessetPersonToPosition(models.Model):
+    source_id = models.IntegerField()
+    person_id = models.IntegerField()
+    position_id = models.IntegerField()
+    knesset_num = models.IntegerField()
+    ministry_id = models.IntegerField()
+    ministry_name = models.CharField(max_length=256)
+    duty_description = models.CharField(max_length=256)
+    faction_id = models.IntegerField()
+    faction_name = models.CharField(max_length=64)
+    gov_num = models.IntegerField()
+    committee_id = models.IntegerField()
+    committee_name = models.CharField(max_length=64)
+    start_update = models.DateTimeField()
+    finish_update = models.DateTimeField()
+    is_current = models.BooleanField()
+    last_update = models.DateTimeField()
+
+
+class KnessetSiteCode(models.Model):
+    source_id = models.IntegerField()
+    kns_id = models.IntegerField()
+    site_id = models.IntegerField()

knesset_data_django/kns_persons/scrapers/person.py

@@ -0,0 +1,56 @@
+from ...common.scrapers.base_datapackage_scraper import BaseDatapackageScraper
+from knesset_data_django.common.exceptions import TooManyObjectsException
+from ..models import KnessetPerson
+
+
+class PersonScraper(BaseDatapackageScraper):
+    DATAPACKAGE_RESOURCE_NAME = "person"
+
+    def _handle_datapackage_item(self, person_data):
+        """
+        updates or create a committee object based on dataservice_person
+        :param person_data: dataservice person object
+        :return: tuple(person, created) the updated or created person model object and True/False if it was created
+                """
+        person_knesset_id = person_data["id"]
+        person_model_data = {
+            "last_name": person_data["last_name"],
+            "first_name": person_data["first_name"],
+            "gender_id": person_data["gender_id"],
+            "gender_description": person_data["gender_description"],
+            "email": person_data["email"],
+            "is_current": person_data["is_current"],
+            "last_update": person_data["last_update"]
+        }
+        person_qs = KnessetPerson.objects.filter(source_id=person_knesset_id)
+        person_qs_count = person_qs.count()
+        if person_qs_count == 1:
+            person = person_qs.first()
+            needs_update = False
+            for attr, scraped_value in person_model_data.iteritems():
+                db_value = getattr(person, attr)
+                if db_value != scraped_value:
+                    needs_update = True
+                    break
+            if needs_update:
+                [setattr(person, k, v) for k, v in person_model_data.iteritems()]
+                created, updated, message = False, True, "detected a change in one of the fields, updating person"
+            else:
+                created, updated, message = False, False, "existing person in DB, no change"
+        elif person_qs_count == 0:
+            person = KnessetPerson(knesset_id=person_knesset_id, **person_model_data)
+            created, updated, message = True, False, "created person"
+        else:
+            raise TooManyObjectsException("committee_knesset_id={}, matching db ids: {}".format(person_knesset_id,
+                                                                                                [c.id for c in
+                                                                                                 person_qs]))
+        if updated or created:
+            person.save()
+        return person, created, updated, message
+
+    def log_return_value(self, person, created, updated, message):
+        prefix = u"person {} - {} {}".format(person.id, person.first_name, person.last_name)
+        if created or updated:
+            self.logger.info(u"{}: {}".format(prefix, message))
+        else:
+            self.logger.debug(u'{}: {}'.format(prefix, message))

knesset_data_django/persons/models.py

@@ -1,2 +1,2 @@
 # currently uses Open Knesset models. TODO: move to a separate data access layer
-from persons.models import Person, PersonAlias
+from persons.models import Person, PersonAlias