diff --git a/knesset_data_django/__init__.py b/knesset_data_django/__init__.py index 2b5bad0..a7bae1c 100644 --- a/knesset_data_django/__init__.py +++ b/knesset_data_django/__init__.py @@ -1,4 +1,5 @@ KNESSET_DATA_DJANGO_APPS = ( 'knesset_data_django.committees', 'knesset_data_django.common', + 'knesset_data_django.kns_persons', ) diff --git a/knesset_data_django/common/scrapers/root_datapackage_scraper.py b/knesset_data_django/common/scrapers/root_datapackage_scraper.py index b112b0c..2df81c6 100644 --- a/knesset_data_django/common/scrapers/root_datapackage_scraper.py +++ b/knesset_data_django/common/scrapers/root_datapackage_scraper.py @@ -178,9 +178,14 @@ def log_scrape_return_value(self, scrape_classes_return_value): scraper_instance, scrape_return_values = scrape_class_return_value if scraper_instance: i = 0 - for scrape_return_value in scrape_return_values: - scraper_instance.log_return_value(*scrape_return_value) - i += 1 + try: + for scrape_return_value in scrape_return_values: + scraper_instance.log_return_value(*scrape_return_value) + i += 1 + except Exception as e: + message = "unexpected exception parsing return value for {}".format(scraper_class) + self.logger.exception(message) + raise Exception(message) self.logger.info("processed {} items for scraper {}".format(i, scraper_class.__name__)) else: self.logger.debug("skipping scraper {}".format(scraper_class.__name__)) diff --git a/knesset_data_django/kns_persons/__init__.py b/knesset_data_django/kns_persons/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/knesset_data_django/kns_persons/models.py b/knesset_data_django/kns_persons/models.py new file mode 100644 index 0000000..b996a5c --- /dev/null +++ b/knesset_data_django/kns_persons/models.py @@ -0,0 +1,45 @@ +from django.db import models + + +class KnessetPerson(models.Model): + source_id = models.IntegerField() + last_name = models.CharField(max_length=64) + first_name = models.CharField(max_length=64) + gender_id = models.IntegerField() + gender_description = models.CharField(max_length=64) + email = models.EmailField(blank=True, null=True) + is_current = models.BooleanField() + last_update = models.DateTimeField() + + +class KnessetPosition(models.Model): + source_id = models.IntegerField() + description = models.CharField(max_length=256) + gender_id = models.IntegerField() + gender_description = models.CharField(max_length=64) + last_update = models.DateTimeField() + + +class KnessetPersonToPosition(models.Model): + source_id = models.IntegerField() + person_id = models.IntegerField() + position_id = models.IntegerField() + knesset_num = models.IntegerField() + ministry_id = models.IntegerField() + ministry_name = models.CharField(max_length=256) + duty_description = models.CharField(max_length=256) + faction_id = models.IntegerField() + faction_name = models.CharField(max_length=64) + gov_num = models.IntegerField() + committee_id = models.IntegerField() + committee_name = models.CharField(max_length=64) + start_update = models.DateTimeField() + finish_update = models.DateTimeField() + is_current = models.BooleanField() + last_update = models.DateTimeField() + + +class KnessetSiteCode(models.Model): + source_id = models.IntegerField() + kns_id = models.IntegerField() + site_id = models.IntegerField() diff --git a/knesset_data_django/kns_persons/scrapers/__init__.py b/knesset_data_django/kns_persons/scrapers/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/knesset_data_django/kns_persons/scrapers/person.py b/knesset_data_django/kns_persons/scrapers/person.py new file mode 100644 index 0000000..9f210a1 --- /dev/null +++ b/knesset_data_django/kns_persons/scrapers/person.py @@ -0,0 +1,56 @@ +from ...common.scrapers.base_datapackage_scraper import BaseDatapackageScraper +from knesset_data_django.common.exceptions import TooManyObjectsException +from ..models import KnessetPerson + + +class PersonScraper(BaseDatapackageScraper): + DATAPACKAGE_RESOURCE_NAME = "person" + + def _handle_datapackage_item(self, person_data): + """ + updates or create a committee object based on dataservice_person + :param person_data: dataservice person object + :return: tuple(person, created) the updated or created person model object and True/False if it was created + """ + person_knesset_id = person_data["id"] + person_model_data = { + "last_name": person_data["last_name"], + "first_name": person_data["first_name"], + "gender_id": person_data["gender_id"], + "gender_description": person_data["gender_description"], + "email": person_data["email"], + "is_current": person_data["is_current"], + "last_update": person_data["last_update"] + } + person_qs = KnessetPerson.objects.filter(source_id=person_knesset_id) + person_qs_count = person_qs.count() + if person_qs_count == 1: + person = person_qs.first() + needs_update = False + for attr, scraped_value in person_model_data.iteritems(): + db_value = getattr(person, attr) + if db_value != scraped_value: + needs_update = True + break + if needs_update: + [setattr(person, k, v) for k, v in person_model_data.iteritems()] + created, updated, message = False, True, "detected a change in one of the fields, updating person" + else: + created, updated, message = False, False, "existing person in DB, no change" + elif person_qs_count == 0: + person = KnessetPerson(knesset_id=person_knesset_id, **person_model_data) + created, updated, message = True, False, "created person" + else: + raise TooManyObjectsException("committee_knesset_id={}, matching db ids: {}".format(person_knesset_id, + [c.id for c in + person_qs])) + if updated or created: + person.save() + return person, created, updated, message + + def log_return_value(self, person, created, updated, message): + prefix = u"person {} - {} {}".format(person.id, person.first_name, person.last_name) + if created or updated: + self.logger.info(u"{}: {}".format(prefix, message)) + else: + self.logger.debug(u'{}: {}'.format(prefix, message)) diff --git a/knesset_data_django/persons/models.py b/knesset_data_django/persons/models.py index df339e2..bc64b57 100644 --- a/knesset_data_django/persons/models.py +++ b/knesset_data_django/persons/models.py @@ -1,2 +1,2 @@ # currently uses Open Knesset models. TODO: move to a separate data access layer -from persons.models import Person, PersonAlias \ No newline at end of file +from persons.models import Person, PersonAlias