committee meeting attendee support

knesset_data_django/__init__.py

@@ -1,4 +1,5 @@
 KNESSET_DATA_DJANGO_APPS = (
     'knesset_data_django.committees',
     'knesset_data_django.common',
+    'knesset_data_django.kns_persons',
 )

knesset_data_django/committees/models.py

@@ -1,2 +1,2 @@
 # currently uses Open Knesset models. TODO: move to a separate data access layer
-from committees.models import Committee, CommitteeMeeting, ProtocolPart
+from committees.models import Committee, CommitteeMeeting, ProtocolPart, CommitteeMeetingAttendee

knesset_data_django/committees/scrapers/committee_meeting_protocols.py

@@ -1,7 +1,7 @@
 from knesset_data_django.committees.protocol_part_builder import CommitteeProtocolPartBuilder
 from ...common.scrapers.base_datapackage_scraper import BaseDatapackageScraper
 from ...mks.utils import get_all_mk_names
-from ..models import CommitteeMeeting, ProtocolPart
+from ..models import CommitteeMeeting, ProtocolPart, CommitteeMeetingAttendee
 from datetime import datetime
 import os
 import csv
@@ -17,7 +17,7 @@ def _get_all_mk_names(self):
             setattr(self, 'loaded_mk_names', True)
         return self.mks, self.mk_names
 
-    def _validate_datapackage_item(self, text_file_path, parts_file_path, meetings_cnt):
+    def _validate_datapackage_item(self, text_file_path, parts_file_path, attendees_file_path, meetings_cnt):
         if meetings_cnt == 0:
             return False, "could not find meeting in DB"
         elif meetings_cnt > 1:
@@ -26,6 +26,8 @@ def _validate_datapackage_item(self, text_file_path, parts_file_path, meetings_c
             return False, "missing text file due to error in datapackage creation, see scraper_errors field in datapackage"
         elif parts_file_path == self._get_datapackage_resource_path():
             return False, "missing parts file due to error in datapackage creation, see scraper_errors field in datapackage"
+        elif attendees_file_path == self._get_datapackage_resource_path():
+            return False, "missing attendees file due to error in datapackage creation, see scraper_errors field in datapackage"
         elif not os.path.exists(text_file_path):
             return False, "missing text file {}".format(text_file_path)
         elif not os.path.exists(parts_file_path):
@@ -95,26 +97,50 @@ def _update_protocol_parts(self, meeting, parts_file_path):
                 attended_ok, attended_message = False, "failed to updated attending members: {}".format(e)
         return protocol_ok, protocol_message, attended_ok, attended_message
 
+    def _save_attendees(self, meeting, attendees):
+        CommitteeMeetingAttendee.objects.bulk_create(attendees)
+        meeting.save()
+
+    def _update_attendees(self, meeting, attendees_file_path):
+        attendees_ok, attendees_message = None, None
+        if meeting.attendees.count() > 0:
+            attendees_ok, attendees_message = False, "meeting has existing attendees, will not reparse"
+        else:
+            attendees_ok = True
+            with open(attendees_file_path) as f:
+                attendees = csv.reader(f)
+                assert attendees.next() == ['name','role','additional_information']
+                committee_attendees = [CommitteeMeetingAttendee(comittee_meeting=meeting,name=attendee[0],role=attendee[1],additional_information=attendee[2] if len(attendee) > 2 else '')
+                                        for attendee in enumerate(attendees)]
+                self._save_attendees(meeting,committee_attendees)
+                attendees_message = "inserted attendees"
+
+        return attendees_ok, attendees_message
+
     def _handle_datapackage_item(self, meeting_data):
         text_updated, text_message = None, None
         parts_updated, parts_message = None, None
         attended_updated, attended_message = None, None
+        #sorry for this redundancy, i don't want to break stuff (@beber)
+        attendees_updated, attendees_message = None, None
         text_file_path = self._get_datapackage_resource_path(meeting_data["text"])
         parts_file_path = self._get_datapackage_resource_path(meeting_data["parts"])
+        attendees_file_path = self._get_datapackage_resource_path(meeting_data["attendees"])
         cnt, qs = self._get_meetings(meeting_data["committee_id"], meeting_data["meeting_id"])
-        ok, error = self._validate_datapackage_item(text_file_path, parts_file_path, cnt)
+        ok, error = self._validate_datapackage_item(text_file_path, parts_file_path, attendees_file_path, cnt)
         if ok:
             meeting = qs.first()
             text_updated, text_message = self._update_protocol_text(meeting, text_file_path)
             if not text_updated:
                 parts_updated, parts_message, attended_updated, attended_message = False, "protocol text not updated, so skipping parts updating as well", None, None
             else:
-                parts_updated, parts_message, attended_updated, attended_message = self._update_protocol_parts(meeting,
-                                                                                                               parts_file_path)
-        return ok, error, meeting_data, text_updated, text_message, parts_updated, parts_message, attended_updated, attended_message
+                parts_updated, parts_message, attended_updated, attended_message = self._update_protocol_parts(meeting, parts_file_path)
+
+            attendees_updated, attendees_message = self._update_attendees(meeting, attendees_file_path)
+        return ok, error, meeting_data, text_updated, text_message, parts_updated, parts_message, attendees_updated, attendees_message
+
+    def log_return_value(self, ok, error, meeting_data, text_updated, text_message, parts_updated, parts_message, attendees_updated, attendees_message):
 
-    def log_return_value(self, ok, error, meeting_data, text_updated, text_message, parts_updated, parts_message,
-                         attended_updated, attended_message):
         if ok:
             self.logger.debug(text_message)
             self.logger.debug(parts_message)
@@ -128,9 +154,11 @@ def log_return_value(self, ok, error, meeting_data, text_updated, text_message,
                 self.logger.error("committee {} meeting {}: updated committee meeting protocol parts only".format(
                     meeting_data["committee_id"], meeting_data["meeting_id"]))
             else:
-                self.logger.debug(
-                    "committee {} meeting {}: no update to protocol text or parts".format(meeting_data["committee_id"],
-                                                                                          meeting_data["meeting_id"]))
-        else:
-            self.logger.info("committee {} meeting {}: error scraping committee meeting protocol: {}".format(
-                meeting_data["committee_id"], meeting_data["meeting_id"], error))
+                self.logger.debug("committee {} meeting {}: no update to protocol text or parts".format(meeting_data["committee_id"], meeting_data["meeting_id"]))
+
+            #fuck this spaghetti (@beber)
+            if attendees_updated:
+                self.logger.info("committee {} meeting {}: updated meeting attendees".format(meeting_data["committee_id"], meeting_data["meeting_id"]))
+            else:
+                self.logger.debug("committee {} meeting {}: no update to meeting attendees".format(meeting_data["committee_id"], meeting_data["meeting_id"]))
+

knesset_data_django/common/scrapers/root_datapackage_scraper.py

@@ -178,9 +178,14 @@ def log_scrape_return_value(self, scrape_classes_return_value):
                 scraper_instance, scrape_return_values = scrape_class_return_value
                 if scraper_instance:
                     i = 0
-                    for scrape_return_value in scrape_return_values:
-                        scraper_instance.log_return_value(*scrape_return_value)
-                        i += 1
+                    try:
+                        for scrape_return_value in scrape_return_values:
+                            scraper_instance.log_return_value(*scrape_return_value)
+                            i += 1
+                    except Exception as e:
+                        message = "unexpected exception parsing return value for {}".format(scraper_class)
+                        self.logger.exception(message)
+                        raise Exception(message)
                     self.logger.info("processed {} items for scraper {}".format(i, scraper_class.__name__))
                 else:
                     self.logger.debug("skipping scraper {}".format(scraper_class.__name__))

knesset_data_django/kns_persons/models.py

@@ -0,0 +1,45 @@
+from django.db import models
+
+
+class KnessetPerson(models.Model):
+    source_id = models.IntegerField()
+    last_name = models.CharField(max_length=64)
+    first_name = models.CharField(max_length=64)
+    gender_id = models.IntegerField()
+    gender_description = models.CharField(max_length=64)
+    email = models.EmailField(blank=True, null=True)
+    is_current = models.BooleanField()
+    last_update = models.DateTimeField()
+
+
+class KnessetPosition(models.Model):
+    source_id = models.IntegerField()
+    description = models.CharField(max_length=256)
+    gender_id = models.IntegerField()
+    gender_description = models.CharField(max_length=64)
+    last_update = models.DateTimeField()
+
+
+class KnessetPersonToPosition(models.Model):
+    source_id = models.IntegerField()
+    person_id = models.IntegerField()
+    position_id = models.IntegerField()
+    knesset_num = models.IntegerField()
+    ministry_id = models.IntegerField()
+    ministry_name = models.CharField(max_length=256)
+    duty_description = models.CharField(max_length=256)
+    faction_id = models.IntegerField()
+    faction_name = models.CharField(max_length=64)
+    gov_num = models.IntegerField()
+    committee_id = models.IntegerField()
+    committee_name = models.CharField(max_length=64)
+    start_update = models.DateTimeField()
+    finish_update = models.DateTimeField()
+    is_current = models.BooleanField()
+    last_update = models.DateTimeField()
+
+
+class KnessetSiteCode(models.Model):
+    source_id = models.IntegerField()
+    kns_id = models.IntegerField()
+    site_id = models.IntegerField()

knesset_data_django/kns_persons/scrapers/person.py

@@ -0,0 +1,56 @@
+from ...common.scrapers.base_datapackage_scraper import BaseDatapackageScraper
+from knesset_data_django.common.exceptions import TooManyObjectsException
+from ..models import KnessetPerson
+
+
+class PersonScraper(BaseDatapackageScraper):
+    DATAPACKAGE_RESOURCE_NAME = "person"
+
+    def _handle_datapackage_item(self, person_data):
+        """
+        updates or create a committee object based on dataservice_person
+        :param person_data: dataservice person object
+        :return: tuple(person, created) the updated or created person model object and True/False if it was created
+                """
+        person_knesset_id = person_data["id"]
+        person_model_data = {
+            "last_name": person_data["last_name"],
+            "first_name": person_data["first_name"],
+            "gender_id": person_data["gender_id"],
+            "gender_description": person_data["gender_description"],
+            "email": person_data["email"],
+            "is_current": person_data["is_current"],
+            "last_update": person_data["last_update"]
+        }
+        person_qs = KnessetPerson.objects.filter(source_id=person_knesset_id)
+        person_qs_count = person_qs.count()
+        if person_qs_count == 1:
+            person = person_qs.first()
+            needs_update = False
+            for attr, scraped_value in person_model_data.iteritems():
+                db_value = getattr(person, attr)
+                if db_value != scraped_value:
+                    needs_update = True
+                    break
+            if needs_update:
+                [setattr(person, k, v) for k, v in person_model_data.iteritems()]
+                created, updated, message = False, True, "detected a change in one of the fields, updating person"
+            else:
+                created, updated, message = False, False, "existing person in DB, no change"
+        elif person_qs_count == 0:
+            person = KnessetPerson(knesset_id=person_knesset_id, **person_model_data)
+            created, updated, message = True, False, "created person"
+        else:
+            raise TooManyObjectsException("committee_knesset_id={}, matching db ids: {}".format(person_knesset_id,
+                                                                                                [c.id for c in
+                                                                                                 person_qs]))
+        if updated or created:
+            person.save()
+        return person, created, updated, message
+
+    def log_return_value(self, person, created, updated, message):
+        prefix = u"person {} - {} {}".format(person.id, person.first_name, person.last_name)
+        if created or updated:
+            self.logger.info(u"{}: {}".format(prefix, message))
+        else:
+            self.logger.debug(u'{}: {}'.format(prefix, message))

knesset_data_django/persons/models.py

@@ -1,2 +1,2 @@
 # currently uses Open Knesset models. TODO: move to a separate data access layer
-from persons.models import Person, PersonAlias
+from persons.models import Person, PersonAlias