Calfeed update

caldav.py

@@ -18,11 +18,11 @@
 import band
 import plan
 import member
-
 import datetime
 import logging
 # from pytz.gae import pytz
 import pytz
+from safetaskqueue import safe_taskqueue_add, check_taskqueue_trust
 
 def make_cal_header(the_title):
     header = """BEGIN:VCALENDAR
@@ -235,52 +235,73 @@ def get(self, *args, **kwargs):
 
         the_member_key = member.member_key_from_urlsafe(mk)
         the_member = the_member_key.get()
+        the_member.last_calfetch = datetime.datetime.now()
+        the_member.put()
 
-        calfeed = None
-        if the_member.cal_feed_dirty is False:
-            calfeed = get_calfeed_for_key("m",the_member.key)
+        calfeed = get_calfeed_for_key("m",the_member.key)
 
         if calfeed is None:
-            logging.info("member cal feed is dirty")
-            the_member.cal_feed_dirty = False
-            the_member.put()
-
-            # construct the calendar feed, since it may have changed lately
-
-            calfeed = u'{0}'.format(make_cal_header(the_member.name))
-
-            the_bands = assoc.get_confirmed_bands_of_member(the_member)
-
-            for a_band in the_bands:
-                a_band_name = a_band.shortname if a_band.shortname else a_band.name
-                all_gigs = gig.get_gigs_for_band_keys(a_band.key, show_past=True)
-                for a_gig in all_gigs:
-                    if not a_gig.is_canceled and not a_gig.hide_from_calendar: # and not a_gig.is_archived:
-                        the_plan = plan.get_plan_for_member_key_for_gig_key(the_member_key, a_gig.key)
-                        if the_plan:
-                            # check member preferences
-                            # include gig if member wants to see all, or if gig is confirmed
-                            if a_gig.is_confirmed or \
-                                the_member.preferences.calendar_show_only_confirmed == False:
-                                # incude gig if member wants to see all, or if has registered
-                                # as maybe or definitely:
-                                if (the_plan.value > 0 and the_plan.value <= 3) or \
-                                    (the_member.preferences.calendar_show_only_committed == False):
-                                    if a_gig.is_confirmed:
-                                        confstr = u'CONFIRMED!'
-                                    else:
-                                        confstr = u'(not confirmed)'
-                                    calfeed = u'{0}{1}'.format(calfeed, \
-                                        make_event(a_gig, a_band, \
-                                        title_format=u'{0}:{{0}} {1}'.format(a_band_name, confstr)))
+            calfeed = u'{0}{1}'.format(make_cal_header(the_member.name), make_cal_footer())
 
-            calfeed = u'{0}{1}'.format(calfeed, make_cal_footer())
-            store_calfeed_for_key("m",the_member.key,calfeed)
         self.response.write(calfeed)
 
-    def post(self):
-        print 'got post request'
+class UpdateCalfeeds(BaseHandler):
+
+    def get(self):
+        """ find members who are using their calfeeds and have dirty calfeeds """
+        member_keys = member.get_member_keys_to_update_calfeed()
+        logging.info("need to make calfeeds for {0} members".format(len(member_keys)))
+        for m in member_keys:
+            safe_taskqueue_add(
+                queue_name='calfeedqueue',
+                url='/update_member_calfeed',
+                params={'the_member':m.urlsafe()}
+                )
+
+class UpdateMemberCalfeed(webapp2.RequestHandler):
 
+    def post(self):
+        check_taskqueue_trust(self.request)
+        murl = self.request.get('the_member')
+        the_member = member.get_member_from_urlsafe_key(murl)
+        make_calfeed_for_member(the_member)
+
+
+def make_calfeed_for_member(the_member):
+    calfeed = None
+    logging.info("making cal feed for {0}".format(the_member.key.urlsafe()))
+    the_member.cal_feed_dirty = False
+    the_member.put()
+
+    # construct the calendar feed, since it may have changed lately
+    calfeed = u'{0}'.format(make_cal_header(the_member.name))
+    the_bands = assoc.get_confirmed_bands_of_member(the_member)
+
+    for a_band in the_bands:
+        a_band_name = a_band.shortname if a_band.shortname else a_band.name
+        all_gigs = gig.get_gigs_for_band_keys(a_band.key, show_past=True)
+        for a_gig in all_gigs:
+            if not a_gig.is_canceled and not a_gig.hide_from_calendar: # and not a_gig.is_archived:
+                the_plan = plan.get_plan_for_member_key_for_gig_key(the_member.key, a_gig.key)
+                if the_plan:
+                    # check member preferences
+                    # include gig if member wants to see all, or if gig is confirmed
+                    if a_gig.is_confirmed or \
+                        the_member.preferences.calendar_show_only_confirmed == False:
+                        # incude gig if member wants to see all, or if has registered
+                        # as maybe or definitely:
+                        if (the_plan.value > 0 and the_plan.value <= 3) or \
+                            (the_member.preferences.calendar_show_only_committed == False):
+                            if a_gig.is_confirmed:
+                                confstr = u'CONFIRMED!'
+                            else:
+                                confstr = u'(not confirmed)'
+                            calfeed = u'{0}{1}'.format(calfeed, \
+                                make_event(a_gig, a_band, \
+                                title_format=u'{0}:{{0}} {1}'.format(a_band_name, confstr)))
+
+    calfeed = u'{0}{1}'.format(calfeed, make_cal_footer())
+    store_calfeed_for_key("m",the_member.key,calfeed)
 
 def store_calfeed_for_key(prefix, the_key, the_feed):
 

cron.yaml

@@ -11,3 +11,7 @@ cron:
 - description: send reminders
   url: /admin_send_reminders
   schedule: every day 05:10
+- description: update cal feeds
+  url: /admin_generate_calfeeds
+  schedule: every 1 hours
+

goemail.py

@@ -4,7 +4,6 @@
 from requestmodel import *
 from google.appengine.api import mail
 from google.appengine.api import users
-from google.appengine.api.taskqueue import taskqueue
 
 import gig
 import member
@@ -14,7 +13,7 @@
 import pickle
 import os
 import stats
-import cryptoutil
+from safetaskqueue import safe_taskqueue_add, check_taskqueue_trust
 
 from webapp2_extras import i18n
 from webapp2_extras.i18n import gettext as _
@@ -146,7 +145,7 @@ def format_details(details, setlist, newline='\n'):
                                                     newline,
                                                     newline.join(setlist.splitlines()))
         else:
-            the_details_string = details
+            the_details_string = newline.join(details.splitlines())
 
         return the_details_string
 
@@ -165,6 +164,7 @@ def format_body(body_format_str, newline='\n'):
     else:
         body = format_body(_('new_gig_email'))
         html = format_body(_('new_gig_email_html'), newline='<br>')
+        print(html)
 
     try:
         ret= _send_admin_mail(the_email_address, u'{0} {1}'.format(title_string, the_gig.title), body, html=html, reply_to=reply_to)
@@ -181,15 +181,16 @@ def announce_new_gig(the_gig, the_gig_url, is_edit=False, is_reminder=False, cha
                                'change_string': change_string,
                                'the_members':   the_members})
 
-    _safe_taskqueue_add(
+    safe_taskqueue_add(
+            queue_name='emailqueue',
             url='/announce_new_gig_handler',
             params={'the_params': the_params
             })
 
 class AnnounceNewGigHandler(webapp2.RequestHandler):
 
     def post(self):
-        _check_taskqueue_trust(self.request)
+        check_taskqueue_trust(self.request)
 
         the_params = pickle.loads(self.request.get('the_params'))
 
@@ -231,7 +232,8 @@ def post(self):
                     'the_member_key': the_member_key
                 })
 
-                _safe_taskqueue_add(
+                safe_taskqueue_add(
+                    queue_name='emailqueue',
                     url='/send_new_gig_handler',
                     params={'the_shared_params': the_shared_params,
                             'the_member_params': the_member_params
@@ -247,7 +249,7 @@ def post(self):
 class SendNewGigHandler(webapp2.RequestHandler):
 
     def post(self):
-        _check_taskqueue_trust(self.request)
+        check_taskqueue_trust(self.request)
 
         the_shared_params = pickle.loads(self.request.get('the_shared_params'))
         the_member_params = pickle.loads(self.request.get('the_member_params'))
@@ -373,7 +375,8 @@ def post(self):
         address = self.request.get('address', None)
 
         if address:
-            _safe_taskqueue_add(
+            safe_taskqueue_add(
+                    queue_name='emailqueue',
                     url='/send_test_email_handler',
                     params={'the_address':address}
                     )
@@ -383,7 +386,7 @@ class SendTestEmailHandler(webapp2.RequestHandler):
 
     def post(self):
 
-        _check_taskqueue_trust(self.request)
+        check_taskqueue_trust(self.request)
         the_address  = self.request.get('the_address', None)
         if the_address:
             _send_admin_mail(the_address, "testing email",
@@ -404,15 +407,16 @@ def post(self):
         else:
             raise ValueError('illegal member key to MemberTestEmail')
 
-        _safe_taskqueue_add(
+        safe_taskqueue_add(
+                queue_name='emailqueue',
                 url='/member_test_email_handler',
                 params={'the_address':the_member.email_address}
                 )
         self.response.write( 200 )
 
 class MemberTestEmailHandler(webapp2.RequestHandler):
     def post(self):
-        _check_taskqueue_trust(self.request)
+        check_taskqueue_trust(self.request)
         the_address  = self.request.get('the_address', None)
         if the_address:
             _send_admin_mail(the_address, "testing email",
@@ -422,13 +426,3 @@ def post(self):
             logging.error('bad request to send member test email from {0}'.format(self.request.remote_addr))
 
         self.response.write( 200 )
-
-def _safe_taskqueue_add(url, params):
-    params['the_key'] = cryptoutil.encrypt_string("Trust Me")
-    taskqueue.add(queue_name='emailqueue', url=url, params=params)
-
-def _check_taskqueue_trust(request):
-    the_key = request.get('the_key','')
-    plain_key = cryptoutil.decrypt_string(the_key).strip()
-    if not plain_key == "Trust Me":
-        raise RuntimeError('bad key to send email from {0}'.format(request.remote_addr))

main.py

@@ -189,6 +189,8 @@
         webapp2.Route('/cal/b/<bk:.+>', caldav.BandRequestHandler),
         webapp2.Route('/cal/m/<mk:.+>', caldav.MemberRequestHandler),
         webapp2.Route('/cal/p/<bk:.+>', caldav.PublicBandGigRequestHandler),
+        webapp2.Route('/admin_generate_calfeeds', caldav.UpdateCalfeeds),
+        webapp2.Route('/update_member_calfeed', caldav.UpdateMemberCalfeed),
         webapp2.Route('/calhelp', caldav.HelpHandler),
         webapp2.Route('/announce_new_gig_handler', goemail.AnnounceNewGigHandler),
         webapp2.Route('/send_new_gig_handler', goemail.SendNewGigHandler),

member.py

@@ -315,9 +315,8 @@ def forget_member_from_key(the_member_key):
     the_member_key.delete()
 
 def get_member_from_urlsafe_key(urlsafe):
-    """take a urlsafe key and cause an ancestor query to happen, to assure previous writes are committed"""
-    untrusted_member=ndb.Key(urlsafe=urlsafe).get()
-    return get_member_from_nickname(untrusted_member.nickname)
+    return ndb.Key(urlsafe=urlsafe).get()
+
 
 
 def get_member(the_member_key):
@@ -469,6 +468,14 @@ def make_member_cal_dirty(the_member_key):
     the_member.cal_feed_dirty = True
     the_member.put()
 
+def get_member_keys_to_update_calfeed():
+    args=[
+        Member.cal_feed_dirty == True,
+        Member.last_calfetch != None
+    ]
+    member_query = Member.lquery(*args)
+    members = member_query.fetch(keys_only=True)
+    return members
 
 def rest_member_info(the_member, include_id=True):
     obj = { k:getattr(the_member,k) for k in ['display_name'] }

queue.yaml

@@ -9,4 +9,9 @@ queue:
   max_concurrent_requests: 1
   retry_parameters:
     task_retry_limit: 0
-
+- name: calfeedqueue
+  rate: 1/m
+  bucket_size: 10
+  max_concurrent_requests: 1
+  retry_parameters:
+    task_retry_limit: 0

templates/calview.html

@@ -48,7 +48,7 @@
                 },
             },
             eventAfterAllRender: calCallback,
-            {% if m and y %}
+            {% if defaultDate %}
                 defaultDate: moment( [ {{ y }}, {{ m }} ] ),
             {% endif %}
         })