diff --git a/admin/notifications/forms.py b/admin/notifications/forms.py index 946754415bb..98ad5c467a2 100644 --- a/admin/notifications/forms.py +++ b/admin/notifications/forms.py @@ -1,8 +1,48 @@ from django import forms -from osf.models import NotificationType +from osf.models import NotificationType, NotificationCampaign +import json class NotificationTypeForm(forms.ModelForm): class Meta: model = NotificationType fields = '__all__' + + +class NotificationCampaignCreateForm(forms.ModelForm): + context = forms.CharField( + required=False, + widget=forms.Textarea(attrs={'rows': 8}), + initial='{}', + ) + + filters = forms.CharField( + required=False, + widget=forms.HiddenInput(), + initial='{}', + ) + + batch_size = forms.IntegerField( + min_value=1, + initial=1000, + ) + + max_retries = forms.IntegerField( + min_value=0, + initial=3, + ) + + class Meta: + model = NotificationCampaign + fields = ( + 'name', + 'notification_type', + ) + + def clean_context(self): + value = self.cleaned_data['context'] or '{}' + return json.loads(value) + + def clean_filters(self): + value = self.cleaned_data['filters'] or '{}' + return json.loads(value) diff --git a/admin/notifications/urls.py b/admin/notifications/urls.py index 236059a577e..a05e60ee284 100644 --- a/admin/notifications/urls.py +++ b/admin/notifications/urls.py @@ -11,4 +11,9 @@ re_path(r'types_preview/(?P\d+)/$', views.NotificationTypePreview.as_view(), name='types_preview'), re_path(r'subscriptions/$', views.NotificationSubscriptionsList.as_view(), name='subscriptions_list'), re_path(r'email_tasks/$', views.EmailTasksList.as_view(), name='email_tasks_list'), + re_path(r'notification_campaigns_list/$', views.NotificationCampaignsList.as_view(), name='notification_campaigns_list'), + re_path(r'notification_campaigns_detail/(?P\d+)/$', views.NotificationCampaignDetail.as_view(), name='notification_campaigns_detail'), + re_path(r'notification_campaigns_create/$', views.NotificationCampaignCreateView.as_view(), name='notification_campaigns_create'), + re_path(r'notification_campaigns_recipients_preview/$', views.NotificationCampaignsRecipientsPreview.as_view(), name='notification_campaigns_recipients_preview'), + re_path(r'notification_campaigns_start/(?P\d+)/$', views.StartNotificationCampaign.as_view(), name='notification_campaigns_start'), ] diff --git a/admin/notifications/views.py b/admin/notifications/views.py index e307b60e8b6..0627063259e 100644 --- a/admin/notifications/views.py +++ b/admin/notifications/views.py @@ -1,17 +1,23 @@ +import re +import json +from collections import defaultdict from django.urls import reverse_lazy from django.db.models import Q -from osf.models import NotificationSubscription, NotificationType, Notification, EmailTask -from django.views.generic import ListView, DetailView, UpdateView +from django.db import models +from django.shortcuts import get_object_or_404, redirect +from django.views.generic import ListView, DetailView, UpdateView, CreateView, View +from django.contrib import messages from django.contrib.auth.mixins import PermissionRequiredMixin +from osf.models import NotificationSubscription, NotificationType, Notification, EmailTask, NotificationCampaign, OSFUser +from osf.models.notification_campaign import NotificationCampaignStatus from django.forms.models import model_to_dict -from .forms import NotificationTypeForm -from osf.email import _render_email_html -import json -from collections import defaultdict +from .forms import NotificationTypeForm, NotificationCampaignCreateForm from mako.lexer import Lexer from mako.parsetree import ControlLine -import re from string import Formatter +from osf.email import _render_email_html +from osf.email.notification_campaign import FILTER_PRESETS, filter_users + def delete_selected_notifications(selected_ids): NotificationSubscription.objects.filter(id__in=selected_ids).delete() @@ -332,3 +338,268 @@ class NotificationTypeChangeForm(PermissionRequiredMixin, UpdateView): def get_success_url(self, *args, **kwargs): return reverse_lazy('notifications:type_display', kwargs={'pk': self.kwargs.get('pk')}) + + +class NotificationCampaignsList(PermissionRequiredMixin, ListView): + paginate_by = 25 + template_name = 'notifications/notification_campaigns_list.html' + ordering = 'name' + permission_required = 'osf.view_notificationcampaign' + raise_exception = True + model = NotificationCampaign + + def get_queryset(self): + qs = NotificationCampaign.objects.all().order_by(self.ordering) + q = self.request.GET.get('q') + if q: + qs = qs.filter( + Q(name__icontains=q) | + Q(status__icontains=q) | + Q(notification_type__name__icontains=q) + ) + return qs + + def get_context_data(self, **kwargs): + context = super().get_context_data(**kwargs) + q = self.request.GET.get('q', '') + context['q'] = q + # append search param to pagination links + if q: + context['extra_query_params'] = f"&q={q}" + else: + context['extra_query_params'] = '' + + context['notification_campaigns'] = context['object_list'] + context['page'] = context['page_obj'] + return context + + +class NotificationCampaignDetail(PermissionRequiredMixin, DetailView): + model = NotificationCampaign + template_name = 'notifications/notification_campaigns_detail.html' + permission_required = 'osf.change_notificationcampaign' + raise_exception = True + + def get_object(self, queryset=None): + return NotificationCampaign.objects.get(id=self.kwargs.get('pk')) + + def get_context_data(self, *args, **kwargs): + notification_campaign = self.get_object() + metadata = notification_campaign.metadata or {} + + context = { + 'notification_campaign': notification_campaign, + 'display_fields': [ + ('Name', notification_campaign.name), + ('Notification Type', notification_campaign.notification_type), + ('Created By', notification_campaign.created_by), + ('Status', notification_campaign.get_status_display()), + ('Recipients', notification_campaign.recipient_count), + ('Sent', notification_campaign.sent_count), + ('Failed', notification_campaign.failed_count), + ('Retries', notification_campaign.retries), + ('Created', notification_campaign.created_at), + ('Started', notification_campaign.started_at), + ('Completed', notification_campaign.completed_at), + ], + 'template': notification_campaign.notification_type.template, + 'metadata': metadata, + 'filters_json': json.dumps(notification_campaign.metadata['filters']), + 'sent_filters_json': json.dumps({ + 'manual': [ + {'field': 'notificationcampaignrecipient', 'value': notification_campaign.id, 'lookup': 'campaign'}, + {'field': 'notificationcampaignrecipient', 'value': 'sent', 'lookup': 'status'} + ] + }), + 'failed_filters_json': json.dumps({ + 'manual': [ + {'field': 'notificationcampaignrecipient', 'value': notification_campaign.id, 'lookup': 'campaign'}, + {'field': 'notificationcampaignrecipient', 'value': 'failed', 'lookup': 'status'} + ] + }), + 'other_metadata': { + k: v + for k, v in metadata.items() + if k not in {'filters', 'context', 'execution', 'template'} + }, + } + + if notification_campaign.status == NotificationCampaignStatus.RUNNING: + context.update({ + 'sent_percent': notification_campaign.sent_count * 100 / notification_campaign.recipient_count if notification_campaign.recipient_count else 0, + 'failed_percent': notification_campaign.failed_count * 100 / notification_campaign.recipient_count if notification_campaign.recipient_count else 0, + }) + + return context + + +LOOKUPS = { + models.CharField: { + 'exact': 'Equals', + 'iexact': 'Equals (case insensitive)', + 'contains': 'Contains', + 'icontains': 'Contains (case insensitive)', + 'startswith': 'Starts with', + 'istartswith': 'Starts with (case insensitive)', + 'endswith': 'Ends with', + 'iendswith': 'Ends with (case insensitive)', + 'in': 'In', + 'isnull': 'Is empty', + }, + models.TextField: { + 'exact': 'Equals', + 'iexact': 'Equals (case insensitive)', + 'contains': 'Contains', + 'icontains': 'Contains (case insensitive)', + 'startswith': 'Starts with', + 'istartswith': 'Starts with (case insensitive)', + 'endswith': 'Ends with', + 'iendswith': 'Ends with (case insensitive)', + 'isnull': 'Is empty', + }, + models.IntegerField: { + 'exact': 'Equals', + 'gt': 'Greater than', + 'gte': 'Greater than or equal to', + 'lt': 'Less than', + 'lte': 'Less than or equal to', + 'in': 'In', + 'isnull': 'Is empty', + }, + models.DateField: { + 'exact': 'On', + 'gt': 'After', + 'gte': 'On or after', + 'lt': 'Before', + 'lte': 'On or before', + 'isnull': 'Is empty', + }, + models.DateTimeField: { + 'exact': 'On', + 'gt': 'After', + 'gte': 'On or after', + 'lt': 'Before', + 'lte': 'On or before', + 'isnull': 'Is empty', + }, + models.BooleanField: { + 'exact': 'Is', + }, +} + + +class NotificationCampaignCreateView(CreateView): + model = NotificationCampaign + form_class = NotificationCampaignCreateForm + template_name = 'notifications/notification_campaing_create.html' + allowed_filters = [ + 'is_active', + 'is_staff', + 'username', + 'last_login', + ] + + def form_valid(self, form): + form.instance.created_by = self.request.user + + form.instance.metadata = { + 'filters': form.cleaned_data['filters'], + 'context': form.cleaned_data['context'], + 'execution': { + 'batch_size': form.cleaned_data['batch_size'], + 'max_retries': form.cleaned_data['max_retries'], + }, + } + + response = super().form_valid(form) + + messages.success( + self.request, + 'Notification campaign created successfully.', + ) + + return response + + def get_success_url(self): + return reverse_lazy( + 'notifications:notification_campaigns_detail', + kwargs={'pk': self.object.pk}, + ) + + def get_context_data(self, **kwargs): + context = super().get_context_data(**kwargs) + context['notification_types'] = NotificationType.objects.order_by('name') + + filter_fields = {} + for field in [f for f in OSFUser._meta.get_fields() if f.name in self.allowed_filters]: + if not field.concrete: + continue + if type(field) not in LOOKUPS.keys(): + continue + filter_fields[field.name] = { + 'label': field.verbose_name, + 'type': field.get_internal_type().lower(), + 'lookups': LOOKUPS.get(type(field), {}) + } + context['filter_fields'] = filter_fields + context['filters'] = [] + context['predefined_filters'] = FILTER_PRESETS.keys() + return context + + +class NotificationCampaignsRecipientsPreview(PermissionRequiredMixin, ListView): + template_name = 'users/list.html' + permission_required = 'osf.view_osfuser' + raise_exception = True + paginate_by = 25 + + def get_queryset(self): + filters = {} + raw_filters = self.request.GET.get('filters', None) + if raw_filters: + json_filters = json.loads(raw_filters) + if predefined := json_filters.get('predefined'): + filters = FILTER_PRESETS.get(predefined, {}) + else: + filters = { + f'{item["field"]}__{item["lookup"]}': item['value'] + for item in json_filters.get('manual', []) + } + + return filter_users(filters) + + def get_context_data(self, **kwargs): + users = self.get_queryset() + + page_size = self.get_paginate_by(users) + paginator, page, query_set, is_paginated = self.paginate_queryset( + users, + page_size, + ) + # append search param to pagination links + kwargs.update({'extra_query_params': f'&filters={self.request.GET.get("filters")}'}) + return super().get_context_data( + **kwargs, + page=page, + users=query_set, + paginator=paginator, + is_paginated=is_paginated, + ) + +class StartNotificationCampaign(PermissionRequiredMixin, View): + permission_required = 'osf.change_notificationtype' + + def post(self, request, *args, **kwargs): + notification_campaign = get_object_or_404( + NotificationCampaign, + pk=kwargs['pk'], + ) + + restart_failed = request.GET.get('restart_failed') == 'true' + + notification_campaign.start(restart_failed=restart_failed) + + return redirect( + 'notifications:notification_campaigns_detail', + pk=notification_campaign.pk, + ) diff --git a/admin/templates/base.html b/admin/templates/base.html index 5f645ffe267..4554c0ff1b5 100644 --- a/admin/templates/base.html +++ b/admin/templates/base.html @@ -289,7 +289,7 @@ {% endif %} {% endif %} - {% if perms.osf.view_notification or perms.osf.view_notificationtype or perms.osf.view_notificationsubscription %} + {% if perms.osf.view_notification or perms.osf.view_notificationtype or perms.osf.view_notificationsubscription or perms.osf.view_notificationcampaign %}
  • Notifications
    • @@ -307,6 +307,9 @@ {% if perms.osf.view_emailtask %}
  • Email Tasks
    • {% endif %} + {% if perms.osf.view_notificationcampaign %} +
  • Notification Campaigns
    • + {% endif %} diff --git a/admin/templates/notifications/notification_campaigns_detail.html b/admin/templates/notifications/notification_campaigns_detail.html new file mode 100644 index 00000000000..fd520e77f79 --- /dev/null +++ b/admin/templates/notifications/notification_campaigns_detail.html @@ -0,0 +1,300 @@ +{% extends "base.html" %} +{% load static %} +{% load render_bundle from webpack_loader %} + +{% block title %} + {{ notification_campaign.name }} +{% endblock title %} + +{% block content %} +
    +
    +
    +

    {{ notification_campaign.name }}

    +

    + Campaign #{{ notification_campaign.id }} +

    +
    +
    + {% if notification_campaign.status == 'running' %} +

    Progress

    + +
    +
    + {{ notification_campaign.sent_count }} +
    +
    + {{ notification_campaign.failed_count }} +
    +
    +

    + {{ notification_campaign.sent_count|add:notification_campaign.failed_count }}/{{ notification_campaign.recipient_count }} processed +

    + {% endif %} +
    + {% csrf_token %} + +
    + + +
    +
    +

    General

    + + {% for field, value in display_fields %} + + + + {% if field == 'Sent' and value != 0 %} + + + {% elif field == 'Failed' and value != 0 %} + + + {% else %} + + + + {% endif %} + + {% endfor %} +
    {{ field }}{{ value|safe }} + + Preview Recipients + + + + Preview Recipients + + +
    + {% csrf_token %} + +
    +
    +
    +
    + + + {% if metadata.filters %} +
    +
    +

    Recipient Filters

    + + {% if not "predefined" in metadata.filters %} + + + + + + + + + + + {% for filter in metadata.filters.manual %} + + + + + + {% empty %} + + + + {% endfor %} + +
    FieldLookupValue
    {{ filter.field }}{{ filter.lookup }}{{ filter.value }}
    + No filters configured. +
    + + {% elif "predefined" in metadata.filters %} + + + + + + +
    Predefined Filter{{ metadata.filters.predefined }}
    + + {% endif %} + + + Preview Recipients + + +
    +
    + {% endif %} + + + {% if metadata.context %} +
    +
    +

    Context

    + + {% for key, value in metadata.context.items %} + + + + + {% endfor %} +
    {{ key }}
    {{ value }}
    +
    +
    + {% endif %} + + + {% if metadata.execution %} +
    +
    +

    Execution

    + + {% for key, value in metadata.execution.items %} + + + + + {% endfor %} +
    {{ key }}{{ value }}
    +
    +
    + {% endif %} + + {% if template or metadata.template %} +
    +
    +

    Email Template

    + + + +
    + + {% if template %} +
    + 
    {{ template }}
    +
    + {% endif %} + + {% if metadata.template %} +
    + 
    {{ metadata.template }}
    +
    + {% endif %} + +
    +
    +
    + {% endif %} + + + {% if other_metadata %} +
    +
    +

    Additional Metadata

    + + {% for key, value in other_metadata.items %} + + + + + {% endfor %} +
    {{ key }}
    {{ value }}
    +
    +
    + {% endif %} + +
    +{% endblock content %} + +{% block bottom_js %} + +{% endblock %} diff --git a/admin/templates/notifications/notification_campaigns_list.html b/admin/templates/notifications/notification_campaigns_list.html new file mode 100644 index 00000000000..494e4584045 --- /dev/null +++ b/admin/templates/notifications/notification_campaigns_list.html @@ -0,0 +1,46 @@ +{% extends "base.html" %} +{% load render_bundle from webpack_loader %} +{% load static %} +{% block title %} + List of Notification Types +{% endblock title %} +{% block content %} +

    List of Notification Campaigns

    +
    +
    +
    + Create Campaign +
    +
    +
    + + {% include "util/pagination.html" with items=page status=status %} +
    + + +
    + + + + + + + + + + + + {% for notification_capmaign in notification_campaigns %} + + + + + + + + + {% endfor %} + +
    NameNotification TypeStatusStarted atCompleted at
    {{ notification_capmaign.name }}{{ notification_capmaign.notification_type.name }}{{ notification_capmaign.status }}{{ notification_capmaign.started_at }}{{ notification_capmaign.completed_at }}
    + +{% endblock content %} diff --git a/admin/templates/notifications/notification_campaing_create.html b/admin/templates/notifications/notification_campaing_create.html new file mode 100644 index 00000000000..bc274bdf4cc --- /dev/null +++ b/admin/templates/notifications/notification_campaing_create.html @@ -0,0 +1,427 @@ +{% extends "base.html" %} +{% load static %} +{% load render_bundle from webpack_loader %} + +{% block title %} + Create Notification Campaign +{% endblock title %} + +{% block content %} +
    + +
    +
    +

    Create Notification Campaign

    +
    +
    + +
    + {% csrf_token %} + + +
    +
    +

    General

    + + + + + + + + + + +
    Campaign Name + +
    Notification Type + +
    +
    +
    + + +
    +
    +

    Recipient Filters

    + +
    + + + +
    + +
    + +
    + + + +
    + + + + + + {{ filter_fields|json_script:"filter-fields" }} + {{ filters|json_script:"initial-filters" }} + +
    +
    + +
    +
    +

    Template Context (JSON)

    + + +
    +
    + + +
    +
    +

    Execution

    + + + + + + + + + + + +
    Batch Size + +
    Max Retries + +
    +
    +
    + + +
    +
    + +
    +
    + +
    + + + +
    +
    + +
    +{% endblock content %} + +{% block bottom_js %} + +{% endblock %} diff --git a/osf/email/notification_campaign.py b/osf/email/notification_campaign.py index 941c02a88a8..e88c41cc01b 100644 --- a/osf/email/notification_campaign.py +++ b/osf/email/notification_campaign.py @@ -1,7 +1,6 @@ import logging -from osf.models import NotificationType, NotificationTypeEnum, OSFUser, UserActivityCounter -from django.db.models import Q -from django.db.models import OuterRef, Subquery, Exists, F +from osf.models import NotificationType, NotificationTypeEnum, OSFUser, UserActivityCounter, Email +from django.db.models import OuterRef, Subquery, Exists, F, Q, Case, When, CharField from django.db.models.functions import Coalesce from framework.celery_tasks import app as celery_app from celery import chord @@ -12,27 +11,45 @@ logger = logging.getLogger(__name__) +first_email_subquery = ( + Email.objects + .filter(user=OuterRef('pk')) + .values('address')[:1] +) + + counter_subquery = ( UserActivityCounter.objects .filter(_id=OuterRef('guids___id')) .values('total')[:1] ) +def filter_users(filters, campaign_id=None, restart_failed=False): + qs = OSFUser.objects.all() + if campaign_id: + if restart_failed: + already_sent_subquery = NotificationCampaignRecipient.objects.filter( + campaign_id=campaign_id, + user_id=OuterRef('pk'), + status__in=['sent', 'pending'] + ) + else: + already_sent_subquery = NotificationCampaignRecipient.objects.filter( + campaign_id=campaign_id, + user_id=OuterRef('pk'), + ) -def get_filtered_batches(filters, batch_size=1000, campaign_id=None): - already_sent_subquery = NotificationCampaignRecipient.objects.filter( - campaign_id=campaign_id, - user_id=OuterRef('pk'), - ) + qs = OSFUser.objects.annotate(already_sent=Exists(already_sent_subquery)).filter(already_sent=False) - qs = ( - OSFUser.objects - .annotate(already_sent=Exists(already_sent_subquery)) - .filter(already_sent=False) - .filter(**filters) - .annotate(activity_total=Coalesce(Subquery(counter_subquery), 0)) - .order_by('-activity_total', '-date_registered', '-id') - ) + qs = qs.filter(**filters) + + return qs + + +def get_filtered_batches(filters, batch_size=1000, campaign_id=None, restart_failed=False): + qs = filter_users(filters, campaign_id, restart_failed=restart_failed) + + qs = qs.annotate(activity_total=Coalesce(Subquery(counter_subquery), 0)).order_by('-activity_total', '-date_registered', '-id') last_total = None last_date = None @@ -117,7 +134,7 @@ def process_campaign_retry(*args, **kwargs): @celery_app.task(name='email.start_notification_campaign') -def start_notification_campaign(campaign_id): +def start_notification_campaign(campaign_id, restart_failed=False): campaign = NotificationCampaign.objects.get(id=campaign_id) filters = campaign.metadata.get('filters', {}) context = campaign.metadata.get('context', {}) @@ -128,11 +145,16 @@ def start_notification_campaign(campaign_id): if predefined_filter_name := filters.get('predefined'): filters = FILTER_PRESETS.get(predefined_filter_name, {}) + else: + filters = { + f'{item["field"]}__{item["lookup"]}': item['value'] + for item in filters.get('manual', []) + } tasks = [] total_recipients = 0 batch_size = campaign.metadata.get('execution', {}).get('batch_size', 1000) - for batch in get_filtered_batches(filters=filters, batch_size=batch_size): + for batch in get_filtered_batches(filters=filters, batch_size=batch_size, campaign_id=campaign_id, restart_failed=restart_failed): tasks.append( send_campaign_batch.s( notification_type_name=notification_type_name, @@ -142,9 +164,9 @@ def start_notification_campaign(campaign_id): ) ) total_recipients += len(batch) - - campaign.recipient_count = total_recipients - campaign.save(update_fields=['recipient_count']) + if not restart_failed: + campaign.recipient_count = total_recipients + campaign.save(update_fields=['recipient_count']) chord(tasks)( process_campaign_retry.s(campaign_id=campaign_id) @@ -182,7 +204,14 @@ def send_campaign_batch(context, recipients_ids, notification_type_name='blank', success_count = 0 failure_count = 0 if campaign.metadata.get('sendgrid_bulk', False): - recipient_emails = list(recipients_qs.values_list('username', flat=True)) + recipients_qs_annotated = recipients_qs.annotate( + recipient_address=Case( + When(username__contains='@', then='username'), + default=Subquery(first_email_subquery), + output_field=CharField(), + ) + ) + recipient_emails = list(recipients_qs_annotated.values_list('recipient_address', flat=True)) send_email_with_send_grid(to_addr=recipient_emails, notification_type=notification_type, context=context) success_count = len(recipient_emails) else: diff --git a/osf/models/__init__.py b/osf/models/__init__.py index 918ca9aa009..90284dd0d2e 100644 --- a/osf/models/__init__.py +++ b/osf/models/__init__.py @@ -67,6 +67,7 @@ from .notification_subscription import NotificationSubscription from .notification_type import NotificationType, NotificationTypeEnum from .notification import Notification +from .notification_campaign import NotificationCampaign, NotificationCampaignRecipient from .oauth import ( ApiOAuth2Application, diff --git a/osf/models/notification_campaign.py b/osf/models/notification_campaign.py index 8c5aa22a3ab..9cce5c3b966 100644 --- a/osf/models/notification_campaign.py +++ b/osf/models/notification_campaign.py @@ -64,18 +64,18 @@ class NotificationCampaign(models.Model): failed_count = models.PositiveIntegerField(default=0) retries = models.PositiveIntegerField(default=0) - def start(self): + def start(self, restart_failed=False): from osf.email.notification_campaign import start_notification_campaign self.status = NotificationCampaignStatus.RUNNING self.started_at = timezone.now() - - self.sent_count = 0 + if not restart_failed: + self.recipient_count = 0 + self.sent_count = 0 self.failed_count = 0 - self.recipient_count = 0 self.retries = 0 self.metadata.update({'template': self.notification_type.template}) self.save() - start_notification_campaign.delay(campaign_id=self.id) + start_notification_campaign.delay(campaign_id=self.id, restart_failed=restart_failed) class NotificationCampaignRecipient(models.Model):