diff --git a/osf/migrations/0045_downloadevent.py b/osf/migrations/0045_downloadevent.py new file mode 100644 index 00000000000..779406da215 --- /dev/null +++ b/osf/migrations/0045_downloadevent.py @@ -0,0 +1,74 @@ +import django.db.models.deletion +from django.db import migrations, models + + +DASHBOARD_GROUP_NAME = 'download_telemetry' + +DASHBOARD_USERS = [ + 'sheredko.andriy@gmail.com', + 'bodintsov@exoft.net', + 'isokhan@exoft.net', + 'ykopka@exoft.net', + 'bgeiger@cos.io', + 'osmand@cos.io', + 'ramya@cos.io', +] + + +def create_dashboard_group(apps, schema_editor): + """Create the allow-list group the dashboard loads against and seed it. + + The group carries no permissions of its own — membership is the only gate. + """ + Group = apps.get_model('auth', 'Group') + OSFUser = apps.get_model('osf', 'OSFUser') + group, _ = Group.objects.get_or_create(name=DASHBOARD_GROUP_NAME) + for username in DASHBOARD_USERS: + user = OSFUser.objects.filter(username=username).first() + if user: + group.user_set.add(user) + + +def remove_dashboard_group(apps, schema_editor): + Group = apps.get_model('auth', 'Group') + Group.objects.filter(name=DASHBOARD_GROUP_NAME).delete() + + +class Migration(migrations.Migration): + + dependencies = [ + ('osf', '0044_notification_scheduled'), + ] + + operations = [ + migrations.CreateModel( + name='DownloadEvent', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('created', models.DateTimeField(auto_now_add=True, db_index=True)), + ('resource_guid', models.CharField(blank=True, db_index=True, default='', max_length=255)), + ('path', models.TextField(blank=True, default='')), + ('download_type', models.CharField(choices=[('file', 'Single file'), ('folder_zip', 'Folder zip'), ('project', 'Whole-project zip')], max_length=16)), + ('zip_completed', models.BooleanField(blank=True, null=True)), + ('size_bytes', models.BigIntegerField(blank=True, null=True)), + ('storage_region', models.CharField(blank=True, default='', max_length=64)), + ('user_region', models.CharField(blank=True, default='', max_length=64)), + ('ip', models.GenericIPAddressField(blank=True, null=True)), + ('source_area', models.CharField(blank=True, default='', max_length=128)), + ('user', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='download_events', to='osf.osfuser')), + ], + ), + migrations.AddIndex( + model_name='downloadevent', + index=models.Index(fields=['created', 'download_type'], name='download_event_crt_type'), + ), + migrations.AddIndex( + model_name='downloadevent', + index=models.Index(fields=['created', 'storage_region'], name='download_event_crt_regn'), + ), + migrations.AddIndex( + model_name='downloadevent', + index=models.Index(fields=['created', 'user_region'], name='download_event_crt_user'), + ), + migrations.RunPython(create_dashboard_group, remove_dashboard_group), + ] diff --git a/osf/models/__init__.py b/osf/models/__init__.py index 918ca9aa009..410f62e6872 100644 --- a/osf/models/__init__.py +++ b/osf/models/__init__.py @@ -12,6 +12,7 @@ from .admin_log_entry import AdminLogEntry from .admin_profile import AdminProfile from .analytics import UserActivityCounter, PageCounter +from .download_event import DownloadEvent from .archive import ArchiveJob, ArchiveTarget from .banner import ScheduledBanner from .base import ( diff --git a/osf/models/download_event.py b/osf/models/download_event.py new file mode 100644 index 00000000000..ad54a37a1e5 --- /dev/null +++ b/osf/models/download_event.py @@ -0,0 +1,64 @@ +from django.db import models + + +class DownloadEvent(models.Model): + """One metadata row per download — never the file contents. + + Foundation for the download-telemetry capture and dashboard. Append-only: + rows are written from the download flow (single files at the osf.io redirect + view, folder/project zips from the WaterButler callback) and read, always + scoped to a time range, by the dashboard. + """ + + FILE = 'file' + FOLDER_ZIP = 'folder_zip' + PROJECT = 'project' + DOWNLOAD_TYPES = ( + (FILE, 'Single file'), + (FOLDER_ZIP, 'Folder zip'), + (PROJECT, 'Whole-project zip'), + ) + + created = models.DateTimeField(auto_now_add=True, db_index=True) + + # what was downloaded + resource_guid = models.CharField(max_length=255, blank=True, default='', db_index=True) + path = models.TextField(blank=True, default='') + download_type = models.CharField(max_length=16, choices=DOWNLOAD_TYPES) + # null for single files (only zips stream through WB, which reports completion) + zip_completed = models.BooleanField(null=True, blank=True) + size_bytes = models.BigIntegerField(null=True, blank=True) + + # storage_region = where the bytes were served from (capacity); + # user_region = roughly where the user is. Kept separate on purpose. + storage_region = models.CharField(max_length=64, blank=True, default='') + user_region = models.CharField(max_length=64, blank=True, default='') + ip = models.GenericIPAddressField(null=True, blank=True) + source_area = models.CharField(max_length=128, blank=True, default='') + + # nullable: anonymous downloads of public files + user = models.ForeignKey( + 'osf.OSFUser', + null=True, + blank=True, + on_delete=models.SET_NULL, + related_name='download_events', + ) + + class Meta: + # `created` is indexed on the field; these cover the dashboard's + # time-range group-bys. + indexes = [ + models.Index(fields=['created', 'download_type'], name='download_event_crt_type'), + models.Index(fields=['created', 'storage_region'], name='download_event_crt_regn'), + models.Index(fields=['created', 'user_region'], name='download_event_crt_user'), + ] + + def __repr__(self): + return ( + f'' + ) + + def __str__(self): + return self.__repr__()