From 30687622418f1ed9b020b459c96724e05fad7172 Mon Sep 17 00:00:00 2001 From: Andriy Sheredko Date: Wed, 15 Jul 2026 17:13:18 +0300 Subject: [PATCH 1/4] feat(osf): ENG-11734 add DownloadEvent model and migration with dashboard access group --- osf/migrations/0045_downloadevent.py | 73 ++++++++++++++++++++++++++++ osf/models/__init__.py | 1 + osf/models/download_event.py | 64 ++++++++++++++++++++++++ 3 files changed, 138 insertions(+) create mode 100644 osf/migrations/0045_downloadevent.py create mode 100644 osf/models/download_event.py diff --git a/osf/migrations/0045_downloadevent.py b/osf/migrations/0045_downloadevent.py new file mode 100644 index 00000000000..bbaaa090076 --- /dev/null +++ b/osf/migrations/0045_downloadevent.py @@ -0,0 +1,73 @@ +import django.db.models.deletion +from django.db import migrations, models + + +DASHBOARD_GROUP_NAME = 'download_telemetry' + +DASHBOARD_USERS = [ + 'sheredko.andriy@gmail.com', + 'bodintsov@exoft.net', + 'isokhan@exoft.net', + 'ykopka@exoft.net', + 'bgeiger@cos.io', + 'osmand@cos.io', +] + + +def create_dashboard_group(apps, schema_editor): + """Create the allow-list group the dashboard loads against and seed it. + + The group carries no permissions of its own — membership is the only gate. + """ + Group = apps.get_model('auth', 'Group') + OSFUser = apps.get_model('osf', 'OSFUser') + group, _ = Group.objects.get_or_create(name=DASHBOARD_GROUP_NAME) + for identifier in DASHBOARD_USERS: + user = ( + OSFUser.objects.filter(username=identifier).first() + or OSFUser.objects.filter(guids___id=identifier).first() + ) + if user: + group.user_set.add(user) + + +def remove_dashboard_group(apps, schema_editor): + Group = apps.get_model('auth', 'Group') + Group.objects.filter(name=DASHBOARD_GROUP_NAME).delete() + + +class Migration(migrations.Migration): + + dependencies = [ + ('osf', '0044_notification_scheduled'), + ] + + operations = [ + migrations.CreateModel( + name='DownloadEvent', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('created', models.DateTimeField(auto_now_add=True, db_index=True)), + ('resource_guid', models.CharField(blank=True, db_index=True, default='', max_length=255)), + ('path', models.TextField(blank=True, default='')), + ('name', models.CharField(blank=True, default='', max_length=255)), + ('download_type', models.CharField(choices=[('file', 'Single file'), ('folder_zip', 'Folder zip'), ('project', 'Whole-project zip')], db_index=True, max_length=16)), + ('zip_completed', models.BooleanField(blank=True, null=True)), + ('size_bytes', models.BigIntegerField(blank=True, null=True)), + ('storage_region', models.CharField(blank=True, db_index=True, default='', max_length=64)), + ('user_region', models.CharField(blank=True, db_index=True, default='', max_length=64)), + ('ip', models.GenericIPAddressField(blank=True, null=True)), + ('source_area', models.CharField(blank=True, default='', max_length=128)), + ('user', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='download_events', to='osf.osfuser')), + ], + ), + migrations.AddIndex( + model_name='downloadevent', + index=models.Index(fields=['created', 'download_type'], name='download_event_crt_type'), + ), + migrations.AddIndex( + model_name='downloadevent', + index=models.Index(fields=['created', 'storage_region'], name='download_event_crt_regn'), + ), + migrations.RunPython(create_dashboard_group, remove_dashboard_group), + ] diff --git a/osf/models/__init__.py b/osf/models/__init__.py index 918ca9aa009..410f62e6872 100644 --- a/osf/models/__init__.py +++ b/osf/models/__init__.py @@ -12,6 +12,7 @@ from .admin_log_entry import AdminLogEntry from .admin_profile import AdminProfile from .analytics import UserActivityCounter, PageCounter +from .download_event import DownloadEvent from .archive import ArchiveJob, ArchiveTarget from .banner import ScheduledBanner from .base import ( diff --git a/osf/models/download_event.py b/osf/models/download_event.py new file mode 100644 index 00000000000..e9bcb69b2b1 --- /dev/null +++ b/osf/models/download_event.py @@ -0,0 +1,64 @@ +from django.db import models + + +class DownloadEvent(models.Model): + """One metadata row per download — never the file contents. + + Foundation for the download-telemetry capture and dashboard. Append-only: + rows are written from the download flow (single files at the osf.io redirect + view, folder/project zips from the WaterButler callback) and read, always + scoped to a time range, by the dashboard. + """ + + FILE = 'file' + FOLDER_ZIP = 'folder_zip' + PROJECT = 'project' + DOWNLOAD_TYPES = ( + (FILE, 'Single file'), + (FOLDER_ZIP, 'Folder zip'), + (PROJECT, 'Whole-project zip'), + ) + + created = models.DateTimeField(auto_now_add=True, db_index=True) + + # what was downloaded + resource_guid = models.CharField(max_length=255, blank=True, default='', db_index=True) + path = models.TextField(blank=True, default='') + name = models.CharField(max_length=255, blank=True, default='') + download_type = models.CharField(max_length=16, choices=DOWNLOAD_TYPES, db_index=True) + # null for single files (only zips stream through WB, which reports completion) + zip_completed = models.BooleanField(null=True, blank=True) + size_bytes = models.BigIntegerField(null=True, blank=True) + + # storage_region = where the bytes were served from (capacity); + # user_region = roughly where the user is. Kept separate on purpose. + storage_region = models.CharField(max_length=64, blank=True, default='', db_index=True) + user_region = models.CharField(max_length=64, blank=True, default='', db_index=True) + ip = models.GenericIPAddressField(null=True, blank=True) + source_area = models.CharField(max_length=128, blank=True, default='') + + # nullable: anonymous downloads of public files + user = models.ForeignKey( + 'osf.OSFUser', + null=True, + blank=True, + on_delete=models.SET_NULL, + related_name='download_events', + ) + + class Meta: + # `created` is indexed on the field; these cover the dashboard's + # time-range group-bys. + indexes = [ + models.Index(fields=['created', 'download_type'], name='download_event_crt_type'), + models.Index(fields=['created', 'storage_region'], name='download_event_crt_regn'), + ] + + def __repr__(self): + return ( + f'' + ) + + def __str__(self): + return self.__repr__() From 6f698554067ff4ae44c48c6e4be9265dc755745f Mon Sep 17 00:00:00 2001 From: Andriy Sheredko Date: Wed, 15 Jul 2026 17:34:00 +0300 Subject: [PATCH 2/4] feat(osf): ENG-11734 add DownloadEvent model and migration with dashboard access group --- osf/migrations/0045_downloadevent.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/osf/migrations/0045_downloadevent.py b/osf/migrations/0045_downloadevent.py index bbaaa090076..3d019bcb74b 100644 --- a/osf/migrations/0045_downloadevent.py +++ b/osf/migrations/0045_downloadevent.py @@ -22,11 +22,8 @@ def create_dashboard_group(apps, schema_editor): Group = apps.get_model('auth', 'Group') OSFUser = apps.get_model('osf', 'OSFUser') group, _ = Group.objects.get_or_create(name=DASHBOARD_GROUP_NAME) - for identifier in DASHBOARD_USERS: - user = ( - OSFUser.objects.filter(username=identifier).first() - or OSFUser.objects.filter(guids___id=identifier).first() - ) + for username in DASHBOARD_USERS: + user = OSFUser.objects.filter(username=username).first() if user: group.user_set.add(user) From ef80d92ff42d968a1d40107f4b4f765249b39920 Mon Sep 17 00:00:00 2001 From: Andriy Sheredko Date: Thu, 16 Jul 2026 15:37:30 +0300 Subject: [PATCH 3/4] feat(osf): ENG-11734 add DownloadEvent model and migration with dashboard access group --- osf/migrations/0045_downloadevent.py | 2 +- osf/models/download_event.py | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/osf/migrations/0045_downloadevent.py b/osf/migrations/0045_downloadevent.py index 3d019bcb74b..391e41852d5 100644 --- a/osf/migrations/0045_downloadevent.py +++ b/osf/migrations/0045_downloadevent.py @@ -11,6 +11,7 @@ 'ykopka@exoft.net', 'bgeiger@cos.io', 'osmand@cos.io', + 'ramya@cos.io', ] @@ -47,7 +48,6 @@ class Migration(migrations.Migration): ('created', models.DateTimeField(auto_now_add=True, db_index=True)), ('resource_guid', models.CharField(blank=True, db_index=True, default='', max_length=255)), ('path', models.TextField(blank=True, default='')), - ('name', models.CharField(blank=True, default='', max_length=255)), ('download_type', models.CharField(choices=[('file', 'Single file'), ('folder_zip', 'Folder zip'), ('project', 'Whole-project zip')], db_index=True, max_length=16)), ('zip_completed', models.BooleanField(blank=True, null=True)), ('size_bytes', models.BigIntegerField(blank=True, null=True)), diff --git a/osf/models/download_event.py b/osf/models/download_event.py index e9bcb69b2b1..ed189a763fd 100644 --- a/osf/models/download_event.py +++ b/osf/models/download_event.py @@ -24,7 +24,6 @@ class DownloadEvent(models.Model): # what was downloaded resource_guid = models.CharField(max_length=255, blank=True, default='', db_index=True) path = models.TextField(blank=True, default='') - name = models.CharField(max_length=255, blank=True, default='') download_type = models.CharField(max_length=16, choices=DOWNLOAD_TYPES, db_index=True) # null for single files (only zips stream through WB, which reports completion) zip_completed = models.BooleanField(null=True, blank=True) From ed3ea81fddd7f19feb8cd991d1fa44cf983a4e4c Mon Sep 17 00:00:00 2001 From: Andriy Sheredko Date: Thu, 16 Jul 2026 15:45:57 +0300 Subject: [PATCH 4/4] feat(osf): ENG-11734 add DownloadEvent model and migration with dashboard access group --- osf/migrations/0045_downloadevent.py | 10 +++++++--- osf/models/download_event.py | 7 ++++--- 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/osf/migrations/0045_downloadevent.py b/osf/migrations/0045_downloadevent.py index 391e41852d5..779406da215 100644 --- a/osf/migrations/0045_downloadevent.py +++ b/osf/migrations/0045_downloadevent.py @@ -48,11 +48,11 @@ class Migration(migrations.Migration): ('created', models.DateTimeField(auto_now_add=True, db_index=True)), ('resource_guid', models.CharField(blank=True, db_index=True, default='', max_length=255)), ('path', models.TextField(blank=True, default='')), - ('download_type', models.CharField(choices=[('file', 'Single file'), ('folder_zip', 'Folder zip'), ('project', 'Whole-project zip')], db_index=True, max_length=16)), + ('download_type', models.CharField(choices=[('file', 'Single file'), ('folder_zip', 'Folder zip'), ('project', 'Whole-project zip')], max_length=16)), ('zip_completed', models.BooleanField(blank=True, null=True)), ('size_bytes', models.BigIntegerField(blank=True, null=True)), - ('storage_region', models.CharField(blank=True, db_index=True, default='', max_length=64)), - ('user_region', models.CharField(blank=True, db_index=True, default='', max_length=64)), + ('storage_region', models.CharField(blank=True, default='', max_length=64)), + ('user_region', models.CharField(blank=True, default='', max_length=64)), ('ip', models.GenericIPAddressField(blank=True, null=True)), ('source_area', models.CharField(blank=True, default='', max_length=128)), ('user', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='download_events', to='osf.osfuser')), @@ -66,5 +66,9 @@ class Migration(migrations.Migration): model_name='downloadevent', index=models.Index(fields=['created', 'storage_region'], name='download_event_crt_regn'), ), + migrations.AddIndex( + model_name='downloadevent', + index=models.Index(fields=['created', 'user_region'], name='download_event_crt_user'), + ), migrations.RunPython(create_dashboard_group, remove_dashboard_group), ] diff --git a/osf/models/download_event.py b/osf/models/download_event.py index ed189a763fd..ad54a37a1e5 100644 --- a/osf/models/download_event.py +++ b/osf/models/download_event.py @@ -24,15 +24,15 @@ class DownloadEvent(models.Model): # what was downloaded resource_guid = models.CharField(max_length=255, blank=True, default='', db_index=True) path = models.TextField(blank=True, default='') - download_type = models.CharField(max_length=16, choices=DOWNLOAD_TYPES, db_index=True) + download_type = models.CharField(max_length=16, choices=DOWNLOAD_TYPES) # null for single files (only zips stream through WB, which reports completion) zip_completed = models.BooleanField(null=True, blank=True) size_bytes = models.BigIntegerField(null=True, blank=True) # storage_region = where the bytes were served from (capacity); # user_region = roughly where the user is. Kept separate on purpose. - storage_region = models.CharField(max_length=64, blank=True, default='', db_index=True) - user_region = models.CharField(max_length=64, blank=True, default='', db_index=True) + storage_region = models.CharField(max_length=64, blank=True, default='') + user_region = models.CharField(max_length=64, blank=True, default='') ip = models.GenericIPAddressField(null=True, blank=True) source_area = models.CharField(max_length=128, blank=True, default='') @@ -51,6 +51,7 @@ class Meta: indexes = [ models.Index(fields=['created', 'download_type'], name='download_event_crt_type'), models.Index(fields=['created', 'storage_region'], name='download_event_crt_regn'), + models.Index(fields=['created', 'user_region'], name='download_event_crt_user'), ] def __repr__(self):