From ed63954c6aa76a6451ffdc14990466e402e71c5f Mon Sep 17 00:00:00 2001 From: Oleh Paduchak Date: Fri, 28 Mar 2025 15:30:33 +0200 Subject: [PATCH 01/65] added external_service, configured_addon, authorized_account --- addon_service/admin/__init__.py | 28 +++++- .../authorized_account/link/__init__.py | 0 .../authorized_account/link/models.py | 24 +++++ .../authorized_account/link/serializers.py | 86 +++++++++++++++++ .../authorized_account/link/views.py | 9 ++ addon_service/common/known_imps.py | 19 ++++ addon_service/common/validators.py | 5 + .../configured_addon/link/__init__.py | 0 addon_service/configured_addon/link/models.py | 18 ++++ .../configured_addon/link/serializers.py | 78 +++++++++++++++ addon_service/configured_addon/link/views.py | 14 +++ .../external_service/link/__init__.py | 2 + addon_service/external_service/link/models.py | 65 +++++++++++++ .../external_service/link/serializers.py | 48 ++++++++++ addon_service/external_service/link/views.py | 9 ++ ...inkaccount_configuredlinkaddon_and_more.py | 94 +++++++++++++++++++ addon_service/models.py | 6 ++ 17 files changed, 503 insertions(+), 2 deletions(-) create mode 100644 addon_service/authorized_account/link/__init__.py create mode 100644 addon_service/authorized_account/link/models.py create mode 100644 addon_service/authorized_account/link/serializers.py create mode 100644 addon_service/authorized_account/link/views.py create mode 100644 addon_service/configured_addon/link/__init__.py create mode 100644 addon_service/configured_addon/link/models.py create mode 100644 addon_service/configured_addon/link/serializers.py create mode 100644 addon_service/configured_addon/link/views.py create mode 100644 addon_service/external_service/link/__init__.py create mode 100644 addon_service/external_service/link/models.py create mode 100644 addon_service/external_service/link/serializers.py create mode 100644 addon_service/external_service/link/views.py create mode 100644 addon_service/migrations/0012_authorizedlinkaccount_configuredlinkaddon_and_more.py diff --git a/addon_service/admin/__init__.py b/addon_service/admin/__init__.py index 45eb02b6..e06f11b1 100644 --- a/addon_service/admin/__init__.py +++ b/addon_service/admin/__init__.py @@ -8,6 +8,7 @@ from addon_service.external_service.storage.models import StorageSupportedFeatures from ..external_service.citation.models import CitationSupportedFeatures +from ..external_service.link.models import SupportedResourceTypes from ._base import GravyvaletModelAdmin from .decorators import linked_many_field @@ -22,7 +23,7 @@ class ExternalStorageServiceAdmin(GravyvaletModelAdmin): ) raw_id_fields = ("oauth2_client_config", "oauth1_client_config") enum_choice_fields = { - "int_addon_imp": known_imps.AddonImpNumbers, + "int_addon_imp": known_imps.StorageAddonImpNumbers, "int_credentials_format": CredentialsFormats, "int_service_type": ServiceTypes, } @@ -41,7 +42,7 @@ class ExternalCitationServiceAdmin(GravyvaletModelAdmin): ) raw_id_fields = ("oauth2_client_config", "oauth1_client_config") enum_choice_fields = { - "int_addon_imp": known_imps.AddonImpNumbers, + "int_addon_imp": known_imps.CitationAddonImpNumbers, "int_credentials_format": CredentialsFormats, "int_service_type": ServiceTypes, } @@ -50,6 +51,25 @@ class ExternalCitationServiceAdmin(GravyvaletModelAdmin): } +@admin.register(models.ExternalLinkService) +class ExternalLinkServiceAdmin(GravyvaletModelAdmin): + list_display = ("display_name", "created", "modified") + readonly_fields = ( + "id", + "created", + "modified", + ) + raw_id_fields = ("oauth2_client_config", "oauth1_client_config") + enum_choice_fields = { + "int_addon_imp": known_imps.AddonImpNumbers, + "int_credentials_format": CredentialsFormats, + "int_service_type": ServiceTypes, + } + enum_multiple_choice_fields = { + "int_supported_resource_types": SupportedResourceTypes, + } + + @admin.register(models.ExternalComputingService) class ExternalComputingServiceAdmin(GravyvaletModelAdmin): list_display = ("display_name", "created", "modified") @@ -72,6 +92,8 @@ class ExternalComputingServiceAdmin(GravyvaletModelAdmin): @admin.register(models.OAuth2ClientConfig) @linked_many_field("external_storage_services") @linked_many_field("external_citation_services") +@linked_many_field("external_link_services") +@linked_many_field("external_computing_services") class OAuth2ClientConfigAdmin(GravyvaletModelAdmin): readonly_fields = ( "id", @@ -83,6 +105,8 @@ class OAuth2ClientConfigAdmin(GravyvaletModelAdmin): @admin.register(models.OAuth1ClientConfig) @linked_many_field("external_storage_services") @linked_many_field("external_citation_services") +@linked_many_field("external_link_services") +@linked_many_field("external_computing_services") class OAuth1ClientConfigAdmin(GravyvaletModelAdmin): readonly_fields = ( "id", diff --git a/addon_service/authorized_account/link/__init__.py b/addon_service/authorized_account/link/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/addon_service/authorized_account/link/models.py b/addon_service/authorized_account/link/models.py new file mode 100644 index 00000000..d3598978 --- /dev/null +++ b/addon_service/authorized_account/link/models.py @@ -0,0 +1,24 @@ +from addon_service.authorized_account.models import AuthorizedAccount +from addon_service.configured_addon.storage.models import ConfiguredStorageAddon + + +class AuthorizedLinkAccount(AuthorizedAccount): + """Model for describing a user's account on an ExternalService. + + This model collects all of the information required to actually perform remote + operations against the service and to aggregate accounts under a known user. + """ + + class Meta: + verbose_name = "Authorized Link Account" + verbose_name_plural = "Authorized Link Accounts" + app_label = "addon_service" + + class JSONAPIMeta: + resource_name = "authorized-storage-accounts" + + @property + def configured_storage_addons(self): + return ConfiguredStorageAddon.objects.filter(base_account=self).select_related( + "authorized_resource" + ) diff --git a/addon_service/authorized_account/link/serializers.py b/addon_service/authorized_account/link/serializers.py new file mode 100644 index 00000000..0d8fbc6c --- /dev/null +++ b/addon_service/authorized_account/link/serializers.py @@ -0,0 +1,86 @@ +from rest_framework_json_api import serializers +from rest_framework_json_api.relations import ( + HyperlinkedRelatedField, + ResourceRelatedField, +) +from rest_framework_json_api.utils import get_resource_type_from_model + +from addon_service.addon_operation.models import AddonOperationModel +from addon_service.authorized_account.serializers import AuthorizedAccountSerializer +from addon_service.common import view_names +from addon_service.common.serializer_fields import ( + DataclassRelatedLinkField, + ReadOnlyResourceRelatedField, +) +from addon_service.models import ( + AuthorizedStorageAccount, + ConfiguredLinkAddon, + ExternalLinkService, + UserReference, +) + + +RESOURCE_TYPE = get_resource_type_from_model(AuthorizedStorageAccount) + + +class AuthorizedLinkAccountSerializer(AuthorizedAccountSerializer): + external_link_service = ResourceRelatedField( + queryset=ExternalLinkService.objects.all(), + many=False, + source="external_service.externallinkservice", + related_link_view_name=view_names.related_view(RESOURCE_TYPE), + ) + configured_link_addons = HyperlinkedRelatedField( + many=True, + source="configured_addons", + queryset=ConfiguredLinkAddon.objects.active(), + related_link_view_name=view_names.related_view(RESOURCE_TYPE), + required=False, + ) + url = serializers.HyperlinkedIdentityField( + view_name=view_names.detail_view(RESOURCE_TYPE), required=False + ) + account_owner = ReadOnlyResourceRelatedField( + many=False, + queryset=UserReference.objects.all(), + related_link_view_name=view_names.related_view(RESOURCE_TYPE), + ) + authorized_operations = DataclassRelatedLinkField( + dataclass_model=AddonOperationModel, + related_link_view_name=view_names.related_view(RESOURCE_TYPE), + ) + + included_serializers = { + "account_owner": "addon_service.serializers.UserReferenceSerializer", + "external_link_service": "addon_service.serializers.ExternalLinkServiceSerializer", + "configured_link_addons": "addon_service.serializers.ConfiguredLinkSerializer", + "authorized_operations": "addon_service.serializers.AddonOperationSerializer", + } + + configured_addons_uris = serializers.SerializerMethodField() + + def get_configured_addons_uris(self, obj): + return obj.configured_link_addons.values_list( + "authorized_resource__resource_uri", flat=True + ) + + class Meta: + model = AuthorizedStorageAccount + fields = [ + "id", + "url", + "display_name", + "account_owner", + "api_base_url", + "auth_url", + "authorized_capabilities", + "authorized_operations", + "authorized_operation_names", + "configured_link_addons", + "credentials", + "default_root_folder", + "external_link_service", + "initiate_oauth", + "credentials_available", + "configured_addons_uris", + ] diff --git a/addon_service/authorized_account/link/views.py b/addon_service/authorized_account/link/views.py new file mode 100644 index 00000000..c8d81e58 --- /dev/null +++ b/addon_service/authorized_account/link/views.py @@ -0,0 +1,9 @@ +from addon_service.authorized_account.views import AuthorizedAccountViewSet + +from .models import AuthorizedLinkAccount +from .serializers import AuthorizedLinkAccountSerializer + + +class AuthorizedStorageAccountViewSet(AuthorizedAccountViewSet): + queryset = AuthorizedLinkAccount.objects.all() + serializer_class = AuthorizedLinkAccountSerializer diff --git a/addon_service/common/known_imps.py b/addon_service/common/known_imps.py index 20e85b37..46f2955b 100644 --- a/addon_service/common/known_imps.py +++ b/addon_service/common/known_imps.py @@ -114,3 +114,22 @@ class AddonImpNumbers(enum.Enum): if __debug__: BLARG = -7 + + +StorageAddonImpNumbers = frozenset( + { + AddonImpNumbers.BOX, + AddonImpNumbers.S3, + AddonImpNumbers.GOOGLEDRIVE, + AddonImpNumbers.DROPBOX, + AddonImpNumbers.FIGSHARE, + AddonImpNumbers.ONEDRIVE, + AddonImpNumbers.OWNCLOUD, + AddonImpNumbers.DATAVERSE, + AddonImpNumbers.GITLAB, + AddonImpNumbers.BITBUCKET, + AddonImpNumbers.GITHUB, + } +) +CitationAddonImpNumbers = frozenset({AddonImpNumbers.ZOTERO, AddonImpNumbers.MENDELEY}) +ComputingAddonImpNumbers = frozenset({AddonImpNumbers.BOA}) diff --git a/addon_service/common/validators.py b/addon_service/common/validators.py index 550ee414..be760a95 100644 --- a/addon_service/common/validators.py +++ b/addon_service/common/validators.py @@ -10,6 +10,7 @@ from addon_toolkit import AddonCapabilities from addon_toolkit.interfaces.citation import CitationAddonImp from addon_toolkit.interfaces.computing import ComputingAddonImp +from addon_toolkit.interfaces.link import LinkAddonImp from addon_toolkit.interfaces.storage import StorageAddonImp from . import known_imps @@ -59,6 +60,10 @@ def validate_storage_imp_number(value): _validate_imp_number(value, StorageAddonImp) +def validate_link_imp_number(value): + _validate_imp_number(value, LinkAddonImp) + + def validate_citation_imp_number(value): _validate_imp_number(value, CitationAddonImp) diff --git a/addon_service/configured_addon/link/__init__.py b/addon_service/configured_addon/link/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/addon_service/configured_addon/link/models.py b/addon_service/configured_addon/link/models.py new file mode 100644 index 00000000..e42ae069 --- /dev/null +++ b/addon_service/configured_addon/link/models.py @@ -0,0 +1,18 @@ +from django.db import models + +from addon_service.configured_addon.models import ConfiguredAddon + + +class ConfiguredLinkAddon(ConfiguredAddon): + + target_uri = models.URLField() + target_id = models.CharField() + int_resource_type = models.IntegerField() + + class Meta: + verbose_name = "Configured Link Addon" + verbose_name_plural = "Configured Link Addons" + app_label = "addon_service" + + class JSONAPIMeta: + resource_name = "configured-link-addons" diff --git a/addon_service/configured_addon/link/serializers.py b/addon_service/configured_addon/link/serializers.py new file mode 100644 index 00000000..db55a035 --- /dev/null +++ b/addon_service/configured_addon/link/serializers.py @@ -0,0 +1,78 @@ +from rest_framework.fields import ( + CharField, + IntegerField, + URLField, +) +from rest_framework_json_api.relations import ResourceRelatedField +from rest_framework_json_api.utils import get_resource_type_from_model + +from addon_service.addon_operation.models import AddonOperationModel +from addon_service.common import view_names +from addon_service.common.serializer_fields import DataclassRelatedLinkField +from addon_service.configured_addon.serializers import ConfiguredAddonSerializer +from addon_service.external_service.link.models import ExternalLinkService +from addon_service.models import ( + AuthorizedLinkAccount, + ConfiguredLinkAddon, +) + + +RESOURCE_TYPE = get_resource_type_from_model(ConfiguredLinkAddon) + + +class ConfiguredLinkAddonSerializer(ConfiguredAddonSerializer): + """api serializer for the `ConfiguredLinkAddon` model""" + + target_uri = URLField() + target_id = CharField() + int_resource_type = IntegerField() + + connected_operations = DataclassRelatedLinkField( + dataclass_model=AddonOperationModel, + related_link_view_name=view_names.related_view(RESOURCE_TYPE), + read_only=True, + ) + base_account = ResourceRelatedField( + queryset=AuthorizedLinkAccount.objects.all(), + many=False, + source="base_account.authorizedlinkaccount", + related_link_view_name=view_names.related_view(RESOURCE_TYPE), + ) + external_link_service = ResourceRelatedField( + many=False, + read_only=True, + model=ExternalLinkService, + source="base_account.external_service.externallinkservice", + related_link_view_name=view_names.related_view(RESOURCE_TYPE), + ) + authorized_resource = ResourceRelatedField( + many=False, + read_only=True, + related_link_view_name=view_names.related_view(RESOURCE_TYPE), + ) + + included_serializers = { + "base_account": ("addon_service.serializers.AuthorizedLinkAccountSerializer"), + "external_link_service": "addon_service.serializers.ExternalLinkServiceSerializer", + "authorized_resource": "addon_service.serializers.ResourceReferenceSerializer", + "connected_operations": "addon_service.serializers.AddonOperationSerializer", + } + + class Meta: + model = ConfiguredLinkAddon + read_only_fields = ["external_link_service"] + fields = [ + "id", + "url", + "display_name", + "root_folder", + "base_account", + "authorized_resource", + "authorized_resource_uri", + "connected_capabilities", + "connected_operations", + "connected_operation_names", + "external_link_service", + "current_user_is_owner", + "external_service_name", + ] diff --git a/addon_service/configured_addon/link/views.py b/addon_service/configured_addon/link/views.py new file mode 100644 index 00000000..ecadab51 --- /dev/null +++ b/addon_service/configured_addon/link/views.py @@ -0,0 +1,14 @@ +from addon_service.configured_addon.views import ConfiguredAddonViewSet + +from .models import ConfiguredLinkAddon +from .serializers import ConfiguredLinkAddonSerializer + + +class ConfiguredLinkAddonViewSet(ConfiguredAddonViewSet): + queryset = ConfiguredLinkAddon.objects.active().select_related( + "base_account__authorizedlinkaccount", + "base_account__account_owner", + "base_account__external_service__externallinkservice", + "authorized_resource", + ) + serializer_class = ConfiguredLinkAddonSerializer diff --git a/addon_service/external_service/link/__init__.py b/addon_service/external_service/link/__init__.py new file mode 100644 index 00000000..6a071ca7 --- /dev/null +++ b/addon_service/external_service/link/__init__.py @@ -0,0 +1,2 @@ +"""addon_service.external_service.storage: represents a third-party storage service +""" diff --git a/addon_service/external_service/link/models.py b/addon_service/external_service/link/models.py new file mode 100644 index 00000000..104549c4 --- /dev/null +++ b/addon_service/external_service/link/models.py @@ -0,0 +1,65 @@ +from enum import ( + Flag, + auto, +) + +from django.db import models + +from addon_service.authorized_account.link.models import AuthorizedLinkAccount +from addon_service.common.validators import ( + _validate_enum_value, + validate_link_imp_number, +) +from addon_service.external_service.models import ExternalService + + +class SupportedResourceTypes(Flag): + ADD_UPDATE_FILES = auto() + ADD_UPDATE_FILES_PARTIAL = auto() + DELETE_FILES = auto() + DELETE_FILES_PARTIAL = auto() + FORKING = auto() + LOGS = auto() + PERMISSIONS = auto() + REGISTERING = auto() + FILE_VERSIONS = auto() + COPY_INTO = auto() + DOWNLOAD_AS_ZIP = auto() + + +def validate_supported_features(value): + _validate_enum_value(SupportedResourceTypes, value) + + +class ExternalLinkService(ExternalService): + int_supported_resource_types = models.IntegerField( + validators=[validate_supported_features], null=True + ) + + @property + def supported_resource_types(self) -> list[SupportedResourceTypes]: + """get the enum representation of int_supported_features""" + return SupportedResourceTypes(self.int_supported_features) + + @supported_resource_types.setter + def supported_resource_types( + self, new_supported_resource_types: SupportedResourceTypes + ): + """set int_authorized_capabilities without caring its int""" + self.int_supported_features = new_supported_resource_types.value + + def clean(self): + super().clean() + validate_link_imp_number(self.int_addon_imp) + + @property + def authorized_link_accounts(self): + return AuthorizedLinkAccount.objects.filter(external_service=self) + + class Meta: + verbose_name = "External Link Service" + verbose_name_plural = "External Link Services" + app_label = "addon_service" + + class JSONAPIMeta: + resource_name = "external-link-services" diff --git a/addon_service/external_service/link/serializers.py b/addon_service/external_service/link/serializers.py new file mode 100644 index 00000000..556a3175 --- /dev/null +++ b/addon_service/external_service/link/serializers.py @@ -0,0 +1,48 @@ +from rest_framework_json_api import serializers +from rest_framework_json_api.utils import get_resource_type_from_model + +from addon_service.addon_imp.models import AddonImpModel +from addon_service.common import view_names +from addon_service.common.enum_serializers import EnumNameMultipleChoiceField +from addon_service.common.serializer_fields import DataclassRelatedDataField +from addon_service.external_service.link.models import SupportedResourceTypes +from addon_service.external_service.serializers import ExternalServiceSerializer + +from .models import ExternalLinkService + + +RESOURCE_TYPE = get_resource_type_from_model(ExternalLinkService) + + +class ExternalLinkServiceSerializer(ExternalServiceSerializer): + """api serializer for the `ExternalService` model""" + + url = serializers.HyperlinkedIdentityField( + view_name=view_names.detail_view(RESOURCE_TYPE) + ) + + addon_imp = DataclassRelatedDataField( + dataclass_model=AddonImpModel, + related_link_view_name=view_names.related_view(RESOURCE_TYPE), + ) + + supported_resource_types = EnumNameMultipleChoiceField( + enum_cls=SupportedResourceTypes, read_only=True + ) + + class Meta: + model = ExternalLinkService + fields = [ + "id", + "addon_imp", + "auth_uri", + "credentials_format", + "display_name", + "url", + "wb_key", + "external_service_name", + "configurable_api_root", + "supported_resource_types", + "icon_url", + "api_base_url_options", + ] diff --git a/addon_service/external_service/link/views.py b/addon_service/external_service/link/views.py new file mode 100644 index 00000000..6eccaf8d --- /dev/null +++ b/addon_service/external_service/link/views.py @@ -0,0 +1,9 @@ +from rest_framework_json_api.views import ReadOnlyModelViewSet + +from .models import ExternalLinkService +from .serializers import ExternalLinkServiceSerializer + + +class ExternalLinkServiceViewSet(ReadOnlyModelViewSet): + queryset = ExternalLinkService.objects.all().select_related("oauth2_client_config") + serializer_class = ExternalLinkServiceSerializer diff --git a/addon_service/migrations/0012_authorizedlinkaccount_configuredlinkaddon_and_more.py b/addon_service/migrations/0012_authorizedlinkaccount_configuredlinkaddon_and_more.py new file mode 100644 index 00000000..707c5f06 --- /dev/null +++ b/addon_service/migrations/0012_authorizedlinkaccount_configuredlinkaddon_and_more.py @@ -0,0 +1,94 @@ +# Generated by Django 4.2.7 on 2025-03-28 12:06 + +import django.db.models.deletion +from django.db import ( + migrations, + models, +) + +import addon_service.external_service.link.models + + +class Migration(migrations.Migration): + + dependencies = [ + ("addon_service", "0011_oauth2tokenmetadata_date_last_refreshed"), + ] + + operations = [ + migrations.CreateModel( + name="AuthorizedLinkAccount", + fields=[ + ( + "authorizedaccount_ptr", + models.OneToOneField( + auto_created=True, + on_delete=django.db.models.deletion.CASCADE, + parent_link=True, + primary_key=True, + serialize=False, + to="addon_service.authorizedaccount", + ), + ), + ], + options={ + "verbose_name": "Authorized Link Account", + "verbose_name_plural": "Authorized Link Accounts", + }, + bases=("addon_service.authorizedaccount",), + ), + migrations.CreateModel( + name="ConfiguredLinkAddon", + fields=[ + ( + "configuredaddon_ptr", + models.OneToOneField( + auto_created=True, + on_delete=django.db.models.deletion.CASCADE, + parent_link=True, + primary_key=True, + serialize=False, + to="addon_service.configuredaddon", + ), + ), + ("target_uri", models.URLField()), + ("target_id", models.CharField()), + ("int_resource_type", models.IntegerField()), + ], + options={ + "verbose_name": "Configured Link Addon", + "verbose_name_plural": "Configured Link Addons", + }, + bases=("addon_service.configuredaddon",), + ), + migrations.CreateModel( + name="ExternalLinkService", + fields=[ + ( + "externalservice_ptr", + models.OneToOneField( + auto_created=True, + on_delete=django.db.models.deletion.CASCADE, + parent_link=True, + primary_key=True, + serialize=False, + to="addon_service.externalservice", + ), + ), + ( + "int_supported_resource_types", + models.IntegerField( + null=True, + validators=[ + addon_service.external_service.link.models.validate_supported_features + ], + ), + ), + ], + options={ + "verbose_name": "External Storage Service", + "verbose_name_plural": "External Storage Services", + }, + bases=("addon_service.externalservice",), + ), + ] diff --git a/addon_service/models.py b/addon_service/models.py index 99e0395e..b350e261 100644 --- a/addon_service/models.py +++ b/addon_service/models.py @@ -5,13 +5,16 @@ from addon_service.addon_operation_invocation.models import AddonOperationInvocation from addon_service.authorized_account.citation.models import AuthorizedCitationAccount from addon_service.authorized_account.computing.models import AuthorizedComputingAccount +from addon_service.authorized_account.link.models import AuthorizedLinkAccount from addon_service.authorized_account.storage.models import AuthorizedStorageAccount from addon_service.configured_addon.citation.models import ConfiguredCitationAddon from addon_service.configured_addon.computing.models import ConfiguredComputingAddon +from addon_service.configured_addon.link.models import ConfiguredLinkAddon from addon_service.configured_addon.storage.models import ConfiguredStorageAddon from addon_service.credentials.models import ExternalCredentials from addon_service.external_service.citation.models import ExternalCitationService from addon_service.external_service.computing.models import ExternalComputingService +from addon_service.external_service.link.models import ExternalLinkService from addon_service.external_service.storage.models import ExternalStorageService from addon_service.oauth1.models import OAuth1ClientConfig from addon_service.oauth2.models import ( @@ -41,4 +44,7 @@ "AuthorizedComputingAccount", "ConfiguredComputingAddon", "ExternalComputingService", + "ExternalLinkService", + "AuthorizedLinkAccount", + "ConfiguredLinkAddon", ) From 535639aae46a490357d42b027c48805cef39048c Mon Sep 17 00:00:00 2001 From: Oleh Paduchak Date: Fri, 28 Mar 2025 15:49:34 +0200 Subject: [PATCH 02/65] fixed leftovers --- addon_service/admin/_base.py | 10 +- .../authorized_account/link/models.py | 8 +- addon_service/common/known_imps.py | 2 +- .../external_service/link/__init__.py | 2 +- addon_toolkit/interfaces/link.py | 141 ++++++++++++++++++ 5 files changed, 149 insertions(+), 14 deletions(-) create mode 100644 addon_toolkit/interfaces/link.py diff --git a/addon_service/admin/_base.py b/addon_service/admin/_base.py index a076e2f0..5df1fe84 100644 --- a/addon_service/admin/_base.py +++ b/addon_service/admin/_base.py @@ -56,10 +56,7 @@ def formfield_for_dbfield(self, db_field, request, **kwargs): label=db_field.verbose_name, choices=[ (None, ""), - *( - (_member.value, _member.name) - for _member in _enum.__members__.values() - ), + *((_member.value, _member.name) for _member in _enum), ], ) if ( @@ -69,10 +66,7 @@ def formfield_for_dbfield(self, db_field, request, **kwargs): _enum = self.enum_multiple_choice_fields[db_field.name] return EnumNameMultipleChoiceField( choices=[ - *( - (_member.value, _member.name) - for _member in _enum.__members__.values() - ), + *((_member.value, _member.name) for _member in _enum), ], widget=forms.CheckboxSelectMultiple, enum_cls=_enum, diff --git a/addon_service/authorized_account/link/models.py b/addon_service/authorized_account/link/models.py index d3598978..615d95ae 100644 --- a/addon_service/authorized_account/link/models.py +++ b/addon_service/authorized_account/link/models.py @@ -1,5 +1,5 @@ from addon_service.authorized_account.models import AuthorizedAccount -from addon_service.configured_addon.storage.models import ConfiguredStorageAddon +from addon_service.configured_addon.link.models import ConfiguredLinkAddon class AuthorizedLinkAccount(AuthorizedAccount): @@ -15,10 +15,10 @@ class Meta: app_label = "addon_service" class JSONAPIMeta: - resource_name = "authorized-storage-accounts" + resource_name = "authorized-link-accounts" @property - def configured_storage_addons(self): - return ConfiguredStorageAddon.objects.filter(base_account=self).select_related( + def configured_link_addons(self): + return ConfiguredLinkAddon.objects.filter(base_account=self).select_related( "authorized_resource" ) diff --git a/addon_service/common/known_imps.py b/addon_service/common/known_imps.py index 46f2955b..7557f953 100644 --- a/addon_service/common/known_imps.py +++ b/addon_service/common/known_imps.py @@ -90,8 +90,8 @@ class KnownAddonImps(enum.Enum): BLARG = my_blarg.MyBlargStorage -@enum.unique @enum_names_same_as(KnownAddonImps) +@enum.unique class AddonImpNumbers(enum.Enum): """Static mapping from each AddonImp name to a unique integer (for database use)""" diff --git a/addon_service/external_service/link/__init__.py b/addon_service/external_service/link/__init__.py index 6a071ca7..4754ef53 100644 --- a/addon_service/external_service/link/__init__.py +++ b/addon_service/external_service/link/__init__.py @@ -1,2 +1,2 @@ -"""addon_service.external_service.storage: represents a third-party storage service +"""addon_service.external_service.link: represents a third-party link service """ diff --git a/addon_toolkit/interfaces/link.py b/addon_toolkit/interfaces/link.py new file mode 100644 index 00000000..3436e7ed --- /dev/null +++ b/addon_toolkit/interfaces/link.py @@ -0,0 +1,141 @@ +"""a static (and still in progress) definition of what composes a link addon""" + +import dataclasses +import enum +import typing +from collections import abc + +from addon_toolkit.addon_operation_declaration import immediate_operation +from addon_toolkit.capabilities import AddonCapabilities +from addon_toolkit.constrained_network.http import HttpRequestor +from addon_toolkit.credentials import Credentials +from addon_toolkit.cursor import Cursor +from addon_toolkit.imp import AddonImp + +from ._base import BaseAddonInterface + + +__all__ = ( + "ItemResult", + "ItemType", + "ItemSampleResult", + "PossibleSingleItemResult", + "LinkAddonInterface", + "LinkAddonImp", + "LinkConfig", +) + + +### +# dataclasses used for operation args and return values + + +@dataclasses.dataclass(frozen=True) +class LinkConfig: + max_upload_mb: int + external_api_url: str + connected_root_id: str | None = None + external_account_id: str | None = None + + +class ItemType(enum.StrEnum): + FILE = enum.auto() + FOLDER = enum.auto() + + +@dataclasses.dataclass +class ItemResult: + item_id: str + item_name: str + item_type: ItemType + can_be_root: bool = None + may_contain_root_candidates: bool = None + item_path: abc.Sequence[typing.Self] | None = None + + def __post_init__(self): + """By default can_be_root and may_contain_root_candidates are bound to item_type""" + if self.can_be_root is None: + self.can_be_root = self.item_type == ItemType.FOLDER + if self.may_contain_root_candidates is None: + self.may_contain_root_candidates = self.item_type == ItemType.FOLDER + + +@dataclasses.dataclass +class PossibleSingleItemResult: + possible_item: ItemResult | None + + +@dataclasses.dataclass +class ItemSampleResult: + """a sample from a possibly-large population of result items""" + + items: abc.Collection[ItemResult] + total_count: int | None = None + this_sample_cursor: str = "" + next_sample_cursor: str | None = None # when None, this is the last page of results + prev_sample_cursor: str | None = None + first_sample_cursor: str = "" + + def with_cursor(self, cursor: Cursor) -> typing.Self: + return dataclasses.replace( + self, + this_sample_cursor=cursor.this_cursor_str, + next_sample_cursor=cursor.next_cursor_str, + prev_sample_cursor=cursor.prev_cursor_str, + first_sample_cursor=cursor.first_cursor_str, + ) + + +### +# declaration of all link addon operations + + +class LinkAddonInterface(BaseAddonInterface, typing.Protocol): + + @immediate_operation(capability=AddonCapabilities.ACCESS) + async def get_item_info(self, item_id: str) -> ItemResult: ... + + @immediate_operation(capability=AddonCapabilities.ACCESS) + async def list_root_items(self, page_cursor: str = "") -> ItemSampleResult: ... + + @immediate_operation(capability=AddonCapabilities.ACCESS) + async def list_child_items( + self, + item_id: str, + page_cursor: str = "", + item_type: ItemType | None = None, + ) -> ItemSampleResult: ... + + +@dataclasses.dataclass +class LinkAddonImp(AddonImp): + """base class for link addon implementations""" + + ADDON_INTERFACE = LinkAddonInterface + + config: LinkConfig + + async def build_wb_config(self) -> dict: + return {} + + +@dataclasses.dataclass +class LinkAddonHttpRequestorImp(LinkAddonImp): + """base class for link addon implementations using GV network""" + + network: HttpRequestor + + +@dataclasses.dataclass +class LinkAddonClientRequestorImp[T](LinkAddonImp): + """base class for link addon with custom clients""" + + client: T = dataclasses.field(init=False) + credentials: dataclasses.InitVar[Credentials] + + def __post_init__(self, credentials): + self.client = self.create_client(credentials) + + @staticmethod + def create_client(credentials) -> T: + raise NotImplementedError From f0fcddded39c3495f235a8889de645e8779cf1ee Mon Sep 17 00:00:00 2001 From: Oleh Paduchak Date: Fri, 28 Mar 2025 15:55:09 +0200 Subject: [PATCH 03/65] cleaned up interface for link addons --- addon_toolkit/interfaces/link.py | 32 +------------------------------- 1 file changed, 1 insertion(+), 31 deletions(-) diff --git a/addon_toolkit/interfaces/link.py b/addon_toolkit/interfaces/link.py index 3436e7ed..4f72b651 100644 --- a/addon_toolkit/interfaces/link.py +++ b/addon_toolkit/interfaces/link.py @@ -19,25 +19,11 @@ "ItemResult", "ItemType", "ItemSampleResult", - "PossibleSingleItemResult", "LinkAddonInterface", "LinkAddonImp", - "LinkConfig", ) -### -# dataclasses used for operation args and return values - - -@dataclasses.dataclass(frozen=True) -class LinkConfig: - max_upload_mb: int - external_api_url: str - connected_root_id: str | None = None - external_account_id: str | None = None - - class ItemType(enum.StrEnum): FILE = enum.auto() FOLDER = enum.auto() @@ -48,21 +34,7 @@ class ItemResult: item_id: str item_name: str item_type: ItemType - can_be_root: bool = None - may_contain_root_candidates: bool = None - item_path: abc.Sequence[typing.Self] | None = None - - def __post_init__(self): - """By default can_be_root and may_contain_root_candidates are bound to item_type""" - if self.can_be_root is None: - self.can_be_root = self.item_type == ItemType.FOLDER - if self.may_contain_root_candidates is None: - self.may_contain_root_candidates = self.item_type == ItemType.FOLDER - - -@dataclasses.dataclass -class PossibleSingleItemResult: - possible_item: ItemResult | None + item_link: str | None = None @dataclasses.dataclass @@ -113,8 +85,6 @@ class LinkAddonImp(AddonImp): ADDON_INTERFACE = LinkAddonInterface - config: LinkConfig - async def build_wb_config(self) -> dict: return {} From 1daba47b06d853a88496c8b9ad5cff9159c1af62 Mon Sep 17 00:00:00 2001 From: Oleh Paduchak Date: Fri, 28 Mar 2025 15:57:39 +0200 Subject: [PATCH 04/65] added resource type --- addon_toolkit/interfaces/link.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/addon_toolkit/interfaces/link.py b/addon_toolkit/interfaces/link.py index 4f72b651..85c5469a 100644 --- a/addon_toolkit/interfaces/link.py +++ b/addon_toolkit/interfaces/link.py @@ -1,5 +1,7 @@ """a static (and still in progress) definition of what composes a link addon""" +from __future__ import annotations + import dataclasses import enum import typing @@ -15,6 +17,9 @@ from ._base import BaseAddonInterface +if typing.TYPE_CHECKING: + from addon_service.external_service.link.models import SupportedResourceTypes + __all__ = ( "ItemResult", "ItemType", @@ -34,6 +39,7 @@ class ItemResult: item_id: str item_name: str item_type: ItemType + resource_type: SupportedResourceTypes item_link: str | None = None From 52995450d4d636c678179762bf097f9627bf5971 Mon Sep 17 00:00:00 2001 From: Oleh Paduchak Date: Mon, 31 Mar 2025 13:23:19 +0300 Subject: [PATCH 05/65] fixed supportedFeatures enum --- addon_service/external_service/link/models.py | 20 +------------------ addon_toolkit/interfaces/link.py | 19 +++++++++++++----- 2 files changed, 15 insertions(+), 24 deletions(-) diff --git a/addon_service/external_service/link/models.py b/addon_service/external_service/link/models.py index 104549c4..41c819ad 100644 --- a/addon_service/external_service/link/models.py +++ b/addon_service/external_service/link/models.py @@ -1,8 +1,3 @@ -from enum import ( - Flag, - auto, -) - from django.db import models from addon_service.authorized_account.link.models import AuthorizedLinkAccount @@ -11,20 +6,7 @@ validate_link_imp_number, ) from addon_service.external_service.models import ExternalService - - -class SupportedResourceTypes(Flag): - ADD_UPDATE_FILES = auto() - ADD_UPDATE_FILES_PARTIAL = auto() - DELETE_FILES = auto() - DELETE_FILES_PARTIAL = auto() - FORKING = auto() - LOGS = auto() - PERMISSIONS = auto() - REGISTERING = auto() - FILE_VERSIONS = auto() - COPY_INTO = auto() - DOWNLOAD_AS_ZIP = auto() +from addon_toolkit.interfaces.link import SupportedResourceTypes def validate_supported_features(value): diff --git a/addon_toolkit/interfaces/link.py b/addon_toolkit/interfaces/link.py index 85c5469a..682057de 100644 --- a/addon_toolkit/interfaces/link.py +++ b/addon_toolkit/interfaces/link.py @@ -1,7 +1,5 @@ """a static (and still in progress) definition of what composes a link addon""" -from __future__ import annotations - import dataclasses import enum import typing @@ -17,9 +15,6 @@ from ._base import BaseAddonInterface -if typing.TYPE_CHECKING: - from addon_service.external_service.link.models import SupportedResourceTypes - __all__ = ( "ItemResult", "ItemType", @@ -34,6 +29,20 @@ class ItemType(enum.StrEnum): FOLDER = enum.auto() +class SupportedResourceTypes(enum.Flag): + ADD_UPDATE_FILES = enum.auto() + ADD_UPDATE_FILES_PARTIAL = enum.auto() + DELETE_FILES = enum.auto() + DELETE_FILES_PARTIAL = enum.auto() + FORKING = enum.auto() + LOGS = enum.auto() + PERMISSIONS = enum.auto() + REGISTERING = enum.auto() + FILE_VERSIONS = enum.auto() + COPY_INTO = enum.auto() + DOWNLOAD_AS_ZIP = enum.auto() + + @dataclasses.dataclass class ItemResult: item_id: str From c44132336c26e7e1fe4a8e130756f09aeb91c810 Mon Sep 17 00:00:00 2001 From: Oleh Paduchak Date: Mon, 31 Mar 2025 13:28:35 +0300 Subject: [PATCH 06/65] added proper resource types --- addon_toolkit/interfaces/link.py | 43 ++++++++++++++++++++++++-------- 1 file changed, 32 insertions(+), 11 deletions(-) diff --git a/addon_toolkit/interfaces/link.py b/addon_toolkit/interfaces/link.py index 682057de..14b12e89 100644 --- a/addon_toolkit/interfaces/link.py +++ b/addon_toolkit/interfaces/link.py @@ -30,17 +30,38 @@ class ItemType(enum.StrEnum): class SupportedResourceTypes(enum.Flag): - ADD_UPDATE_FILES = enum.auto() - ADD_UPDATE_FILES_PARTIAL = enum.auto() - DELETE_FILES = enum.auto() - DELETE_FILES_PARTIAL = enum.auto() - FORKING = enum.auto() - LOGS = enum.auto() - PERMISSIONS = enum.auto() - REGISTERING = enum.auto() - FILE_VERSIONS = enum.auto() - COPY_INTO = enum.auto() - DOWNLOAD_AS_ZIP = enum.auto() + AUDIOVISUAL = enum.auto() + AWARD = enum.auto() + BOOK = enum.auto() + BOOK_CHAPTER = enum.auto() + COLLECTION = enum.auto() + COMPUTATIONAL_NOTEBOOK = enum.auto() + CONFERENCE_PAPER = enum.auto() + CONFERENCE_PROCEEDING = enum.auto() + DATA_PAPER = enum.auto() + DATASET = enum.auto() + DISSERTATION = enum.auto() + EVENT = enum.auto() + IMAGE = enum.auto() + INSTRUMENT = enum.auto() + INTERACTIVE_RESOURCE = enum.auto() + JOURNAL = enum.auto() + JOURNAL_ARTICLE = enum.auto() + MODEL = enum.auto() + OUTPUT_MANAGEMENT_PLAN = enum.auto() + PEER_REVIEW = enum.auto() + PHYSICAL_OBJECT = enum.auto() + PREPRINT = enum.auto() + PROJECT = enum.auto() + REPORT = enum.auto() + SERVICE = enum.auto() + SOFTWARE = enum.auto() + SOUND = enum.auto() + STANDARD = enum.auto() + STUDY_REGISTRATION = enum.auto() + TEXT = enum.auto() + WORKFLOW = enum.auto() + OTHER = enum.auto() @dataclasses.dataclass From 434e1506005b3b3d94e1833cfaf9197269b6170c Mon Sep 17 00:00:00 2001 From: Oleh Paduchak Date: Mon, 31 Mar 2025 13:37:23 +0300 Subject: [PATCH 07/65] added validator to resource type --- addon_service/configured_addon/link/models.py | 11 ++++++++++- addon_service/configured_addon/link/serializers.py | 2 +- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/addon_service/configured_addon/link/models.py b/addon_service/configured_addon/link/models.py index e42ae069..8282dd4a 100644 --- a/addon_service/configured_addon/link/models.py +++ b/addon_service/configured_addon/link/models.py @@ -1,13 +1,22 @@ +from django.core.exceptions import ValidationError from django.db import models from addon_service.configured_addon.models import ConfiguredAddon +from addon_toolkit.interfaces.link import SupportedResourceTypes + + +def is_supported_resource_type(resource_type: int): + try: + SupportedResourceTypes(resource_type) + except ValueError: + raise ValidationError("Invalid resource type") class ConfiguredLinkAddon(ConfiguredAddon): target_uri = models.URLField() target_id = models.CharField() - int_resource_type = models.IntegerField() + int_resource_type = models.IntegerField(validators=[is_supported_resource_type]) class Meta: verbose_name = "Configured Link Addon" diff --git a/addon_service/configured_addon/link/serializers.py b/addon_service/configured_addon/link/serializers.py index db55a035..77369f3b 100644 --- a/addon_service/configured_addon/link/serializers.py +++ b/addon_service/configured_addon/link/serializers.py @@ -52,7 +52,7 @@ class ConfiguredLinkAddonSerializer(ConfiguredAddonSerializer): ) included_serializers = { - "base_account": ("addon_service.serializers.AuthorizedLinkAccountSerializer"), + "base_account": "addon_service.serializers.AuthorizedLinkAccountSerializer", "external_link_service": "addon_service.serializers.ExternalLinkServiceSerializer", "authorized_resource": "addon_service.serializers.ResourceReferenceSerializer", "connected_operations": "addon_service.serializers.AddonOperationSerializer", From 30ea16aa5805703da6c76756d984ea3bd44d095f Mon Sep 17 00:00:00 2001 From: Oleh Paduchak Date: Tue, 1 Apr 2025 17:02:10 +0300 Subject: [PATCH 08/65] fixed some things whcih vere left over --- addon_service/addon_operation_invocation/views.py | 10 ++++++++++ .../authorized_account/polymorphic_serializers.py | 4 ++++ addon_service/configured_addon/link/models.py | 8 +++++++- addon_service/configured_addon/link/serializers.py | 8 ++++++-- .../configured_addon/polymorphic_serializers.py | 4 ++++ 5 files changed, 31 insertions(+), 3 deletions(-) diff --git a/addon_service/addon_operation_invocation/views.py b/addon_service/addon_operation_invocation/views.py index da7736d3..36d29b2d 100644 --- a/addon_service/addon_operation_invocation/views.py +++ b/addon_service/addon_operation_invocation/views.py @@ -19,10 +19,12 @@ from ..authorized_account.computing.serializers import ( AuthorizedComputingAccountSerializer, ) +from ..authorized_account.link.serializers import AuthorizedLinkAccountSerializer from ..authorized_account.models import AuthorizedAccount from ..authorized_account.storage.serializers import AuthorizedStorageAccountSerializer from ..configured_addon.citation.serializers import ConfiguredCitationAddonSerializer from ..configured_addon.computing.serializers import ConfiguredComputingAddonSerializer +from ..configured_addon.link.serializers import ConfiguredLinkAddonSerializer from ..configured_addon.models import ConfiguredAddon from ..configured_addon.storage.serializers import ConfiguredStorageAddonSerializer from .models import AddonOperationInvocation @@ -63,6 +65,10 @@ def retrieve_related(self, request, *args, **kwargs): serializer = AuthorizedComputingAccountSerializer( instance, context={"request": request} ) + elif hasattr(instance, "authorizedlinkaccount"): + serializer = AuthorizedLinkAccountSerializer( + instance, context={"request": request} + ) else: raise ValueError("unknown authorized account type") elif isinstance(instance, ConfiguredAddon): @@ -78,6 +84,10 @@ def retrieve_related(self, request, *args, **kwargs): serializer = ConfiguredComputingAddonSerializer( instance, context={"request": request} ) + elif hasattr(instance, "configuredcomputingaddon"): + serializer = ConfiguredLinkAddonSerializer( + instance, context={"request": request} + ) else: raise ValueError("unknown configured addon type") else: diff --git a/addon_service/authorized_account/polymorphic_serializers.py b/addon_service/authorized_account/polymorphic_serializers.py index 773c63d0..9c3eea90 100644 --- a/addon_service/authorized_account/polymorphic_serializers.py +++ b/addon_service/authorized_account/polymorphic_serializers.py @@ -6,6 +6,9 @@ from addon_service.authorized_account.computing.serializers import ( AuthorizedComputingAccountSerializer, ) +from addon_service.authorized_account.link.serializers import ( + AuthorizedLinkAccountSerializer, +) from addon_service.authorized_account.models import AuthorizedAccount from addon_service.authorized_account.storage.serializers import ( AuthorizedStorageAccountSerializer, @@ -17,6 +20,7 @@ class AuthorizedAccountPolymorphicSerializer(serializers.PolymorphicModelSeriali AuthorizedCitationAccountSerializer, AuthorizedComputingAccountSerializer, AuthorizedStorageAccountSerializer, + AuthorizedLinkAccountSerializer, ] class Meta: diff --git a/addon_service/configured_addon/link/models.py b/addon_service/configured_addon/link/models.py index 8282dd4a..b97426f7 100644 --- a/addon_service/configured_addon/link/models.py +++ b/addon_service/configured_addon/link/models.py @@ -7,7 +7,9 @@ def is_supported_resource_type(resource_type: int): try: - SupportedResourceTypes(resource_type) + resource_type = SupportedResourceTypes(resource_type) + if len(resource_type) != 1: + raise ValidationError("One may select only one resource type") except ValueError: raise ValidationError("Invalid resource type") @@ -18,6 +20,10 @@ class ConfiguredLinkAddon(ConfiguredAddon): target_id = models.CharField() int_resource_type = models.IntegerField(validators=[is_supported_resource_type]) + @property + def resource_type(self) -> str: + return SupportedResourceTypes(self.int_resource_type).name + class Meta: verbose_name = "Configured Link Addon" verbose_name_plural = "Configured Link Addons" diff --git a/addon_service/configured_addon/link/serializers.py b/addon_service/configured_addon/link/serializers.py index 77369f3b..f339f7c1 100644 --- a/addon_service/configured_addon/link/serializers.py +++ b/addon_service/configured_addon/link/serializers.py @@ -1,6 +1,5 @@ from rest_framework.fields import ( CharField, - IntegerField, URLField, ) from rest_framework_json_api.relations import ResourceRelatedField @@ -8,6 +7,7 @@ from addon_service.addon_operation.models import AddonOperationModel from addon_service.common import view_names +from addon_service.common.enum_serializers import EnumNameChoiceField from addon_service.common.serializer_fields import DataclassRelatedLinkField from addon_service.configured_addon.serializers import ConfiguredAddonSerializer from addon_service.external_service.link.models import ExternalLinkService @@ -15,6 +15,7 @@ AuthorizedLinkAccount, ConfiguredLinkAddon, ) +from addon_toolkit.interfaces.link import SupportedResourceTypes RESOURCE_TYPE = get_resource_type_from_model(ConfiguredLinkAddon) @@ -25,7 +26,7 @@ class ConfiguredLinkAddonSerializer(ConfiguredAddonSerializer): target_uri = URLField() target_id = CharField() - int_resource_type = IntegerField() + resource_type = EnumNameChoiceField(enum_cls=SupportedResourceTypes) connected_operations = DataclassRelatedLinkField( dataclass_model=AddonOperationModel, @@ -75,4 +76,7 @@ class Meta: "external_link_service", "current_user_is_owner", "external_service_name", + "resource_type", + "target_uri", + "target_id", ] diff --git a/addon_service/configured_addon/polymorphic_serializers.py b/addon_service/configured_addon/polymorphic_serializers.py index 28b58d13..26fb44cb 100644 --- a/addon_service/configured_addon/polymorphic_serializers.py +++ b/addon_service/configured_addon/polymorphic_serializers.py @@ -6,6 +6,9 @@ from addon_service.configured_addon.computing.serializers import ( ConfiguredComputingAddonSerializer, ) +from addon_service.configured_addon.link.serializers import ( + ConfiguredLinkAddonSerializer, +) from addon_service.configured_addon.models import ConfiguredAddon from addon_service.configured_addon.storage.serializers import ( ConfiguredStorageAddonSerializer, @@ -17,6 +20,7 @@ class ConfiguredAddonPolymorphicSerializer(serializers.PolymorphicModelSerialize ConfiguredCitationAddonSerializer, ConfiguredComputingAddonSerializer, ConfiguredStorageAddonSerializer, + ConfiguredLinkAddonSerializer, ] class Meta: From 47c34ed23649efc61e07a0d846f8e031043a0346 Mon Sep 17 00:00:00 2001 From: Oleh Paduchak <158075011+opaduchak@users.noreply.github.com> Date: Wed, 2 Apr 2025 18:46:54 +0300 Subject: [PATCH 09/65] Apply suggestions from code review Co-authored-by: Yuhuai Liu --- addon_service/addon_operation_invocation/views.py | 2 +- addon_service/authorized_account/link/serializers.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/addon_service/addon_operation_invocation/views.py b/addon_service/addon_operation_invocation/views.py index 36d29b2d..579c44f7 100644 --- a/addon_service/addon_operation_invocation/views.py +++ b/addon_service/addon_operation_invocation/views.py @@ -84,7 +84,7 @@ def retrieve_related(self, request, *args, **kwargs): serializer = ConfiguredComputingAddonSerializer( instance, context={"request": request} ) - elif hasattr(instance, "configuredcomputingaddon"): + elif hasattr(instance, "configuredlinkaddon"): serializer = ConfiguredLinkAddonSerializer( instance, context={"request": request} ) diff --git a/addon_service/authorized_account/link/serializers.py b/addon_service/authorized_account/link/serializers.py index 0d8fbc6c..68c22d6e 100644 --- a/addon_service/authorized_account/link/serializers.py +++ b/addon_service/authorized_account/link/serializers.py @@ -20,7 +20,7 @@ ) -RESOURCE_TYPE = get_resource_type_from_model(AuthorizedStorageAccount) +RESOURCE_TYPE = get_resource_type_from_model(AuthorizedLinkAccount) class AuthorizedLinkAccountSerializer(AuthorizedAccountSerializer): From d625575224139d7cfafbe65da5a7d499d2921fee Mon Sep 17 00:00:00 2001 From: Oleh Paduchak Date: Fri, 4 Apr 2025 14:19:06 +0300 Subject: [PATCH 10/65] more complete implementation of link structures --- addon_imps/link/__init__.py | 0 addon_imps/link/dataverse.py | 195 ++++++++++++++++++ addon_service/addon_imp/instantiation.py | 33 +++ addon_service/admin/__init__.py | 4 +- .../authorized_account/link/serializers.py | 7 +- .../authorized_account/link/views.py | 2 +- addon_service/authorized_account/utils.py | 4 +- addon_service/common/enum_serializers.py | 2 + addon_service/common/known_imps.py | 39 ++-- addon_service/configured_addon/link/models.py | 15 +- .../configured_addon/link/serializers.py | 6 +- addon_service/configured_addon/serializers.py | 4 +- addon_service/credentials/serializers.py | 4 + addon_service/external_service/link/models.py | 6 +- ...er_externallinkservice_options_and_more.py | 49 +++++ addon_service/resource_reference/models.py | 11 + .../resource_reference/serializers.py | 9 + addon_service/serializers.py | 12 ++ addon_service/urls.py | 3 + addon_service/user_reference/models.py | 7 + addon_service/user_reference/serializers.py | 10 + addon_service/views.py | 6 + addon_toolkit/interfaces/__init__.py | 2 + addon_toolkit/interfaces/link.py | 11 +- 24 files changed, 399 insertions(+), 42 deletions(-) create mode 100644 addon_imps/link/__init__.py create mode 100644 addon_imps/link/dataverse.py create mode 100644 addon_service/migrations/0013_alter_externallinkservice_options_and_more.py diff --git a/addon_imps/link/__init__.py b/addon_imps/link/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/addon_imps/link/dataverse.py b/addon_imps/link/dataverse.py new file mode 100644 index 00000000..7a98e9af --- /dev/null +++ b/addon_imps/link/dataverse.py @@ -0,0 +1,195 @@ +from __future__ import annotations + +import asyncio +import re +from dataclasses import dataclass + +from django.core.exceptions import ValidationError + +from addon_toolkit.interfaces.link import ( + ItemResult, + ItemSampleResult, + ItemType, + LinkAddonHttpRequestorImp, +) + + +DATAVERSE_REGEX = re.compile(r"^dataverse/(?P\d*)$") +DATASET_REGEX = re.compile(r"^dataset/(?P.*)$") + + +@dataclass +class DataverseLinkImp(LinkAddonHttpRequestorImp): + """storage on dataverse + + see https://guides.dataverse.org/en/latest/api/native-api.html + """ + + async def build_url_for_id(self, item_id: str) -> str: + dataset_id = DATASET_REGEX.match(item_id)["id"] + dataset = await self._fetch_dataset(dataset_id) + return dataset.item_id + + async def get_external_account_id(self, _: dict[str, str]) -> str: + try: + async with self.network.GET("api/v1/users/:me") as response: + if not response.http_status.is_success: + raise ValidationError( + "Could not get dataverse account id, check your API Token" + ) + content = await response.json_content() + return content.get("data", {}).get("id") + except ValueError as exc: + if "relative url may not alter the base url" in str(exc).lower(): + raise ValidationError( + "Invalid host URL. Please check your Dataverse base URL." + ) + raise + + async def list_root_items(self, page_cursor: str = "") -> ItemSampleResult: + async with self.network.GET( + "api/mydata/retrieve", + query=[ + ["selected_page", page_cursor], + *[("role_ids", role) for role in range(1, 9)], + ("dvobject_types", "Dataverse"), + *[ + ("published_states", state) + for state in [ + "Unpublished", + "Published", + "Draft", + "Deaccessioned", + "In+Review", + ] + ], + ], + ) as response: + content = await response.json_content() + if resp_data := content.get("data"): + return parse_mydata(resp_data) + return ItemSampleResult(items=[], total_count=0) + + async def get_item_info(self, item_id: str) -> ItemResult: + if not item_id: + return ItemResult(item_id="", item_name="", item_type=ItemType.FOLDER) + elif match := DATAVERSE_REGEX.match(item_id): + entity = await self._fetch_dataverse(match["id"]) + elif match := DATASET_REGEX.match(item_id): + entity = await self._fetch_dataset(persistent_id=match["id"]) + else: + raise ValueError(f"Invalid item id: {item_id}") + + return entity + + async def list_child_items( + self, + item_id: str, + page_cursor: str = "", + item_type: ItemType | None = None, + ) -> ItemSampleResult: + if not item_id: + return await self.list_root_items(page_cursor) + elif match := DATAVERSE_REGEX.match(item_id): + items = await self._fetch_dataverse_items(match["id"]) + return ItemSampleResult( + items=items, + total_count=len(items), + ) + else: + return ItemSampleResult(items=[], total_count=0) + + async def _fetch_dataverse_items(self, dataverse_id) -> list[ItemResult]: + async with self.network.GET( + f"api/dataverses/{dataverse_id}/contents" + ) as response: + response_content = await response.json_content() + return await asyncio.gather( + *[ + self.get_dataverse_or_dataset_item(item) + for item in response_content["data"] + ] + ) + + async def get_dataverse_or_dataset_item(self, item: dict): + match item["type"]: + case "dataset": + return await self._fetch_dataset(dataset_id=item["id"]) + case "dataverse": + return parse_dataverse_as_subitem(item) + raise ValueError(f"Invalid item type: {item['type']}") + + async def _fetch_dataverse(self, dataverse_id) -> ItemResult: + async with self.network.GET(f"api/dataverses/{dataverse_id}") as response: + return parse_dataverse(await response.json_content()) + + async def _fetch_dataset( + self, dataset_id: str = None, persistent_id: str = None + ) -> ItemResult: + url = f"api/datasets/{':persistentId' if persistent_id else dataset_id}" + query = {"persistentId": persistent_id} if persistent_id else {} + # async with self.network.GET("api/datasets/:persistentId/", query={'persistentId': dataset_id}) as response: + async with self.network.GET(url, query=query) as response: + return parse_dataset(await response.json_content()) + + +### +# module-local helpers + + +def parse_dataverse_as_subitem(data: dict): + return ItemResult( + item_type=ItemType.FOLDER, + item_name=data["title"], + item_id=f'dataverse/{data["id"]}', + ) + + +def parse_dataverse(data: dict): + if data.get("data"): + data = data["data"] + return ItemResult( + item_type=ItemType.FOLDER, + item_name=data["name"], + item_id=f'dataverse/{data["id"]}', + ) + + +def parse_mydata(data: dict): + if data.get("data"): + data = data["data"] + return ItemSampleResult( + items=[ + ItemResult( + item_id=f"dataverse/{file['entity_id']}", + item_name=file["name"], + item_type=ItemType.FOLDER, + ) + for file in data["items"] + ], + total_count=data["total_count"], + next_sample_cursor=( + data["pagination"]["nextPageNumber"] + if data["pagination"]["hasNextPageNumber"] + else None + ), + ) + + +def parse_dataset(data: dict) -> ItemResult: + if data.get("data"): + data = data["data"] + try: + return ItemResult( + item_id=f'dataset/{data['latestVersion']["datasetPersistentId"]}', + item_name=[ + item + for item in data["latestVersion"]["metadataBlocks"]["citation"][ + "fields" + ] + if item["typeName"] == "title" + ][0]["value"], + item_type=ItemType.FOLDER, + ) + except (KeyError, IndexError) as e: + raise ValueError(f"Invalid dataset response: {e=}") diff --git a/addon_service/addon_imp/instantiation.py b/addon_service/addon_imp/instantiation.py index 427220cc..9466a031 100644 --- a/addon_service/addon_imp/instantiation.py +++ b/addon_service/addon_imp/instantiation.py @@ -17,6 +17,11 @@ ComputingAddonImp, ComputingConfig, ) +from addon_toolkit.interfaces.link import ( + LinkAddonClientRequestorImp, + LinkAddonHttpRequestorImp, + LinkAddonImp, +) from addon_toolkit.interfaces.storage import ( StorageAddonClientRequestorImp, StorageAddonHttpRequestorImp, @@ -26,6 +31,7 @@ if TYPE_CHECKING: + from addon_service.authorized_account.link.models import AuthorizedLinkAccount from addon_service.authorized_account.models import AuthorizedAccount from addon_service.models import ( AuthorizedCitationAccount, @@ -45,6 +51,8 @@ async def get_addon_instance( return await get_citation_addon_instance(imp_cls, account, config) elif issubclass(imp_cls, ComputingAddonImp): return await get_computing_addon_instance(imp_cls, account, config) + elif issubclass(imp_cls, LinkAddonImp): + return await get_link_addon_instance(imp_cls, account) raise ValueError(f"unknown addon type {imp_cls}") @@ -134,3 +142,28 @@ async def get_computing_addon_instance( get_computing_addon_instance__blocking = async_to_sync(get_computing_addon_instance) + + +async def get_link_addon_instance( + imp_cls: type[LinkAddonImp], + account: AuthorizedLinkAccount, +) -> LinkAddonImp: + """create an instance of a `linkAddonImp`""" + + assert issubclass(imp_cls, AddonImp) + assert imp_cls is not LinkAddonImp, "Addons shouldn't directly extend LinkAddonImp" + if issubclass(imp_cls, LinkAddonHttpRequestorImp): + imp = imp_cls( + network=GravyvaletHttpRequestor( + client_session=await get_singleton_client_session(), + prefix_url=account.external_service.api_base_url, + account=account, + ), + ) + if issubclass(imp_cls, LinkAddonClientRequestorImp): + imp = imp_cls(credentials=await account.get_credentials__async()) + + return imp + + +get_link_addon_instance__blocking = async_to_sync(get_link_addon_instance) diff --git a/addon_service/admin/__init__.py b/addon_service/admin/__init__.py index e06f11b1..59347d5c 100644 --- a/addon_service/admin/__init__.py +++ b/addon_service/admin/__init__.py @@ -61,7 +61,7 @@ class ExternalLinkServiceAdmin(GravyvaletModelAdmin): ) raw_id_fields = ("oauth2_client_config", "oauth1_client_config") enum_choice_fields = { - "int_addon_imp": known_imps.AddonImpNumbers, + "int_addon_imp": known_imps.LinkAddonImpNumbers, "int_credentials_format": CredentialsFormats, "int_service_type": ServiceTypes, } @@ -80,7 +80,7 @@ class ExternalComputingServiceAdmin(GravyvaletModelAdmin): ) raw_id_fields = ("oauth2_client_config",) enum_choice_fields = { - "int_addon_imp": known_imps.AddonImpNumbers, + "int_addon_imp": known_imps.ComputingAddonImpNumbers, "int_credentials_format": CredentialsFormats, "int_service_type": ServiceTypes, } diff --git a/addon_service/authorized_account/link/serializers.py b/addon_service/authorized_account/link/serializers.py index 68c22d6e..8de8350f 100644 --- a/addon_service/authorized_account/link/serializers.py +++ b/addon_service/authorized_account/link/serializers.py @@ -6,6 +6,7 @@ from rest_framework_json_api.utils import get_resource_type_from_model from addon_service.addon_operation.models import AddonOperationModel +from addon_service.authorized_account.link.models import AuthorizedLinkAccount from addon_service.authorized_account.serializers import AuthorizedAccountSerializer from addon_service.common import view_names from addon_service.common.serializer_fields import ( @@ -13,7 +14,6 @@ ReadOnlyResourceRelatedField, ) from addon_service.models import ( - AuthorizedStorageAccount, ConfiguredLinkAddon, ExternalLinkService, UserReference, @@ -53,7 +53,7 @@ class AuthorizedLinkAccountSerializer(AuthorizedAccountSerializer): included_serializers = { "account_owner": "addon_service.serializers.UserReferenceSerializer", "external_link_service": "addon_service.serializers.ExternalLinkServiceSerializer", - "configured_link_addons": "addon_service.serializers.ConfiguredLinkSerializer", + "configured_link_addons": "addon_service.serializers.ConfiguredLinkAddonSerializer", "authorized_operations": "addon_service.serializers.AddonOperationSerializer", } @@ -65,7 +65,7 @@ def get_configured_addons_uris(self, obj): ) class Meta: - model = AuthorizedStorageAccount + model = AuthorizedLinkAccount fields = [ "id", "url", @@ -78,7 +78,6 @@ class Meta: "authorized_operation_names", "configured_link_addons", "credentials", - "default_root_folder", "external_link_service", "initiate_oauth", "credentials_available", diff --git a/addon_service/authorized_account/link/views.py b/addon_service/authorized_account/link/views.py index c8d81e58..c6ea9ab6 100644 --- a/addon_service/authorized_account/link/views.py +++ b/addon_service/authorized_account/link/views.py @@ -4,6 +4,6 @@ from .serializers import AuthorizedLinkAccountSerializer -class AuthorizedStorageAccountViewSet(AuthorizedAccountViewSet): +class AuthorizedLinkAccountViewSet(AuthorizedAccountViewSet): queryset = AuthorizedLinkAccount.objects.all() serializer_class = AuthorizedLinkAccountSerializer diff --git a/addon_service/authorized_account/utils.py b/addon_service/authorized_account/utils.py index 5dfd4c60..2f691dd9 100644 --- a/addon_service/authorized_account/utils.py +++ b/addon_service/authorized_account/utils.py @@ -4,6 +4,7 @@ from addon_toolkit.interfaces.citation import CitationAddonImp from addon_toolkit.interfaces.computing import ComputingAddonImp +from addon_toolkit.interfaces.link import LinkAddonImp from addon_toolkit.interfaces.storage import StorageAddonImp @@ -18,5 +19,6 @@ def get_config_for_account(account: AuthorizedAccount): return account.authorizedcitationaccount.config elif issubclass(account.imp_cls, ComputingAddonImp): return account.authorizedcomputingaccount.config - + elif issubclass(account.imp_cls, LinkAddonImp): + return None raise ValueError(f"this function implementation does not support {account.imp_cls}") diff --git a/addon_service/common/enum_serializers.py b/addon_service/common/enum_serializers.py index dcc929a3..6ea1453a 100644 --- a/addon_service/common/enum_serializers.py +++ b/addon_service/common/enum_serializers.py @@ -41,6 +41,8 @@ def to_internal_value(self, data) -> enum.Enum: return self.enum_cls[_name] def to_representation(self, value: enum.Enum): + if isinstance(value, str): + value = self.enum_cls[value] return super().to_representation(value).name diff --git a/addon_service/common/known_imps.py b/addon_service/common/known_imps.py index 7557f953..e6b49653 100644 --- a/addon_service/common/known_imps.py +++ b/addon_service/common/known_imps.py @@ -10,6 +10,7 @@ zotero_org, ) from addon_imps.computing import boa +from addon_imps.link import dataverse as link_dataverse from addon_imps.storage import ( bitbucket, box_dot_com, @@ -25,6 +26,10 @@ ) from addon_service.common.enum_decorators import enum_names_same_as from addon_toolkit import AddonImp +from addon_toolkit.interfaces.citation import CitationAddonImp +from addon_toolkit.interfaces.computing import ComputingAddonImp +from addon_toolkit.interfaces.link import LinkAddonImp +from addon_toolkit.interfaces.storage import StorageAddonImp if __debug__: @@ -79,7 +84,7 @@ class KnownAddonImps(enum.Enum): BITBUCKET = bitbucket.BitbucketStorageImp DATAVERSE = dataverse.DataverseStorageImp OWNCLOUD = owncloud.OwnCloudStorageImp - + LINK_DATAVERSE = link_dataverse.DataverseLinkImp GITHUB = github.GitHubStorageImp GITLAB = gitlab.GitlabStorageImp DROPBOX = dropbox.DropboxStorageImp @@ -108,6 +113,7 @@ class AddonImpNumbers(enum.Enum): GITLAB = 1011 BITBUCKET = 1012 + LINK_DATAVERSE = 1030 GITHUB = 1013 BOA = 1020 @@ -116,20 +122,17 @@ class AddonImpNumbers(enum.Enum): BLARG = -7 -StorageAddonImpNumbers = frozenset( - { - AddonImpNumbers.BOX, - AddonImpNumbers.S3, - AddonImpNumbers.GOOGLEDRIVE, - AddonImpNumbers.DROPBOX, - AddonImpNumbers.FIGSHARE, - AddonImpNumbers.ONEDRIVE, - AddonImpNumbers.OWNCLOUD, - AddonImpNumbers.DATAVERSE, - AddonImpNumbers.GITLAB, - AddonImpNumbers.BITBUCKET, - AddonImpNumbers.GITHUB, - } -) -CitationAddonImpNumbers = frozenset({AddonImpNumbers.ZOTERO, AddonImpNumbers.MENDELEY}) -ComputingAddonImpNumbers = frozenset({AddonImpNumbers.BOA}) +def filter_addons_by_type(addon_type): + return frozenset( + { + AddonImpNumbers[item.name] + for item in KnownAddonImps + if issubclass(item.value, addon_type) + } + ) + + +StorageAddonImpNumbers = filter_addons_by_type(StorageAddonImp) +CitationAddonImpNumbers = filter_addons_by_type(CitationAddonImp) +ComputingAddonImpNumbers = filter_addons_by_type(ComputingAddonImp) +LinkAddonImpNumbers = filter_addons_by_type(LinkAddonImp) diff --git a/addon_service/configured_addon/link/models.py b/addon_service/configured_addon/link/models.py index b97426f7..13af2118 100644 --- a/addon_service/configured_addon/link/models.py +++ b/addon_service/configured_addon/link/models.py @@ -1,3 +1,4 @@ +from asgiref.sync import async_to_sync from django.core.exceptions import ValidationError from django.db import models @@ -16,7 +17,6 @@ def is_supported_resource_type(resource_type: int): class ConfiguredLinkAddon(ConfiguredAddon): - target_uri = models.URLField() target_id = models.CharField() int_resource_type = models.IntegerField(validators=[is_supported_resource_type]) @@ -24,6 +24,19 @@ class ConfiguredLinkAddon(ConfiguredAddon): def resource_type(self) -> str: return SupportedResourceTypes(self.int_resource_type).name + @resource_type.setter + def resource_type(self, value) -> None: + self.int_resource_type = value.value + + def target_url(self): + if self.target_id: + from addon_service.addon_imp.instantiation import ( + get_link_addon_instance__blocking, + ) + + addon = get_link_addon_instance__blocking(self.imp_cls, self.base_account) + return async_to_sync(addon.build_url_for_id)(self.target_id) + class Meta: verbose_name = "Configured Link Addon" verbose_name_plural = "Configured Link Addons" diff --git a/addon_service/configured_addon/link/serializers.py b/addon_service/configured_addon/link/serializers.py index f339f7c1..fea9efc3 100644 --- a/addon_service/configured_addon/link/serializers.py +++ b/addon_service/configured_addon/link/serializers.py @@ -24,7 +24,7 @@ class ConfiguredLinkAddonSerializer(ConfiguredAddonSerializer): """api serializer for the `ConfiguredLinkAddon` model""" - target_uri = URLField() + target_url = URLField(read_only=True) target_id = CharField() resource_type = EnumNameChoiceField(enum_cls=SupportedResourceTypes) @@ -64,9 +64,8 @@ class Meta: read_only_fields = ["external_link_service"] fields = [ "id", - "url", "display_name", - "root_folder", + "target_url", "base_account", "authorized_resource", "authorized_resource_uri", @@ -77,6 +76,5 @@ class Meta: "current_user_is_owner", "external_service_name", "resource_type", - "target_uri", "target_id", ] diff --git a/addon_service/configured_addon/serializers.py b/addon_service/configured_addon/serializers.py index 05c9e865..ce6d92ad 100644 --- a/addon_service/configured_addon/serializers.py +++ b/addon_service/configured_addon/serializers.py @@ -6,7 +6,7 @@ from addon_toolkit import AddonCapabilities -REQUIRED_FIELDS = frozenset(["url", "connected_operations", "authorized_resource"]) +REQUIRED_FIELDS = frozenset(["connected_operations", "authorized_resource"]) class ConfiguredAddonSerializer(serializers.HyperlinkedModelSerializer): @@ -14,7 +14,7 @@ def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) if not REQUIRED_FIELDS.issubset(set(self.fields.keys())): raise Exception( - f"{self.__class__.__name__} requires {self.REQUIRED_FIELDS} to be instantiated" + f"{self.__class__.__name__} requires {REQUIRED_FIELDS} to be instantiated" ) connected_capabilities = EnumNameMultipleChoiceField(enum_cls=AddonCapabilities) diff --git a/addon_service/credentials/serializers.py b/addon_service/credentials/serializers.py index 23b4ece1..cb5e8863 100644 --- a/addon_service/credentials/serializers.py +++ b/addon_service/credentials/serializers.py @@ -1,3 +1,5 @@ +import json + from rest_framework.serializers import ( JSONField, ValidationError, @@ -21,6 +23,8 @@ def __init__(self, write_only=True, required=False, *args, **kwargs): def to_internal_value(self, data): # this issue still hasn't been fixed on FE, so keeping this for now + if not isinstance(data, dict): + data = json.loads(data) if not data or not any(data.values()): return None # consider empty {} same as omitting the field # No access to the credentials format here, so just try all of them diff --git a/addon_service/external_service/link/models.py b/addon_service/external_service/link/models.py index 41c819ad..dd2bfa92 100644 --- a/addon_service/external_service/link/models.py +++ b/addon_service/external_service/link/models.py @@ -14,21 +14,21 @@ def validate_supported_features(value): class ExternalLinkService(ExternalService): - int_supported_resource_types = models.IntegerField( + int_supported_resource_types = models.BigIntegerField( validators=[validate_supported_features], null=True ) @property def supported_resource_types(self) -> list[SupportedResourceTypes]: """get the enum representation of int_supported_features""" - return SupportedResourceTypes(self.int_supported_features) + return SupportedResourceTypes(self.int_supported_resource_types) @supported_resource_types.setter def supported_resource_types( self, new_supported_resource_types: SupportedResourceTypes ): """set int_authorized_capabilities without caring its int""" - self.int_supported_features = new_supported_resource_types.value + self.int_supported_resource_types = new_supported_resource_types.value def clean(self): super().clean() diff --git a/addon_service/migrations/0013_alter_externallinkservice_options_and_more.py b/addon_service/migrations/0013_alter_externallinkservice_options_and_more.py new file mode 100644 index 00000000..4ae8041a --- /dev/null +++ b/addon_service/migrations/0013_alter_externallinkservice_options_and_more.py @@ -0,0 +1,49 @@ +# Generated by Django 4.2.7 on 2025-04-03 10:10 + +from django.db import ( + migrations, + models, +) + +import addon_service.configured_addon.link.models +import addon_service.external_service.link.models + + +class Migration(migrations.Migration): + + dependencies = [ + ("addon_service", "0012_authorizedlinkaccount_configuredlinkaddon_and_more"), + ] + + operations = [ + migrations.AlterModelOptions( + name="externallinkservice", + options={ + "verbose_name": "External Link Service", + "verbose_name_plural": "External Link Services", + }, + ), + migrations.RemoveField( + model_name="configuredlinkaddon", + name="target_uri", + ), + migrations.AlterField( + model_name="configuredlinkaddon", + name="int_resource_type", + field=models.IntegerField( + validators=[ + addon_service.configured_addon.link.models.is_supported_resource_type + ] + ), + ), + migrations.AlterField( + model_name="externallinkservice", + name="int_supported_resource_types", + field=models.BigIntegerField( + null=True, + validators=[ + addon_service.external_service.link.models.validate_supported_features + ], + ), + ), + ] diff --git a/addon_service/resource_reference/models.py b/addon_service/resource_reference/models.py index d9004be6..b83021f8 100644 --- a/addon_service/resource_reference/models.py +++ b/addon_service/resource_reference/models.py @@ -4,6 +4,7 @@ from addon_service.common.base_model import AddonsServiceBaseModel from addon_service.configured_addon.citation.models import ConfiguredCitationAddon from addon_service.configured_addon.computing.models import ConfiguredComputingAddon +from addon_service.configured_addon.link.models import ConfiguredLinkAddon from addon_service.configured_addon.storage.models import ConfiguredStorageAddon @@ -22,6 +23,16 @@ def configured_storage_addons(self): .order_by(Lower("_display_name")) ) + @property + def configured_link_addons(self): + return ConfiguredLinkAddon.objects.filter( + authorized_resource=self + ).select_related( + "base_account__external_service__externallinkservice", + "base_account__authorizedlinkaccount", + "base_account__account_owner", + ) + @property def configured_citation_addons(self): return ConfiguredCitationAddon.objects.filter( diff --git a/addon_service/resource_reference/serializers.py b/addon_service/resource_reference/serializers.py index dccf363a..eb1a79d3 100644 --- a/addon_service/resource_reference/serializers.py +++ b/addon_service/resource_reference/serializers.py @@ -30,6 +30,11 @@ class ResourceReferenceSerializer(serializers.HyperlinkedModelSerializer): queryset=ConfiguredCitationAddon.objects.active(), related_link_view_name=view_names.related_view(RESOURCE_TYPE), ) + configured_link_addons = HyperlinkedRelatedField( + many=True, + queryset=ConfiguredCitationAddon.objects.active(), + related_link_view_name=view_names.related_view(RESOURCE_TYPE), + ) configured_computing_addons = HyperlinkedRelatedField( many=True, queryset=ConfiguredComputingAddon.objects.active(), @@ -43,6 +48,9 @@ class ResourceReferenceSerializer(serializers.HyperlinkedModelSerializer): "configured_citation_addons": ( "addon_service.serializers.ConfiguredCitationAddonSerializer" ), + "configured_link_addons": ( + "addon_service.serializers.ConfiguredComputingAddonSerializer" + ), "configured_computing_addons": ( "addon_service.serializers.ConfiguredComputingAddonSerializer" ), @@ -55,6 +63,7 @@ class Meta: "url", "resource_uri", "configured_storage_addons", + "configured_link_addons", "configured_citation_addons", "configured_computing_addons", ] diff --git a/addon_service/serializers.py b/addon_service/serializers.py index e3f15b46..6cebd0f7 100644 --- a/addon_service/serializers.py +++ b/addon_service/serializers.py @@ -11,6 +11,9 @@ from addon_service.authorized_account.computing.serializers import ( AuthorizedComputingAccountSerializer, ) +from addon_service.authorized_account.link.serializers import ( + AuthorizedLinkAccountSerializer, +) from addon_service.authorized_account.polymorphic_serializers import ( AuthorizedAccountPolymorphicSerializer, ) @@ -24,6 +27,9 @@ from addon_service.configured_addon.computing.serializers import ( ConfiguredComputingAddonSerializer, ) +from addon_service.configured_addon.link.serializers import ( + ConfiguredLinkAddonSerializer, +) from addon_service.configured_addon.polymorphic_serializers import ( ConfiguredAddonPolymorphicSerializer, ) @@ -37,6 +43,9 @@ from addon_service.external_service.computing.serializers import ( ExternalComputingServiceSerializer, ) +from addon_service.external_service.link.serializers import ( + ExternalLinkServiceSerializer, +) from addon_service.external_service.serializers import ExternalServiceSerializer from addon_service.external_service.storage.serializers import ( ExternalStorageServiceSerializer, @@ -67,4 +76,7 @@ "AuthorizedAccountSerializer", "AuthorizedAccountPolymorphicSerializer", "ConfiguredAddonPolymorphicSerializer", + "ConfiguredLinkAddonSerializer", + "AuthorizedLinkAccountSerializer", + "ExternalLinkServiceSerializer", ) diff --git a/addon_service/urls.py b/addon_service/urls.py index d6b404bf..ab89009d 100644 --- a/addon_service/urls.py +++ b/addon_service/urls.py @@ -57,6 +57,9 @@ def _register_viewset(viewset): _register_viewset(views.AuthorizedCitationAccountViewSet) _register_viewset(views.ConfiguredCitationAddonViewSet) _register_viewset(views.ExternalCitationServiceViewSet) +_register_viewset(views.AuthorizedLinkAccountViewSet) +_register_viewset(views.ConfiguredLinkAddonViewSet) +_register_viewset(views.ExternalLinkServiceViewSet) _register_viewset(views.AuthorizedComputingAccountViewSet) _register_viewset(views.ConfiguredComputingAddonViewSet) _register_viewset(views.ExternalComputingServiceViewSet) diff --git a/addon_service/user_reference/models.py b/addon_service/user_reference/models.py index e54ad38d..3b2b39c4 100644 --- a/addon_service/user_reference/models.py +++ b/addon_service/user_reference/models.py @@ -3,6 +3,7 @@ from addon_service.authorized_account.citation.models import AuthorizedCitationAccount from addon_service.authorized_account.computing.models import AuthorizedComputingAccount +from addon_service.authorized_account.link.models import AuthorizedLinkAccount from addon_service.authorized_account.storage.models import AuthorizedStorageAccount from addon_service.common.base_model import AddonsServiceBaseModel from addon_service.configured_addon.computing.models import ConfiguredComputingAddon @@ -55,6 +56,12 @@ def authorized_computing_accounts(self): account_owner=self ).select_related("external_service") + @property + def authorized_link_accounts(self): + return AuthorizedLinkAccount.objects.filter(account_owner=self).select_related( + "external_service" + ) + class Meta: verbose_name = "User Reference" verbose_name_plural = "User References" diff --git a/addon_service/user_reference/serializers.py b/addon_service/user_reference/serializers.py index b45204e5..70df89e8 100644 --- a/addon_service/user_reference/serializers.py +++ b/addon_service/user_reference/serializers.py @@ -34,6 +34,12 @@ class UserReferenceSerializer(serializers.HyperlinkedModelSerializer): related_link_view_name=view_names.related_view(RESOURCE_TYPE), ) + authorized_link_accounts = HyperlinkedRelatedField( + many=True, + queryset=AuthorizedCitationAccount.objects.all(), + related_link_view_name=view_names.related_view(RESOURCE_TYPE), + ) + authorized_computing_accounts = HyperlinkedRelatedField( many=True, queryset=AuthorizedComputingAccount.objects.all(), @@ -56,6 +62,9 @@ class UserReferenceSerializer(serializers.HyperlinkedModelSerializer): "authorized_computing_accounts": ( "addon_service.serializers.AuthorizedComputingAccountSerializer" ), + "authorized_link_accounts": ( + "addon_service.serializers.AuthorizedComputingAccountSerializer" + ), "configured_resources": ( "addon_service.serializers.ResourceReferenceSerializer" ), @@ -70,5 +79,6 @@ class Meta: "authorized_storage_accounts", "authorized_citation_accounts", "authorized_computing_accounts", + "authorized_link_accounts", "configured_resources", ] diff --git a/addon_service/views.py b/addon_service/views.py index 4f7078a8..f72ee948 100644 --- a/addon_service/views.py +++ b/addon_service/views.py @@ -16,6 +16,7 @@ from addon_service.authorized_account.computing.views import ( AuthorizedComputingAccountViewSet, ) +from addon_service.authorized_account.link.views import AuthorizedLinkAccountViewSet from addon_service.authorized_account.storage.views import ( AuthorizedStorageAccountViewSet, ) @@ -23,11 +24,13 @@ from addon_service.configured_addon.computing.views import ( ConfiguredComputingAddonViewSet, ) +from addon_service.configured_addon.link.views import ConfiguredLinkAddonViewSet from addon_service.configured_addon.storage.views import ConfiguredStorageAddonViewSet from addon_service.external_service.citation.views import ExternalCitationServiceViewSet from addon_service.external_service.computing.views import ( ExternalComputingServiceViewSet, ) +from addon_service.external_service.link.views import ExternalLinkServiceViewSet from addon_service.external_service.storage.views import ExternalStorageServiceViewSet from addon_service.oauth1.views import oauth1_callback_view from addon_service.oauth2.views import oauth2_callback_view @@ -61,6 +64,9 @@ async def status(request): "AuthorizedComputingAccountViewSet", "ConfiguredComputingAddonViewSet", "ExternalComputingServiceViewSet", + "AuthorizedLinkAccountViewSet", + "ConfiguredLinkAddonViewSet", + "ExternalLinkServiceViewSet", "AuthorizedStorageAccountViewSet", "ConfiguredStorageAddonViewSet", "ExternalStorageServiceViewSet", diff --git a/addon_toolkit/interfaces/__init__.py b/addon_toolkit/interfaces/__init__.py index 2b878dd7..e00b0158 100644 --- a/addon_toolkit/interfaces/__init__.py +++ b/addon_toolkit/interfaces/__init__.py @@ -3,6 +3,7 @@ from . import ( citation, computing, + link, storage, ) from ._base import BaseAddonInterface @@ -21,3 +22,4 @@ class AllAddonInterfaces(enum.Enum): STORAGE = storage.StorageAddonInterface CITATION = citation.CitationServiceInterface COMPUTING = computing.ComputingAddonInterface + LINK = link.LinkAddonInterface diff --git a/addon_toolkit/interfaces/link.py b/addon_toolkit/interfaces/link.py index 14b12e89..82e203d2 100644 --- a/addon_toolkit/interfaces/link.py +++ b/addon_toolkit/interfaces/link.py @@ -25,7 +25,7 @@ class ItemType(enum.StrEnum): - FILE = enum.auto() + RESOURCE = enum.auto() FOLDER = enum.auto() @@ -69,7 +69,7 @@ class ItemResult: item_id: str item_name: str item_type: ItemType - resource_type: SupportedResourceTypes + resource_type: SupportedResourceTypes | None = None item_link: str | None = None @@ -100,6 +100,8 @@ def with_cursor(self, cursor: Cursor) -> typing.Self: class LinkAddonInterface(BaseAddonInterface, typing.Protocol): + async def build_url_for_id(self, item_id: str) -> str: ... + @immediate_operation(capability=AddonCapabilities.ACCESS) async def get_item_info(self, item_id: str) -> ItemResult: ... @@ -121,12 +123,9 @@ class LinkAddonImp(AddonImp): ADDON_INTERFACE = LinkAddonInterface - async def build_wb_config(self) -> dict: - return {} - @dataclasses.dataclass -class LinkAddonHttpRequestorImp(LinkAddonImp): +class LinkAddonHttpRequestorImp(LinkAddonImp, LinkAddonInterface): """base class for link addon implementations using GV network""" network: HttpRequestor From 9a32d59e1626b9fca5047d0e54c49b8195d1300e Mon Sep 17 00:00:00 2001 From: Oleh Paduchak Date: Fri, 4 Apr 2025 15:20:02 +0300 Subject: [PATCH 11/65] squashed migrations, fixed unit tests fixed resource type for configured link addon --- addon_service/configured_addon/link/models.py | 2 +- ...inkaccount_configuredlinkaddon_and_more.py | 19 ++++--- ...er_externallinkservice_options_and_more.py | 49 ------------------- .../test_by_type/test_resource_reference.py | 1 + .../tests/test_by_type/test_user_reference.py | 2 + 5 files changed, 17 insertions(+), 56 deletions(-) delete mode 100644 addon_service/migrations/0013_alter_externallinkservice_options_and_more.py diff --git a/addon_service/configured_addon/link/models.py b/addon_service/configured_addon/link/models.py index 13af2118..7dbfbe2a 100644 --- a/addon_service/configured_addon/link/models.py +++ b/addon_service/configured_addon/link/models.py @@ -18,7 +18,7 @@ def is_supported_resource_type(resource_type: int): class ConfiguredLinkAddon(ConfiguredAddon): target_id = models.CharField() - int_resource_type = models.IntegerField(validators=[is_supported_resource_type]) + int_resource_type = models.BigIntegerField(validators=[is_supported_resource_type]) @property def resource_type(self) -> str: diff --git a/addon_service/migrations/0012_authorizedlinkaccount_configuredlinkaddon_and_more.py b/addon_service/migrations/0012_authorizedlinkaccount_configuredlinkaddon_and_more.py index 707c5f06..87df7bb6 100644 --- a/addon_service/migrations/0012_authorizedlinkaccount_configuredlinkaddon_and_more.py +++ b/addon_service/migrations/0012_authorizedlinkaccount_configuredlinkaddon_and_more.py @@ -1,4 +1,4 @@ -# Generated by Django 4.2.7 on 2025-03-28 12:06 +# Generated by Django 4.2.7 on 2025-04-04 12:12 import django.db.models.deletion from django.db import ( @@ -6,6 +6,7 @@ models, ) +import addon_service.configured_addon.link.models import addon_service.external_service.link.models @@ -51,9 +52,15 @@ class Migration(migrations.Migration): to="addon_service.configuredaddon", ), ), - ("target_uri", models.URLField()), ("target_id", models.CharField()), - ("int_resource_type", models.IntegerField()), + ( + "int_resource_type", + models.BigIntegerField( + validators=[ + addon_service.configured_addon.link.models.is_supported_resource_type + ] + ), + ), ], options={ "verbose_name": "Configured Link Addon", @@ -77,7 +84,7 @@ class Migration(migrations.Migration): ), ( "int_supported_resource_types", - models.IntegerField( + models.BigIntegerField( null=True, validators=[ addon_service.external_service.link.models.validate_supported_features @@ -86,8 +93,8 @@ class Migration(migrations.Migration): ), ], options={ - "verbose_name": "External Storage Service", - "verbose_name_plural": "External Storage Services", + "verbose_name": "External Link Service", + "verbose_name_plural": "External Link Services", }, bases=("addon_service.externalservice",), ), diff --git a/addon_service/migrations/0013_alter_externallinkservice_options_and_more.py b/addon_service/migrations/0013_alter_externallinkservice_options_and_more.py deleted file mode 100644 index 4ae8041a..00000000 --- a/addon_service/migrations/0013_alter_externallinkservice_options_and_more.py +++ /dev/null @@ -1,49 +0,0 @@ -# Generated by Django 4.2.7 on 2025-04-03 10:10 - -from django.db import ( - migrations, - models, -) - -import addon_service.configured_addon.link.models -import addon_service.external_service.link.models - - -class Migration(migrations.Migration): - - dependencies = [ - ("addon_service", "0012_authorizedlinkaccount_configuredlinkaddon_and_more"), - ] - - operations = [ - migrations.AlterModelOptions( - name="externallinkservice", - options={ - "verbose_name": "External Link Service", - "verbose_name_plural": "External Link Services", - }, - ), - migrations.RemoveField( - model_name="configuredlinkaddon", - name="target_uri", - ), - migrations.AlterField( - model_name="configuredlinkaddon", - name="int_resource_type", - field=models.IntegerField( - validators=[ - addon_service.configured_addon.link.models.is_supported_resource_type - ] - ), - ), - migrations.AlterField( - model_name="externallinkservice", - name="int_supported_resource_types", - field=models.BigIntegerField( - null=True, - validators=[ - addon_service.external_service.link.models.validate_supported_features - ], - ), - ), - ] diff --git a/addon_service/tests/test_by_type/test_resource_reference.py b/addon_service/tests/test_by_type/test_resource_reference.py index a9af0a4f..2a5f71e4 100644 --- a/addon_service/tests/test_by_type/test_resource_reference.py +++ b/addon_service/tests/test_by_type/test_resource_reference.py @@ -165,6 +165,7 @@ def test_get(self): "configured_storage_addons", "configured_citation_addons", "configured_computing_addons", + "configured_link_addons", }, ) diff --git a/addon_service/tests/test_by_type/test_user_reference.py b/addon_service/tests/test_by_type/test_user_reference.py index de0eeb56..9b3c79b9 100644 --- a/addon_service/tests/test_by_type/test_user_reference.py +++ b/addon_service/tests/test_by_type/test_user_reference.py @@ -67,6 +67,7 @@ def test_get(self): "authorized_storage_accounts", "authorized_citation_accounts", "authorized_computing_accounts", + "authorized_link_accounts", "configured_resources", }, ) @@ -202,6 +203,7 @@ def test_get(self): "authorized_storage_accounts", "authorized_citation_accounts", "authorized_computing_accounts", + "authorized_link_accounts", "configured_resources", }, ) From 2c1974f36e5091afd7971997482fe90d9d867a0b Mon Sep 17 00:00:00 2001 From: Yuhuai Liu Date: Fri, 7 Feb 2025 20:40:03 -0500 Subject: [PATCH 12/65] enable sharing sessions between GV and OSF --- addon_service/authentication.py | 1 - addon_service/common/osf.py | 7 +------ app/env.py | 3 +++ app/middleware.py | 35 +++++++++++++++++++++++++++++++++ app/settings.py | 22 +++++++++++++++++++-- poetry.lock | 30 +++++++++++++++++++++++++++- pyproject.toml | 2 ++ 7 files changed, 90 insertions(+), 10 deletions(-) create mode 100644 app/middleware.py diff --git a/addon_service/authentication.py b/addon_service/authentication.py index 82d2cbb6..e9dbe28d 100644 --- a/addon_service/authentication.py +++ b/addon_service/authentication.py @@ -12,7 +12,6 @@ def authenticate(self, request: DrfRequest): _user_uri = osf.get_osf_user_uri(request) if _user_uri: UserReference.objects.get_or_create(user_uri=_user_uri) - request.session["user_reference_uri"] = _user_uri return True, None return None # unauthenticated diff --git a/addon_service/common/osf.py b/addon_service/common/osf.py index dc8668a4..2a017519 100644 --- a/addon_service/common/osf.py +++ b/addon_service/common/osf.py @@ -58,12 +58,7 @@ async def get_osf_user_uri(request: django_http.HttpRequest) -> str | None: _auth_headers = _get_osf_auth_headers(request) if not _auth_headers: return None - _client = await get_singleton_client_session() - async with _client.get(_osfapi_me_url(), headers=_auth_headers) as _response: - if HTTPStatus(_response.status).is_client_error: - return None - _response_content = await _response.json() - return _iri_from_osfapi_resource(_response_content["data"]) + return request.session["user_reference_uri"] @async_to_sync diff --git a/app/env.py b/app/env.py index 51b493e6..6ba78efd 100644 --- a/app/env.py +++ b/app/env.py @@ -36,6 +36,8 @@ OSFDB_CONN_MAX_AGE = os.environ.get("OSFDB_CONN_MAX_AGE", 0) OSFDB_SSLMODE = os.environ.get("OSFDB_SSLMODE", "prefer") +REDIS_HOST = os.environ.get("REDIS_HOST", "redis://192.168.168.167:6379") + ### # for interacting with osf @@ -46,6 +48,7 @@ OSF_BASE_URL = os.environ.get("OSF_BASE_URL", "https://osf.example") OSF_API_BASE_URL = os.environ.get("OSF_API_BASE_URL", "https://api.osf.example") OSF_AUTH_COOKIE_NAME = os.environ.get("OSF_AUTH_COOKIE_NAME", "osf_staging") +OSF_AUTH_COOKIE_SECRET = os.environ.get("OSF_AUTH_COOKIE_SECRET", "CHANGEME") ### # amqp/celery diff --git a/app/middleware.py b/app/middleware.py new file mode 100644 index 00000000..927a868d --- /dev/null +++ b/app/middleware.py @@ -0,0 +1,35 @@ +from importlib import import_module + +import itsdangerous +from django.conf import settings +from django.contrib.sessions.middleware import SessionMiddleware + + +SessionStore = import_module(settings.SESSION_ENGINE).SessionStore + + +def ensure_str(value): + if isinstance(value, bytes): + return value.decode() + return value + + +class UnsignCookieSessionMiddleware(SessionMiddleware): + """ + Overrides the process_request hook of SessionMiddleware + to retrieve the session key for finding the correct session + by unsigning the cookie value using server secret + """ + + def process_request(self, request): + cookie = request.COOKIES.get(settings.SESSION_COOKIE_NAME) + if cookie: + try: + session_key = ensure_str( + itsdangerous.Signer(settings.OSF_AUTH_COOKIE_SECRET).unsign(cookie) + ) + except itsdangerous.BadSignature: + return None + request.session = SessionStore(session_key=session_key) + else: + request.session = SessionStore() diff --git a/app/settings.py b/app/settings.py index efd9c845..9159ab53 100644 --- a/app/settings.py +++ b/app/settings.py @@ -66,7 +66,24 @@ OSF_BASE_URL = env.OSF_BASE_URL.rstrip("/") OSF_API_BASE_URL = env.OSF_API_BASE_URL.rstrip("/") ALLOWED_RESOURCE_URI_PREFIXES = {OSF_BASE_URL} -SESSION_COOKIE_DOMAIN = env.SESSION_COOKIE_DOMAIN +# SESSION_COOKIE_DOMAIN = env.SESSION_COOKIE_DOMAIN +SESSION_ENGINE = "django.contrib.sessions.backends.cache" +SESSION_COOKIE_NAME = env.OSF_AUTH_COOKIE_NAME +OSF_AUTH_COOKIE_SECRET = env.OSF_AUTH_COOKIE_SECRET +REDIS_HOST = env.REDIS_HOST + +CACHES = { + "default": { + "BACKEND": "django.core.cache.backends.redis.RedisCache", + "LOCATION": REDIS_HOST, + # 'OPTIONS': { + # 'CLIENT_CLASS': 'django_redis.client.DefaultClient', + # 'CONNECTION_POOL_KWARGS': { + # 'max_connections': 100, + # }, + # }, + }, +} if DEBUG: # allow for local osf shenanigans @@ -102,9 +119,10 @@ MIDDLEWARE = [ "corsheaders.middleware.CorsMiddleware", "django.middleware.security.SecurityMiddleware", - "django.contrib.sessions.middleware.SessionMiddleware", "django.middleware.common.CommonMiddleware", "django.middleware.csrf.CsrfViewMiddleware", + "django.contrib.sessions.middleware.SessionMiddleware", + "app.middleware.UnsignCookieSessionMiddleware", "django.contrib.auth.middleware.AuthenticationMiddleware", "django.contrib.messages.middleware.MessageMiddleware", "django.middleware.clickjacking.XFrameOptionsMiddleware", diff --git a/poetry.lock b/poetry.lock index f35a593a..b0020063 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1138,6 +1138,18 @@ files = [ [package.extras] colors = ["colorama (>=0.4.6)"] +[[package]] +name = "itsdangerous" +version = "2.2.0" +description = "Safely pass data to untrusted environments and back." +optional = false +python-versions = ">=3.8" +groups = ["main"] +files = [ + {file = "itsdangerous-2.2.0-py3-none-any.whl", hash = "sha256:c6242fc49e35958c8b15141343aa660db5fc54d4f13a1db01a3f5891b98700ef"}, + {file = "itsdangerous-2.2.0.tar.gz", hash = "sha256:e0050c0b7da1eea53ffaf149c0cfbb5c6e2e2b69c4bef22c81fa6eb73e5f6173"}, +] + [[package]] name = "jinja2" version = "3.1.4" @@ -1803,6 +1815,22 @@ files = [ {file = "pyyaml-6.0.2.tar.gz", hash = "sha256:d584d9ec91ad65861cc08d42e834324ef890a082e591037abe114850ff7bbc3e"}, ] +[[package]] +name = "redis" +version = "5.2.1" +description = "Python client for Redis database and key-value store" +optional = false +python-versions = ">=3.8" +groups = ["main"] +files = [ + {file = "redis-5.2.1-py3-none-any.whl", hash = "sha256:ee7e1056b9aea0f04c6c2ed59452947f34c4940ee025f5dd83e6a6418b6989e4"}, + {file = "redis-5.2.1.tar.gz", hash = "sha256:16f2e22dff21d5125e8481515e386711a34cbec50f0e44413dd7d9c060a54e0f"}, +] + +[package.extras] +hiredis = ["hiredis (>=3.0.0)"] +ocsp = ["cryptography (>=36.0.1)", "pyopenssl (==23.2.1)", "requests (>=2.31.0)"] + [[package]] name = "referencing" version = "0.35.1" @@ -2437,4 +2465,4 @@ testing = ["coverage (>=5.0.3)", "zope.event", "zope.testing"] [metadata] lock-version = "2.1" python-versions = "^3.12" -content-hash = "0a5d1b9ae431553e15534618f985ea8f476c492bbf617b5be6b0c1fde45b5d0c" +content-hash = "9f3d39b29f00131619e3d005ac8687d5c6189ef2b3e95dc7c3adb18714202b27" diff --git a/pyproject.toml b/pyproject.toml index 6263c1c8..bc79b1b0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -26,6 +26,8 @@ boa-api = "^0.1.14" django-silk = "^5.3.2" django-celery-beat = "^2.7.0" tqdm = "^4.67.1" +itsdangerous = "^2.2.0" +redis = "^5.2.1" [tool.poetry.group.dev.dependencies] From f86e13e6d7595c4a00f135142dbe640a6b1d8482 Mon Sep 17 00:00:00 2001 From: Oleh Paduchak Date: Tue, 8 Apr 2025 17:56:39 +0300 Subject: [PATCH 13/65] wip --- addon_service/authentication.py | 5 +++++ addon_service/tasks/update_datacite.py | 2 ++ app/celery.py | 7 +++++++ 3 files changed, 14 insertions(+) create mode 100644 addon_service/tasks/update_datacite.py diff --git a/addon_service/authentication.py b/addon_service/authentication.py index 82d2cbb6..05e66552 100644 --- a/addon_service/authentication.py +++ b/addon_service/authentication.py @@ -3,12 +3,17 @@ from addon_service.common import osf from addon_service.models import UserReference +from app.celery import app class GVCombinedAuthentication(drf_authentication.BaseAuthentication): """Authentication supporting session, basic, and token methods.""" def authenticate(self, request: DrfRequest): + app.send_task( + "osf.framework.analytics.do_datacite_stuff", + kwargs={"datacite_id": "456", "node_id": "123"}, + ) _user_uri = osf.get_osf_user_uri(request) if _user_uri: UserReference.objects.get_or_create(user_uri=_user_uri) diff --git a/addon_service/tasks/update_datacite.py b/addon_service/tasks/update_datacite.py new file mode 100644 index 00000000..b427f62a --- /dev/null +++ b/addon_service/tasks/update_datacite.py @@ -0,0 +1,2 @@ +def update_datacite(): + pass diff --git a/app/celery.py b/app/celery.py index bcb43b76..c51c63fe 100644 --- a/app/celery.py +++ b/app/celery.py @@ -28,6 +28,12 @@ def queue_name(self) -> str: gv_chill_queue = Queue(TaskUrgency.CHILL.queue_name()) gv_reactive_queue = Queue(TaskUrgency.REACTIVE.queue_name()) gv_interactive_queue = Queue(TaskUrgency.INTERACTIVE.queue_name()) +osf_high_queue = Queue( + "external_high", no_declare=True +) # this queue must be declared by osf's celery +osf_low_queue = Queue( + "external_low", no_declare=True +) # this queue must be declared by osf's celery app = Celery( broker_url=AMQP_BROKER_URL, @@ -44,6 +50,7 @@ def queue_name(self) -> str: "addon_service.management.commands.refresh_addon_tokens.*": { "queue": gv_chill_queue }, + "osf.*": {"queue": osf_high_queue}, }, include=[ "addon_service.tasks", From bb425947cecd84f52baae9237d4c17d3878599b1 Mon Sep 17 00:00:00 2001 From: Oleh Paduchak Date: Wed, 9 Apr 2025 15:21:32 +0300 Subject: [PATCH 14/65] implemented celery communication from GV to OSF --- addon_service/authentication.py | 5 ----- addon_service/configured_addon/models.py | 28 ++++++++++++++++++++++++ 2 files changed, 28 insertions(+), 5 deletions(-) diff --git a/addon_service/authentication.py b/addon_service/authentication.py index 05e66552..82d2cbb6 100644 --- a/addon_service/authentication.py +++ b/addon_service/authentication.py @@ -3,17 +3,12 @@ from addon_service.common import osf from addon_service.models import UserReference -from app.celery import app class GVCombinedAuthentication(drf_authentication.BaseAuthentication): """Authentication supporting session, basic, and token methods.""" def authenticate(self, request: DrfRequest): - app.send_task( - "osf.framework.analytics.do_datacite_stuff", - kwargs={"datacite_id": "456", "node_id": "123"}, - ) _user_uri = osf.get_osf_user_uri(request) if _user_uri: UserReference.objects.get_or_create(user_uri=_user_uri) diff --git a/addon_service/configured_addon/models.py b/addon_service/configured_addon/models.py index cf006702..ce91544c 100644 --- a/addon_service/configured_addon/models.py +++ b/addon_service/configured_addon/models.py @@ -11,6 +11,7 @@ AddonCapabilities, AddonImp, ) +from app.celery import app class ConnectedAddonManager(models.Manager): @@ -88,6 +89,20 @@ def connected_operations(self) -> list[AddonOperationModel]: ) ] + def save(self, *args, full_clean=True, **kwargs): + id_ = self.pk + super().save(*args, full_clean=full_clean, **kwargs) + if not id_: # If instance is created, not updated + app.send_task( + "osf.tasks.log_gv_addon", + kwargs={ + "node_url": self.resource_uri, + "user_url": self.owner_uri, + "addon": self.display_name, + "action": "addon_added", + }, + ) + @property def connected_operation_names(self): return [ @@ -97,6 +112,19 @@ def connected_operation_names(self): ) ] + def delete(self, *args, **kwargs): + result = super().delete(*args, **kwargs) + if result: + app.send_task( + "osf.tasks.log_gv_addon", + kwargs={ + "node_url": self.resource_uri, + "user_url": self.owner_uri, + "addon": self.display_name, + "action": "addon_removed", + }, + ) + @property def credentials(self): return self.base_account.credentials From 2c993972e946548e0d09af074178f74eb1f80cce Mon Sep 17 00:00:00 2001 From: Oleh Paduchak Date: Wed, 9 Apr 2025 15:30:08 +0300 Subject: [PATCH 15/65] removed unnecessary stuff --- addon_service/tasks/update_datacite.py | 2 -- 1 file changed, 2 deletions(-) delete mode 100644 addon_service/tasks/update_datacite.py diff --git a/addon_service/tasks/update_datacite.py b/addon_service/tasks/update_datacite.py deleted file mode 100644 index b427f62a..00000000 --- a/addon_service/tasks/update_datacite.py +++ /dev/null @@ -1,2 +0,0 @@ -def update_datacite(): - pass From 5515b346f49f2d95bc6d380107209aaef12dcc01 Mon Sep 17 00:00:00 2001 From: Oleh Paduchak Date: Wed, 9 Apr 2025 15:40:56 +0300 Subject: [PATCH 16/65] added rabbitme to tests --- .github/workflows/run_gravyvalet_tests.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/run_gravyvalet_tests.yml b/.github/workflows/run_gravyvalet_tests.yml index 180f92e5..e9fc8c2e 100644 --- a/.github/workflows/run_gravyvalet_tests.yml +++ b/.github/workflows/run_gravyvalet_tests.yml @@ -14,6 +14,10 @@ jobs: postgres-version: ['15'] runs-on: ubuntu-latest services: + rabbitmq: + image: rabbitmq:latest + ports: + - 5672:5672 postgres: image: postgres:${{ matrix.postgres-version }} env: @@ -54,6 +58,7 @@ jobs: run: poetry run python -Werror manage.py test env: DEBUG: 1 + AMQP_BROKER_URL: "amqp://guest:guest@rabbitmq:5672" POSTGRES_HOST: localhost POSTGRES_DB: gravyvalettest POSTGRES_USER: postgres From f84c286dccd6bd482300da613ba16d417bdd7544 Mon Sep 17 00:00:00 2001 From: Oleh Paduchak Date: Wed, 9 Apr 2025 15:47:46 +0300 Subject: [PATCH 17/65] fixed host for rabbitmq --- .github/workflows/run_gravyvalet_tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/run_gravyvalet_tests.yml b/.github/workflows/run_gravyvalet_tests.yml index e9fc8c2e..22556a6a 100644 --- a/.github/workflows/run_gravyvalet_tests.yml +++ b/.github/workflows/run_gravyvalet_tests.yml @@ -58,7 +58,7 @@ jobs: run: poetry run python -Werror manage.py test env: DEBUG: 1 - AMQP_BROKER_URL: "amqp://guest:guest@rabbitmq:5672" + AMQP_BROKER_URL: "amqp://guest:guest@localhost:5672" POSTGRES_HOST: localhost POSTGRES_DB: gravyvalettest POSTGRES_USER: postgres From 9fbde27fe2d910e98d468429b10650f5648dbd2e Mon Sep 17 00:00:00 2001 From: Yuhuai Liu Date: Wed, 9 Apr 2025 10:05:44 -0400 Subject: [PATCH 18/65] add redis to CI workflow --- .github/workflows/run_gravyvalet_tests.yml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/.github/workflows/run_gravyvalet_tests.yml b/.github/workflows/run_gravyvalet_tests.yml index 180f92e5..779fe9ea 100644 --- a/.github/workflows/run_gravyvalet_tests.yml +++ b/.github/workflows/run_gravyvalet_tests.yml @@ -14,6 +14,16 @@ jobs: postgres-version: ['15'] runs-on: ubuntu-latest services: + redis: + image: redis + # Set health checks to wait until redis has started + options: >- + --health-cmd "redis-cli ping" + --health-interval 10s + --health-timeout 5s + --health-retries 5 + ports: + - 6379:6379 postgres: image: postgres:${{ matrix.postgres-version }} env: From 236377c6834becc0ba9062bb0b44ae19cfee592d Mon Sep 17 00:00:00 2001 From: Yuhuai Liu Date: Wed, 9 Apr 2025 10:56:56 -0400 Subject: [PATCH 19/65] add REDIS_HOST env var --- .github/workflows/run_gravyvalet_tests.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/run_gravyvalet_tests.yml b/.github/workflows/run_gravyvalet_tests.yml index 779fe9ea..2267f0cd 100644 --- a/.github/workflows/run_gravyvalet_tests.yml +++ b/.github/workflows/run_gravyvalet_tests.yml @@ -68,3 +68,4 @@ jobs: POSTGRES_DB: gravyvalettest POSTGRES_USER: postgres SECRET_KEY: oh-so-secret + REDIS_HOST: localhost:6379 From 5e379fa930d542ad47a4093b4b95c6712f9c4870 Mon Sep 17 00:00:00 2001 From: Yuhuai Liu Date: Wed, 9 Apr 2025 11:01:15 -0400 Subject: [PATCH 20/65] fix --- .github/workflows/run_gravyvalet_tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/run_gravyvalet_tests.yml b/.github/workflows/run_gravyvalet_tests.yml index 2267f0cd..8494121d 100644 --- a/.github/workflows/run_gravyvalet_tests.yml +++ b/.github/workflows/run_gravyvalet_tests.yml @@ -68,4 +68,4 @@ jobs: POSTGRES_DB: gravyvalettest POSTGRES_USER: postgres SECRET_KEY: oh-so-secret - REDIS_HOST: localhost:6379 + REDIS_HOST: redis://localhost:6379 From 1890581a5fde938eec439c7265775e4d1ff4da9d Mon Sep 17 00:00:00 2001 From: Yuhuai Liu Date: Wed, 9 Apr 2025 15:16:34 -0400 Subject: [PATCH 21/65] fix tests --- addon_service/tests/_helpers.py | 14 ++++++------- .../tests/e2e_tests/test_oauth_flow.py | 5 ++--- .../test_authorized_storage_account.py | 12 ++++------- .../test_configured_storage_addon.py | 4 +++- .../test_by_type/test_resource_reference.py | 21 +++++++++---------- .../tests/test_by_type/test_user_reference.py | 21 ++++++++----------- 6 files changed, 35 insertions(+), 42 deletions(-) diff --git a/addon_service/tests/_helpers.py b/addon_service/tests/_helpers.py index 4a036338..4b0aeaf3 100644 --- a/addon_service/tests/_helpers.py +++ b/addon_service/tests/_helpers.py @@ -20,7 +20,6 @@ ) from asgiref.sync import sync_to_async -from django.conf import settings from django.contrib.sessions.backends.db import SessionStore from django.urls import reverse from rest_framework.test import APIRequestFactory @@ -80,11 +79,13 @@ def configure_user_role(self, user_uri, resource_uri, role): def configure_resource_visibility(self, resource_uri, *, public=True): self._permissions[resource_uri]["public"] = public - def _get_assumed_caller(self, cookies=None): + def _get_assumed_caller(self, request): if self._configured_caller_uri: + request.session["user_reference_uri"] = self._configured_caller_uri return self._configured_caller_uri - if cookies is not None: - return cookies.get(settings.USER_REFERENCE_COOKIE) + + if request.COOKIES is not None: + return request.session.get("user_reference_uri") return None def _get_user_permissions(self, user_uri, resource_uri): @@ -103,8 +104,7 @@ def _get_user_permissions(self, user_uri, resource_uri): def _mock_user_check(self, request) -> tuple[Any, Any] | None: # replaces `authenticate` on a custom rest_framework authenticator: # https://www.django-rest-framework.org/api-guide/authentication/#custom-authentication - caller_uri = self._get_assumed_caller(cookies=request.COOKIES) - request.session["user_reference_uri"] = caller_uri + caller_uri = self._get_assumed_caller(request) return ( (None, None) # success! return a tuple (values here yet unused) if caller_uri @@ -112,7 +112,7 @@ def _mock_user_check(self, request) -> tuple[Any, Any] | None: ) def _mock_resource_check(self, request, uri, required_permission, *args, **kwargs): - caller = self._get_assumed_caller(cookies=request.COOKIES) + caller = self._get_assumed_caller(request) permissions = self._get_user_permissions(user_uri=caller, resource_uri=uri) return bool(required_permission.lower() in permissions) diff --git a/addon_service/tests/e2e_tests/test_oauth_flow.py b/addon_service/tests/e2e_tests/test_oauth_flow.py index cd9b5954..64caf5a5 100644 --- a/addon_service/tests/e2e_tests/test_oauth_flow.py +++ b/addon_service/tests/e2e_tests/test_oauth_flow.py @@ -1,7 +1,6 @@ import secrets from asgiref.sync import async_to_sync -from django.conf import settings from django.urls import reverse from rest_framework.test import APITestCase @@ -55,8 +54,8 @@ def setUp(self): access=self.MOCK_ACCESS_TOKEN, refresh=self.MOCK_REFRESH_TOKEN ) - self.client.cookies[settings.USER_REFERENCE_COOKIE] = self._user.user_uri self._mock_osf = _helpers.MockOSF() + self._mock_osf.configure_assumed_caller(self._user.user_uri) self.enterContext(self._mock_osf.mocking()) def test_oauth_account_setup(self): @@ -119,8 +118,8 @@ def setUp(self): _static_verifier=self.MOCK_VERIFIER, ) - self.client.cookies[settings.USER_REFERENCE_COOKIE] = self._user.user_uri self._mock_osf = _helpers.MockOSF() + self._mock_osf.configure_assumed_caller(self._user.user_uri) self.enterContext(self._mock_osf.mocking()) self.enterContext(self._mock_service.mocking()) diff --git a/addon_service/tests/test_by_type/test_authorized_storage_account.py b/addon_service/tests/test_by_type/test_authorized_storage_account.py index c7c995ca..8f3a174f 100644 --- a/addon_service/tests/test_by_type/test_authorized_storage_account.py +++ b/addon_service/tests/test_by_type/test_authorized_storage_account.py @@ -1,7 +1,6 @@ import urllib from http import HTTPStatus -from django.conf import settings from django.core.exceptions import ValidationError from django.test import TestCase from django.urls import reverse @@ -100,8 +99,8 @@ def setUpTestData(cls): def setUp(self): super().setUp() - self.client.cookies[settings.USER_REFERENCE_COOKIE] = self._user.user_uri self._mock_osf = MockOSF() + self._mock_osf.configure_assumed_caller(self._user.user_uri) self.enterContext(self._mock_osf.mocking()) @property @@ -488,13 +487,12 @@ def setUpTestData(cls): def setUp(self): super().setUp() self._mock_osf = MockOSF() + self._mock_osf.configure_assumed_caller(self._user.user_uri) self.enterContext(self._mock_osf.mocking()) def test_get_related__empty(self): _resp = self._related_view( - get_test_request( - cookies={settings.USER_REFERENCE_COOKIE: self._user.user_uri} - ), + get_test_request(), pk=self._asa.pk, related_field="configured_storage_addons", ) @@ -507,9 +505,7 @@ def test_get_related__several(self): base_account=self._asa, ) _resp = self._related_view( - get_test_request( - cookies={settings.USER_REFERENCE_COOKIE: self._user.user_uri} - ), + get_test_request(), pk=self._asa.pk, related_field="configured_storage_addons", ) diff --git a/addon_service/tests/test_by_type/test_configured_storage_addon.py b/addon_service/tests/test_by_type/test_configured_storage_addon.py index 5d4b08c1..ad4efbcd 100644 --- a/addon_service/tests/test_by_type/test_configured_storage_addon.py +++ b/addon_service/tests/test_by_type/test_configured_storage_addon.py @@ -38,8 +38,8 @@ def setUpTestData(cls): def setUp(self): super().setUp() - self.client.cookies[settings.USER_REFERENCE_COOKIE] = self._user.user_uri self._mock_osf = MockOSF() + self._mock_osf.configure_assumed_caller(self._user.user_uri) self._mock_osf.configure_user_role( self._user.user_uri, self._configured_storage_addon.resource_uri, "admin" ) @@ -127,6 +127,8 @@ def test_viewset_retrieve(self): def test_unauthorized_user(self): self.set_auth_header("session") + unauthorized_user = test_factories.UserReferenceFactory() + self._mock_osf.configure_assumed_caller(unauthorized_user.user_uri) response = self.client.get(self.related_url("base_account")) self.assertEqual(response.status_code, HTTPStatus.FORBIDDEN) diff --git a/addon_service/tests/test_by_type/test_resource_reference.py b/addon_service/tests/test_by_type/test_resource_reference.py index 2a5f71e4..aa3c25ba 100644 --- a/addon_service/tests/test_by_type/test_resource_reference.py +++ b/addon_service/tests/test_by_type/test_resource_reference.py @@ -26,13 +26,13 @@ def setUpTestData(cls): def setUp(self): super().setUp() - self.client.cookies[settings.USER_REFERENCE_COOKIE] = self._user.user_uri self._mock_osf = MockOSF() self._mock_osf.configure_user_role( user_uri=self._user.user_uri, resource_uri=self._resource.resource_uri, role="admin", ) + self._mock_osf.configure_assumed_caller(self._user.user_uri) self.enterContext(self._mock_osf.mocking()) @property @@ -74,7 +74,7 @@ def test_list__wrong_filter(self): def test_list__wrong_user(self): other_user = _factories.UserReferenceFactory() - self.client.cookies[settings.USER_REFERENCE_COOKIE] = other_user.user_uri + self._mock_osf.configure_assumed_caller(other_user) _resp = self.client.get( self._list_path, {"filter[resource_uri]": self._resource.resource_uri} ) @@ -142,13 +142,12 @@ def setUp(self): self._mock_osf.configure_user_role( self._user.user_uri, self._resource.resource_uri, "admin" ) + self._mock_osf.configure_assumed_caller(self._user.user_uri) self.enterContext(self._mock_osf.mocking()) def test_get(self): _resp = self._view( - get_test_request( - cookies={settings.USER_REFERENCE_COOKIE: self._user.user_uri} - ), + get_test_request(), pk=self._resource.pk, ) self.assertEqual(_resp.status_code, HTTPStatus.OK) @@ -173,6 +172,7 @@ def test_unauthorized__private_resource(self): self._mock_osf.configure_resource_visibility( self._resource.resource_uri, public=False ) + self._mock_osf.configure_assumed_caller(None) _anon_resp = self._view(get_test_request(), pk=self._resource.pk) self.assertEqual(_anon_resp.status_code, HTTPStatus.UNAUTHORIZED) @@ -187,10 +187,10 @@ def test_wrong_user__pivate_resource(self): self._mock_osf.configure_resource_visibility( self._resource.resource_uri, public=False ) + wrong_user = _factories.UserReferenceFactory() + self._mock_osf.configure_assumed_caller(wrong_user) _resp = self._view( - get_test_request( - cookies={settings.USER_REFERENCE_COOKIE: "this is wrong user auth"} - ), + get_test_request(), pk=self._resource.pk, ) self.assertEqual(_resp.status_code, HTTPStatus.FORBIDDEN) @@ -222,6 +222,7 @@ def setUpTestData(cls): def setUp(self): self._mock_osf = MockOSF() + self._mock_osf.configure_assumed_caller(self._user.user_uri) self._mock_osf.configure_user_role( self._user.user_uri, self._resource.resource_uri, "admin" ) @@ -231,9 +232,7 @@ def test_get_related__empty(self): self._csa.delete() _resp = self._related_view( - get_test_request( - cookies={settings.USER_REFERENCE_COOKIE: self._user.user_uri} - ), + get_test_request(), pk=self._resource.pk, related_field="configured_storage_addons", ) diff --git a/addon_service/tests/test_by_type/test_user_reference.py b/addon_service/tests/test_by_type/test_user_reference.py index 9b3c79b9..b344fb81 100644 --- a/addon_service/tests/test_by_type/test_user_reference.py +++ b/addon_service/tests/test_by_type/test_user_reference.py @@ -1,7 +1,6 @@ import json from http import HTTPStatus -from django.conf import settings from django.core.exceptions import ValidationError from django.db.models.query import QuerySet from django.test import TestCase @@ -24,8 +23,8 @@ def setUpTestData(cls): def setUp(self): super().setUp() - self.client.cookies[settings.USER_REFERENCE_COOKIE] = self._user.user_uri self._mock_osf = MockOSF() + self._mock_osf.configure_assumed_caller(self._user.user_uri) self.enterContext(self._mock_osf.mocking()) @property @@ -180,13 +179,12 @@ def setUpTestData(cls): def setUp(self): super().setUp() self._mock_osf = MockOSF() + self._mock_osf.configure_assumed_caller(self._user.user_uri) self.enterContext(self._mock_osf.mocking()) def test_get(self): _resp = self._view( - get_test_request( - cookies={settings.USER_REFERENCE_COOKIE: self._user.user_uri}, - ), + get_test_request(), pk=self._user.pk, ) self.assertEqual(_resp.status_code, HTTPStatus.OK) @@ -209,15 +207,16 @@ def test_get(self): ) def test_wrong_user(self): + wrong_user = _factories.UserReferenceFactory() + self._mock_osf.configure_assumed_caller(wrong_user.user_uri) _resp = self._view( - get_test_request( - cookies={settings.USER_REFERENCE_COOKIE: "this is the wrong cookie"} - ), + get_test_request(), pk=self._user.pk, ) self.assertEqual(_resp.status_code, HTTPStatus.FORBIDDEN) def test_unauthorized(self): + self._mock_osf.configure_assumed_caller(None) _anon_resp = self._view(get_test_request(), pk=self._user.pk) self.assertEqual(_anon_resp.status_code, HTTPStatus.UNAUTHORIZED) @@ -231,11 +230,9 @@ def setUpTestData(cls): def setUp(self): super().setUp() self._mock_osf = MockOSF() + self._mock_osf.configure_assumed_caller(self._user.user_uri) self.enterContext(self._mock_osf.mocking()) - self.request = get_test_request( - user=self._user, - cookies={settings.USER_REFERENCE_COOKIE: self._user.user_uri}, - ) + self.request = get_test_request() def test_get_related__empty(self): _resp = self._related_view( From 5e3ef39b4dfa7a610631588f6300863c215ca2d2 Mon Sep 17 00:00:00 2001 From: Oleh Paduchak Date: Wed, 9 Apr 2025 19:45:19 +0300 Subject: [PATCH 22/65] Implemented dataverse for VRL --- addon_imps/link/dataverse.py | 114 ++++++++++++------ addon_service/addon_imp/instantiation.py | 11 +- .../authorized_account/link/models.py | 7 ++ addon_service/authorized_account/utils.py | 2 +- addon_service/configured_addon/link/models.py | 13 +- addon_service/configured_addon/utils.py | 4 +- addon_toolkit/interfaces/link.py | 7 ++ 7 files changed, 112 insertions(+), 46 deletions(-) diff --git a/addon_imps/link/dataverse.py b/addon_imps/link/dataverse.py index 7a98e9af..308945d5 100644 --- a/addon_imps/link/dataverse.py +++ b/addon_imps/link/dataverse.py @@ -15,7 +15,8 @@ DATAVERSE_REGEX = re.compile(r"^dataverse/(?P\d*)$") -DATASET_REGEX = re.compile(r"^dataset/(?P.*)$") +DATASET_REGEX = re.compile(r"^dataset/(?P\d*)(?P.*)$") +FILE_REGEX = re.compile(r"^file/(?P.*)$") @dataclass @@ -26,9 +27,13 @@ class DataverseLinkImp(LinkAddonHttpRequestorImp): """ async def build_url_for_id(self, item_id: str) -> str: - dataset_id = DATASET_REGEX.match(item_id)["id"] - dataset = await self._fetch_dataset(dataset_id) - return dataset.item_id + match = DATASET_REGEX.match(item_id) + if not (persistent_id := match["persistent_id"]): + dataset = await self._fetch_dataset(match["id"]) + return dataset.item_link + return ( + f"{self.config.external_api_url}/dataset.xhtml?persistentId={persistent_id}" + ) async def get_external_account_id(self, _: dict[str, str]) -> str: try: @@ -53,16 +58,7 @@ async def list_root_items(self, page_cursor: str = "") -> ItemSampleResult: ["selected_page", page_cursor], *[("role_ids", role) for role in range(1, 9)], ("dvobject_types", "Dataverse"), - *[ - ("published_states", state) - for state in [ - "Unpublished", - "Published", - "Draft", - "Deaccessioned", - "In+Review", - ] - ], + ("published_states", "Published"), ], ) as response: content = await response.json_content() @@ -76,7 +72,11 @@ async def get_item_info(self, item_id: str) -> ItemResult: elif match := DATAVERSE_REGEX.match(item_id): entity = await self._fetch_dataverse(match["id"]) elif match := DATASET_REGEX.match(item_id): - entity = await self._fetch_dataset(persistent_id=match["id"]) + entity = await self._fetch_dataset( + dataset_id=match["id"], persistent_id=match["persistent_id"] + ) + elif match := FILE_REGEX.match(item_id): + entity = await self._fetch_file(match["persistent_id"]) else: raise ValueError(f"Invalid item id: {item_id}") @@ -96,6 +96,14 @@ async def list_child_items( items=items, total_count=len(items), ) + elif match := DATASET_REGEX.match(item_id): + items = await self._fetch_dataset_files( + dataset_id=match["id"], persistent_id=match["persistent_id"] + ) + return ItemSampleResult( + items=items, + total_count=len(items), + ) else: return ItemSampleResult(items=[], total_count=0) @@ -119,6 +127,12 @@ async def get_dataverse_or_dataset_item(self, item: dict): return parse_dataverse_as_subitem(item) raise ValueError(f"Invalid item type: {item['type']}") + async def _fetch_file(self, dataverse_id) -> ItemResult: + async with self.network.GET( + "api/files/:persistentId", query={"persistentId": dataverse_id} + ) as response: + return self._parse_datafile(await response.json_content()) + async def _fetch_dataverse(self, dataverse_id) -> ItemResult: async with self.network.GET(f"api/dataverses/{dataverse_id}") as response: return parse_dataverse(await response.json_content()) @@ -126,11 +140,56 @@ async def _fetch_dataverse(self, dataverse_id) -> ItemResult: async def _fetch_dataset( self, dataset_id: str = None, persistent_id: str = None ) -> ItemResult: - url = f"api/datasets/{':persistentId' if persistent_id else dataset_id}" + url = f"api/datasets/{':persistentId' if persistent_id else dataset_id}/versions/:latest-published" + query = {"persistentId": persistent_id} if persistent_id else {} + async with self.network.GET(url, query=query) as response: + return self._parse_dataset(await response.json_content()) + + async def _fetch_dataset_files( + self, dataset_id: str = None, persistent_id: str = None + ) -> list[ItemResult]: + url = f"api/datasets/{':persistentId' if persistent_id else dataset_id}/versions/:latest-published" query = {"persistentId": persistent_id} if persistent_id else {} - # async with self.network.GET("api/datasets/:persistentId/", query={'persistentId': dataset_id}) as response: async with self.network.GET(url, query=query) as response: - return parse_dataset(await response.json_content()) + return self._parse_dataset_files(await response.json_content()) + + def _parse_datafile(self, data: dict): + if data.get("data"): + data = data["data"] + + return ItemResult( + item_id=f"file/{data['dataFile']['persistentId']}", + item_name=data["label"], + item_type=ItemType.RESOURCE, + item_link=f'{self.config.external_api_url}/file.xhtml?persistentId={data['dataFile']["persistentId"]}', + doi=data["dataFile"]["persistentId"], + ) + + def _parse_dataset(self, data: dict) -> ItemResult: + if data.get("data"): + data = data["data"] + try: + return ItemResult( + item_id=f'dataset/{data["datasetPersistentId"]}', + item_name=[ + item + for item in data["metadataBlocks"]["citation"]["fields"] + if item["typeName"] == "title" + ][0]["value"], + item_type=ItemType.FOLDER, + item_link=f'{self.config.external_api_url}/dataset.xhtml?persistentId={data["datasetPersistentId"]}', + doi=data["datasetPersistentId"], + ) + except (KeyError, IndexError) as e: + raise ValueError(f"Invalid dataset response: {e=}") + + def _parse_dataset_files(self, data: dict) -> list[ItemResult]: + if data.get("data"): + data = data["data"] + try: + return [self._parse_datafile(file) for file in data["files"]] + except (KeyError, IndexError) as e: + raise ValueError(f"Invalid dataset response:{e=}") ### @@ -174,22 +233,3 @@ def parse_mydata(data: dict): else None ), ) - - -def parse_dataset(data: dict) -> ItemResult: - if data.get("data"): - data = data["data"] - try: - return ItemResult( - item_id=f'dataset/{data['latestVersion']["datasetPersistentId"]}', - item_name=[ - item - for item in data["latestVersion"]["metadataBlocks"]["citation"][ - "fields" - ] - if item["typeName"] == "title" - ][0]["value"], - item_type=ItemType.FOLDER, - ) - except (KeyError, IndexError) as e: - raise ValueError(f"Invalid dataset response: {e=}") diff --git a/addon_service/addon_imp/instantiation.py b/addon_service/addon_imp/instantiation.py index 9466a031..5b32500d 100644 --- a/addon_service/addon_imp/instantiation.py +++ b/addon_service/addon_imp/instantiation.py @@ -21,6 +21,7 @@ LinkAddonClientRequestorImp, LinkAddonHttpRequestorImp, LinkAddonImp, + LinkConfig, ) from addon_toolkit.interfaces.storage import ( StorageAddonClientRequestorImp, @@ -43,7 +44,7 @@ async def get_addon_instance( imp_cls: type[AddonImp], account: AuthorizedAccount, - config: StorageConfig | CitationConfig | ComputingConfig, + config: StorageConfig | CitationConfig | ComputingConfig | LinkConfig, ) -> AddonImp: if issubclass(imp_cls, StorageAddonImp): return await get_storage_addon_instance(imp_cls, account, config) @@ -52,7 +53,7 @@ async def get_addon_instance( elif issubclass(imp_cls, ComputingAddonImp): return await get_computing_addon_instance(imp_cls, account, config) elif issubclass(imp_cls, LinkAddonImp): - return await get_link_addon_instance(imp_cls, account) + return await get_link_addon_instance(imp_cls, account, config) raise ValueError(f"unknown addon type {imp_cls}") @@ -145,8 +146,7 @@ async def get_computing_addon_instance( async def get_link_addon_instance( - imp_cls: type[LinkAddonImp], - account: AuthorizedLinkAccount, + imp_cls: type[LinkAddonImp], account: AuthorizedLinkAccount, config: LinkConfig ) -> LinkAddonImp: """create an instance of a `linkAddonImp`""" @@ -159,9 +159,10 @@ async def get_link_addon_instance( prefix_url=account.external_service.api_base_url, account=account, ), + config=config, ) if issubclass(imp_cls, LinkAddonClientRequestorImp): - imp = imp_cls(credentials=await account.get_credentials__async()) + imp = imp_cls(credentials=await account.get_credentials__async(), config=config) return imp diff --git a/addon_service/authorized_account/link/models.py b/addon_service/authorized_account/link/models.py index 615d95ae..addc6ac9 100644 --- a/addon_service/authorized_account/link/models.py +++ b/addon_service/authorized_account/link/models.py @@ -1,5 +1,6 @@ from addon_service.authorized_account.models import AuthorizedAccount from addon_service.configured_addon.link.models import ConfiguredLinkAddon +from addon_toolkit.interfaces.link import LinkConfig class AuthorizedLinkAccount(AuthorizedAccount): @@ -22,3 +23,9 @@ def configured_link_addons(self): return ConfiguredLinkAddon.objects.filter(base_account=self).select_related( "authorized_resource" ) + + @property + def config(self) -> LinkConfig: + return LinkConfig( + external_api_url=self.api_base_url, + ) diff --git a/addon_service/authorized_account/utils.py b/addon_service/authorized_account/utils.py index 2f691dd9..21c7fb49 100644 --- a/addon_service/authorized_account/utils.py +++ b/addon_service/authorized_account/utils.py @@ -20,5 +20,5 @@ def get_config_for_account(account: AuthorizedAccount): elif issubclass(account.imp_cls, ComputingAddonImp): return account.authorizedcomputingaccount.config elif issubclass(account.imp_cls, LinkAddonImp): - return None + return account.authorizedlinkaccount.config raise ValueError(f"this function implementation does not support {account.imp_cls}") diff --git a/addon_service/configured_addon/link/models.py b/addon_service/configured_addon/link/models.py index 7dbfbe2a..0b5a411a 100644 --- a/addon_service/configured_addon/link/models.py +++ b/addon_service/configured_addon/link/models.py @@ -3,7 +3,10 @@ from django.db import models from addon_service.configured_addon.models import ConfiguredAddon -from addon_toolkit.interfaces.link import SupportedResourceTypes +from addon_toolkit.interfaces.link import ( + LinkConfig, + SupportedResourceTypes, +) def is_supported_resource_type(resource_type: int): @@ -34,9 +37,15 @@ def target_url(self): get_link_addon_instance__blocking, ) - addon = get_link_addon_instance__blocking(self.imp_cls, self.base_account) + addon = get_link_addon_instance__blocking( + self.imp_cls, self.base_account, self.config + ) return async_to_sync(addon.build_url_for_id)(self.target_id) + @property + def config(self) -> LinkConfig: + return self.base_account.authorizedlinkaccount.config + class Meta: verbose_name = "Configured Link Addon" verbose_name_plural = "Configured Link Addons" diff --git a/addon_service/configured_addon/utils.py b/addon_service/configured_addon/utils.py index 0daad9df..ca4bc07a 100644 --- a/addon_service/configured_addon/utils.py +++ b/addon_service/configured_addon/utils.py @@ -4,6 +4,7 @@ from addon_toolkit.interfaces.citation import CitationAddonImp from addon_toolkit.interfaces.computing import ComputingAddonImp +from addon_toolkit.interfaces.link import LinkAddonImp from addon_toolkit.interfaces.storage import StorageAddonImp @@ -18,5 +19,6 @@ def get_config_for_addon(addon: ConfiguredAddon): return addon.configuredcitationaddon.config elif issubclass(addon.imp_cls, ComputingAddonImp): return addon.configuredcomputingaddon.config - + elif issubclass(addon.imp_cls, LinkAddonImp): + return addon.configuredlinkaddon.config raise ValueError(f"this function implementation does not support {addon.imp_cls}") diff --git a/addon_toolkit/interfaces/link.py b/addon_toolkit/interfaces/link.py index 82e203d2..25a23d37 100644 --- a/addon_toolkit/interfaces/link.py +++ b/addon_toolkit/interfaces/link.py @@ -71,6 +71,12 @@ class ItemResult: item_type: ItemType resource_type: SupportedResourceTypes | None = None item_link: str | None = None + doi: str | None = None + + +@dataclasses.dataclass(frozen=True) +class LinkConfig: + external_api_url: str @dataclasses.dataclass @@ -122,6 +128,7 @@ class LinkAddonImp(AddonImp): """base class for link addon implementations""" ADDON_INTERFACE = LinkAddonInterface + config: LinkConfig @dataclasses.dataclass From bc4a201c9d7591933c5cd04f16cb5bab865aaca4 Mon Sep 17 00:00:00 2001 From: Oleh Paduchak Date: Fri, 11 Apr 2025 15:30:45 +0300 Subject: [PATCH 23/65] cleaned dataverse implementation up --- addon_imps/link/dataverse.py | 64 +++++++++++++++++++++--------------- 1 file changed, 38 insertions(+), 26 deletions(-) diff --git a/addon_imps/link/dataverse.py b/addon_imps/link/dataverse.py index 308945d5..77c1f78b 100644 --- a/addon_imps/link/dataverse.py +++ b/addon_imps/link/dataverse.py @@ -15,7 +15,7 @@ DATAVERSE_REGEX = re.compile(r"^dataverse/(?P\d*)$") -DATASET_REGEX = re.compile(r"^dataset/(?P\d*)(?P.*)$") +DATASET_REGEX = re.compile(r"^dataset/(?P.*)$") FILE_REGEX = re.compile(r"^file/(?P.*)$") @@ -28,12 +28,12 @@ class DataverseLinkImp(LinkAddonHttpRequestorImp): async def build_url_for_id(self, item_id: str) -> str: match = DATASET_REGEX.match(item_id) - if not (persistent_id := match["persistent_id"]): - dataset = await self._fetch_dataset(match["id"]) - return dataset.item_link - return ( - f"{self.config.external_api_url}/dataset.xhtml?persistentId={persistent_id}" - ) + if match: + persistent_id = match["persistent_id"] + return f"{self.config.external_api_url}/dataset.xhtml?persistentId={persistent_id}" + elif match := FILE_REGEX.match(item_id): + persistent_id = match["persistent_id"] + return f"{self.config.external_api_url}/file.xhtml?persistentId={persistent_id}" async def get_external_account_id(self, _: dict[str, str]) -> str: try: @@ -57,7 +57,10 @@ async def list_root_items(self, page_cursor: str = "") -> ItemSampleResult: query=[ ["selected_page", page_cursor], *[("role_ids", role) for role in range(1, 9)], - ("dvobject_types", "Dataverse"), + ( + "dvobject_types", + "Dataverse", + ), # only published dataverses may contain published datasets ("published_states", "Published"), ], ) as response: @@ -114,12 +117,12 @@ async def _fetch_dataverse_items(self, dataverse_id) -> list[ItemResult]: response_content = await response.json_content() return await asyncio.gather( *[ - self.get_dataverse_or_dataset_item(item) + self._get_dataverse_or_dataset_item(item) for item in response_content["data"] ] ) - async def get_dataverse_or_dataset_item(self, item: dict): + async def _get_dataverse_or_dataset_item(self, item: dict): match item["type"]: case "dataset": return await self._fetch_dataset(dataset_id=item["id"]) @@ -137,21 +140,30 @@ async def _fetch_dataverse(self, dataverse_id) -> ItemResult: async with self.network.GET(f"api/dataverses/{dataverse_id}") as response: return parse_dataverse(await response.json_content()) - async def _fetch_dataset( - self, dataset_id: str = None, persistent_id: str = None - ) -> ItemResult: + async def _fetch_dataset_with_parser( + self, + dataset_id: str = None, + persistent_id: str = None, + parser=None, + ) -> ItemResult | list[ItemResult]: url = f"api/datasets/{':persistentId' if persistent_id else dataset_id}/versions/:latest-published" query = {"persistentId": persistent_id} if persistent_id else {} async with self.network.GET(url, query=query) as response: - return self._parse_dataset(await response.json_content()) + return parser(await response.json_content()) + + async def _fetch_dataset( + self, dataset_id: str = None, persistent_id: str = None + ) -> ItemResult: + return await self._fetch_dataset_with_parser( + dataset_id, persistent_id, parser=self._parse_dataset + ) async def _fetch_dataset_files( self, dataset_id: str = None, persistent_id: str = None ) -> list[ItemResult]: - url = f"api/datasets/{':persistentId' if persistent_id else dataset_id}/versions/:latest-published" - query = {"persistentId": persistent_id} if persistent_id else {} - async with self.network.GET(url, query=query) as response: - return self._parse_dataset_files(await response.json_content()) + return await self._fetch_dataset_with_parser( + dataset_id, persistent_id, parser=self._parse_dataset_files + ) def _parse_datafile(self, data: dict): if data.get("data"): @@ -165,6 +177,14 @@ def _parse_datafile(self, data: dict): doi=data["dataFile"]["persistentId"], ) + def _parse_dataset_files(self, data: dict) -> list[ItemResult]: + if data.get("data"): + data = data["data"] + try: + return [self._parse_datafile(file) for file in data["files"]] + except (KeyError, IndexError) as e: + raise ValueError(f"Invalid dataset response:{e=}") + def _parse_dataset(self, data: dict) -> ItemResult: if data.get("data"): data = data["data"] @@ -183,14 +203,6 @@ def _parse_dataset(self, data: dict) -> ItemResult: except (KeyError, IndexError) as e: raise ValueError(f"Invalid dataset response: {e=}") - def _parse_dataset_files(self, data: dict) -> list[ItemResult]: - if data.get("data"): - data = data["data"] - try: - return [self._parse_datafile(file) for file in data["files"]] - except (KeyError, IndexError) as e: - raise ValueError(f"Invalid dataset response:{e=}") - ### # module-local helpers From 793a22f6c06bab4c90ddd0cbe887f54a44960f84 Mon Sep 17 00:00:00 2001 From: Yuhuai Liu Date: Thu, 17 Apr 2025 10:22:12 -0400 Subject: [PATCH 24/65] CR follwup --- addon_service/common/osf.py | 22 ++++++++++++++----- .../test_configured_storage_addon.py | 4 ++-- .../test_by_type/test_resource_reference.py | 4 ++-- app/env.py | 1 - app/settings.py | 9 +------- docker-compose.yml | 1 - 6 files changed, 21 insertions(+), 20 deletions(-) diff --git a/addon_service/common/osf.py b/addon_service/common/osf.py index 2a017519..2669c965 100644 --- a/addon_service/common/osf.py +++ b/addon_service/common/osf.py @@ -54,12 +54,22 @@ async def get_osf_user_uri(request: django_http.HttpRequest) -> str | None: raise PermissionDenied(e) except hmac_utils.NotUsingHmac: pass # the only acceptable hmac-related error is not using hmac at all - # not hmac -- ask osf - _auth_headers = _get_osf_auth_headers(request) + + # if not hmac, check to see if it's in shared session + _user_referernce_uri = request.session.get["user_reference_uri"] + if _user_referernce_uri: + return _user_referernce_uri + + # if not in shared session, check to see if there is personal access token + _auth_headers = _osf_token_auth_headers(request) if not _auth_headers: return None - return request.session["user_reference_uri"] - + _client = await get_singleton_client_session() + async with _client.get(_osfapi_me_url(), headers=_auth_headers) as _response: + if HTTPStatus(_response.status).is_client_error: + return None + _response_content = await _response.json() + return _iri_from_osfapi_resource(_response_content["data"]) @async_to_sync async def has_osf_permission_on_resource( @@ -168,9 +178,9 @@ def _get_osf_auth_headers(request: django_http.HttpRequest) -> _HeaderList: def _osf_cookie_auth_headers(request: django_http.HttpRequest) -> _HeaderList: - _osf_user_cookie = request.COOKIES.get(settings.USER_REFERENCE_COOKIE) + _osf_user_cookie = request.COOKIES.get(settings.OSF_AUTH_COOKIE_NAME) return ( - [("Cookie", f"{settings.USER_REFERENCE_COOKIE}={_osf_user_cookie}")] + [("Cookie", f"{settings.OSF_AUTH_COOKIE_NAME}={_osf_user_cookie}")] if _osf_user_cookie else [] ) diff --git a/addon_service/tests/test_by_type/test_configured_storage_addon.py b/addon_service/tests/test_by_type/test_configured_storage_addon.py index ad4efbcd..115c97b2 100644 --- a/addon_service/tests/test_by_type/test_configured_storage_addon.py +++ b/addon_service/tests/test_by_type/test_configured_storage_addon.py @@ -26,7 +26,7 @@ def set_auth_header(self, auth_type): credentials = base64.b64encode(b"admin:password").decode() self.client.credentials(HTTP_AUTHORIZATION=f"Basic {credentials}") elif auth_type == "session": - self.client.cookies[settings.USER_REFERENCE_COOKIE] = "some auth" + self.client.cookies[settings.OSF_AUTH_COOKIE_NAME] = "some auth" elif auth_type == "no_auth": self.client.cookies.clear() self.client.credentials() @@ -210,7 +210,7 @@ def test_get_waterbutler_credentials(self): def test_get_waterbutler_credentials__error__no_headers(self): # credentials request requires HMAC-signed headers # Cookie + OSF-side permissions will not suffice - self.client.cookies[settings.USER_REFERENCE_COOKIE] = "some auth" + self.client.cookies[settings.OSF_AUTH_COOKIE_NAME] = "some auth" _mock_osf = MockOSF() _mock_osf.configure_user_role( self._user.user_uri, self._configured_storage_addon.resource_uri, "admin" diff --git a/addon_service/tests/test_by_type/test_resource_reference.py b/addon_service/tests/test_by_type/test_resource_reference.py index aa3c25ba..35fe21b5 100644 --- a/addon_service/tests/test_by_type/test_resource_reference.py +++ b/addon_service/tests/test_by_type/test_resource_reference.py @@ -201,7 +201,7 @@ def test_wrong_user__public_resource(self): ) _resp = self._view( get_test_request( - cookies={settings.USER_REFERENCE_COOKIE: "this is wrong user auth"} + cookies={settings.OSF_AUTH_COOKIE_NAME: "this is wrong user auth"} ), pk=self._resource.pk, ) @@ -246,7 +246,7 @@ def test_get_related__several(self): ) + [self._csa] _resp = self._related_view( get_test_request( - cookies={settings.USER_REFERENCE_COOKIE: self._user.user_uri} + cookies={settings.OSF_AUTH_COOKIE_NAME: self._user.user_uri} ), pk=self._resource.pk, related_field="configured_storage_addons", diff --git a/app/env.py b/app/env.py index 6ba78efd..c821233a 100644 --- a/app/env.py +++ b/app/env.py @@ -16,7 +16,6 @@ SECURE_PROXY_SSL_HEADER = os.environ.get("SECURE_PROXY_SSL_HEADER") NEW_RELIC_CONFIG_FILE = os.environ.get("NEW_RELIC_CONFIG_FILE") NEW_RELIC_ENVIRONMENT = os.environ.get("NEW_RELIC_ENVIRONMENT") -SESSION_COOKIE_DOMAIN = os.environ.get("SESSION_COOKIE_DOMAIN") ### diff --git a/app/settings.py b/app/settings.py index 9159ab53..814f2c82 100644 --- a/app/settings.py +++ b/app/settings.py @@ -62,11 +62,10 @@ DEBUG = env.DEBUG SILKY_PYTHON_PROFILER = env.SILKY_PYTHON_PROFILER -USER_REFERENCE_COOKIE = env.OSF_AUTH_COOKIE_NAME +OSF_AUTH_COOKIE_NAME = env.OSF_AUTH_COOKIE_NAME OSF_BASE_URL = env.OSF_BASE_URL.rstrip("/") OSF_API_BASE_URL = env.OSF_API_BASE_URL.rstrip("/") ALLOWED_RESOURCE_URI_PREFIXES = {OSF_BASE_URL} -# SESSION_COOKIE_DOMAIN = env.SESSION_COOKIE_DOMAIN SESSION_ENGINE = "django.contrib.sessions.backends.cache" SESSION_COOKIE_NAME = env.OSF_AUTH_COOKIE_NAME OSF_AUTH_COOKIE_SECRET = env.OSF_AUTH_COOKIE_SECRET @@ -76,12 +75,6 @@ "default": { "BACKEND": "django.core.cache.backends.redis.RedisCache", "LOCATION": REDIS_HOST, - # 'OPTIONS': { - # 'CLIENT_CLASS': 'django_redis.client.DefaultClient', - # 'CONNECTION_POOL_KWARGS': { - # 'max_connections': 100, - # }, - # }, }, } diff --git a/docker-compose.yml b/docker-compose.yml index 70bcfe26..2f2c5b4a 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -22,7 +22,6 @@ services: OSF_BASE_URL: "http://192.168.168.167:5000" OSF_API_BASE_URL: "http://192.168.168.167:8000" OSF_AUTH_COOKIE_NAME: osf - SESSION_COOKIE_DOMAIN: localhost AMQP_BROKER_URL: "amqp://guest:guest@rabbitmq:5672" GRAVYVALET_ENCRYPT_SECRET: "this is fine" ports: From 3dca7b826bbf499872fa25e5ce380fd1998cb608 Mon Sep 17 00:00:00 2001 From: Yuhuai Liu Date: Thu, 17 Apr 2025 10:55:42 -0400 Subject: [PATCH 25/65] fix linting --- addon_service/common/osf.py | 1 + 1 file changed, 1 insertion(+) diff --git a/addon_service/common/osf.py b/addon_service/common/osf.py index 2669c965..8650c982 100644 --- a/addon_service/common/osf.py +++ b/addon_service/common/osf.py @@ -71,6 +71,7 @@ async def get_osf_user_uri(request: django_http.HttpRequest) -> str | None: _response_content = await _response.json() return _iri_from_osfapi_resource(_response_content["data"]) + @async_to_sync async def has_osf_permission_on_resource( request: django_http.HttpRequest, From cbdba0b64c716bf771bf54344c7315515f1d8452 Mon Sep 17 00:00:00 2001 From: Oleh Paduchak Date: Fri, 18 Apr 2025 12:00:07 +0300 Subject: [PATCH 26/65] fixed link config to have both web url and api url --- addon_imps/link/dataverse.py | 35 ++++++++++++------- addon_service/addon_imp/instantiation.py | 2 +- .../authorized_account/link/models.py | 2 ++ .../authorized_account/link/serializers.py | 1 - addon_service/common/serializer_fields.py | 4 +++ addon_service/external_service/link/models.py | 6 ++++ ...allinkservice_browser_base_url_and_more.py | 21 +++++++++++ .../resource_reference/serializers.py | 5 +-- addon_toolkit/interfaces/link.py | 1 + 9 files changed, 60 insertions(+), 17 deletions(-) create mode 100644 addon_service/migrations/0013_externallinkservice_browser_base_url_and_more.py diff --git a/addon_imps/link/dataverse.py b/addon_imps/link/dataverse.py index 77c1f78b..9f6ee80d 100644 --- a/addon_imps/link/dataverse.py +++ b/addon_imps/link/dataverse.py @@ -3,6 +3,7 @@ import asyncio import re from dataclasses import dataclass +from typing import Literal from django.core.exceptions import ValidationError @@ -27,13 +28,19 @@ class DataverseLinkImp(LinkAddonHttpRequestorImp): """ async def build_url_for_id(self, item_id: str) -> str: - match = DATASET_REGEX.match(item_id) - if match: - persistent_id = match["persistent_id"] - return f"{self.config.external_api_url}/dataset.xhtml?persistentId={persistent_id}" + if match := DATASET_REGEX.match(item_id): + entity_type = "dataset" elif match := FILE_REGEX.match(item_id): - persistent_id = match["persistent_id"] - return f"{self.config.external_api_url}/file.xhtml?persistentId={persistent_id}" + entity_type = "file" + else: + raise ValidationError(f"Invalid {item_id=}") + + persistent_id = match["persistent_id"] + + return self._make_url(entity_type, persistent_id) + + def _make_url(self, entity_type: Literal["file", "dataset"], persistent_id): + return f"{self.config.external_web_url}/{entity_type}.xhtml?persistentId={persistent_id}" async def get_external_account_id(self, _: dict[str, str]) -> str: try: @@ -101,7 +108,7 @@ async def list_child_items( ) elif match := DATASET_REGEX.match(item_id): items = await self._fetch_dataset_files( - dataset_id=match["id"], persistent_id=match["persistent_id"] + persistent_id=match["persistent_id"] ) return ItemSampleResult( items=items, @@ -169,12 +176,13 @@ def _parse_datafile(self, data: dict): if data.get("data"): data = data["data"] + doi = data["dataFile"]["persistentId"] return ItemResult( - item_id=f"file/{data['dataFile']['persistentId']}", + item_id=f"file/{doi}", item_name=data["label"], item_type=ItemType.RESOURCE, - item_link=f'{self.config.external_api_url}/file.xhtml?persistentId={data['dataFile']["persistentId"]}', - doi=data["dataFile"]["persistentId"], + item_link=self._make_url("file", doi), + doi=doi, ) def _parse_dataset_files(self, data: dict) -> list[ItemResult]: @@ -189,16 +197,17 @@ def _parse_dataset(self, data: dict) -> ItemResult: if data.get("data"): data = data["data"] try: + doi = data["datasetPersistentId"] return ItemResult( - item_id=f'dataset/{data["datasetPersistentId"]}', + item_id=f"dataset/{doi}", item_name=[ item for item in data["metadataBlocks"]["citation"]["fields"] if item["typeName"] == "title" ][0]["value"], item_type=ItemType.FOLDER, - item_link=f'{self.config.external_api_url}/dataset.xhtml?persistentId={data["datasetPersistentId"]}', - doi=data["datasetPersistentId"], + item_link=self._make_url("dataset", doi), + doi=doi, ) except (KeyError, IndexError) as e: raise ValueError(f"Invalid dataset response: {e=}") diff --git a/addon_service/addon_imp/instantiation.py b/addon_service/addon_imp/instantiation.py index 5b32500d..cad416fa 100644 --- a/addon_service/addon_imp/instantiation.py +++ b/addon_service/addon_imp/instantiation.py @@ -156,7 +156,7 @@ async def get_link_addon_instance( imp = imp_cls( network=GravyvaletHttpRequestor( client_session=await get_singleton_client_session(), - prefix_url=account.external_service.api_base_url, + prefix_url=config.external_api_url, account=account, ), config=config, diff --git a/addon_service/authorized_account/link/models.py b/addon_service/authorized_account/link/models.py index addc6ac9..6ac6880a 100644 --- a/addon_service/authorized_account/link/models.py +++ b/addon_service/authorized_account/link/models.py @@ -28,4 +28,6 @@ def configured_link_addons(self): def config(self) -> LinkConfig: return LinkConfig( external_api_url=self.api_base_url, + external_web_url=self.external_service.externallinkservice.browser_base_url + or self.api_base_url, ) diff --git a/addon_service/authorized_account/link/serializers.py b/addon_service/authorized_account/link/serializers.py index 8de8350f..cc2fe709 100644 --- a/addon_service/authorized_account/link/serializers.py +++ b/addon_service/authorized_account/link/serializers.py @@ -32,7 +32,6 @@ class AuthorizedLinkAccountSerializer(AuthorizedAccountSerializer): ) configured_link_addons = HyperlinkedRelatedField( many=True, - source="configured_addons", queryset=ConfiguredLinkAddon.objects.active(), related_link_view_name=view_names.related_view(RESOURCE_TYPE), required=False, diff --git a/addon_service/common/serializer_fields.py b/addon_service/common/serializer_fields.py index 1d63488d..83638d0a 100644 --- a/addon_service/common/serializer_fields.py +++ b/addon_service/common/serializer_fields.py @@ -64,6 +64,8 @@ def to_representation(self, value): data["type"] = "authorized-storage-accounts" elif hasattr(value, "authorizedcomputingaccount"): data["type"] = "authorized-computing-accounts" + elif hasattr(value, "authorizedlinkaccount"): + data["type"] = "authorized-link-accounts" elif isinstance(value, ConfiguredAddon): if hasattr(value, "configuredcitationaddon"): data["type"] = "configured-citation-addons" @@ -71,4 +73,6 @@ def to_representation(self, value): data["type"] = "configured-storage-addons" elif hasattr(value, "configuredcomputingaddon"): data["type"] = "configured-computing-addons" + elif hasattr(value, "configuredlinkaddon"): + data["type"] = "configured-link-addons" return data diff --git a/addon_service/external_service/link/models.py b/addon_service/external_service/link/models.py index dd2bfa92..8ebd148d 100644 --- a/addon_service/external_service/link/models.py +++ b/addon_service/external_service/link/models.py @@ -1,3 +1,4 @@ +from django.core.exceptions import ValidationError from django.db import models from addon_service.authorized_account.link.models import AuthorizedLinkAccount @@ -14,6 +15,7 @@ def validate_supported_features(value): class ExternalLinkService(ExternalService): + browser_base_url = models.URLField(blank=True, default="") int_supported_resource_types = models.BigIntegerField( validators=[validate_supported_features], null=True ) @@ -33,6 +35,10 @@ def supported_resource_types( def clean(self): super().clean() validate_link_imp_number(self.int_addon_imp) + if not self.api_base_url and not self.browser_base_url: + raise ValidationError( + "External Link Service should provide at least one of `browser_base_url` or `api_base_url`" + ) @property def authorized_link_accounts(self): diff --git a/addon_service/migrations/0013_externallinkservice_browser_base_url_and_more.py b/addon_service/migrations/0013_externallinkservice_browser_base_url_and_more.py new file mode 100644 index 00000000..fbe304cb --- /dev/null +++ b/addon_service/migrations/0013_externallinkservice_browser_base_url_and_more.py @@ -0,0 +1,21 @@ +# Generated by Django 4.2.7 on 2025-04-17 16:44 + +from django.db import ( + migrations, + models, +) + + +class Migration(migrations.Migration): + + dependencies = [ + ("addon_service", "0012_authorizedlinkaccount_configuredlinkaddon_and_more"), + ] + + operations = [ + migrations.AddField( + model_name="externallinkservice", + name="browser_base_url", + field=models.URLField(blank=True, default=""), + ) + ] diff --git a/addon_service/resource_reference/serializers.py b/addon_service/resource_reference/serializers.py index eb1a79d3..48461d21 100644 --- a/addon_service/resource_reference/serializers.py +++ b/addon_service/resource_reference/serializers.py @@ -5,6 +5,7 @@ from addon_service.common import view_names from addon_service.configured_addon.citation.models import ConfiguredCitationAddon from addon_service.configured_addon.computing.models import ConfiguredComputingAddon +from addon_service.configured_addon.link.models import ConfiguredLinkAddon from addon_service.models import ( ConfiguredStorageAddon, ResourceReference, @@ -32,7 +33,7 @@ class ResourceReferenceSerializer(serializers.HyperlinkedModelSerializer): ) configured_link_addons = HyperlinkedRelatedField( many=True, - queryset=ConfiguredCitationAddon.objects.active(), + queryset=ConfiguredLinkAddon.objects.active(), related_link_view_name=view_names.related_view(RESOURCE_TYPE), ) configured_computing_addons = HyperlinkedRelatedField( @@ -49,7 +50,7 @@ class ResourceReferenceSerializer(serializers.HyperlinkedModelSerializer): "addon_service.serializers.ConfiguredCitationAddonSerializer" ), "configured_link_addons": ( - "addon_service.serializers.ConfiguredComputingAddonSerializer" + "addon_service.serializers.ConfiguredLinkAddonSerializer" ), "configured_computing_addons": ( "addon_service.serializers.ConfiguredComputingAddonSerializer" diff --git a/addon_toolkit/interfaces/link.py b/addon_toolkit/interfaces/link.py index 25a23d37..cfab8570 100644 --- a/addon_toolkit/interfaces/link.py +++ b/addon_toolkit/interfaces/link.py @@ -77,6 +77,7 @@ class ItemResult: @dataclasses.dataclass(frozen=True) class LinkConfig: external_api_url: str + external_web_url: str @dataclasses.dataclass From 9c7a55c7174ba06d5423504e0832ae2f103f0491 Mon Sep 17 00:00:00 2001 From: Oleh Paduchak Date: Tue, 22 Apr 2025 16:43:27 +0300 Subject: [PATCH 27/65] added PoC to get all verified links for given node --- .../configured_addon/link/serializers.py | 22 +++++++++++++++ addon_service/configured_addon/link/views.py | 27 ++++++++++++++++++- addon_service/configured_addon/views.py | 2 ++ 3 files changed, 50 insertions(+), 1 deletion(-) diff --git a/addon_service/configured_addon/link/serializers.py b/addon_service/configured_addon/link/serializers.py index fea9efc3..839d34b0 100644 --- a/addon_service/configured_addon/link/serializers.py +++ b/addon_service/configured_addon/link/serializers.py @@ -2,6 +2,7 @@ CharField, URLField, ) +from rest_framework_json_api import serializers from rest_framework_json_api.relations import ResourceRelatedField from rest_framework_json_api.utils import get_resource_type_from_model @@ -78,3 +79,24 @@ class Meta: "resource_type", "target_id", ] + + +class VerifiedLink(serializers.Serializer): + """Serialize ConfiguredStorageAddon information required by WaterButler. + + The returned data should share a shape with the existing `serialize_waterbutler_credentials` + and `serialize_waterbutler_settings` functions used by the OSF-based Addons. + + NB: The Boa addon needs credentials from GV, and this seems like a good short-term + place to hang it. Boa doesn't actually connect with WB, so this will probably be + refactored in the future. But for now, any configured_storage_addon could actually + be a configured_computed_addon. + """ + + class JSONAPIMeta: + resource_name = "verified-link" + + target_url = URLField(read_only=True) + target_id = CharField() + resource_type = EnumNameChoiceField(enum_cls=SupportedResourceTypes) + service_name = CharField(source="external_service.external_service_name") diff --git a/addon_service/configured_addon/link/views.py b/addon_service/configured_addon/link/views.py index ecadab51..fc64394c 100644 --- a/addon_service/configured_addon/link/views.py +++ b/addon_service/configured_addon/link/views.py @@ -1,7 +1,16 @@ +from http import HTTPMethod + +from rest_framework.decorators import action +from rest_framework.response import Response + from addon_service.configured_addon.views import ConfiguredAddonViewSet +from app.settings import ALLOWED_RESOURCE_URI_PREFIXES from .models import ConfiguredLinkAddon -from .serializers import ConfiguredLinkAddonSerializer +from .serializers import ( + ConfiguredLinkAddonSerializer, + VerifiedLink, +) class ConfiguredLinkAddonViewSet(ConfiguredAddonViewSet): @@ -12,3 +21,19 @@ class ConfiguredLinkAddonViewSet(ConfiguredAddonViewSet): "authorized_resource", ) serializer_class = ConfiguredLinkAddonSerializer + + @action( + detail=True, + methods=[HTTPMethod.GET], + url_name="verified-link", + url_path="verified-link", + ) + def verified_link(self, request, pk=None): + addons: ConfiguredLinkAddon = ConfiguredLinkAddon.objects.filter( + authorized_resource__resource_uri__in=[ + f"{prefix}/{pk}" for prefix in ALLOWED_RESOURCE_URI_PREFIXES + ], + ).select_related("base_account__external_service") + self.resource_name = "verified-link" + + return Response(VerifiedLink(addons, many=True).data) diff --git a/addon_service/configured_addon/views.py b/addon_service/configured_addon/views.py index fe1189c3..3b1cfaec 100644 --- a/addon_service/configured_addon/views.py +++ b/addon_service/configured_addon/views.py @@ -21,6 +21,8 @@ def get_permissions(self): return [IsAuthenticated(), SessionUserMayConnectAddon()] case "get_wb_credentials": return [IsValidHMACSignedRequest()] + case "verified_link": + return [IsAuthenticated()] case None: return super().get_permissions() case _: From 8bff195c58aea0ab1f64728152aa8cc0ccc8e102 Mon Sep 17 00:00:00 2001 From: Oleh Paduchak Date: Tue, 22 Apr 2025 16:45:20 +0300 Subject: [PATCH 28/65] fixed permissions --- addon_service/configured_addon/views.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/addon_service/configured_addon/views.py b/addon_service/configured_addon/views.py index 3b1cfaec..da331eea 100644 --- a/addon_service/configured_addon/views.py +++ b/addon_service/configured_addon/views.py @@ -19,10 +19,8 @@ def get_permissions(self): return [IsAuthenticated(), SessionUserIsOwnerOrResourceAdmin()] case "create": return [IsAuthenticated(), SessionUserMayConnectAddon()] - case "get_wb_credentials": + case "get_wb_credentials" | "verified_link": return [IsValidHMACSignedRequest()] - case "verified_link": - return [IsAuthenticated()] case None: return super().get_permissions() case _: From 44815121890bf6912a2035d35f0988d7ab2220dd Mon Sep 17 00:00:00 2001 From: Yuhuai Liu Date: Wed, 23 Apr 2025 00:26:48 -0400 Subject: [PATCH 29/65] fix test --- addon_service/common/osf.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/addon_service/common/osf.py b/addon_service/common/osf.py index 8650c982..aa77f610 100644 --- a/addon_service/common/osf.py +++ b/addon_service/common/osf.py @@ -56,7 +56,7 @@ async def get_osf_user_uri(request: django_http.HttpRequest) -> str | None: pass # the only acceptable hmac-related error is not using hmac at all # if not hmac, check to see if it's in shared session - _user_referernce_uri = request.session.get["user_reference_uri"] + _user_referernce_uri = request.session.get("user_reference_uri") if _user_referernce_uri: return _user_referernce_uri From ef532989620f8a28d53142482ff1cf46c52f081e Mon Sep 17 00:00:00 2001 From: Oleh Paduchak Date: Thu, 24 Apr 2025 13:25:03 +0300 Subject: [PATCH 30/65] fixed comments --- addon_service/configured_addon/link/serializers.py | 11 +---------- addon_service/configured_addon/link/views.py | 6 +++--- 2 files changed, 4 insertions(+), 13 deletions(-) diff --git a/addon_service/configured_addon/link/serializers.py b/addon_service/configured_addon/link/serializers.py index 839d34b0..ccb84cc1 100644 --- a/addon_service/configured_addon/link/serializers.py +++ b/addon_service/configured_addon/link/serializers.py @@ -82,16 +82,7 @@ class Meta: class VerifiedLink(serializers.Serializer): - """Serialize ConfiguredStorageAddon information required by WaterButler. - - The returned data should share a shape with the existing `serialize_waterbutler_credentials` - and `serialize_waterbutler_settings` functions used by the OSF-based Addons. - - NB: The Boa addon needs credentials from GV, and this seems like a good short-term - place to hang it. Boa doesn't actually connect with WB, so this will probably be - refactored in the future. But for now, any configured_storage_addon could actually - be a configured_computed_addon. - """ + """Serialize VerifiedLinks information required by OSF.""" class JSONAPIMeta: resource_name = "verified-link" diff --git a/addon_service/configured_addon/link/views.py b/addon_service/configured_addon/link/views.py index fc64394c..b5825479 100644 --- a/addon_service/configured_addon/link/views.py +++ b/addon_service/configured_addon/link/views.py @@ -25,10 +25,10 @@ class ConfiguredLinkAddonViewSet(ConfiguredAddonViewSet): @action( detail=True, methods=[HTTPMethod.GET], - url_name="verified-link", - url_path="verified-link", + url_name="verified-links", + url_path="verified-links", ) - def verified_link(self, request, pk=None): + def get_verified_links(self, request, pk=None): addons: ConfiguredLinkAddon = ConfiguredLinkAddon.objects.filter( authorized_resource__resource_uri__in=[ f"{prefix}/{pk}" for prefix in ALLOWED_RESOURCE_URI_PREFIXES From 0231a3690b936db20ca3087e2f389d92b7817962 Mon Sep 17 00:00:00 2001 From: Oleh Paduchak Date: Thu, 24 Apr 2025 13:29:26 +0300 Subject: [PATCH 31/65] fixed comment for verified link class --- addon_service/configured_addon/link/serializers.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/addon_service/configured_addon/link/serializers.py b/addon_service/configured_addon/link/serializers.py index ccb84cc1..7ffb4691 100644 --- a/addon_service/configured_addon/link/serializers.py +++ b/addon_service/configured_addon/link/serializers.py @@ -82,7 +82,11 @@ class Meta: class VerifiedLink(serializers.Serializer): - """Serialize VerifiedLinks information required by OSF.""" + """Serialize ConfiguredLinkAddon information required by OSF. + + The information is shaped for osf to be able to update datacite and share metadata + with minimal performance footprint + """ class JSONAPIMeta: resource_name = "verified-link" From e5ff0d16e3eb714c09ba3cb521c2b379627d0f5a Mon Sep 17 00:00:00 2001 From: Oleh Paduchak Date: Thu, 24 Apr 2025 15:51:29 +0300 Subject: [PATCH 32/65] fixed permissions --- addon_service/configured_addon/views.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/addon_service/configured_addon/views.py b/addon_service/configured_addon/views.py index da331eea..eb0bf0c4 100644 --- a/addon_service/configured_addon/views.py +++ b/addon_service/configured_addon/views.py @@ -19,7 +19,7 @@ def get_permissions(self): return [IsAuthenticated(), SessionUserIsOwnerOrResourceAdmin()] case "create": return [IsAuthenticated(), SessionUserMayConnectAddon()] - case "get_wb_credentials" | "verified_link": + case "get_wb_credentials" | "get_verified_links": return [IsValidHMACSignedRequest()] case None: return super().get_permissions() From 7d5f81b6704161492961babb712ea5140edfb67d Mon Sep 17 00:00:00 2001 From: Oleh Paduchak Date: Thu, 24 Apr 2025 16:16:28 +0300 Subject: [PATCH 33/65] implemented sending celery task when links change --- addon_service/common/regex.py | 4 ++++ addon_service/configured_addon/link/models.py | 9 ++++++++- addon_service/configured_addon/models.py | 4 ++-- addon_service/resource_reference/models.py | 8 ++++++++ addon_service/user_reference/models.py | 5 +++++ app/celery.py | 1 + 6 files changed, 28 insertions(+), 3 deletions(-) create mode 100644 addon_service/common/regex.py diff --git a/addon_service/common/regex.py b/addon_service/common/regex.py new file mode 100644 index 00000000..2dfe2127 --- /dev/null +++ b/addon_service/common/regex.py @@ -0,0 +1,4 @@ +import re + + +uri_regex = re.compile(r"http://[^/]+/(?P\w{5})/?") diff --git a/addon_service/configured_addon/link/models.py b/addon_service/configured_addon/link/models.py index 0b5a411a..2cbc0f51 100644 --- a/addon_service/configured_addon/link/models.py +++ b/addon_service/configured_addon/link/models.py @@ -7,6 +7,7 @@ LinkConfig, SupportedResourceTypes, ) +from app.celery import app def is_supported_resource_type(resource_type: int): @@ -19,7 +20,6 @@ def is_supported_resource_type(resource_type: int): class ConfiguredLinkAddon(ConfiguredAddon): - target_id = models.CharField() int_resource_type = models.BigIntegerField(validators=[is_supported_resource_type]) @@ -31,6 +31,13 @@ def resource_type(self) -> str: def resource_type(self, value) -> None: self.int_resource_type = value.value + def save(self, *args, full_clean=True, **kwargs): + super().save(*args, full_clean=full_clean, **kwargs) + app.send_task( + "website.identifiers.tasks.task__update_doi_metadata_with_verified_links", + kwargs={"target_guid": self.authorized_resource.guid}, + ) + def target_url(self): if self.target_id: from addon_service.addon_imp.instantiation import ( diff --git a/addon_service/configured_addon/models.py b/addon_service/configured_addon/models.py index ce91544c..be3727e5 100644 --- a/addon_service/configured_addon/models.py +++ b/addon_service/configured_addon/models.py @@ -96,8 +96,8 @@ def save(self, *args, full_clean=True, **kwargs): app.send_task( "osf.tasks.log_gv_addon", kwargs={ - "node_url": self.resource_uri, - "user_url": self.owner_uri, + "node_guid": self.authorized_resource.guid, + "user_guid": self.account_owner.guid, "addon": self.display_name, "action": "addon_added", }, diff --git a/addon_service/resource_reference/models.py b/addon_service/resource_reference/models.py index b83021f8..5e447bd2 100644 --- a/addon_service/resource_reference/models.py +++ b/addon_service/resource_reference/models.py @@ -2,6 +2,7 @@ from django.db.models.functions import Lower from addon_service.common.base_model import AddonsServiceBaseModel +from addon_service.common.regex import uri_regex from addon_service.configured_addon.citation.models import ConfiguredCitationAddon from addon_service.configured_addon.computing.models import ConfiguredComputingAddon from addon_service.configured_addon.link.models import ConfiguredLinkAddon @@ -53,6 +54,13 @@ def configured_computing_addons(self): "base_account__account_owner", ) + @property + def guid(self) -> str | None: + match = uri_regex.match(self.resource_uri) + if match: + return match["id"] + return None + class Meta: verbose_name = "Resource Reference" verbose_name_plural = "Resource References" diff --git a/addon_service/user_reference/models.py b/addon_service/user_reference/models.py index 3b2b39c4..ad189e98 100644 --- a/addon_service/user_reference/models.py +++ b/addon_service/user_reference/models.py @@ -6,6 +6,7 @@ from addon_service.authorized_account.link.models import AuthorizedLinkAccount from addon_service.authorized_account.storage.models import AuthorizedStorageAccount from addon_service.common.base_model import AddonsServiceBaseModel +from addon_service.common.regex import uri_regex from addon_service.configured_addon.computing.models import ConfiguredComputingAddon from addon_service.configured_addon.storage.models import ConfiguredStorageAddon from addon_service.resource_reference.models import ResourceReference @@ -74,6 +75,10 @@ class JSONAPIMeta: def owner_uri(self) -> str: return self.user_uri + @property + def guid(self) -> str: + return uri_regex.match(self.user_uri) + def deactivate(self): self.deactivated = timezone.now() self.save() diff --git a/app/celery.py b/app/celery.py index c51c63fe..c972607d 100644 --- a/app/celery.py +++ b/app/celery.py @@ -51,6 +51,7 @@ def queue_name(self) -> str: "queue": gv_chill_queue }, "osf.*": {"queue": osf_high_queue}, + "website.*": {"queue": osf_high_queue}, }, include=[ "addon_service.tasks", From e17fa44860ab00b22ea62a7cc0ad5375b9a56e71 Mon Sep 17 00:00:00 2001 From: Oleh Paduchak Date: Thu, 24 Apr 2025 16:18:28 +0300 Subject: [PATCH 34/65] fixed user guid --- addon_service/user_reference/models.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/addon_service/user_reference/models.py b/addon_service/user_reference/models.py index ad189e98..ef004e56 100644 --- a/addon_service/user_reference/models.py +++ b/addon_service/user_reference/models.py @@ -76,8 +76,11 @@ def owner_uri(self) -> str: return self.user_uri @property - def guid(self) -> str: - return uri_regex.match(self.user_uri) + def guid(self) -> str | None: + match = uri_regex.match(self.user_uri) + if match: + return match["id"] + return None def deactivate(self): self.deactivated = timezone.now() From bd48926a0ca96ebbb6b6acf53e003a9669200bfc Mon Sep 17 00:00:00 2001 From: Oleh Paduchak Date: Fri, 25 Apr 2025 17:13:04 +0300 Subject: [PATCH 35/65] fixed Resource types to be consistent with datacite --- addon_toolkit/interfaces/link.py | 69 +++++++++++++++++--------------- 1 file changed, 37 insertions(+), 32 deletions(-) diff --git a/addon_toolkit/interfaces/link.py b/addon_toolkit/interfaces/link.py index cfab8570..f55e48a3 100644 --- a/addon_toolkit/interfaces/link.py +++ b/addon_toolkit/interfaces/link.py @@ -30,38 +30,43 @@ class ItemType(enum.StrEnum): class SupportedResourceTypes(enum.Flag): - AUDIOVISUAL = enum.auto() - AWARD = enum.auto() - BOOK = enum.auto() - BOOK_CHAPTER = enum.auto() - COLLECTION = enum.auto() - COMPUTATIONAL_NOTEBOOK = enum.auto() - CONFERENCE_PAPER = enum.auto() - CONFERENCE_PROCEEDING = enum.auto() - DATA_PAPER = enum.auto() - DATASET = enum.auto() - DISSERTATION = enum.auto() - EVENT = enum.auto() - IMAGE = enum.auto() - INSTRUMENT = enum.auto() - INTERACTIVE_RESOURCE = enum.auto() - JOURNAL = enum.auto() - JOURNAL_ARTICLE = enum.auto() - MODEL = enum.auto() - OUTPUT_MANAGEMENT_PLAN = enum.auto() - PEER_REVIEW = enum.auto() - PHYSICAL_OBJECT = enum.auto() - PREPRINT = enum.auto() - PROJECT = enum.auto() - REPORT = enum.auto() - SERVICE = enum.auto() - SOFTWARE = enum.auto() - SOUND = enum.auto() - STANDARD = enum.auto() - STUDY_REGISTRATION = enum.auto() - TEXT = enum.auto() - WORKFLOW = enum.auto() - OTHER = enum.auto() + """ + Even though this is not conventional for python enums to have upper camel case, we opt for it to follow + https://datacite-metadata-schema.readthedocs.io/en/4.6/appendices/appendix-1/resourceTypeGeneral/ + """ + + Audiovisual = enum.auto() + Award = enum.auto() + Book = enum.auto() + BookChapter = enum.auto() + Collection = enum.auto() + ComputationalNotebook = enum.auto() + ConferencePaper = enum.auto() + ConferenceProceeding = enum.auto() + DataPaper = enum.auto() + Dataset = enum.auto() + Dissertation = enum.auto() + Event = enum.auto() + Image = enum.auto() + Instrument = enum.auto() + InteractiveResource = enum.auto() + Journal = enum.auto() + JournalArticle = enum.auto() + Model = enum.auto() + OutputManagement_plan = enum.auto() + PeerReview = enum.auto() + PhysicalObject = enum.auto() + Preprint = enum.auto() + Project = enum.auto() + Report = enum.auto() + Service = enum.auto() + Software = enum.auto() + Sound = enum.auto() + Standard = enum.auto() + StudyRegistration = enum.auto() + Text = enum.auto() + Workflow = enum.auto() + Other = enum.auto() @dataclasses.dataclass From 37db09c78842a3e3cf3e988d377f116c8b84fecb Mon Sep 17 00:00:00 2001 From: Oleh Paduchak Date: Fri, 25 Apr 2025 17:39:05 +0300 Subject: [PATCH 36/65] fixed access token auth for user-references --- addon_service/authentication.py | 1 + addon_service/common/permissions.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/addon_service/authentication.py b/addon_service/authentication.py index e9dbe28d..7d1d19ac 100644 --- a/addon_service/authentication.py +++ b/addon_service/authentication.py @@ -12,6 +12,7 @@ def authenticate(self, request: DrfRequest): _user_uri = osf.get_osf_user_uri(request) if _user_uri: UserReference.objects.get_or_create(user_uri=_user_uri) + request.user_uri = _user_uri return True, None return None # unauthenticated diff --git a/addon_service/common/permissions.py b/addon_service/common/permissions.py index 6bdf7e40..5bb14e3c 100644 --- a/addon_service/common/permissions.py +++ b/addon_service/common/permissions.py @@ -19,7 +19,7 @@ class SessionUserIsOwner(permissions.BasePermission): """for object permissions on objects with `owner_uri`""" def has_object_permission(self, request, view, obj): - session_user_uri = request.session.get("user_reference_uri") + session_user_uri = request.session.get("user_reference_uri", request.user_uri) if session_user_uri: return session_user_uri == obj.owner_uri return False From 06df870585ac98511961c7b0c9f56ac0c351f27c Mon Sep 17 00:00:00 2001 From: Oleh Paduchak Date: Fri, 25 Apr 2025 17:45:49 +0300 Subject: [PATCH 37/65] fixed tests --- addon_service/common/permissions.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/addon_service/common/permissions.py b/addon_service/common/permissions.py index 5bb14e3c..69b8e9d4 100644 --- a/addon_service/common/permissions.py +++ b/addon_service/common/permissions.py @@ -19,7 +19,9 @@ class SessionUserIsOwner(permissions.BasePermission): """for object permissions on objects with `owner_uri`""" def has_object_permission(self, request, view, obj): - session_user_uri = request.session.get("user_reference_uri", request.user_uri) + session_user_uri = request.session.get( + "user_reference_uri", getattr(request, "user_uri", None) + ) if session_user_uri: return session_user_uri == obj.owner_uri return False From 9cf0e9546cb82c9af15ab90cf22ef56049c489af Mon Sep 17 00:00:00 2001 From: Oleh Paduchak Date: Fri, 25 Apr 2025 18:08:27 +0300 Subject: [PATCH 38/65] fixed PAT for IsAuthenticated checks --- addon_service/common/permissions.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/addon_service/common/permissions.py b/addon_service/common/permissions.py index 69b8e9d4..3a0a3895 100644 --- a/addon_service/common/permissions.py +++ b/addon_service/common/permissions.py @@ -12,7 +12,12 @@ class IsAuthenticated(permissions.BasePermission): """allow any logged-in user""" def has_permission(self, request, view): - return request.session.get("user_reference_uri") is not None + return ( + request.session.get( + "user_reference_uri", getattr(request, "user_uri", None) + ) + is not None + ) class SessionUserIsOwner(permissions.BasePermission): From 2a44bb2022ca151776ef8838d2da2cd883d2eaaf Mon Sep 17 00:00:00 2001 From: Oleh Paduchak Date: Mon, 28 Apr 2025 18:18:19 +0300 Subject: [PATCH 39/65] final fix for PAT --- .../addon_operation_invocation/serializers.py | 6 ++---- .../authorized_account/serializers.py | 3 ++- addon_service/common/get_user_uri.py | 8 ++++++++ addon_service/common/osf.py | 3 ++- addon_service/common/permissions.py | 20 +++++++------------ addon_service/configured_addon/serializers.py | 5 ++--- 6 files changed, 23 insertions(+), 22 deletions(-) create mode 100644 addon_service/common/get_user_uri.py diff --git a/addon_service/addon_operation_invocation/serializers.py b/addon_service/addon_operation_invocation/serializers.py index 80be67af..d6f97d4e 100644 --- a/addon_service/addon_operation_invocation/serializers.py +++ b/addon_service/addon_operation_invocation/serializers.py @@ -9,6 +9,7 @@ ) from addon_service.common import view_names from addon_service.common.enum_serializers import EnumNameChoiceField +from addon_service.common.get_user_uri import get_user_uri from addon_service.common.invocation_status import InvocationStatus from addon_service.common.serializer_fields import ( CustomPolymorphicResourceRelatedField, @@ -113,10 +114,7 @@ def create(self, validated_data): _imp_cls = _thru_account.imp_cls _operation = _imp_cls.get_operation_declaration(_operation_name) _request = self.context["request"] - _user_uri = ( - _request.session.get("user_reference_uri") - or f"{settings.OSF_BASE_URL}/anonymous" - ) + _user_uri = get_user_uri(_request) or f"{settings.OSF_BASE_URL}/anonymous" _user, _ = UserReference.objects.get_or_create(user_uri=_user_uri) return AddonOperationInvocation( operation=AddonOperationModel(_imp_cls.ADDON_INTERFACE, _operation), diff --git a/addon_service/authorized_account/serializers.py b/addon_service/authorized_account/serializers.py index 6367a6d4..40b85c59 100644 --- a/addon_service/authorized_account/serializers.py +++ b/addon_service/authorized_account/serializers.py @@ -8,6 +8,7 @@ from addon_service.authorized_account.models import AuthorizedAccount from addon_service.common.credentials_formats import CredentialsFormats +from addon_service.common.get_user_uri import get_user_uri from addon_service.external_service.models import ExternalService from addon_service.osf_models.fields import encrypt_string from addon_service.serializer_fields import ( @@ -72,7 +73,7 @@ def create_authorized_account( api_base_url: str = "", **kwargs, ) -> AuthorizedAccount: - session_user_uri = self.context["request"].session.get("user_reference_uri") + session_user_uri = get_user_uri(self.context["request"]) account_owner, _ = UserReference.objects.get_or_create( user_uri=session_user_uri ) diff --git a/addon_service/common/get_user_uri.py b/addon_service/common/get_user_uri.py new file mode 100644 index 00000000..7c35add2 --- /dev/null +++ b/addon_service/common/get_user_uri.py @@ -0,0 +1,8 @@ +def get_user_uri(request) -> str | None: + """ + Helper function which returns user's uri + It supports both session auth and PAT, which must not store any info in session (because they may be revoked later) + """ + return request.session.get("user_reference_uri") or getattr( + request, "user_uri", None + ) diff --git a/addon_service/common/osf.py b/addon_service/common/osf.py index aa77f610..4104bff8 100644 --- a/addon_service/common/osf.py +++ b/addon_service/common/osf.py @@ -11,6 +11,7 @@ from addon_service.common import hmac as hmac_utils from addon_service.common.aiohttp_session import get_singleton_client_session +from addon_service.common.get_user_uri import get_user_uri from addon_toolkit import AddonCapabilities @@ -56,7 +57,7 @@ async def get_osf_user_uri(request: django_http.HttpRequest) -> str | None: pass # the only acceptable hmac-related error is not using hmac at all # if not hmac, check to see if it's in shared session - _user_referernce_uri = request.session.get("user_reference_uri") + _user_referernce_uri = get_user_uri(request) if _user_referernce_uri: return _user_referernce_uri diff --git a/addon_service/common/permissions.py b/addon_service/common/permissions.py index 3a0a3895..f9b4294c 100644 --- a/addon_service/common/permissions.py +++ b/addon_service/common/permissions.py @@ -6,27 +6,21 @@ from addon_service.common import hmac as hmac_utils from addon_service.common import osf +from addon_service.common.get_user_uri import get_user_uri class IsAuthenticated(permissions.BasePermission): """allow any logged-in user""" def has_permission(self, request, view): - return ( - request.session.get( - "user_reference_uri", getattr(request, "user_uri", None) - ) - is not None - ) + return get_user_uri(request) is not None class SessionUserIsOwner(permissions.BasePermission): """for object permissions on objects with `owner_uri`""" def has_object_permission(self, request, view, obj): - session_user_uri = request.session.get( - "user_reference_uri", getattr(request, "user_uri", None) - ) + session_user_uri = get_user_uri(request) if session_user_uri: return session_user_uri == obj.owner_uri return False @@ -38,7 +32,7 @@ class SessionUserIsOwnerOrResourceAdmin(permissions.BasePermission): message = "Permission denied, to perform this action you should be the one who created this addon or project admin" def has_object_permission(self, request, view, obj): - _user_uri = request.session.get("user_reference_uri") + _user_uri = get_user_uri(request) return (_user_uri == obj.owner_uri) or osf.has_osf_permission_on_resource( request, obj.resource_uri, @@ -61,7 +55,7 @@ class SessionUserMayConnectAddon(permissions.BasePermission): """for object permissions on objects with `owner_uri` and `resource_uri`""" def has_object_permission(self, request, view, obj): - _user_uri = request.session.get("user_reference_uri") + _user_uri = get_user_uri(request) return ( # must be account owner (_user_uri == obj.owner_uri) @@ -78,7 +72,7 @@ class SessionUserMayAccessInvocation(permissions.BasePermission): """for object permissions on `addon_service.models.AddonOperationInvocation`""" def has_object_permission(self, request, view, obj): - _user_uri = request.session.get("user_reference_uri") + _user_uri = get_user_uri(request) return bool( # must be the invoker: (_user_uri == obj.by_user.user_uri) @@ -97,7 +91,7 @@ class SessionUserMayPerformInvocation(permissions.BasePermission): """for object permissions on `addon_service.models.AddonOperationInvocation`""" def has_object_permission(self, request, view, obj): - _user_uri = request.session.get("user_reference_uri") + _user_uri = get_user_uri(request) _thru_addon = obj.thru_addon _thru_account = obj.thru_account if _thru_addon is None: diff --git a/addon_service/configured_addon/serializers.py b/addon_service/configured_addon/serializers.py index ce6d92ad..ded2328e 100644 --- a/addon_service/configured_addon/serializers.py +++ b/addon_service/configured_addon/serializers.py @@ -2,6 +2,7 @@ from rest_framework_json_api import serializers from addon_service.common.enum_serializers import EnumNameMultipleChoiceField +from addon_service.common.get_user_uri import get_user_uri from addon_service.configured_addon.models import ConfiguredAddon from addon_toolkit import AddonCapabilities @@ -33,9 +34,7 @@ def __init__(self, *args, **kwargs): current_user_is_owner = serializers.SerializerMethodField() def get_current_user_is_owner(self, configured_addon: ConfiguredAddon): - return configured_addon.owner_uri == self.context["request"].session.get( - "user_reference_uri" - ) + return configured_addon.owner_uri == get_user_uri(self.context["request"]) def create(self, validated_data): validated_data = self.fix_dotted_base_account(validated_data) From 2f2ef0e2960089667d035b51f2d6d767ee9cdc0f Mon Sep 17 00:00:00 2001 From: Longze Chen Date: Tue, 29 Apr 2025 13:03:36 -0400 Subject: [PATCH 40/65] Re-run lock and update poetry.lock --- poetry.lock | 81 ++++++++++++++++++++++++++--------------------------- 1 file changed, 39 insertions(+), 42 deletions(-) diff --git a/poetry.lock b/poetry.lock index 64548921..e4a9d38d 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 2.0.1 and should not be changed by hand. +# This file is automatically @generated by Poetry 2.1.2 and should not be changed by hand. [[package]] name = "aiohappyeyeballs" @@ -113,7 +113,7 @@ propcache = ">=0.2.0" yarl = ">=1.17.0,<2.0" [package.extras] -speedups = ["Brotli", "aiodns (>=3.2.0)", "brotlicffi"] +speedups = ["Brotli ; platform_python_implementation == \"CPython\"", "aiodns (>=3.2.0) ; sys_platform == \"linux\" or sys_platform == \"darwin\"", "brotlicffi ; platform_python_implementation != \"CPython\""] [[package]] name = "aiosignal" @@ -173,12 +173,12 @@ files = [ ] [package.extras] -benchmark = ["cloudpickle", "hypothesis", "mypy (>=1.11.1)", "pympler", "pytest (>=4.3.0)", "pytest-codspeed", "pytest-mypy-plugins", "pytest-xdist[psutil]"] -cov = ["cloudpickle", "coverage[toml] (>=5.3)", "hypothesis", "mypy (>=1.11.1)", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins", "pytest-xdist[psutil]"] -dev = ["cloudpickle", "hypothesis", "mypy (>=1.11.1)", "pre-commit", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins", "pytest-xdist[psutil]"] +benchmark = ["cloudpickle ; platform_python_implementation == \"CPython\"", "hypothesis", "mypy (>=1.11.1) ; platform_python_implementation == \"CPython\" and python_version >= \"3.9\"", "pympler", "pytest (>=4.3.0)", "pytest-codspeed", "pytest-mypy-plugins ; platform_python_implementation == \"CPython\" and python_version >= \"3.9\" and python_version < \"3.13\"", "pytest-xdist[psutil]"] +cov = ["cloudpickle ; platform_python_implementation == \"CPython\"", "coverage[toml] (>=5.3)", "hypothesis", "mypy (>=1.11.1) ; platform_python_implementation == \"CPython\" and python_version >= \"3.9\"", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins ; platform_python_implementation == \"CPython\" and python_version >= \"3.9\" and python_version < \"3.13\"", "pytest-xdist[psutil]"] +dev = ["cloudpickle ; platform_python_implementation == \"CPython\"", "hypothesis", "mypy (>=1.11.1) ; platform_python_implementation == \"CPython\" and python_version >= \"3.9\"", "pre-commit", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins ; platform_python_implementation == \"CPython\" and python_version >= \"3.9\" and python_version < \"3.13\"", "pytest-xdist[psutil]"] docs = ["cogapp", "furo", "myst-parser", "sphinx", "sphinx-notfound-page", "sphinxcontrib-towncrier", "towncrier (<24.7)"] -tests = ["cloudpickle", "hypothesis", "mypy (>=1.11.1)", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins", "pytest-xdist[psutil]"] -tests-mypy = ["mypy (>=1.11.1)", "pytest-mypy-plugins"] +tests = ["cloudpickle ; platform_python_implementation == \"CPython\"", "hypothesis", "mypy (>=1.11.1) ; platform_python_implementation == \"CPython\" and python_version >= \"3.9\"", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins ; platform_python_implementation == \"CPython\" and python_version >= \"3.9\" and python_version < \"3.13\"", "pytest-xdist[psutil]"] +tests-mypy = ["mypy (>=1.11.1) ; platform_python_implementation == \"CPython\" and python_version >= \"3.9\"", "pytest-mypy-plugins ; platform_python_implementation == \"CPython\" and python_version >= \"3.9\" and python_version < \"3.13\""] [[package]] name = "autobahn" @@ -199,13 +199,13 @@ setuptools = "*" txaio = ">=21.2.1" [package.extras] -all = ["PyGObject (>=3.40.0)", "argon2-cffi (>=20.1.0)", "attrs (>=20.3.0)", "base58 (>=2.1.0)", "bitarray (>=2.7.5)", "cbor2 (>=5.2.0)", "cffi (>=1.14.5)", "click (>=8.1.2)", "ecdsa (>=0.16.1)", "eth-abi (>=4.0.0)", "flatbuffers (>=22.12.6)", "hkdf (>=0.0.3)", "jinja2 (>=2.11.3)", "mnemonic (>=0.19)", "msgpack (>=1.0.2)", "passlib (>=1.7.4)", "py-ecc (>=5.1.0)", "py-eth-sig-utils (>=0.4.0)", "py-multihash (>=2.0.1)", "py-ubjson (>=0.16.1)", "pynacl (>=1.4.0)", "pyopenssl (>=20.0.1)", "python-snappy (>=0.6.0)", "pytrie (>=0.4.0)", "qrcode (>=7.3.1)", "rlp (>=2.0.1)", "service-identity (>=18.1.0)", "spake2 (>=0.8)", "twisted (>=20.3.0)", "twisted (>=24.3.0)", "u-msgpack-python (>=2.1)", "ujson (>=4.0.2)", "web3[ipfs] (>=6.0.0)", "xbr (>=21.2.1)", "yapf (==0.29.0)", "zlmdb (>=21.2.1)", "zope.interface (>=5.2.0)"] +all = ["PyGObject (>=3.40.0)", "argon2-cffi (>=20.1.0)", "attrs (>=20.3.0)", "base58 (>=2.1.0)", "bitarray (>=2.7.5)", "cbor2 (>=5.2.0)", "cffi (>=1.14.5)", "click (>=8.1.2)", "ecdsa (>=0.16.1)", "eth-abi (>=4.0.0)", "flatbuffers (>=22.12.6)", "hkdf (>=0.0.3)", "jinja2 (>=2.11.3)", "mnemonic (>=0.19)", "msgpack (>=1.0.2) ; platform_python_implementation == \"CPython\"", "passlib (>=1.7.4)", "py-ecc (>=5.1.0)", "py-eth-sig-utils (>=0.4.0)", "py-multihash (>=2.0.1)", "py-ubjson (>=0.16.1)", "pynacl (>=1.4.0)", "pyopenssl (>=20.0.1)", "python-snappy (>=0.6.0)", "pytrie (>=0.4.0)", "qrcode (>=7.3.1)", "rlp (>=2.0.1)", "service-identity (>=18.1.0)", "spake2 (>=0.8)", "twisted (>=20.3.0)", "twisted (>=24.3.0)", "u-msgpack-python (>=2.1) ; platform_python_implementation != \"CPython\"", "ujson (>=4.0.2) ; platform_python_implementation == \"CPython\"", "web3[ipfs] (>=6.0.0)", "xbr (>=21.2.1)", "yapf (==0.29.0)", "zlmdb (>=21.2.1)", "zope.interface (>=5.2.0)"] compress = ["python-snappy (>=0.6.0)"] -dev = ["backports.tempfile (>=1.0)", "build (>=1.2.1)", "bumpversion (>=0.5.3)", "codecov (>=2.0.15)", "flake8 (<5)", "humanize (>=0.5.1)", "mypy (>=0.610)", "passlib", "pep8-naming (>=0.3.3)", "pip (>=9.0.1)", "pyenchant (>=1.6.6)", "pyflakes (>=1.0.0)", "pyinstaller (>=4.2)", "pylint (>=1.9.2)", "pytest (>=3.4.2)", "pytest-aiohttp", "pytest-asyncio (>=0.14.0)", "pytest-runner (>=2.11.1)", "pyyaml (>=4.2b4)", "qualname", "sphinx (>=1.7.1)", "sphinx-autoapi (>=1.7.0)", "sphinx-rtd-theme (>=0.1.9)", "sphinxcontrib-images (>=0.9.1)", "tox (>=4.2.8)", "tox-gh-actions (>=2.2.0)", "twine (>=3.3.0)", "twisted (>=22.10.0)", "txaio (>=20.4.1)", "watchdog (>=0.8.3)", "wheel (>=0.36.2)", "yapf (==0.29.0)"] +dev = ["backports.tempfile (>=1.0)", "build (>=1.2.1)", "bumpversion (>=0.5.3)", "codecov (>=2.0.15)", "flake8 (<5)", "humanize (>=0.5.1)", "mypy (>=0.610) ; python_version >= \"3.4\" and platform_python_implementation != \"PyPy\"", "passlib", "pep8-naming (>=0.3.3)", "pip (>=9.0.1)", "pyenchant (>=1.6.6)", "pyflakes (>=1.0.0)", "pyinstaller (>=4.2)", "pylint (>=1.9.2)", "pytest (>=3.4.2)", "pytest-aiohttp", "pytest-asyncio (>=0.14.0)", "pytest-runner (>=2.11.1)", "pyyaml (>=4.2b4)", "qualname", "sphinx (>=1.7.1)", "sphinx-autoapi (>=1.7.0)", "sphinx-rtd-theme (>=0.1.9)", "sphinxcontrib-images (>=0.9.1)", "tox (>=4.2.8)", "tox-gh-actions (>=2.2.0)", "twine (>=3.3.0)", "twisted (>=22.10.0)", "txaio (>=20.4.1)", "watchdog (>=0.8.3)", "wheel (>=0.36.2)", "yapf (==0.29.0)"] encryption = ["pynacl (>=1.4.0)", "pyopenssl (>=20.0.1)", "pytrie (>=0.4.0)", "qrcode (>=7.3.1)", "service-identity (>=18.1.0)"] nvx = ["cffi (>=1.14.5)"] scram = ["argon2-cffi (>=20.1.0)", "cffi (>=1.14.5)", "passlib (>=1.7.4)"] -serialization = ["cbor2 (>=5.2.0)", "flatbuffers (>=22.12.6)", "msgpack (>=1.0.2)", "py-ubjson (>=0.16.1)", "u-msgpack-python (>=2.1)", "ujson (>=4.0.2)"] +serialization = ["cbor2 (>=5.2.0)", "flatbuffers (>=22.12.6)", "msgpack (>=1.0.2) ; platform_python_implementation == \"CPython\"", "py-ubjson (>=0.16.1)", "u-msgpack-python (>=2.1) ; platform_python_implementation != \"CPython\"", "ujson (>=4.0.2) ; platform_python_implementation == \"CPython\""] twisted = ["attrs (>=20.3.0)", "twisted (>=24.3.0)", "zope.interface (>=5.2.0)"] ui = ["PyGObject (>=3.40.0)"] xbr = ["base58 (>=2.1.0)", "bitarray (>=2.7.5)", "cbor2 (>=5.2.0)", "click (>=8.1.2)", "ecdsa (>=0.16.1)", "eth-abi (>=4.0.0)", "hkdf (>=0.0.3)", "jinja2 (>=2.11.3)", "mnemonic (>=0.19)", "py-ecc (>=5.1.0)", "py-eth-sig-utils (>=0.4.0)", "py-multihash (>=2.0.1)", "rlp (>=2.0.1)", "spake2 (>=0.8)", "twisted (>=20.3.0)", "web3[ipfs] (>=6.0.0)", "xbr (>=21.2.1)", "yapf (==0.29.0)", "zlmdb (>=21.2.1)"] @@ -330,33 +330,33 @@ vine = ">=5.1.0,<6.0" arangodb = ["pyArango (>=2.0.2)"] auth = ["cryptography (==44.0.2)"] azureblockblob = ["azure-identity (>=1.19.0)", "azure-storage-blob (>=12.15.0)"] -brotli = ["brotli (>=1.0.0)", "brotlipy (>=0.7.0)"] +brotli = ["brotli (>=1.0.0) ; platform_python_implementation == \"CPython\"", "brotlipy (>=0.7.0) ; platform_python_implementation == \"PyPy\""] cassandra = ["cassandra-driver (>=3.25.0,<4)"] consul = ["python-consul2 (==0.1.5)"] cosmosdbsql = ["pydocumentdb (==2.3.5)"] -couchbase = ["couchbase (>=3.0.0)"] +couchbase = ["couchbase (>=3.0.0) ; platform_python_implementation != \"PyPy\" and (platform_system != \"Windows\" or python_version < \"3.10\")"] couchdb = ["pycouchdb (==1.16.0)"] django = ["Django (>=2.2.28)"] dynamodb = ["boto3 (>=1.26.143)"] elasticsearch = ["elastic-transport (<=8.17.1)", "elasticsearch (<=8.17.2)"] -eventlet = ["eventlet (>=0.32.0)"] +eventlet = ["eventlet (>=0.32.0) ; python_version < \"3.10\""] gcs = ["google-cloud-firestore (==2.20.1)", "google-cloud-storage (>=2.10.0)", "grpcio (==1.67.0)"] gevent = ["gevent (>=1.5.0)"] -librabbitmq = ["librabbitmq (>=2.0.0)"] -memcache = ["pylibmc (==1.6.3)"] +librabbitmq = ["librabbitmq (>=2.0.0) ; python_version < \"3.11\""] +memcache = ["pylibmc (==1.6.3) ; platform_system != \"Windows\""] mongodb = ["pymongo (==4.10.1)"] msgpack = ["msgpack (==1.1.0)"] pydantic = ["pydantic (>=2.4)"] pymemcache = ["python-memcached (>=1.61)"] -pyro = ["pyro4 (==4.82)"] +pyro = ["pyro4 (==4.82) ; python_version < \"3.11\""] pytest = ["pytest-celery[all] (>=1.2.0,<1.3.0)"] redis = ["redis (>=4.5.2,!=4.5.5,<6.0.0)"] s3 = ["boto3 (>=1.26.143)"] slmq = ["softlayer_messaging (>=1.0.3)"] -solar = ["ephem (==4.2)"] +solar = ["ephem (==4.2) ; platform_python_implementation != \"PyPy\""] sqlalchemy = ["sqlalchemy (>=1.4.48,<2.1)"] sqs = ["boto3 (>=1.26.143)", "kombu[sqs] (>=5.3.4)", "urllib3 (>=1.26.16)"] -tblib = ["tblib (>=1.3.0)", "tblib (>=1.5.0)"] +tblib = ["tblib (>=1.3.0) ; python_version < \"3.8.0\"", "tblib (>=1.5.0) ; python_version >= \"3.8.0\""] yaml = ["PyYAML (>=3.10)"] zookeeper = ["kazoo (>=1.3.1)"] zstd = ["zstandard (==0.23.0)"] @@ -718,10 +718,10 @@ files = [ cffi = {version = ">=1.12", markers = "platform_python_implementation != \"PyPy\""} [package.extras] -docs = ["sphinx (>=5.3.0)", "sphinx-rtd-theme (>=3.0.0)"] +docs = ["sphinx (>=5.3.0)", "sphinx-rtd-theme (>=3.0.0) ; python_version >= \"3.8\""] docstest = ["pyenchant (>=3)", "readme-renderer (>=30.0)", "sphinxcontrib-spelling (>=7.3.1)"] -nox = ["nox (>=2024.4.15)", "nox[uv] (>=2024.3.2)"] -pep8test = ["check-sdist", "click (>=8.0.1)", "mypy (>=1.4)", "ruff (>=0.3.6)"] +nox = ["nox (>=2024.4.15)", "nox[uv] (>=2024.3.2) ; python_version >= \"3.8\""] +pep8test = ["check-sdist ; python_version >= \"3.8\"", "click (>=8.0.1)", "mypy (>=1.4)", "ruff (>=0.3.6)"] sdist = ["build (>=1.0.0)"] ssh = ["bcrypt (>=3.1.5)"] test = ["certifi (>=2024)", "cryptography-vectors (==44.0.1)", "pretend (>=0.7)", "pytest (>=7.4.0)", "pytest-benchmark (>=4.0)", "pytest-cov (>=2.10.1)", "pytest-xdist (>=3.5.0)"] @@ -950,7 +950,7 @@ files = [ [package.extras] docs = ["furo (>=2023.9.10)", "sphinx (>=7.2.6)", "sphinx-autodoc-typehints (>=1.25.2)"] testing = ["covdefaults (>=2.3)", "coverage (>=7.3.2)", "diff-cover (>=8.0.1)", "pytest (>=7.4.3)", "pytest-asyncio (>=0.21)", "pytest-cov (>=4.1)", "pytest-mock (>=3.12)", "pytest-timeout (>=2.2)", "virtualenv (>=20.26.2)"] -typing = ["typing-extensions (>=4.8)"] +typing = ["typing-extensions (>=4.8) ; python_version < \"3.11\""] [[package]] name = "flake8" @@ -1257,7 +1257,7 @@ azurestoragequeues = ["azure-identity (>=1.12.0)", "azure-storage-queue (>=12.6. confluentkafka = ["confluent-kafka (>=2.2.0)"] consul = ["python-consul2 (==0.1.5)"] gcpubsub = ["google-cloud-monitoring (>=2.16.0)", "google-cloud-pubsub (>=2.18.4)", "grpcio (==1.67.0)", "protobuf (==4.25.5)"] -librabbitmq = ["librabbitmq (>=2.0.0)"] +librabbitmq = ["librabbitmq (>=2.0.0) ; python_version < \"3.11\""] mongodb = ["pymongo (>=4.1.1)"] msgpack = ["msgpack (==1.1.0)"] pyro = ["pyro4 (==4.82)"] @@ -1703,8 +1703,8 @@ typing-extensions = {version = ">=4.6", markers = "python_version < \"3.13\""} tzdata = {version = "*", markers = "sys_platform == \"win32\""} [package.extras] -binary = ["psycopg-binary (==3.2.6)"] -c = ["psycopg-c (==3.2.6)"] +binary = ["psycopg-binary (==3.2.6) ; implementation_name != \"pypy\""] +c = ["psycopg-c (==3.2.6) ; implementation_name != \"pypy\""] dev = ["ast-comments (>=1.1.2)", "black (>=24.1.0)", "codespell (>=2.2)", "dnspython (>=2.1)", "flake8 (>=4.0)", "isort-psycopg", "isort[colors] (>=6.0)", "mypy (>=1.14)", "pre-commit (>=4.0.1)", "types-setuptools (>=57.4)", "wheel (>=0.37)"] docs = ["Sphinx (>=5.0)", "furo (==2022.6.21)", "sphinx-autobuild (>=2021.3.14)", "sphinx-autodoc-typehints (>=1.12)"] pool = ["psycopg-pool"] @@ -2080,7 +2080,7 @@ requests = ">=2.30.0,<3.0" urllib3 = ">=1.25.10,<3.0" [package.extras] -tests = ["coverage (>=6.0.0)", "flake8", "mypy", "pytest (>=7.0.0)", "pytest-asyncio", "pytest-cov", "pytest-httpserver", "tomli", "tomli-w", "types-PyYAML", "types-requests"] +tests = ["coverage (>=6.0.0)", "flake8", "mypy", "pytest (>=7.0.0)", "pytest-asyncio", "pytest-cov", "pytest-httpserver", "tomli ; python_version < \"3.11\"", "tomli-w", "types-PyYAML", "types-requests"] [[package]] name = "rpds-py" @@ -2302,9 +2302,9 @@ files = [ ] [package.extras] -core = ["importlib-metadata (>=6)", "importlib-resources (>=5.10.2)", "jaraco.text (>=3.7)", "more-itertools (>=8.8)", "ordered-set (>=3.1.1)", "packaging (>=24)", "platformdirs (>=2.6.2)", "tomli (>=2.0.1)", "wheel (>=0.43.0)"] +core = ["importlib-metadata (>=6) ; python_version < \"3.10\"", "importlib-resources (>=5.10.2) ; python_version < \"3.9\"", "jaraco.text (>=3.7)", "more-itertools (>=8.8)", "ordered-set (>=3.1.1)", "packaging (>=24)", "platformdirs (>=2.6.2)", "tomli (>=2.0.1) ; python_version < \"3.11\"", "wheel (>=0.43.0)"] doc = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "pygments-github-lexers (==0.0.5)", "pyproject-hooks (!=1.1)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-favicon", "sphinx-inline-tabs", "sphinx-lint", "sphinx-notfound-page (>=1,<2)", "sphinx-reredirects", "sphinxcontrib-towncrier", "towncrier (<24.7)"] -test = ["build[virtualenv] (>=1.0.3)", "filelock (>=3.4.0)", "importlib-metadata", "ini2toml[lite] (>=0.14)", "jaraco.develop (>=7.21)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "jaraco.test", "mypy (==1.11.*)", "packaging (>=23.2)", "pip (>=19.1)", "pyproject-hooks (!=1.1)", "pytest (>=6,!=8.1.*)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=2.2)", "pytest-home (>=0.5)", "pytest-mypy", "pytest-perf", "pytest-ruff (<0.4)", "pytest-ruff (>=0.2.1)", "pytest-ruff (>=0.3.2)", "pytest-subprocess", "pytest-timeout", "pytest-xdist (>=3)", "tomli", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel"] +test = ["build[virtualenv] (>=1.0.3)", "filelock (>=3.4.0)", "importlib-metadata", "ini2toml[lite] (>=0.14)", "jaraco.develop (>=7.21) ; python_version >= \"3.9\" and sys_platform != \"cygwin\"", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "jaraco.test", "mypy (==1.11.*)", "packaging (>=23.2)", "pip (>=19.1)", "pyproject-hooks (!=1.1)", "pytest (>=6,!=8.1.*)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=2.2)", "pytest-home (>=0.5)", "pytest-mypy", "pytest-perf ; sys_platform != \"cygwin\"", "pytest-ruff (<0.4) ; platform_system == \"Windows\"", "pytest-ruff (>=0.2.1) ; sys_platform != \"cygwin\"", "pytest-ruff (>=0.3.2) ; sys_platform != \"cygwin\"", "pytest-subprocess", "pytest-timeout", "pytest-xdist (>=3)", "tomli", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel"] [[package]] name = "six" @@ -2381,19 +2381,19 @@ typing-extensions = ">=4.2.0" zope-interface = ">=5" [package.extras] -all-non-platform = ["appdirs (>=1.4.0)", "appdirs (>=1.4.0)", "bcrypt (>=3.1.3)", "bcrypt (>=3.1.3)", "cryptography (>=3.3)", "cryptography (>=3.3)", "cython-test-exception-raiser (>=1.0.2,<2)", "cython-test-exception-raiser (>=1.0.2,<2)", "h2 (>=3.0,<5.0)", "h2 (>=3.0,<5.0)", "hypothesis (>=6.56)", "hypothesis (>=6.56)", "idna (>=2.4)", "idna (>=2.4)", "priority (>=1.1.0,<2.0)", "priority (>=1.1.0,<2.0)", "pyhamcrest (>=2)", "pyhamcrest (>=2)", "pyopenssl (>=21.0.0)", "pyopenssl (>=21.0.0)", "pyserial (>=3.0)", "pyserial (>=3.0)", "pywin32 (!=226)", "pywin32 (!=226)", "service-identity (>=18.1.0)", "service-identity (>=18.1.0)"] +all-non-platform = ["appdirs (>=1.4.0)", "appdirs (>=1.4.0)", "bcrypt (>=3.1.3)", "bcrypt (>=3.1.3)", "cryptography (>=3.3)", "cryptography (>=3.3)", "cython-test-exception-raiser (>=1.0.2,<2)", "cython-test-exception-raiser (>=1.0.2,<2)", "h2 (>=3.0,<5.0)", "h2 (>=3.0,<5.0)", "hypothesis (>=6.56)", "hypothesis (>=6.56)", "idna (>=2.4)", "idna (>=2.4)", "priority (>=1.1.0,<2.0)", "priority (>=1.1.0,<2.0)", "pyhamcrest (>=2)", "pyhamcrest (>=2)", "pyopenssl (>=21.0.0)", "pyopenssl (>=21.0.0)", "pyserial (>=3.0)", "pyserial (>=3.0)", "pywin32 (!=226) ; platform_system == \"Windows\"", "pywin32 (!=226) ; platform_system == \"Windows\"", "service-identity (>=18.1.0)", "service-identity (>=18.1.0)"] conch = ["appdirs (>=1.4.0)", "bcrypt (>=3.1.3)", "cryptography (>=3.3)"] dev = ["coverage (>=7.5,<8.0)", "cython-test-exception-raiser (>=1.0.2,<2)", "hypothesis (>=6.56)", "pydoctor (>=23.9.0,<23.10.0)", "pyflakes (>=2.2,<3.0)", "pyhamcrest (>=2)", "python-subunit (>=1.4,<2.0)", "sphinx (>=6,<7)", "sphinx-rtd-theme (>=1.3,<2.0)", "towncrier (>=23.6,<24.0)", "twistedchecker (>=0.7,<1.0)"] dev-release = ["pydoctor (>=23.9.0,<23.10.0)", "pydoctor (>=23.9.0,<23.10.0)", "sphinx (>=6,<7)", "sphinx (>=6,<7)", "sphinx-rtd-theme (>=1.3,<2.0)", "sphinx-rtd-theme (>=1.3,<2.0)", "towncrier (>=23.6,<24.0)", "towncrier (>=23.6,<24.0)"] -gtk-platform = ["appdirs (>=1.4.0)", "appdirs (>=1.4.0)", "bcrypt (>=3.1.3)", "bcrypt (>=3.1.3)", "cryptography (>=3.3)", "cryptography (>=3.3)", "cython-test-exception-raiser (>=1.0.2,<2)", "cython-test-exception-raiser (>=1.0.2,<2)", "h2 (>=3.0,<5.0)", "h2 (>=3.0,<5.0)", "hypothesis (>=6.56)", "hypothesis (>=6.56)", "idna (>=2.4)", "idna (>=2.4)", "priority (>=1.1.0,<2.0)", "priority (>=1.1.0,<2.0)", "pygobject", "pygobject", "pyhamcrest (>=2)", "pyhamcrest (>=2)", "pyopenssl (>=21.0.0)", "pyopenssl (>=21.0.0)", "pyserial (>=3.0)", "pyserial (>=3.0)", "pywin32 (!=226)", "pywin32 (!=226)", "service-identity (>=18.1.0)", "service-identity (>=18.1.0)"] +gtk-platform = ["appdirs (>=1.4.0)", "appdirs (>=1.4.0)", "bcrypt (>=3.1.3)", "bcrypt (>=3.1.3)", "cryptography (>=3.3)", "cryptography (>=3.3)", "cython-test-exception-raiser (>=1.0.2,<2)", "cython-test-exception-raiser (>=1.0.2,<2)", "h2 (>=3.0,<5.0)", "h2 (>=3.0,<5.0)", "hypothesis (>=6.56)", "hypothesis (>=6.56)", "idna (>=2.4)", "idna (>=2.4)", "priority (>=1.1.0,<2.0)", "priority (>=1.1.0,<2.0)", "pygobject", "pygobject", "pyhamcrest (>=2)", "pyhamcrest (>=2)", "pyopenssl (>=21.0.0)", "pyopenssl (>=21.0.0)", "pyserial (>=3.0)", "pyserial (>=3.0)", "pywin32 (!=226) ; platform_system == \"Windows\"", "pywin32 (!=226) ; platform_system == \"Windows\"", "service-identity (>=18.1.0)", "service-identity (>=18.1.0)"] http2 = ["h2 (>=3.0,<5.0)", "priority (>=1.1.0,<2.0)"] -macos-platform = ["appdirs (>=1.4.0)", "appdirs (>=1.4.0)", "bcrypt (>=3.1.3)", "bcrypt (>=3.1.3)", "cryptography (>=3.3)", "cryptography (>=3.3)", "cython-test-exception-raiser (>=1.0.2,<2)", "cython-test-exception-raiser (>=1.0.2,<2)", "h2 (>=3.0,<5.0)", "h2 (>=3.0,<5.0)", "hypothesis (>=6.56)", "hypothesis (>=6.56)", "idna (>=2.4)", "idna (>=2.4)", "priority (>=1.1.0,<2.0)", "priority (>=1.1.0,<2.0)", "pyhamcrest (>=2)", "pyhamcrest (>=2)", "pyobjc-core", "pyobjc-core", "pyobjc-framework-cfnetwork", "pyobjc-framework-cfnetwork", "pyobjc-framework-cocoa", "pyobjc-framework-cocoa", "pyopenssl (>=21.0.0)", "pyopenssl (>=21.0.0)", "pyserial (>=3.0)", "pyserial (>=3.0)", "pywin32 (!=226)", "pywin32 (!=226)", "service-identity (>=18.1.0)", "service-identity (>=18.1.0)"] -mypy = ["appdirs (>=1.4.0)", "bcrypt (>=3.1.3)", "coverage (>=7.5,<8.0)", "cryptography (>=3.3)", "cython-test-exception-raiser (>=1.0.2,<2)", "h2 (>=3.0,<5.0)", "hypothesis (>=6.56)", "idna (>=2.4)", "mypy (>=1.8,<2.0)", "mypy-zope (>=1.0.3,<1.1.0)", "priority (>=1.1.0,<2.0)", "pydoctor (>=23.9.0,<23.10.0)", "pyflakes (>=2.2,<3.0)", "pyhamcrest (>=2)", "pyopenssl (>=21.0.0)", "pyserial (>=3.0)", "python-subunit (>=1.4,<2.0)", "pywin32 (!=226)", "service-identity (>=18.1.0)", "sphinx (>=6,<7)", "sphinx-rtd-theme (>=1.3,<2.0)", "towncrier (>=23.6,<24.0)", "twistedchecker (>=0.7,<1.0)", "types-pyopenssl", "types-setuptools"] -osx-platform = ["appdirs (>=1.4.0)", "appdirs (>=1.4.0)", "bcrypt (>=3.1.3)", "bcrypt (>=3.1.3)", "cryptography (>=3.3)", "cryptography (>=3.3)", "cython-test-exception-raiser (>=1.0.2,<2)", "cython-test-exception-raiser (>=1.0.2,<2)", "h2 (>=3.0,<5.0)", "h2 (>=3.0,<5.0)", "hypothesis (>=6.56)", "hypothesis (>=6.56)", "idna (>=2.4)", "idna (>=2.4)", "priority (>=1.1.0,<2.0)", "priority (>=1.1.0,<2.0)", "pyhamcrest (>=2)", "pyhamcrest (>=2)", "pyobjc-core", "pyobjc-core", "pyobjc-framework-cfnetwork", "pyobjc-framework-cfnetwork", "pyobjc-framework-cocoa", "pyobjc-framework-cocoa", "pyopenssl (>=21.0.0)", "pyopenssl (>=21.0.0)", "pyserial (>=3.0)", "pyserial (>=3.0)", "pywin32 (!=226)", "pywin32 (!=226)", "service-identity (>=18.1.0)", "service-identity (>=18.1.0)"] -serial = ["pyserial (>=3.0)", "pywin32 (!=226)"] +macos-platform = ["appdirs (>=1.4.0)", "appdirs (>=1.4.0)", "bcrypt (>=3.1.3)", "bcrypt (>=3.1.3)", "cryptography (>=3.3)", "cryptography (>=3.3)", "cython-test-exception-raiser (>=1.0.2,<2)", "cython-test-exception-raiser (>=1.0.2,<2)", "h2 (>=3.0,<5.0)", "h2 (>=3.0,<5.0)", "hypothesis (>=6.56)", "hypothesis (>=6.56)", "idna (>=2.4)", "idna (>=2.4)", "priority (>=1.1.0,<2.0)", "priority (>=1.1.0,<2.0)", "pyhamcrest (>=2)", "pyhamcrest (>=2)", "pyobjc-core", "pyobjc-core", "pyobjc-framework-cfnetwork", "pyobjc-framework-cfnetwork", "pyobjc-framework-cocoa", "pyobjc-framework-cocoa", "pyopenssl (>=21.0.0)", "pyopenssl (>=21.0.0)", "pyserial (>=3.0)", "pyserial (>=3.0)", "pywin32 (!=226) ; platform_system == \"Windows\"", "pywin32 (!=226) ; platform_system == \"Windows\"", "service-identity (>=18.1.0)", "service-identity (>=18.1.0)"] +mypy = ["appdirs (>=1.4.0)", "bcrypt (>=3.1.3)", "coverage (>=7.5,<8.0)", "cryptography (>=3.3)", "cython-test-exception-raiser (>=1.0.2,<2)", "h2 (>=3.0,<5.0)", "hypothesis (>=6.56)", "idna (>=2.4)", "mypy (>=1.8,<2.0)", "mypy-zope (>=1.0.3,<1.1.0)", "priority (>=1.1.0,<2.0)", "pydoctor (>=23.9.0,<23.10.0)", "pyflakes (>=2.2,<3.0)", "pyhamcrest (>=2)", "pyopenssl (>=21.0.0)", "pyserial (>=3.0)", "python-subunit (>=1.4,<2.0)", "pywin32 (!=226) ; platform_system == \"Windows\"", "service-identity (>=18.1.0)", "sphinx (>=6,<7)", "sphinx-rtd-theme (>=1.3,<2.0)", "towncrier (>=23.6,<24.0)", "twistedchecker (>=0.7,<1.0)", "types-pyopenssl", "types-setuptools"] +osx-platform = ["appdirs (>=1.4.0)", "appdirs (>=1.4.0)", "bcrypt (>=3.1.3)", "bcrypt (>=3.1.3)", "cryptography (>=3.3)", "cryptography (>=3.3)", "cython-test-exception-raiser (>=1.0.2,<2)", "cython-test-exception-raiser (>=1.0.2,<2)", "h2 (>=3.0,<5.0)", "h2 (>=3.0,<5.0)", "hypothesis (>=6.56)", "hypothesis (>=6.56)", "idna (>=2.4)", "idna (>=2.4)", "priority (>=1.1.0,<2.0)", "priority (>=1.1.0,<2.0)", "pyhamcrest (>=2)", "pyhamcrest (>=2)", "pyobjc-core", "pyobjc-core", "pyobjc-framework-cfnetwork", "pyobjc-framework-cfnetwork", "pyobjc-framework-cocoa", "pyobjc-framework-cocoa", "pyopenssl (>=21.0.0)", "pyopenssl (>=21.0.0)", "pyserial (>=3.0)", "pyserial (>=3.0)", "pywin32 (!=226) ; platform_system == \"Windows\"", "pywin32 (!=226) ; platform_system == \"Windows\"", "service-identity (>=18.1.0)", "service-identity (>=18.1.0)"] +serial = ["pyserial (>=3.0)", "pywin32 (!=226) ; platform_system == \"Windows\""] test = ["cython-test-exception-raiser (>=1.0.2,<2)", "hypothesis (>=6.56)", "pyhamcrest (>=2)"] tls = ["idna (>=2.4)", "pyopenssl (>=21.0.0)", "service-identity (>=18.1.0)"] -windows-platform = ["appdirs (>=1.4.0)", "appdirs (>=1.4.0)", "bcrypt (>=3.1.3)", "bcrypt (>=3.1.3)", "cryptography (>=3.3)", "cryptography (>=3.3)", "cython-test-exception-raiser (>=1.0.2,<2)", "cython-test-exception-raiser (>=1.0.2,<2)", "h2 (>=3.0,<5.0)", "h2 (>=3.0,<5.0)", "hypothesis (>=6.56)", "hypothesis (>=6.56)", "idna (>=2.4)", "idna (>=2.4)", "priority (>=1.1.0,<2.0)", "priority (>=1.1.0,<2.0)", "pyhamcrest (>=2)", "pyhamcrest (>=2)", "pyopenssl (>=21.0.0)", "pyopenssl (>=21.0.0)", "pyserial (>=3.0)", "pyserial (>=3.0)", "pywin32 (!=226)", "pywin32 (!=226)", "pywin32 (!=226)", "pywin32 (!=226)", "service-identity (>=18.1.0)", "service-identity (>=18.1.0)", "twisted-iocpsupport (>=1.0.2)", "twisted-iocpsupport (>=1.0.2)"] +windows-platform = ["appdirs (>=1.4.0)", "appdirs (>=1.4.0)", "bcrypt (>=3.1.3)", "bcrypt (>=3.1.3)", "cryptography (>=3.3)", "cryptography (>=3.3)", "cython-test-exception-raiser (>=1.0.2,<2)", "cython-test-exception-raiser (>=1.0.2,<2)", "h2 (>=3.0,<5.0)", "h2 (>=3.0,<5.0)", "hypothesis (>=6.56)", "hypothesis (>=6.56)", "idna (>=2.4)", "idna (>=2.4)", "priority (>=1.1.0,<2.0)", "priority (>=1.1.0,<2.0)", "pyhamcrest (>=2)", "pyhamcrest (>=2)", "pyopenssl (>=21.0.0)", "pyopenssl (>=21.0.0)", "pyserial (>=3.0)", "pyserial (>=3.0)", "pywin32 (!=226)", "pywin32 (!=226)", "pywin32 (!=226) ; platform_system == \"Windows\"", "pywin32 (!=226) ; platform_system == \"Windows\"", "service-identity (>=18.1.0)", "service-identity (>=18.1.0)", "twisted-iocpsupport (>=1.0.2)", "twisted-iocpsupport (>=1.0.2)"] [[package]] name = "txaio" @@ -2423,7 +2423,7 @@ files = [ {file = "typing_extensions-4.12.2-py3-none-any.whl", hash = "sha256:04e5ca0351e0f3f85c6853954072df659d0d13fac324d0072316b67d7794700d"}, {file = "typing_extensions-4.12.2.tar.gz", hash = "sha256:1a7ead55c7e559dd4dee8856e3a88b41225abfe1ce8df57b7c13915fe121ffb8"}, ] -markers = {dev = "python_version < \"3.13\"", release = "python_version < \"3.13\""} +markers = {dev = "python_version == \"3.12\"", release = "python_version == \"3.12\""} [[package]] name = "tzdata" @@ -2451,7 +2451,7 @@ files = [ ] [package.extras] -brotli = ["brotli (>=1.0.9)", "brotlicffi (>=0.8.0)"] +brotli = ["brotli (>=1.0.9) ; platform_python_implementation == \"CPython\"", "brotlicffi (>=0.8.0) ; platform_python_implementation != \"CPython\""] h2 = ["h2 (>=4,<5)"] socks = ["pysocks (>=1.5.6,!=1.5.7,<2.0)"] zstd = ["zstandard (>=0.18.0)"] @@ -2487,7 +2487,7 @@ platformdirs = ">=3.9.1,<5" [package.extras] docs = ["furo (>=2023.7.26)", "proselint (>=0.13)", "sphinx (>=7.1.2,!=7.3)", "sphinx-argparse (>=0.4)", "sphinxcontrib-towncrier (>=0.2.1a0)", "towncrier (>=23.6)"] -test = ["covdefaults (>=2.3)", "coverage (>=7.2.7)", "coverage-enable-subprocess (>=1)", "flaky (>=3.7)", "packaging (>=23.1)", "pytest (>=7.4)", "pytest-env (>=0.8.2)", "pytest-freezer (>=0.4.8)", "pytest-mock (>=3.11.1)", "pytest-randomly (>=3.12)", "pytest-timeout (>=2.1)", "setuptools (>=68)", "time-machine (>=2.10)"] +test = ["covdefaults (>=2.3)", "coverage (>=7.2.7)", "coverage-enable-subprocess (>=1)", "flaky (>=3.7)", "packaging (>=23.1)", "pytest (>=7.4)", "pytest-env (>=0.8.2)", "pytest-freezer (>=0.4.8) ; platform_python_implementation == \"PyPy\" or platform_python_implementation == \"CPython\" and sys_platform == \"win32\" and python_version >= \"3.13\"", "pytest-mock (>=3.11.1)", "pytest-randomly (>=3.12)", "pytest-timeout (>=2.1)", "setuptools (>=68)", "time-machine (>=2.10) ; platform_python_implementation == \"CPython\""] [[package]] name = "wcwidth" @@ -2653,7 +2653,4 @@ testing = ["coverage (>=5.0.3)", "zope.event", "zope.testing"] [metadata] lock-version = "2.1" python-versions = "^3.12" -# feature/verified-resource-linking -content-hash = "9f3d39b29f00131619e3d005ac8687d5c6189ef2b3e95dc7c3adb18714202b27" -# upstream/develop -# content-hash = "580d926a21f66415a86daa4f9442ffd47d0bf7a0bc2446948ece4349b53db600" +content-hash = "bc1ceda74871d00d3d7b3777b6ba9edfb18c1a3af3497451a2800815818e272a" From 92fd2c02011364d0e5e27707f9cbec9178847bbb Mon Sep 17 00:00:00 2001 From: Longze Chen Date: Wed, 30 Apr 2025 09:02:31 -0400 Subject: [PATCH 41/65] Fix migration conflicts from develop --- ...allinkservice_browser_base_url_and_more.py | 21 ------------------- ...nkaccount_configuredlinkaddon_and_more.py} | 5 +++-- 2 files changed, 3 insertions(+), 23 deletions(-) delete mode 100644 addon_service/migrations/0013_externallinkservice_browser_base_url_and_more.py rename addon_service/migrations/{0012_authorizedlinkaccount_configuredlinkaddon_and_more.py => 0014_authorizedlinkaccount_configuredlinkaddon_and_more.py} (93%) diff --git a/addon_service/migrations/0013_externallinkservice_browser_base_url_and_more.py b/addon_service/migrations/0013_externallinkservice_browser_base_url_and_more.py deleted file mode 100644 index fbe304cb..00000000 --- a/addon_service/migrations/0013_externallinkservice_browser_base_url_and_more.py +++ /dev/null @@ -1,21 +0,0 @@ -# Generated by Django 4.2.7 on 2025-04-17 16:44 - -from django.db import ( - migrations, - models, -) - - -class Migration(migrations.Migration): - - dependencies = [ - ("addon_service", "0012_authorizedlinkaccount_configuredlinkaddon_and_more"), - ] - - operations = [ - migrations.AddField( - model_name="externallinkservice", - name="browser_base_url", - field=models.URLField(blank=True, default=""), - ) - ] diff --git a/addon_service/migrations/0012_authorizedlinkaccount_configuredlinkaddon_and_more.py b/addon_service/migrations/0014_authorizedlinkaccount_configuredlinkaddon_and_more.py similarity index 93% rename from addon_service/migrations/0012_authorizedlinkaccount_configuredlinkaddon_and_more.py rename to addon_service/migrations/0014_authorizedlinkaccount_configuredlinkaddon_and_more.py index 87df7bb6..a8ff71fe 100644 --- a/addon_service/migrations/0012_authorizedlinkaccount_configuredlinkaddon_and_more.py +++ b/addon_service/migrations/0014_authorizedlinkaccount_configuredlinkaddon_and_more.py @@ -1,4 +1,4 @@ -# Generated by Django 4.2.7 on 2025-04-04 12:12 +# Generated by Django 4.2.7 on 2025-04-30 12:43 import django.db.models.deletion from django.db import ( @@ -13,7 +13,7 @@ class Migration(migrations.Migration): dependencies = [ - ("addon_service", "0011_oauth2tokenmetadata_date_last_refreshed"), + ("addon_service", "0013_externalcredentials_int_credentials_format_and_more"), ] operations = [ @@ -82,6 +82,7 @@ class Migration(migrations.Migration): to="addon_service.externalservice", ), ), + ("browser_base_url", models.URLField(blank=True, default="")), ( "int_supported_resource_types", models.BigIntegerField( From 75a14ba36e49841e9be0c26afde3dd487c8020b6 Mon Sep 17 00:00:00 2001 From: Longze Chen Date: Wed, 30 Apr 2025 12:06:37 -0400 Subject: [PATCH 42/65] Fix resource_type and enum serializer --- addon_service/common/enum_serializers.py | 4 +--- addon_service/configured_addon/link/models.py | 4 ++-- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/addon_service/common/enum_serializers.py b/addon_service/common/enum_serializers.py index 6ea1453a..24d9bfdd 100644 --- a/addon_service/common/enum_serializers.py +++ b/addon_service/common/enum_serializers.py @@ -41,9 +41,7 @@ def to_internal_value(self, data) -> enum.Enum: return self.enum_cls[_name] def to_representation(self, value: enum.Enum): - if isinstance(value, str): - value = self.enum_cls[value] - return super().to_representation(value).name + return value.name class EnumNameMultipleChoiceField( diff --git a/addon_service/configured_addon/link/models.py b/addon_service/configured_addon/link/models.py index 2cbc0f51..40564aee 100644 --- a/addon_service/configured_addon/link/models.py +++ b/addon_service/configured_addon/link/models.py @@ -24,8 +24,8 @@ class ConfiguredLinkAddon(ConfiguredAddon): int_resource_type = models.BigIntegerField(validators=[is_supported_resource_type]) @property - def resource_type(self) -> str: - return SupportedResourceTypes(self.int_resource_type).name + def resource_type(self) -> SupportedResourceTypes: + return SupportedResourceTypes(self.int_resource_type) @resource_type.setter def resource_type(self, value) -> None: From 3af8f7f25f06547dcf5a232099ccaad9243871c8 Mon Sep 17 00:00:00 2001 From: Longze Chen Date: Mon, 5 May 2025 16:23:34 -0400 Subject: [PATCH 43/65] Both target_id and resource_type can be blank or null --- addon_service/configured_addon/link/models.py | 40 +++++++++++-------- ...redlinkaddon_int_resource_type_and_more.py | 35 ++++++++++++++++ 2 files changed, 59 insertions(+), 16 deletions(-) create mode 100644 addon_service/migrations/0015_alter_configuredlinkaddon_int_resource_type_and_more.py diff --git a/addon_service/configured_addon/link/models.py b/addon_service/configured_addon/link/models.py index 40564aee..01888c48 100644 --- a/addon_service/configured_addon/link/models.py +++ b/addon_service/configured_addon/link/models.py @@ -11,6 +11,8 @@ def is_supported_resource_type(resource_type: int): + if not resource_type: + return try: resource_type = SupportedResourceTypes(resource_type) if len(resource_type) != 1: @@ -20,12 +22,16 @@ def is_supported_resource_type(resource_type: int): class ConfiguredLinkAddon(ConfiguredAddon): - target_id = models.CharField() - int_resource_type = models.BigIntegerField(validators=[is_supported_resource_type]) + target_id = models.CharField(default=None, null=True, blank=True) + int_resource_type = models.BigIntegerField( + default=None, null=True, blank=True, validators=[is_supported_resource_type] + ) @property - def resource_type(self) -> SupportedResourceTypes: - return SupportedResourceTypes(self.int_resource_type) + def resource_type(self) -> SupportedResourceTypes | None: + if not self.int_resource_type: + return SupportedResourceTypes(self.int_resource_type) + return None @resource_type.setter def resource_type(self, value) -> None: @@ -33,21 +39,23 @@ def resource_type(self, value) -> None: def save(self, *args, full_clean=True, **kwargs): super().save(*args, full_clean=full_clean, **kwargs) - app.send_task( - "website.identifiers.tasks.task__update_doi_metadata_with_verified_links", - kwargs={"target_guid": self.authorized_resource.guid}, - ) + if self.resource_type and self.target_id: + app.send_task( + "website.identifiers.tasks.task__update_doi_metadata_with_verified_links", + kwargs={"target_guid": self.authorized_resource.guid}, + ) def target_url(self): - if self.target_id: - from addon_service.addon_imp.instantiation import ( - get_link_addon_instance__blocking, - ) + from addon_service.addon_imp.instantiation import ( + get_link_addon_instance__blocking, + ) - addon = get_link_addon_instance__blocking( - self.imp_cls, self.base_account, self.config - ) - return async_to_sync(addon.build_url_for_id)(self.target_id) + if not self.target_id: + return None + addon = get_link_addon_instance__blocking( + self.imp_cls, self.base_account, self.config + ) + return async_to_sync(addon.build_url_for_id)(self.target_id) @property def config(self) -> LinkConfig: diff --git a/addon_service/migrations/0015_alter_configuredlinkaddon_int_resource_type_and_more.py b/addon_service/migrations/0015_alter_configuredlinkaddon_int_resource_type_and_more.py new file mode 100644 index 00000000..887ff3e5 --- /dev/null +++ b/addon_service/migrations/0015_alter_configuredlinkaddon_int_resource_type_and_more.py @@ -0,0 +1,35 @@ +# Generated by Django 4.2.7 on 2025-05-05 20:22 + +from django.db import ( + migrations, + models, +) + +import addon_service.configured_addon.link.models + + +class Migration(migrations.Migration): + + dependencies = [ + ("addon_service", "0014_authorizedlinkaccount_configuredlinkaddon_and_more"), + ] + + operations = [ + migrations.AlterField( + model_name="configuredlinkaddon", + name="int_resource_type", + field=models.BigIntegerField( + blank=True, + default=None, + null=True, + validators=[ + addon_service.configured_addon.link.models.is_supported_resource_type + ], + ), + ), + migrations.AlterField( + model_name="configuredlinkaddon", + name="target_id", + field=models.CharField(blank=True, default=None, null=True), + ), + ] From 9e3b900e1ad2ec4811e40d87a129ee1022334d42 Mon Sep 17 00:00:00 2001 From: Longze Chen Date: Mon, 5 May 2025 16:26:52 -0400 Subject: [PATCH 44/65] Update ConfiguredLinkAddonSerializer: allow null and blank --- addon_service/configured_addon/link/serializers.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/addon_service/configured_addon/link/serializers.py b/addon_service/configured_addon/link/serializers.py index 7ffb4691..d07ee669 100644 --- a/addon_service/configured_addon/link/serializers.py +++ b/addon_service/configured_addon/link/serializers.py @@ -25,9 +25,11 @@ class ConfiguredLinkAddonSerializer(ConfiguredAddonSerializer): """api serializer for the `ConfiguredLinkAddon` model""" - target_url = URLField(read_only=True) - target_id = CharField() - resource_type = EnumNameChoiceField(enum_cls=SupportedResourceTypes) + target_url = URLField(allow_null=True, allow_blank=True, read_only=True) + target_id = CharField(allow_null=True, allow_blank=True) + resource_type = EnumNameChoiceField( + allow_null=True, allow_blank=True, enum_cls=SupportedResourceTypes + ) connected_operations = DataclassRelatedLinkField( dataclass_model=AddonOperationModel, From 706b7a0ff768ed83936d43b319c04f970238779e Mon Sep 17 00:00:00 2001 From: Longze Chen Date: Mon, 5 May 2025 16:29:04 -0400 Subject: [PATCH 45/65] Update VerifiedLinkSerializer * Fix serializer naming * All fields are read-only --- addon_service/configured_addon/link/serializers.py | 10 ++++++---- addon_service/configured_addon/link/views.py | 4 ++-- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/addon_service/configured_addon/link/serializers.py b/addon_service/configured_addon/link/serializers.py index d07ee669..e3f4db2a 100644 --- a/addon_service/configured_addon/link/serializers.py +++ b/addon_service/configured_addon/link/serializers.py @@ -83,7 +83,7 @@ class Meta: ] -class VerifiedLink(serializers.Serializer): +class VerifiedLinkSerializer(serializers.Serializer): """Serialize ConfiguredLinkAddon information required by OSF. The information is shaped for osf to be able to update datacite and share metadata @@ -94,6 +94,8 @@ class JSONAPIMeta: resource_name = "verified-link" target_url = URLField(read_only=True) - target_id = CharField() - resource_type = EnumNameChoiceField(enum_cls=SupportedResourceTypes) - service_name = CharField(source="external_service.external_service_name") + target_id = CharField(read_only=True) + resource_type = EnumNameChoiceField(read_only=True, enum_cls=SupportedResourceTypes) + service_name = CharField( + read_only=True, source="external_service.external_service_name" + ) diff --git a/addon_service/configured_addon/link/views.py b/addon_service/configured_addon/link/views.py index b5825479..bf1a886b 100644 --- a/addon_service/configured_addon/link/views.py +++ b/addon_service/configured_addon/link/views.py @@ -9,7 +9,7 @@ from .models import ConfiguredLinkAddon from .serializers import ( ConfiguredLinkAddonSerializer, - VerifiedLink, + VerifiedLinkSerializer, ) @@ -36,4 +36,4 @@ def get_verified_links(self, request, pk=None): ).select_related("base_account__external_service") self.resource_name = "verified-link" - return Response(VerifiedLink(addons, many=True).data) + return Response(VerifiedLinkSerializer(addons, many=True).data) From 16521c47af98ba05ac6c5679fe82eff1094e3dd1 Mon Sep 17 00:00:00 2001 From: Oleh Paduchak Date: Tue, 6 May 2025 13:18:20 +0300 Subject: [PATCH 46/65] fixed Hybrid service type not being displayed in admin --- addon_service/admin/_base.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/addon_service/admin/_base.py b/addon_service/admin/_base.py index 5df1fe84..ff84a979 100644 --- a/addon_service/admin/_base.py +++ b/addon_service/admin/_base.py @@ -56,7 +56,7 @@ def formfield_for_dbfield(self, db_field, request, **kwargs): label=db_field.verbose_name, choices=[ (None, ""), - *((_member.value, _member.name) for _member in _enum), + *self.iter_enum_members(_enum), ], ) if ( @@ -65,11 +65,17 @@ def formfield_for_dbfield(self, db_field, request, **kwargs): ): _enum = self.enum_multiple_choice_fields[db_field.name] return EnumNameMultipleChoiceField( - choices=[ - *((_member.value, _member.name) for _member in _enum), - ], + choices=self.iter_enum_members(_enum), widget=forms.CheckboxSelectMultiple, enum_cls=_enum, ) return super().formfield_for_dbfield(db_field, request, **kwargs) + + def iter_enum_members(self, _enum): + if hasattr(_enum, "__members__"): + iterator = _enum.__members__.values() + else: + iterator = _enum + + return [(_member.value, _member.name) for _member in iterator] From 034a34dcb909f4f379c33ab52060a87d7dea8cc1 Mon Sep 17 00:00:00 2001 From: Oleh Paduchak Date: Tue, 6 May 2025 13:38:04 +0300 Subject: [PATCH 47/65] fixed helper to be private --- addon_service/admin/_base.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/addon_service/admin/_base.py b/addon_service/admin/_base.py index ff84a979..c1454aca 100644 --- a/addon_service/admin/_base.py +++ b/addon_service/admin/_base.py @@ -56,7 +56,7 @@ def formfield_for_dbfield(self, db_field, request, **kwargs): label=db_field.verbose_name, choices=[ (None, ""), - *self.iter_enum_members(_enum), + *self._list_enum_members(_enum), ], ) if ( @@ -65,14 +65,14 @@ def formfield_for_dbfield(self, db_field, request, **kwargs): ): _enum = self.enum_multiple_choice_fields[db_field.name] return EnumNameMultipleChoiceField( - choices=self.iter_enum_members(_enum), + choices=self._list_enum_members(_enum), widget=forms.CheckboxSelectMultiple, enum_cls=_enum, ) return super().formfield_for_dbfield(db_field, request, **kwargs) - def iter_enum_members(self, _enum): + def _list_enum_members(self, _enum): if hasattr(_enum, "__members__"): iterator = _enum.__members__.values() else: From 9b204309900693f6dfc07195368625ccc02a054f Mon Sep 17 00:00:00 2001 From: Longze Chen Date: Tue, 6 May 2025 09:44:08 -0400 Subject: [PATCH 48/65] Fix typo in condition check Co-authored-by: Yuhuai Liu --- addon_service/configured_addon/link/models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/addon_service/configured_addon/link/models.py b/addon_service/configured_addon/link/models.py index 01888c48..fd5d42d9 100644 --- a/addon_service/configured_addon/link/models.py +++ b/addon_service/configured_addon/link/models.py @@ -29,7 +29,7 @@ class ConfiguredLinkAddon(ConfiguredAddon): @property def resource_type(self) -> SupportedResourceTypes | None: - if not self.int_resource_type: + if self.int_resource_type: return SupportedResourceTypes(self.int_resource_type) return None From f044698688343233b257ec5ace4889c7946fc588 Mon Sep 17 00:00:00 2001 From: Longze Chen Date: Tue, 6 May 2025 09:51:19 -0400 Subject: [PATCH 49/65] Use required=False for target_id and resource_type in serializer Co-authored-by: Yuhuai Liu --- addon_service/configured_addon/link/serializers.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/addon_service/configured_addon/link/serializers.py b/addon_service/configured_addon/link/serializers.py index e3f4db2a..aab1d534 100644 --- a/addon_service/configured_addon/link/serializers.py +++ b/addon_service/configured_addon/link/serializers.py @@ -26,9 +26,10 @@ class ConfiguredLinkAddonSerializer(ConfiguredAddonSerializer): """api serializer for the `ConfiguredLinkAddon` model""" target_url = URLField(allow_null=True, allow_blank=True, read_only=True) - target_id = CharField(allow_null=True, allow_blank=True) + target_id = CharField(required=False) resource_type = EnumNameChoiceField( - allow_null=True, allow_blank=True, enum_cls=SupportedResourceTypes + required=False, + enum_cls=SupportedResourceTypes, ) connected_operations = DataclassRelatedLinkField( From e3f0127a55cf4bd0b81a67a06893cf0f50f0bcc9 Mon Sep 17 00:00:00 2001 From: Longze Chen Date: Tue, 6 May 2025 11:30:16 -0400 Subject: [PATCH 50/65] Add comments on why we eased restrictions due to FE limitation --- addon_service/configured_addon/link/models.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/addon_service/configured_addon/link/models.py b/addon_service/configured_addon/link/models.py index fd5d42d9..085a1394 100644 --- a/addon_service/configured_addon/link/models.py +++ b/addon_service/configured_addon/link/models.py @@ -22,6 +22,10 @@ def is_supported_resource_type(resource_type: int): class ConfiguredLinkAddon(ConfiguredAddon): + # Note: ideally, `target_id` and `int_resource_type` are both required and + # should be present on first save. However, due to a limitation on how our + # FE workflow is currently wired, a `ConfiguredLinkAddon` instance must be + # saved before both attributes can be set. Thus, we allow null and blank. target_id = models.CharField(default=None, null=True, blank=True) int_resource_type = models.BigIntegerField( default=None, null=True, blank=True, validators=[is_supported_resource_type] @@ -39,6 +43,7 @@ def resource_type(self, value) -> None: def save(self, *args, full_clean=True, **kwargs): super().save(*args, full_clean=full_clean, **kwargs) + # Only send for OSF task after the addon has been fully configured if self.resource_type and self.target_id: app.send_task( "website.identifiers.tasks.task__update_doi_metadata_with_verified_links", From dcf47fa7268c5fd11c3589d59073f102596533a1 Mon Sep 17 00:00:00 2001 From: Longze Chen Date: Tue, 6 May 2025 11:33:46 -0400 Subject: [PATCH 51/65] Skip allow_null and allow_blank for read-only fields in serializer --- addon_service/configured_addon/link/serializers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/addon_service/configured_addon/link/serializers.py b/addon_service/configured_addon/link/serializers.py index aab1d534..ae273f69 100644 --- a/addon_service/configured_addon/link/serializers.py +++ b/addon_service/configured_addon/link/serializers.py @@ -25,7 +25,7 @@ class ConfiguredLinkAddonSerializer(ConfiguredAddonSerializer): """api serializer for the `ConfiguredLinkAddon` model""" - target_url = URLField(allow_null=True, allow_blank=True, read_only=True) + target_url = URLField(read_only=True) target_id = CharField(required=False) resource_type = EnumNameChoiceField( required=False, From b9983ebc960123ffaac2574235ddbd58a4053a0c Mon Sep 17 00:00:00 2001 From: Oleh Paduchak Date: Wed, 7 May 2025 11:45:10 +0300 Subject: [PATCH 52/65] fixed requests failing when dataverses contain unpublished datasets --- addon_imps/link/dataverse.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/addon_imps/link/dataverse.py b/addon_imps/link/dataverse.py index 9f6ee80d..23f513ae 100644 --- a/addon_imps/link/dataverse.py +++ b/addon_imps/link/dataverse.py @@ -3,6 +3,7 @@ import asyncio import re from dataclasses import dataclass +from http import HTTPStatus from typing import Literal from django.core.exceptions import ValidationError @@ -122,12 +123,13 @@ async def _fetch_dataverse_items(self, dataverse_id) -> list[ItemResult]: f"api/dataverses/{dataverse_id}/contents" ) as response: response_content = await response.json_content() - return await asyncio.gather( + items = await asyncio.gather( *[ self._get_dataverse_or_dataset_item(item) for item in response_content["data"] ] ) + return [item for item in items if item] async def _get_dataverse_or_dataset_item(self, item: dict): match item["type"]: @@ -152,10 +154,12 @@ async def _fetch_dataset_with_parser( dataset_id: str = None, persistent_id: str = None, parser=None, - ) -> ItemResult | list[ItemResult]: + ) -> ItemResult | list[ItemResult] | None: url = f"api/datasets/{':persistentId' if persistent_id else dataset_id}/versions/:latest-published" query = {"persistentId": persistent_id} if persistent_id else {} async with self.network.GET(url, query=query) as response: + if response.http_status == HTTPStatus.NOT_FOUND: + return None return parser(await response.json_content()) async def _fetch_dataset( From 01a27860a61eada635b73d559b639b1d5f8124bb Mon Sep 17 00:00:00 2001 From: Yuhuai Liu Date: Sun, 11 May 2025 21:00:45 -0400 Subject: [PATCH 53/65] fix --- addon_service/configured_addon/link/models.py | 3 ++- addon_service/configured_addon/link/serializers.py | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/addon_service/configured_addon/link/models.py b/addon_service/configured_addon/link/models.py index 085a1394..a33efa4b 100644 --- a/addon_service/configured_addon/link/models.py +++ b/addon_service/configured_addon/link/models.py @@ -39,7 +39,8 @@ def resource_type(self) -> SupportedResourceTypes | None: @resource_type.setter def resource_type(self, value) -> None: - self.int_resource_type = value.value + if value is not None: + self.int_resource_type = value.value def save(self, *args, full_clean=True, **kwargs): super().save(*args, full_clean=full_clean, **kwargs) diff --git a/addon_service/configured_addon/link/serializers.py b/addon_service/configured_addon/link/serializers.py index ae273f69..54558e37 100644 --- a/addon_service/configured_addon/link/serializers.py +++ b/addon_service/configured_addon/link/serializers.py @@ -26,9 +26,10 @@ class ConfiguredLinkAddonSerializer(ConfiguredAddonSerializer): """api serializer for the `ConfiguredLinkAddon` model""" target_url = URLField(read_only=True) - target_id = CharField(required=False) + target_id = CharField(required=False, allow_null=True) resource_type = EnumNameChoiceField( required=False, + allow_null=True, enum_cls=SupportedResourceTypes, ) From 7d311e1a2afbad2bebc2c079f1d6e3d7c4b6f15e Mon Sep 17 00:00:00 2001 From: Longze Chen Date: Mon, 12 May 2025 12:24:53 -0400 Subject: [PATCH 54/65] Fix naming for OutputManagementPlan --- addon_toolkit/interfaces/link.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/addon_toolkit/interfaces/link.py b/addon_toolkit/interfaces/link.py index f55e48a3..0085b35c 100644 --- a/addon_toolkit/interfaces/link.py +++ b/addon_toolkit/interfaces/link.py @@ -53,7 +53,7 @@ class SupportedResourceTypes(enum.Flag): Journal = enum.auto() JournalArticle = enum.auto() Model = enum.auto() - OutputManagement_plan = enum.auto() + OutputManagementPlan = enum.auto() PeerReview = enum.auto() PhysicalObject = enum.auto() Preprint = enum.auto() From 5a228fa9f5e932138add5ef80a3c1b60ed356de6 Mon Sep 17 00:00:00 2001 From: Oleh Paduchak Date: Thu, 15 May 2025 15:41:59 +0300 Subject: [PATCH 55/65] fixed invalid serializer for AuthorizedLinkAccounts accessed from UserReference --- addon_service/user_reference/serializers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/addon_service/user_reference/serializers.py b/addon_service/user_reference/serializers.py index 70df89e8..5d5e4215 100644 --- a/addon_service/user_reference/serializers.py +++ b/addon_service/user_reference/serializers.py @@ -63,7 +63,7 @@ class UserReferenceSerializer(serializers.HyperlinkedModelSerializer): "addon_service.serializers.AuthorizedComputingAccountSerializer" ), "authorized_link_accounts": ( - "addon_service.serializers.AuthorizedComputingAccountSerializer" + "addon_service.serializers.AuthorizedLinkAccountSerializer" ), "configured_resources": ( "addon_service.serializers.ResourceReferenceSerializer" From 387b79cafdc6171b042b51e1e83467ea4ee67d56 Mon Sep 17 00:00:00 2001 From: Oleh Paduchak Date: Fri, 16 May 2025 13:06:51 +0300 Subject: [PATCH 56/65] made EnumNameMultipleChoiceField sorted --- addon_service/common/enum_serializers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/addon_service/common/enum_serializers.py b/addon_service/common/enum_serializers.py index 24d9bfdd..476531c9 100644 --- a/addon_service/common/enum_serializers.py +++ b/addon_service/common/enum_serializers.py @@ -61,4 +61,4 @@ def to_internal_value(self, data) -> enum.Flag: def to_representation(self, value): _member_list = super().to_representation(value) - return [_member.name for _member in _member_list] + return sorted([_member.name for _member in _member_list]) From c876d883a572e98c97da01a60a3392a3c8a2ad41 Mon Sep 17 00:00:00 2001 From: Oleh Paduchak Date: Tue, 20 May 2025 13:21:25 +0300 Subject: [PATCH 57/65] added dataverse to fill_external_services script --- .../commands/fill_external_services.py | 2 ++ .../commands/providers/providers.csv | 31 ++++++++++--------- 2 files changed, 18 insertions(+), 15 deletions(-) diff --git a/addon_service/management/commands/fill_external_services.py b/addon_service/management/commands/fill_external_services.py index 081e986d..d4de9690 100644 --- a/addon_service/management/commands/fill_external_services.py +++ b/addon_service/management/commands/fill_external_services.py @@ -6,6 +6,7 @@ from addon_service.external_service.citation.models import ExternalCitationService from addon_service.external_service.computing.models import ExternalComputingService +from addon_service.external_service.link.models import ExternalLinkService from addon_service.external_service.storage.models import ExternalStorageService from addon_service.oauth1 import OAuth1ClientConfig from addon_service.oauth2 import OAuth2ClientConfig @@ -20,6 +21,7 @@ "storage": ExternalStorageService, "citation": ExternalCitationService, "computing": ExternalComputingService, + "link": ExternalLinkService, } icons_path = Path(__file__).parent.parent.parent / "static/provider_icons" diff --git a/addon_service/management/commands/providers/providers.csv b/addon_service/management/commands/providers/providers.csv index 27c077c4..62594c38 100644 --- a/addon_service/management/commands/providers/providers.csv +++ b/addon_service/management/commands/providers/providers.csv @@ -1,15 +1,16 @@ -id,display_name,int_credentials_format,int_service_type,int_addon_imp,supported_scopes,api_base_url,wb_key,oauth1_client_config_id,oauth2_client_config_id,icon_name,api_base_url_options,max_concurrent_downloads,max_upload_mb,int_supported_features,service_type -d424bec8-811b-4d23-bf1c-f4e8219e0454,Mendeley,1,1,1004,{all},https://api.mendeley.com,mendeley,,b3fd76fc-3368-47b4-9053-ff4b7ca4baa2,mendeley.svg,,,,3,citation -fce6ada4-9a6e-4986-94c8-9a378170f5fb,Zotero,5,1,1002,{all},https://api.zotero.org,zotero_org,e3663c71-3520-4fce-b0ff-e56c81a73d54,,zotero.svg,,,,3,citation -455cfb5f-6ca0-4d60-8fc8-279f7b229da2,Boa,3,1,1020,{all},https://boa.com,boa,,,boa.png,,,,1,computing -d5ad0101-b732-4a08-8306-384d66beef0e,Gitlab,4,1,1011,{},https://gitlab.com,gitlab,,,gitlab.svg,{},10,10,511,storage -e3cb1bc9-83ec-49fa-86de-1d04d3f94834,Dataverse,6,3,1010,{},https://demo.dataverse.org,dataverse,,,dataverse.svg,,10,10,511,storage -4a2aa5c7-c5be-4e3f-abd8-04fb34928bd3,s3,2,1,1003,{},https://api.github.com,s3,,,s3.svg,,10,10,511,storage -4854b243-593a-4c6f-96e8-5bb2a2df9134,BOX,1,1,1001,{root_readwrite},https://api.box.com/2.0/,box,,4258d732-db77-42aa-b275-8da841adf77f,box.svg,,10,10,2047,storage -9e7148ae-4919-4676-bb44-a7a6662c6dd0,Dropbox,1,1,1006,{files.metadata.write files.content.write files.content.read files.metadata.read},https://api.dropboxapi.com/2/,dropbox,,90b44f3d-66e3-444c-9261-534b9e37b21a,dropbox.svg,,10,10,511,storage -15b66b0c-4db9-4425-b2e0-dfd701a55b05,OneDrive,1,1,1008,{User.Read offline_access Files.Read Files.Read.All Files.ReadWrite},https://graph.microsoft.com/v1.0/,onedrive,,a64e222c-a37b-4f34-9f27-83611d851f5c,onedrive.svg,,10,10,2047,storage -babfcfcc-0e47-4c8d-9ca1-3b335bb8cf55,Github,1,1,1013,{repo contents},https://api.github.com,github,,285f28ee-15b5-429e-b7ba-3371a19678a8,github.svg,,10,10,2047,storage -b670cf7a-752f-49ed-8367-8238cf8a24d1,Figshare,1,1,1007,{all},https://api.figshare.com/v2/,figshare,,5c161353-7444-478e-881f-3b8aec3a002a,figshare.svg,,10,10,511,storage -2acd3e09-5e27-4e5b-a3d1-579f16663cfc,Bitbucket,1,1,1012,{account repository team},https://api.bitbucket.org/2.0/,bitbucket,,1b2d71cc-425e-4aa6-886c-e2ff247c4f2c,bitbucket.svg,,10,10,511,storage -102c6356-e31a-4316-a9fe-8b2bcb20fdd6,Google Drive,1,1,1005,{https://www.googleapis.com/auth/drive},https://www.googleapis.com/,googledrive,,5cf93c06-c27f-4446-b8ac-ffac209ba465,google_drive.svg,,10,10,511,storage -5bc440c8-a03a-4aa6-bd9e-856aee9883e9,Owncloud,3,2,1009,{},,owncloud,,,owncloud.svg,,10,10,511,storage +id,display_name,int_credentials_format,int_service_type,int_addon_imp,supported_scopes,api_base_url,wb_key,oauth1_client_config_id,oauth2_client_config_id,icon_name,api_base_url_options,max_concurrent_downloads,max_upload_mb,int_supported_features,int_supported_resource_types,browser_base_url,service_type +d424bec8-811b-4d23-bf1c-f4e8219e0454,Mendeley,1,1,1004,{all},https://api.mendeley.com,mendeley,,b3fd76fc-3368-47b4-9053-ff4b7ca4baa2,mendeley.svg,,,,3,,,citation +fce6ada4-9a6e-4986-94c8-9a378170f5fb,Zotero,5,1,1002,{all},https://api.zotero.org,zotero_org,e3663c71-3520-4fce-b0ff-e56c81a73d54,,zotero.svg,,,,3,,,citation +455cfb5f-6ca0-4d60-8fc8-279f7b229da2,Boa,3,1,1020,{all},https://boa.com,boa,,,boa.png,,,,1,,,computing +d5ad0101-b732-4a08-8306-384d66beef0e,Gitlab,4,1,1011,{},https://gitlab.com,gitlab,,,gitlab.svg,{},10,10,511,,,storage +e3cb1bc9-83ec-49fa-86de-1d04d3f94834,Dataverse,6,3,1010,{},https://demo.dataverse.org,dataverse,,,dataverse.svg,,10,10,511,,,storage +4a2aa5c7-c5be-4e3f-abd8-04fb34928bd3,s3,2,1,1003,{},https://api.github.com,s3,,,s3.svg,,10,10,511,,,storage +4854b243-593a-4c6f-96e8-5bb2a2df9134,BOX,1,1,1001,{root_readwrite},https://api.box.com/2.0/,box,,4258d732-db77-42aa-b275-8da841adf77f,box.svg,,10,10,2047,,,storage +9e7148ae-4919-4676-bb44-a7a6662c6dd0,Dropbox,1,1,1006,{files.metadata.write files.content.write files.content.read files.metadata.read},https://api.dropboxapi.com/2/,dropbox,,90b44f3d-66e3-444c-9261-534b9e37b21a,dropbox.svg,,10,10,511,,,storage +15b66b0c-4db9-4425-b2e0-dfd701a55b05,OneDrive,1,1,1008,{User.Read offline_access Files.Read Files.Read.All Files.ReadWrite},https://graph.microsoft.com/v1.0/,onedrive,,a64e222c-a37b-4f34-9f27-83611d851f5c,onedrive.svg,,10,10,2047,,,storage +babfcfcc-0e47-4c8d-9ca1-3b335bb8cf55,Github,1,1,1013,{repo contents},https://api.github.com,github,,285f28ee-15b5-429e-b7ba-3371a19678a8,github.svg,,10,10,2047,,,storage +b670cf7a-752f-49ed-8367-8238cf8a24d1,Figshare,1,1,1007,{all},https://api.figshare.com/v2/,figshare,,5c161353-7444-478e-881f-3b8aec3a002a,figshare.svg,,10,10,511,,,storage +2acd3e09-5e27-4e5b-a3d1-579f16663cfc,Bitbucket,1,1,1012,{account repository team},https://api.bitbucket.org/2.0/,bitbucket,,1b2d71cc-425e-4aa6-886c-e2ff247c4f2c,bitbucket.svg,,10,10,511,,,storage +102c6356-e31a-4316-a9fe-8b2bcb20fdd6,Google Drive,1,1,1005,{https://www.googleapis.com/auth/drive},https://www.googleapis.com/,googledrive,,5cf93c06-c27f-4446-b8ac-ffac209ba465,google_drive.svg,,10,10,511,,,storage +5bc440c8-a03a-4aa6-bd9e-856aee9883e9,Owncloud,3,2,1009,{},,owncloud,,,owncloud.svg,,10,10,511,,,storage +edf57d0c-ae0a-4c85-b759-017fc8b009f0,Link dataverse,6,2,1030,{},https://demo.dataverse.org,,,,dataverse.svg,{},link,,,4294967295,,link From 338020b25cdeed72aa760aeca31407f78b5ab096 Mon Sep 17 00:00:00 2001 From: Longze Chen Date: Fri, 23 May 2025 09:50:04 -0400 Subject: [PATCH 58/65] Reorder KnownAddonImps and AddonImpNumbers --- addon_service/common/known_imps.py | 47 +++++++++++++++++++++--------- 1 file changed, 33 insertions(+), 14 deletions(-) diff --git a/addon_service/common/known_imps.py b/addon_service/common/known_imps.py index e6b49653..8692e99e 100644 --- a/addon_service/common/known_imps.py +++ b/addon_service/common/known_imps.py @@ -72,25 +72,34 @@ def get_imp_number(imp: type[AddonImp]) -> int: @enum.unique class KnownAddonImps(enum.Enum): - """Static mapping from API-facing name for an AddonImp to the Imp itself""" + """Static mapping from API-facing name for an AddonImp to the Imp itself. + Note: Grouped by type and then ordered by respective AddonImpNumbers. + """ + + # Type: Storage BOX = box_dot_com.BoxDotComStorageImp S3 = s3.S3StorageImp - ONEDRIVE = onedrive.OneDriveStorageImp - ZOTERO = zotero_org.ZoteroOrgCitationImp GOOGLEDRIVE = google_drive.GoogleDriveStorageImp + DROPBOX = dropbox.DropboxStorageImp FIGSHARE = figshare.FigshareStorageImp - MENDELEY = mendeley.MendeleyCitationImp - BITBUCKET = bitbucket.BitbucketStorageImp - DATAVERSE = dataverse.DataverseStorageImp + ONEDRIVE = onedrive.OneDriveStorageImp OWNCLOUD = owncloud.OwnCloudStorageImp - LINK_DATAVERSE = link_dataverse.DataverseLinkImp - GITHUB = github.GitHubStorageImp + DATAVERSE = dataverse.DataverseStorageImp GITLAB = gitlab.GitlabStorageImp - DROPBOX = dropbox.DropboxStorageImp + BITBUCKET = bitbucket.BitbucketStorageImp + GITHUB = github.GitHubStorageImp + # Type: Citation + ZOTERO = zotero_org.ZoteroOrgCitationImp + MENDELEY = mendeley.MendeleyCitationImp + + # Type: Cloud Computing BOA = boa.BoaComputingImp + # Type: Link + LINK_DATAVERSE = link_dataverse.DataverseLinkImp + if __debug__: BLARG = my_blarg.MyBlargStorage @@ -98,12 +107,16 @@ class KnownAddonImps(enum.Enum): @enum_names_same_as(KnownAddonImps) @enum.unique class AddonImpNumbers(enum.Enum): - """Static mapping from each AddonImp name to a unique integer (for database use)""" + """Static mapping from each AddonImp name to a unique integer (for database use) + Note: Ideally, we should "prefix" the number by type and take future scalability into account. + e.g. Storage type uses 10xx, Citation type uses 11xx, Cloud Computing type uses 12xx and LINK type uses 13xx. + Consider this a future improvement when we run out of 1001~1019 for both storage and citation addons. + """ + + # Type: Storage BOX = 1001 - ZOTERO = 1002 S3 = 1003 - MENDELEY = 1004 GOOGLEDRIVE = 1005 DROPBOX = 1006 FIGSHARE = 1007 @@ -112,12 +125,18 @@ class AddonImpNumbers(enum.Enum): DATAVERSE = 1010 GITLAB = 1011 BITBUCKET = 1012 - - LINK_DATAVERSE = 1030 GITHUB = 1013 + # Type: Citation + ZOTERO = 1002 + MENDELEY = 1004 + + # Type: Cloud Computing BOA = 1020 + # Type: Link + LINK_DATAVERSE = 1030 + if __debug__: BLARG = -7 From 1b3516f2699e21852c1b6dd0b9532f1d9fc78327 Mon Sep 17 00:00:00 2001 From: Longze Chen Date: Tue, 27 May 2025 10:06:40 -0400 Subject: [PATCH 59/65] Add int_supported_features to link type external service --- addon_service/admin/__init__.py | 6 ++- addon_service/external_service/link/models.py | 41 ++++++++++++++++++- .../external_service/link/serializers.py | 10 ++++- ...service_int_supported_features_and_more.py | 38 +++++++++++++++++ 4 files changed, 91 insertions(+), 4 deletions(-) create mode 100644 addon_service/migrations/0016_externallinkservice_int_supported_features_and_more.py diff --git a/addon_service/admin/__init__.py b/addon_service/admin/__init__.py index 59347d5c..10389538 100644 --- a/addon_service/admin/__init__.py +++ b/addon_service/admin/__init__.py @@ -8,7 +8,10 @@ from addon_service.external_service.storage.models import StorageSupportedFeatures from ..external_service.citation.models import CitationSupportedFeatures -from ..external_service.link.models import SupportedResourceTypes +from ..external_service.link.models import ( + LinkSupportedFeatures, + SupportedResourceTypes, +) from ._base import GravyvaletModelAdmin from .decorators import linked_many_field @@ -66,6 +69,7 @@ class ExternalLinkServiceAdmin(GravyvaletModelAdmin): "int_service_type": ServiceTypes, } enum_multiple_choice_fields = { + "int_supported_features": LinkSupportedFeatures, "int_supported_resource_types": SupportedResourceTypes, } diff --git a/addon_service/external_service/link/models.py b/addon_service/external_service/link/models.py index 8ebd148d..41453389 100644 --- a/addon_service/external_service/link/models.py +++ b/addon_service/external_service/link/models.py @@ -1,3 +1,8 @@ +from enum import ( + Flag, + auto, +) + from django.core.exceptions import ValidationError from django.db import models @@ -10,19 +15,51 @@ from addon_toolkit.interfaces.link import SupportedResourceTypes +# TODO: need to trim down the features +class LinkSupportedFeatures(Flag): + ADD_UPDATE_FILES = auto() + ADD_UPDATE_FILES_PARTIAL = auto() + DELETE_FILES = auto() + DELETE_FILES_PARTIAL = auto() + FORKING = auto() + LOGS = auto() + PERMISSIONS = auto() + REGISTERING = auto() + FILE_VERSIONS = auto() + COPY_INTO = auto() + DOWNLOAD_AS_ZIP = auto() + + def validate_supported_features(value): + _validate_enum_value(LinkSupportedFeatures, value) + + +def validate_supported_resource_types(value): _validate_enum_value(SupportedResourceTypes, value) class ExternalLinkService(ExternalService): browser_base_url = models.URLField(blank=True, default="") - int_supported_resource_types = models.BigIntegerField( + int_supported_features = models.IntegerField( validators=[validate_supported_features], null=True ) + int_supported_resource_types = models.BigIntegerField( + validators=[validate_supported_resource_types], null=True + ) @property - def supported_resource_types(self) -> list[SupportedResourceTypes]: + def supported_features(self) -> list[LinkSupportedFeatures]: """get the enum representation of int_supported_features""" + return LinkSupportedFeatures(self.int_supported_features) + + @supported_features.setter + def supported_features(self, new_supported_features: LinkSupportedFeatures): + """set int_authorized_capabilities without caring its int""" + self.int_supported_features = new_supported_features.value + + @property + def supported_resource_types(self) -> list[SupportedResourceTypes]: + """get the enum representation of int_supported_resource_types""" return SupportedResourceTypes(self.int_supported_resource_types) @supported_resource_types.setter diff --git a/addon_service/external_service/link/serializers.py b/addon_service/external_service/link/serializers.py index 556a3175..c42a0a5a 100644 --- a/addon_service/external_service/link/serializers.py +++ b/addon_service/external_service/link/serializers.py @@ -5,7 +5,10 @@ from addon_service.common import view_names from addon_service.common.enum_serializers import EnumNameMultipleChoiceField from addon_service.common.serializer_fields import DataclassRelatedDataField -from addon_service.external_service.link.models import SupportedResourceTypes +from addon_service.external_service.link.models import ( + LinkSupportedFeatures, + SupportedResourceTypes, +) from addon_service.external_service.serializers import ExternalServiceSerializer from .models import ExternalLinkService @@ -26,6 +29,10 @@ class ExternalLinkServiceSerializer(ExternalServiceSerializer): related_link_view_name=view_names.related_view(RESOURCE_TYPE), ) + supported_features = EnumNameMultipleChoiceField( + enum_cls=LinkSupportedFeatures, read_only=True + ) + supported_resource_types = EnumNameMultipleChoiceField( enum_cls=SupportedResourceTypes, read_only=True ) @@ -43,6 +50,7 @@ class Meta: "external_service_name", "configurable_api_root", "supported_resource_types", + "supported_features", "icon_url", "api_base_url_options", ] diff --git a/addon_service/migrations/0016_externallinkservice_int_supported_features_and_more.py b/addon_service/migrations/0016_externallinkservice_int_supported_features_and_more.py new file mode 100644 index 00000000..e75123b2 --- /dev/null +++ b/addon_service/migrations/0016_externallinkservice_int_supported_features_and_more.py @@ -0,0 +1,38 @@ +# Generated by Django 4.2.20 on 2025-05-27 13:57 + +from django.db import ( + migrations, + models, +) + +import addon_service.external_service.link.models + + +class Migration(migrations.Migration): + + dependencies = [ + ("addon_service", "0015_alter_configuredlinkaddon_int_resource_type_and_more"), + ] + + operations = [ + migrations.AddField( + model_name="externallinkservice", + name="int_supported_features", + field=models.IntegerField( + null=True, + validators=[ + addon_service.external_service.link.models.validate_supported_features + ], + ), + ), + migrations.AlterField( + model_name="externallinkservice", + name="int_supported_resource_types", + field=models.BigIntegerField( + null=True, + validators=[ + addon_service.external_service.link.models.validate_supported_resource_types + ], + ), + ), + ] From 2b53b15633989e44bc200bc7c36145f5501d8a27 Mon Sep 17 00:00:00 2001 From: Oleh Paduchak Date: Tue, 27 May 2025 17:38:29 +0300 Subject: [PATCH 60/65] renamed task__update_doi_metadata_with_verified_links --- addon_service/configured_addon/link/models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/addon_service/configured_addon/link/models.py b/addon_service/configured_addon/link/models.py index a33efa4b..c16f4dbf 100644 --- a/addon_service/configured_addon/link/models.py +++ b/addon_service/configured_addon/link/models.py @@ -47,7 +47,7 @@ def save(self, *args, full_clean=True, **kwargs): # Only send for OSF task after the addon has been fully configured if self.resource_type and self.target_id: app.send_task( - "website.identifiers.tasks.task__update_doi_metadata_with_verified_links", + "website.identifiers.tasks.task__update_verified_links", kwargs={"target_guid": self.authorized_resource.guid}, ) From f6c088b8ba376e7728eb8c445da8071f04b66b74 Mon Sep 17 00:00:00 2001 From: Longze Chen Date: Tue, 27 May 2025 14:47:04 -0400 Subject: [PATCH 61/65] Update supported features for LINK external service --- addon_service/external_service/link/models.py | 13 ++----------- 1 file changed, 2 insertions(+), 11 deletions(-) diff --git a/addon_service/external_service/link/models.py b/addon_service/external_service/link/models.py index 41453389..35b627e0 100644 --- a/addon_service/external_service/link/models.py +++ b/addon_service/external_service/link/models.py @@ -15,19 +15,10 @@ from addon_toolkit.interfaces.link import SupportedResourceTypes -# TODO: need to trim down the features class LinkSupportedFeatures(Flag): - ADD_UPDATE_FILES = auto() - ADD_UPDATE_FILES_PARTIAL = auto() - DELETE_FILES = auto() - DELETE_FILES_PARTIAL = auto() - FORKING = auto() - LOGS = auto() + FORKING_PARTIAL = auto() + LOGS_PARTIAL = auto() PERMISSIONS = auto() - REGISTERING = auto() - FILE_VERSIONS = auto() - COPY_INTO = auto() - DOWNLOAD_AS_ZIP = auto() def validate_supported_features(value): From b7fb8cf8674e8a553241014ea0f095e58d6f730d Mon Sep 17 00:00:00 2001 From: Longze Chen Date: Wed, 28 May 2025 12:10:52 -0400 Subject: [PATCH 62/65] Fix supported features for link type external service --- addon_service/external_service/link/models.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/addon_service/external_service/link/models.py b/addon_service/external_service/link/models.py index 35b627e0..ef510025 100644 --- a/addon_service/external_service/link/models.py +++ b/addon_service/external_service/link/models.py @@ -16,9 +16,15 @@ class LinkSupportedFeatures(Flag): + ADD_UPDATE_FILES = auto() + DELETE_FILES = auto() + FORKING = auto() FORKING_PARTIAL = auto() + LOGS = auto() LOGS_PARTIAL = auto() PERMISSIONS = auto() + REGISTERING = auto() + FILE_VERSIONS = auto() def validate_supported_features(value): From 3ca14831b65a25e5c60ce9c92f2dab4c9a164dd0 Mon Sep 17 00:00:00 2001 From: Oleh Paduchak Date: Wed, 18 Jun 2025 16:24:04 +0300 Subject: [PATCH 63/65] fixed dataverse provider to make dataverse work as link --- addon_imps/link/dataverse.py | 38 ++++++++++++++++++++++++------------ 1 file changed, 25 insertions(+), 13 deletions(-) diff --git a/addon_imps/link/dataverse.py b/addon_imps/link/dataverse.py index 23f513ae..84d42fb8 100644 --- a/addon_imps/link/dataverse.py +++ b/addon_imps/link/dataverse.py @@ -16,7 +16,7 @@ ) -DATAVERSE_REGEX = re.compile(r"^dataverse/(?P\d*)$") +DATAVERSE_REGEX = re.compile(r"^dataverse/(?P.*)$") DATASET_REGEX = re.compile(r"^dataset/(?P.*)$") FILE_REGEX = re.compile(r"^file/(?P.*)$") @@ -29,19 +29,31 @@ class DataverseLinkImp(LinkAddonHttpRequestorImp): """ async def build_url_for_id(self, item_id: str) -> str: - if match := DATASET_REGEX.match(item_id): + if match := DATAVERSE_REGEX.match(item_id): + entity_type = "dataverse" + elif match := DATASET_REGEX.match(item_id): entity_type = "dataset" elif match := FILE_REGEX.match(item_id): entity_type = "file" else: raise ValidationError(f"Invalid {item_id=}") - persistent_id = match["persistent_id"] + return self._make_url(entity_type, **match.groupdict()) - return self._make_url(entity_type, persistent_id) + def _make_url( + self, + entity_type: Literal["file", "dataset", "dataverse"], + persistent_id: str = None, + id_: str = None, + ): + if bool(id_) == bool(persistent_id): + raise ValueError("You must pass either id or persistent_id to build url") - def _make_url(self, entity_type: Literal["file", "dataset"], persistent_id): - return f"{self.config.external_web_url}/{entity_type}.xhtml?persistentId={persistent_id}" + if persistent_id: + suffix = f".xhtml?persistentId={persistent_id}" + else: + suffix = f"/{id_}" + return f"{self.config.external_web_url}/{entity_type}{suffix}" async def get_external_account_id(self, _: dict[str, str]) -> str: try: @@ -81,7 +93,7 @@ async def get_item_info(self, item_id: str) -> ItemResult: if not item_id: return ItemResult(item_id="", item_name="", item_type=ItemType.FOLDER) elif match := DATAVERSE_REGEX.match(item_id): - entity = await self._fetch_dataverse(match["id"]) + entity = await self._fetch_dataverse(match["id_"]) elif match := DATASET_REGEX.match(item_id): entity = await self._fetch_dataset( dataset_id=match["id"], persistent_id=match["persistent_id"] @@ -102,7 +114,7 @@ async def list_child_items( if not item_id: return await self.list_root_items(page_cursor) elif match := DATAVERSE_REGEX.match(item_id): - items = await self._fetch_dataverse_items(match["id"]) + items = await self._fetch_dataverse_items(match["id_"]) return ItemSampleResult( items=items, total_count=len(items), @@ -136,7 +148,7 @@ async def _get_dataverse_or_dataset_item(self, item: dict): case "dataset": return await self._fetch_dataset(dataset_id=item["id"]) case "dataverse": - return parse_dataverse_as_subitem(item) + return await self._fetch_dataverse(item["id"]) raise ValueError(f"Invalid item type: {item['type']}") async def _fetch_file(self, dataverse_id) -> ItemResult: @@ -235,7 +247,7 @@ def parse_dataverse(data: dict): return ItemResult( item_type=ItemType.FOLDER, item_name=data["name"], - item_id=f'dataverse/{data["id"]}', + item_id=f'dataverse/{data["alias"]}', ) @@ -245,11 +257,11 @@ def parse_mydata(data: dict): return ItemSampleResult( items=[ ItemResult( - item_id=f"dataverse/{file['entity_id']}", - item_name=file["name"], + item_id=f"dataverse/{item['identifier']}", + item_name=item["name"], item_type=ItemType.FOLDER, ) - for file in data["items"] + for item in data["items"] ], total_count=data["total_count"], next_sample_cursor=( From 8a3091e6ad8ceb6f4d94cac1a3928c086cd02ec9 Mon Sep 17 00:00:00 2001 From: Longze Chen Date: Thu, 19 Jun 2025 20:23:56 -0400 Subject: [PATCH 64/65] Fix DATASET_REGEX.match in get_item_info() for linked services --- addon_imps/link/dataverse.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/addon_imps/link/dataverse.py b/addon_imps/link/dataverse.py index 84d42fb8..ba39a140 100644 --- a/addon_imps/link/dataverse.py +++ b/addon_imps/link/dataverse.py @@ -95,9 +95,7 @@ async def get_item_info(self, item_id: str) -> ItemResult: elif match := DATAVERSE_REGEX.match(item_id): entity = await self._fetch_dataverse(match["id_"]) elif match := DATASET_REGEX.match(item_id): - entity = await self._fetch_dataset( - dataset_id=match["id"], persistent_id=match["persistent_id"] - ) + entity = await self._fetch_dataset(persistent_id=match["persistent_id"]) elif match := FILE_REGEX.match(item_id): entity = await self._fetch_file(match["persistent_id"]) else: From d7091cc347f8dd53b18818d21059dc24e3ff4e75 Mon Sep 17 00:00:00 2001 From: Vlad0n20 Date: Fri, 20 Jun 2025 18:07:00 +0300 Subject: [PATCH 65/65] Fix params for log_gv_addon task --- addon_service/configured_addon/models.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/addon_service/configured_addon/models.py b/addon_service/configured_addon/models.py index be3727e5..ce91544c 100644 --- a/addon_service/configured_addon/models.py +++ b/addon_service/configured_addon/models.py @@ -96,8 +96,8 @@ def save(self, *args, full_clean=True, **kwargs): app.send_task( "osf.tasks.log_gv_addon", kwargs={ - "node_guid": self.authorized_resource.guid, - "user_guid": self.account_owner.guid, + "node_url": self.resource_uri, + "user_url": self.owner_uri, "addon": self.display_name, "action": "addon_added", },