diff --git a/osf/external/spam/tasks.py b/osf/external/spam/tasks.py index d67b21912dc..8c175831254 100644 --- a/osf/external/spam/tasks.py +++ b/osf/external/spam/tasks.py @@ -17,7 +17,7 @@ DOMAIN_REGEX = re.compile(r'\W*(?P\w+://)?(?Pwww\.)?(?P([\w-]+\.)+[a-zA-Z]+)(?P[/\-\.\w]*)?\W*') REDIRECT_CODES = {301, 302, 303, 307, 308} - +NOTABLE_POST_NOMINALS = ['m.sc.', 'msc.', 'b.sc.', 'bsc.', 'd.sc.', 'dsc.', 'phd.', 'ph.d.', 'msc.pt', 'pt.', 'prof.', 'dr.', 'md.', 'jd.', 'esq.'] @celery_app.task() def reclassify_domain_references(notable_domain_id, current_note, previous_note): diff --git a/osf/models/mixins.py b/osf/models/mixins.py index 7851bce18e1..1fe75420fa2 100644 --- a/osf/models/mixins.py +++ b/osf/models/mixins.py @@ -34,7 +34,7 @@ from .nodelog import NodeLog from .subject import Subject from .spam import SpamMixin, SpamStatus -from .validators import validate_title +from .validators import validate_title, has_domain_in_user_fields_for_names from .tag import Tag from osf.utils import sanitize from .validators import validate_subject_hierarchy, validate_email, expand_subject_hierarchy @@ -1581,6 +1581,9 @@ def add_unregistered_contributor( except BlockedEmailError: raise ValidationError('Unregistered contributor email address domain is blocked.') + if has_domain_in_user_fields_for_names(fullname): + raise ValidationError('Invalid personal information.') + # Create a new user record if you weren't passed an existing user contributor = existing_user if existing_user else OSFUser.create_unregistered(fullname=fullname, email=email) diff --git a/osf/models/user.py b/osf/models/user.py index 5739813cd48..0218af34692 100644 --- a/osf/models/user.py +++ b/osf/models/user.py @@ -57,13 +57,12 @@ from .spam import SpamMixin from .session import UserSessionMap from .tag import Tag -from .validators import validate_email, validate_social, validate_history_item, has_domain_in_user_fields_for_names +from .validators import validate_email, validate_social, validate_history_item from osf.utils.datetime_aware_jsonfield import DateTimeAwareJSONField from osf.utils.fields import NonNaiveDateTimeField, LowercaseEmailField, ensure_str from osf.utils.names import impute_names from osf.utils.requests import check_select_for_update from osf.utils.permissions import API_CONTRIBUTOR_PERMISSIONS, MANAGER, MEMBER, ADMIN -from osf.exceptions import ValidationError from website import settings as website_settings from website import filters from website.project import new_bookmark_collection @@ -1054,23 +1053,12 @@ def update_is_active(self): def save(self, *args, **kwargs): from website import mailchimp_utils - was_creating = self._state.adding - has_domain = False - if not self.is_spammy: - has_domain = has_domain_in_user_fields_for_names(self) - - if has_domain and was_creating: - raise ValidationError('Invalid personal information.') - self.update_is_active() self.username = self.username.lower().strip() if self.username else None dirty_fields = self.get_dirty_fields(check_relationship=True) ret = super().save(*args, **kwargs) # must save BEFORE spam check, as user needs guid. - if has_domain and not was_creating: - self.confirm_spam() - if set(self.SPAM_USER_PROFILE_FIELDS.keys()).intersection(dirty_fields): request = get_current_request() headers = string_type_request_headers(request) diff --git a/osf/models/validators.py b/osf/models/validators.py index c5acafd3ea7..8411731d788 100644 --- a/osf/models/validators.py +++ b/osf/models/validators.py @@ -1,6 +1,6 @@ import re import waffle - +from django.db.models import Q from jsonschema import ValidationError as JsonSchemaValidationError, SchemaError, Draft7Validator, validate, validators from django.conf import settings from django.core.validators import URLValidator, validate_email as django_validate_email @@ -11,9 +11,10 @@ from osf.utils.registrations import FILE_VIEW_URL_REGEX from osf.utils.sanitize import strip_html from osf.exceptions import ValidationError, ValidationValueError, reraise_django_validation_errors, BlockedEmailError -from osf.external.spam.tasks import DOMAIN_REGEX +from osf.external.spam.tasks import DOMAIN_REGEX, NOTABLE_POST_NOMINALS from website.language import SWITCH_VALIDATOR_ERROR +from urlextract import URLExtract def validate_history_item(items): @@ -113,10 +114,31 @@ def validate_email(value): raise BlockedEmailError('Invalid Email') -def has_domain_in_user_fields_for_names(user): - name_content = ' '.join([getattr(user, field) or '' for field in user.DOMAIN_VALIDATION_FIELDS]) - return True if DOMAIN_REGEX.search(name_content) else False - +def has_domain_in_user_fields_for_names(fullname): + from osf.models import NotableDomain + notable_domain_list = set( + NotableDomain.objects.filter( + Q(note=NotableDomain.Note.ASSUME_HAM_UNTIL_REPORTED) | + Q(note=NotableDomain.Note.IGNORED) + ) + .values_list('domain', flat=True) + .distinct() + ) + for match in DOMAIN_REGEX.finditer(fullname): + domain = match.group('domain') + if domain in notable_domain_list: + continue + if looks_like_url(match): + return True + return False + +def looks_like_url(candidate): + candidate = candidate.group(0).strip() + words = re.findall(r'[A-Za-z.]+', candidate.lower()) + if any(word in NOTABLE_POST_NOMINALS for word in words): + return False + extractor = URLExtract() + return bool(extractor.find_urls(candidate)) def validate_subject_hierarchy_length(parent): from osf.models import Subject diff --git a/osf_tests/test_user.py b/osf_tests/test_user.py index 43ccf47ffcf..27a447d6681 100644 --- a/osf_tests/test_user.py +++ b/osf_tests/test_user.py @@ -2081,11 +2081,42 @@ def test_validate_domains_field(self): self.user.save() self.user.refresh_from_db() - assert self.user.is_spammy is True + assert self.user.is_spammy is False self.user.unspam(save=True) setattr(self.user, field, 'not a url') self.user.save() + def test_validate_domain_fields_unregistered_contributor(self): + project = ProjectFactory(creator=self.user) + with pytest.raises(ValidationError): + project.add_unregistered_contributor( + fullname='Visit google.com for fun spam action', + email='spammy@cos.io', + auth=Auth(self.user), + notification_type=False, + ) + assert not OSFUser.objects.filter(username='spammy@cos.io').exists() + + def test_validate_domain_fields_notable_domain_unregistered_contributor(self): + NotableDomain.objects.get_or_create(domain='google.com', note=NotableDomain.Note.IGNORED) + project = ProjectFactory(creator=self.user) + contributor = project.add_unregistered_contributor( + fullname='google.com', + email='notspammy@cos.io', + auth=Auth(self.user), + notification_type=False, + ) + assert contributor.is_spammy is False + + def test_validate_domain_fields_no_domain(self): + project = ProjectFactory(creator=self.user) + contributor = project.add_unregistered_contributor( + fullname='Sandhya N.Sathesh', + email='sandhya@cos.io', + auth=Auth(self.user), + notification_type=False, + ) + assert contributor.is_spammy is False class TestUserGdprDelete: diff --git a/osf_tests/test_validators.py b/osf_tests/test_validators.py index dc561389356..7e9baa86d72 100644 --- a/osf_tests/test_validators.py +++ b/osf_tests/test_validators.py @@ -1,9 +1,11 @@ import pytest from osf.exceptions import ValidationValueError -from osf.models import validators +from osf.models import validators, NotableDomain +from osf.models.validators import has_domain_in_user_fields_for_names from osf_tests.factories import SubjectFactory + # Ported from tests/framework/test_mongo.py def test_string_required_passes_with_string(): @@ -50,3 +52,49 @@ def test_validate_expand_subject_hierarchy(): subject_list = [fruit._id, '12345_bad_id'] with pytest.raises(ValidationValueError): validators.expand_subject_hierarchy(subject_list) + +@pytest.mark.parametrize( + 'fullname', + [ + 'Judith Sarah Preuss, M.Sc.', + 'J.H. van Hateren', + 'Sami-Egil Ahonen', + 'Giovanni Luca Ciampaglia', + 'Joseph P.R.O. Orgel', + 'Andrew Daoust', + 'Aidan G.C. Wright', + 'Guillermo Perez Algorta', + 'Sarah Wojkowski, MSc.PT, PhD.', + 'Brockmann, L.C. (Leon)', + 'Gragnolati, G.M. (Gaia Mariavittoria)', + 'F.H. Leeuwis', + 'Grauss, S.E. (Sophie)', + 'Sandhya N.Sathesh', + 'John Doe', + 'John Doe, B.Sc.', + 'John Doe, D.Sc.', + ] +) +def test_has_domain_in_user_fields(fullname): + assert has_domain_in_user_fields_for_names(fullname) is False + +@pytest.mark.parametrize( + 'fullname', + [ + 'Judith Sarah Visit https://www.google.com today', + 'J.H. https://google.com', + 'Judith Sarah www.google.com', + 'Judith Hateren google.com', + ] +) +def test_has_domain_in_user_fields_fail(fullname): + assert has_domain_in_user_fields_for_names(fullname) is True + +def test_has_notable_domain_in_user_fields(): + NotableDomain.objects.get_or_create(domain='osf.io', note=NotableDomain.Note.IGNORED) + assert has_domain_in_user_fields_for_names('Judith Sarah osf.io') is False + +def test_has_no_notable_domain_in_user_fields(): + NotableDomain.objects.get_or_create(domain='google.com', note=NotableDomain.Note.IGNORED) + assert has_domain_in_user_fields_for_names('Judith Sarah osf.io') is True + assert has_domain_in_user_fields_for_names('Sarah google.com buy-pills.example.com') is True diff --git a/poetry.lock b/poetry.lock index 67311478796..6a2bc3e2d44 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1556,7 +1556,7 @@ version = "3.16.1" description = "A platform independent file lock." optional = false python-versions = ">=3.8" -groups = ["dev"] +groups = ["main", "dev"] files = [ {file = "filelock-3.16.1-py3-none-any.whl", hash = "sha256:2082e5703d51fbf98ea75855d9d5527e33d8ff23099bec374a134febee6946b0"}, {file = "filelock-3.16.1.tar.gz", hash = "sha256:c249fbfcd5db47e5e2d6d62198e565475ee65e4831e2561c8e313fa7eb961435"}, @@ -2972,7 +2972,7 @@ version = "4.3.6" description = "A small Python package for determining appropriate platform-specific dirs, e.g. a `user data dir`." optional = false python-versions = ">=3.8" -groups = ["dev"] +groups = ["main", "dev"] files = [ {file = "platformdirs-4.3.6-py3-none-any.whl", hash = "sha256:73e575e1408ab8103900836b97580d5307456908a03e92031bab39e4554cc3fb"}, {file = "platformdirs-4.3.6.tar.gz", hash = "sha256:357fb2acbc885b0419afd3ce3ed34564c13c9b95c89360cd9563f73aa5e2b907"}, @@ -4449,6 +4449,36 @@ files = [ {file = "uritemplate-4.1.1.tar.gz", hash = "sha256:4346edfc5c3b79f694bccd6d6099a322bbeb628dbf2cd86eea55a456ce5124f0"}, ] +[[package]] +name = "uritools" +version = "6.1.2" +description = "URI parsing, classification and composition" +optional = false +python-versions = ">=3.10" +groups = ["main"] +files = [ + {file = "uritools-6.1.2-py3-none-any.whl", hash = "sha256:5682f8c58e96e379224fd72aec227a45432740c1fa860a06b3024d47c2968601"}, + {file = "uritools-6.1.2.tar.gz", hash = "sha256:fa60028843a8be651699a1ee2b399066eeaef349224b32a177efa4aeba463f00"}, +] + +[[package]] +name = "urlextract" +version = "1.9.0" +description = "Collects and extracts URLs from given text." +optional = false +python-versions = "*" +groups = ["main"] +files = [ + {file = "urlextract-1.9.0-py3-none-any.whl", hash = "sha256:f88963532488b1c7c405e21bd162ae97871754ea04b60e18d33ee075b19b82fd"}, + {file = "urlextract-1.9.0.tar.gz", hash = "sha256:70508e02ba9df372e25cf0642db367cece273e8712cd0ce78178fc5dd7ea00db"}, +] + +[package.dependencies] +filelock = "*" +idna = "*" +platformdirs = "*" +uritools = "*" + [[package]] name = "urllib3" version = "1.26.18" @@ -4727,4 +4757,4 @@ testing = ["coverage (>=5.0.3)", "zope.event", "zope.testing"] [metadata] lock-version = "2.1" python-versions = "^3.12" -content-hash = "9b2f29b9b4958b47e4c428f4225d04bbd12f239c02f9afc186989a49547b83ed" +content-hash = "5599dfc677ced71d0e9097822fb1d1f6f0e22320eacca87299d1542b89e9f286" diff --git a/pyproject.toml b/pyproject.toml index b789987d9a9..2112ccf307a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -102,6 +102,7 @@ djangorestframework-csv = "3.0.2" gevent = "24.2.1" packaging = "^24.0" flask-cors = "6.0.1" +urlextract = "^1.9.0" [tool.poetry.group.dev.dependencies] pytest = "7.4.4"