[ENG-10340] Check for flag

osf/external/spam/tasks.py

@@ -17,7 +17,7 @@
 
 DOMAIN_REGEX = re.compile(r'\W*(?P<protocol>\w+://)?(?P<www>www\.)?(?P<domain>([\w-]+\.)+[a-zA-Z]+)(?P<path>[/\-\.\w]*)?\W*')
 REDIRECT_CODES = {301, 302, 303, 307, 308}
-
+NOTABLE_POST_NOMINALS = ['m.sc.', 'msc.', 'b.sc.', 'bsc.', 'd.sc.', 'dsc.', 'phd.', 'ph.d.', 'msc.pt', 'pt.', 'prof.', 'dr.', 'md.', 'jd.', 'esq.']
 
 @celery_app.task()
 def reclassify_domain_references(notable_domain_id, current_note, previous_note):

osf/models/mixins.py

@@ -34,7 +34,7 @@
 from .nodelog import NodeLog
 from .subject import Subject
 from .spam import SpamMixin, SpamStatus
-from .validators import validate_title
+from .validators import validate_title, has_domain_in_user_fields_for_names
 from .tag import Tag
 from osf.utils import sanitize
 from .validators import validate_subject_hierarchy, validate_email, expand_subject_hierarchy
@@ -1581,6 +1581,9 @@ def add_unregistered_contributor(
             except BlockedEmailError:
                 raise ValidationError('Unregistered contributor email address domain is blocked.')
 
+        if has_domain_in_user_fields_for_names(fullname):
+            raise ValidationError('Invalid personal information.')
+
         # Create a new user record if you weren't passed an existing user
         contributor = existing_user if existing_user else OSFUser.create_unregistered(fullname=fullname, email=email)
 

osf/models/user.py

@@ -57,13 +57,12 @@
 from .spam import SpamMixin
 from .session import UserSessionMap
 from .tag import Tag
-from .validators import validate_email, validate_social, validate_history_item, has_domain_in_user_fields_for_names
+from .validators import validate_email, validate_social, validate_history_item
 from osf.utils.datetime_aware_jsonfield import DateTimeAwareJSONField
 from osf.utils.fields import NonNaiveDateTimeField, LowercaseEmailField, ensure_str
 from osf.utils.names import impute_names
 from osf.utils.requests import check_select_for_update
 from osf.utils.permissions import API_CONTRIBUTOR_PERMISSIONS, MANAGER, MEMBER, ADMIN
-from osf.exceptions import ValidationError
 from website import settings as website_settings
 from website import filters
 from website.project import new_bookmark_collection
@@ -1054,23 +1053,12 @@ def update_is_active(self):
     def save(self, *args, **kwargs):
         from website import mailchimp_utils
 
-        was_creating = self._state.adding
-        has_domain = False
-        if not self.is_spammy:
-            has_domain = has_domain_in_user_fields_for_names(self)
-
-        if has_domain and was_creating:
-            raise ValidationError('Invalid personal information.')
-
         self.update_is_active()
         self.username = self.username.lower().strip() if self.username else None
 
         dirty_fields = self.get_dirty_fields(check_relationship=True)
         ret = super().save(*args, **kwargs)  # must save BEFORE spam check, as user needs guid.
 
-        if has_domain and not was_creating:
-            self.confirm_spam()
-
         if set(self.SPAM_USER_PROFILE_FIELDS.keys()).intersection(dirty_fields):
             request = get_current_request()
             headers = string_type_request_headers(request)

osf/models/validators.py

@@ -1,6 +1,6 @@
 import re
 import waffle
-
+from django.db.models import Q
 from jsonschema import ValidationError as JsonSchemaValidationError, SchemaError, Draft7Validator, validate, validators
 from django.conf import settings
 from django.core.validators import URLValidator, validate_email as django_validate_email
@@ -11,9 +11,10 @@
 from osf.utils.registrations import FILE_VIEW_URL_REGEX
 from osf.utils.sanitize import strip_html
 from osf.exceptions import ValidationError, ValidationValueError, reraise_django_validation_errors, BlockedEmailError
-from osf.external.spam.tasks import DOMAIN_REGEX
+from osf.external.spam.tasks import DOMAIN_REGEX, NOTABLE_POST_NOMINALS
 
 from website.language import SWITCH_VALIDATOR_ERROR
+from urlextract import URLExtract
 
 
 def validate_history_item(items):
@@ -113,10 +114,31 @@ def validate_email(value):
         raise BlockedEmailError('Invalid Email')
 
 
-def has_domain_in_user_fields_for_names(user):
-    name_content = ' '.join([getattr(user, field) or '' for field in user.DOMAIN_VALIDATION_FIELDS])
-    return True if DOMAIN_REGEX.search(name_content) else False
-
+def has_domain_in_user_fields_for_names(fullname):
+    from osf.models import NotableDomain
+    notable_domain_list = set(
+        NotableDomain.objects.filter(
+            Q(note=NotableDomain.Note.ASSUME_HAM_UNTIL_REPORTED) |
+            Q(note=NotableDomain.Note.IGNORED)
+        )
+        .values_list('domain', flat=True)
+        .distinct()
+    )
+    for match in DOMAIN_REGEX.finditer(fullname):
+        domain = match.group('domain')
+        if domain in notable_domain_list:
+            continue
+        if looks_like_url(match):
+            return True
+    return False
+
+def looks_like_url(candidate):
+    candidate = candidate.group(0).strip()
+    words = re.findall(r'[A-Za-z.]+', candidate.lower())
+    if any(word in NOTABLE_POST_NOMINALS for word in words):
+        return False
+    extractor = URLExtract()
+    return bool(extractor.find_urls(candidate))
 
 def validate_subject_hierarchy_length(parent):
     from osf.models import Subject

osf_tests/test_user.py

@@ -2081,11 +2081,42 @@ def test_validate_domains_field(self):
             self.user.save()
             self.user.refresh_from_db()
 
-            assert self.user.is_spammy is True
+            assert self.user.is_spammy is False
             self.user.unspam(save=True)
             setattr(self.user, field, 'not a url')
             self.user.save()
 
+    def test_validate_domain_fields_unregistered_contributor(self):
+        project = ProjectFactory(creator=self.user)
+        with pytest.raises(ValidationError):
+            project.add_unregistered_contributor(
+                fullname='Visit google.com for fun spam action',
+                email='spammy@cos.io',
+                auth=Auth(self.user),
+                notification_type=False,
+            )
+        assert not OSFUser.objects.filter(username='spammy@cos.io').exists()
+
+    def test_validate_domain_fields_notable_domain_unregistered_contributor(self):
+        NotableDomain.objects.get_or_create(domain='google.com', note=NotableDomain.Note.IGNORED)
+        project = ProjectFactory(creator=self.user)
+        contributor = project.add_unregistered_contributor(
+            fullname='google.com',
+            email='notspammy@cos.io',
+            auth=Auth(self.user),
+            notification_type=False,
+        )
+        assert contributor.is_spammy is False
+
+    def test_validate_domain_fields_no_domain(self):
+        project = ProjectFactory(creator=self.user)
+        contributor = project.add_unregistered_contributor(
+            fullname='Sandhya N.Sathesh',
+            email='sandhya@cos.io',
+            auth=Auth(self.user),
+            notification_type=False,
+        )
+        assert contributor.is_spammy is False
 
 class TestUserGdprDelete:
 

osf_tests/test_validators.py

@@ -1,9 +1,11 @@
 import pytest
 from osf.exceptions import ValidationValueError
 
-from osf.models import validators
+from osf.models import validators, NotableDomain
+from osf.models.validators import has_domain_in_user_fields_for_names
 from osf_tests.factories import SubjectFactory
 
+
 # Ported from tests/framework/test_mongo.py
 
 def test_string_required_passes_with_string():
@@ -50,3 +52,49 @@ def test_validate_expand_subject_hierarchy():
     subject_list = [fruit._id, '12345_bad_id']
     with pytest.raises(ValidationValueError):
         validators.expand_subject_hierarchy(subject_list)
+
+@pytest.mark.parametrize(
+    'fullname',
+    [
+        'Judith Sarah Preuss, M.Sc.',
+        'J.H. van Hateren',
+        'Sami-Egil Ahonen',
+        'Giovanni Luca Ciampaglia',
+        'Joseph P.R.O. Orgel',
+        'Andrew Daoust',
+        'Aidan G.C. Wright',
+        'Guillermo Perez Algorta',
+        'Sarah Wojkowski, MSc.PT, PhD.',
+        'Brockmann, L.C. (Leon)',
+        'Gragnolati, G.M. (Gaia Mariavittoria)',
+        'F.H. Leeuwis',
+        'Grauss, S.E. (Sophie)',
+        'Sandhya N.Sathesh',
+        'John Doe',
+        'John Doe, B.Sc.',
+        'John Doe, D.Sc.',
+    ]
+)
+def test_has_domain_in_user_fields(fullname):
+    assert has_domain_in_user_fields_for_names(fullname) is False
+
+@pytest.mark.parametrize(
+    'fullname',
+    [
+        'Judith Sarah Visit https://www.google.com today',
+        'J.H. https://google.com',
+        'Judith Sarah www.google.com',
+        'Judith Hateren google.com',
+    ]
+)
+def test_has_domain_in_user_fields_fail(fullname):
+    assert has_domain_in_user_fields_for_names(fullname) is True
+
+def test_has_notable_domain_in_user_fields():
+    NotableDomain.objects.get_or_create(domain='osf.io', note=NotableDomain.Note.IGNORED)
+    assert has_domain_in_user_fields_for_names('Judith Sarah osf.io') is False
+
+def test_has_no_notable_domain_in_user_fields():
+    NotableDomain.objects.get_or_create(domain='google.com', note=NotableDomain.Note.IGNORED)
+    assert has_domain_in_user_fields_for_names('Judith Sarah osf.io') is True
+    assert has_domain_in_user_fields_for_names('Sarah google.com buy-pills.example.com') is True

pyproject.toml

@@ -102,6 +102,7 @@ djangorestframework-csv = "3.0.2"
 gevent = "24.2.1"
 packaging = "^24.0"
 flask-cors = "6.0.1"
+urlextract = "^1.9.0"
 
 [tool.poetry.group.dev.dependencies]
 pytest = "7.4.4"