From e094d8cbb7bb789382264ba2278e021576710119 Mon Sep 17 00:00:00 2001 From: Richard Jones Date: Thu, 30 Apr 2026 17:23:40 +0100 Subject: [PATCH 1/5] add support for user attributes on user accounts --- portality/constants.py | 21 ++ portality/forms/validate.py | 20 +- .../migrate/4317_user_attributes/__init__.py | 0 .../apply_user_attributes.py | 185 ++++++++++++++++++ portality/models/account.py | 79 ++++++++ .../_account/includes/_edit_form_js.html | 67 ++++++- .../_account/includes/_edit_user_form.html | 57 +++++- portality/templates-v2/management/base.html | 12 ++ portality/view/account.py | 50 ++++- 9 files changed, 473 insertions(+), 18 deletions(-) create mode 100644 portality/migrate/4317_user_attributes/__init__.py create mode 100644 portality/migrate/4317_user_attributes/apply_user_attributes.py diff --git a/portality/constants.py b/portality/constants.py index 4ed8d7c3a5..a8569d71f3 100644 --- a/portality/constants.py +++ b/portality/constants.py @@ -113,6 +113,27 @@ # TODO add ultra_bulk_delete and refactor view to use constants ROLE_ADMIN_REPORT_WITH_NOTES = "ultra_admin_reports_with_notes" # MUST start with ultra_ so that superusers don't gain +USER_ATTR__WORKFLOW = "workflow" +USER_ATTR__LANGUAGE = "language" +USER_ATTR__COUNTRY = "country" +USER_ATTR__TAG = "tag" + +USER_ATTR__ALL = [ + USER_ATTR__WORKFLOW, + USER_ATTR__LANGUAGE, + USER_ATTR__COUNTRY, + USER_ATTR__TAG +] + +EWF__TRIAGE = "Triage" +EWF__QUICK_FAIL = "Quick Fail" +EWF__QUALITY_REVIEW = "Quality Review" + +EWF__ALL_STAGES = [ + EWF__TRIAGE, + EWF__QUICK_FAIL, + EWF__QUALITY_REVIEW +] CRON_NEVER = {"month": "2", "day": "31", "day_of_week": "*", "hour": "*", "minute": "*"} diff --git a/portality/forms/validate.py b/portality/forms/validate.py index 228ca2b803..e3afc275c3 100644 --- a/portality/forms/validate.py +++ b/portality/forms/validate.py @@ -670,9 +670,13 @@ def __init__(self, message=None): def __call__(self, form, field): if field.data is not None and field.data != '': - check = get_currency_code(field.data, fail_if_not_found=True) - if check is None: - raise validators.ValidationError(self.message) + value = field.data + if not isinstance(value, list): + value = [value] + for v in value: + check = get_currency_code(v, fail_if_not_found=True) + if check is None: + raise validators.ValidationError(self.message) class CurrentISOLanguage(object): @@ -683,6 +687,10 @@ def __init__(self, message=None): def __call__(self, form, field): if field.data is not None and field.data != '': - check = isolang.find(field.data) - if check is None: - raise validators.ValidationError(self.message) + value = field.data + if not isinstance(value, list): + value = [value] + for v in value: + check = isolang.find(v) + if check is None: + raise validators.ValidationError(self.message) diff --git a/portality/migrate/4317_user_attributes/__init__.py b/portality/migrate/4317_user_attributes/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/portality/migrate/4317_user_attributes/apply_user_attributes.py b/portality/migrate/4317_user_attributes/apply_user_attributes.py new file mode 100644 index 0000000000..dde8fd1a38 --- /dev/null +++ b/portality/migrate/4317_user_attributes/apply_user_attributes.py @@ -0,0 +1,185 @@ +"""Migration script to add user attributes (country and language) based on Editor Group CSVs. + +Usage: + python -m portality.migrate.4317_user_attributes /path/to/countries.csv /path/to/languages.csv + +The first CSV should have two columns with headers: the first is the country name, the second the Editor Group name. +The second CSV should have two columns with headers: the first is the language name, the second the Editor Group name. + +For each row this script finds the EditorGroup by name, collects the accounts attached to the group +(maned, editor, associates) and schedules the appropriate ISO code to be added to each user's attributes +('country' or 'language'). All validations are done against the portality datasets (pycountry/isolang). + +The script collates all labels for each user and performs a single save per user at the end. +""" + +import csv +import sys +import argparse +from collections import defaultdict + +from portality.models.editors import EditorGroup +from portality.models.account import Account +from portality import datasets, constants +from portality.lib import isolang +from portality.core import app + + +def resolve_country_code(name: str): + """Return a 2-character country code for a given country name, or None if not found.""" + if not name: + return None + code = datasets.get_country_code(name, fail_if_not_found=True) + return code + + +def resolve_language_code(name: str): + """Return a 2-character language code (alpha2) for a given language name, or None if not found.""" + if not name: + return None + lang = isolang.find(name) + if not lang: + return None + # isolang.find returns a dict with 'alpha2' key + return lang.get('alpha2') + + +def read_pairs_from_csv(path): + """Read CSV and return list of (label, editor_group_name) pairs using the first two columns. + + Expects two columns with headers; will take the first and second header values. + """ + pairs = [] + with open(path, newline='') as fh: + reader = csv.DictReader(fh) + headers = reader.fieldnames + if not headers or len(headers) < 2: + raise ValueError(f"CSV at {path} must have at least two columns with headers") + col_label = headers[0] + col_group = headers[1] + for row in reader: + label = (row.get(col_label) or '').strip() + group = (row.get(col_group) or '').strip() + if label and group: + pairs.append((label, group)) + return pairs + + +def collate_attributes(country_pairs, language_pairs): + """Process pairs and collate desired attributes per account id. + + Returns mapping: account_id -> { 'country': set(...), 'language': set(...) } + """ + account_attrs = defaultdict(lambda: {'country': set(), 'language': set()}) + + def _collect_for_group(group_name, code, attr_type): + eg = EditorGroup.group_by_name(group_name) + if eg is None: + app.logger.warning(f"EditorGroup not found: '{group_name}'") + print(f"WARNING: EditorGroup not found: '{group_name}'") + return + + # collect maned, editor, associates + acct_ids = set() + if eg.maned: + acct_ids.add(eg.maned) + if eg.editor: + acct_ids.add(eg.editor) + for a in eg.associates or []: + acct_ids.add(a) + + for aid in acct_ids: + if not aid: + continue + account_attrs[aid][attr_type].add(code) + + # countries + for country_name, group_name in country_pairs: + code = resolve_country_code(country_name) + if code is None: + app.logger.warning(f"Unable to resolve country '{country_name}'") + print(f"WARNING: Unable to resolve country '{country_name}'") + continue + _collect_for_group(group_name, code.upper(), 'country') + + # languages + for lang_name, group_name in language_pairs: + code = resolve_language_code(lang_name) + if code is None: + app.logger.warning(f"Unable to resolve language '{lang_name}'") + print(f"WARNING: Unable to resolve language '{lang_name}'") + continue + _collect_for_group(group_name, code.upper(), 'language') + + return account_attrs + + +def apply_attributes(account_attrs, dry_run=False): + """Apply collated attributes to Account objects and save them once each. + + Returns a summary dict. + """ + summary = {'updated': 0, 'skipped_missing_account': 0} + for aid, attrs in account_attrs.items(): + acc = Account.pull(aid) + if acc is None: + summary['skipped_missing_account'] += 1 + app.logger.warning(f"Account not found: {aid}") + print(f"WARNING: Account not found: {aid}") + continue + + changed = False + # countries + for c in sorted(attrs.get('country', [])): + try: + if not acc.has_attribute(constants.USER_ATTR__COUNTRY, c): + acc.add_attribute(constants.USER_ATTR__COUNTRY, c) + changed = True + except Exception: + app.logger.exception(f"Failed to add country attribute '{c}' to account {aid}") + + for l in sorted(attrs.get('language', [])): + try: + if not acc.has_attribute(constants.USER_ATTR__LANGUAGE, l): + acc.add_attribute(constants.USER_ATTR__LANGUAGE, l) + changed = True + except Exception: + app.logger.exception(f"Failed to add language attribute '{l}' to account {aid}") + + if changed: + if dry_run: + print(f"DRY RUN: Would save account {aid} with new attributes: {attrs}") + else: + acc.save() + print(f"Saved account {aid} (added {len(attrs.get('country', []))} country(s), {len(attrs.get('language', []))} language(s))") + summary['updated'] += 1 + + return summary + + +def main(): + parser = argparse.ArgumentParser(description='Add user attributes from Editor Group CSVs') + parser.add_argument('countries_csv', help='CSV: first col country name, second col Editor Group name') + parser.add_argument('languages_csv', help='CSV: first col language name, second col Editor Group name') + parser.add_argument('--dry-run', action='store_true', help='Do not save changes, just report') + + args = parser.parse_args() + + country_pairs = read_pairs_from_csv(args.countries_csv) + language_pairs = read_pairs_from_csv(args.languages_csv) + + print(f"Read {len(country_pairs)} country rows and {len(language_pairs)} language rows") + + account_attrs = collate_attributes(country_pairs, language_pairs) + + print(f"Collated attributes for {len(account_attrs)} accounts") + + summary = apply_attributes(account_attrs, dry_run=args.dry_run) + + print("Done. Summary:") + print(summary) + + +if __name__ == '__main__': + main() + diff --git a/portality/models/account.py b/portality/models/account.py index bd733d98f2..714917a77a 100644 --- a/portality/models/account.py +++ b/portality/models/account.py @@ -219,6 +219,56 @@ def set_role(self, role): role = [role] self.data["role"] = role + ############################### + ## user attributes + + @property + def attributes(self): + return self.data.get("attributes") + + @attributes.deleter + def attributes(self): + if "attributes" in self.data: + del self.data["attributes"] + + @property + def attribute_workflow(self): + return self.data.get("attributes", {}).get(constants.USER_ATTR__WORKFLOW, []) + + @property + def attribute_language(self): + return self.data.get("attributes", {}).get(constants.USER_ATTR__LANGUAGE, []) + + @property + def attribute_country(self): + return self.data.get("attributes", {}).get(constants.USER_ATTR__COUNTRY, []) + + @property + def attribute_tag(self): + return self.data.get("attributes", {}).get(constants.USER_ATTR__TAG, []) + + def add_attribute(self, attribute_type, value): + if attribute_type not in constants.USER_ATTR__ALL: + raise ValueError("Unknown user attribute type: {}".format(attribute_type)) + if "attributes" not in self.data: + self.data["attributes"] = {} + if attribute_type not in self.data["attributes"]: + self.data["attributes"][attribute_type] = [] + if value not in self.data["attributes"][attribute_type]: + self.data["attributes"][attribute_type].append(value) + + def has_attribute(self, attribute_type, value): + if attribute_type not in constants.USER_ATTR__ALL: + raise ValueError("Unknown user attribute type: {}".format(attribute_type)) + return value in self.data.get("attributes", {}).get(attribute_type, []) + + def get_attributes(self, attribute_type): + if attribute_type not in constants.USER_ATTR__ALL: + raise ValueError("Unknown user attribute type: {}".format(attribute_type)) + return self.data.get("attributes", {}).get(attribute_type, []) + + ####################### + def prep(self): self.data['last_updated'] = dates.now_str() @@ -273,3 +323,32 @@ def is_enable_publisher_email(cls) -> bool: # TODO: in the long run this needs to move out to the user's email preferences but for now it # is here to replicate the behaviour in the code it replaces return app.config.get("ENABLE_PUBLISHER_EMAIL", False) + + @classmethod + def find_by_attributes(cls, attribute_types_and_values:list[tuple[str, str]], limit=1000): + q = AttributesQuery(attribute_types_and_values, limit) + return cls.object_query(q.query()) + + +class AttributesQuery: + def __init__(self, attribute_types_and_values, page_size): + self._tup = attribute_types_and_values + self._size = page_size + + def query(self): + musts = [] + for t, v in self._tup.items(): + if not isinstance(v, list): + v = [v] + f = {"terms": {f"attribute.{t}.exact": v}} + musts.append(f) + + q = { + "query": { + "bool": { + "must": musts + } + }, + "size": self._size + } + return q diff --git a/portality/templates-v2/_account/includes/_edit_form_js.html b/portality/templates-v2/_account/includes/_edit_form_js.html index bb6cf3ebfe..d313f79361 100644 --- a/portality/templates-v2/_account/includes/_edit_form_js.html +++ b/portality/templates-v2/_account/includes/_edit_form_js.html @@ -2,8 +2,71 @@