From 138f2e8a9c8c15ced1da45696a200d5165a1b92d Mon Sep 17 00:00:00 2001 From: Martin Rehr Date: Mon, 26 Jan 2026 14:30:56 +0100 Subject: [PATCH 01/14] Added support for 'grid_accounting' daemon that generate user storage accounting based on 'home', 'vgrid' and 'peers' usage. --- bin/showaccounting.py | 191 +++++ mig/install/MiGserver-template.conf | 6 + mig/install/migrid-init.d-deb-template | 53 +- mig/install/migrid-init.d-rh-template | 50 +- mig/lib/accounting.py | 709 ++++++++++++++++++ mig/server/grid_accounting.py | 1 + mig/shared/configuration.py | 23 +- mig/shared/install.py | 6 + sbin/grid_accounting.py | 132 ++++ tests/fixture/confs-stdlocal/MiGserver.conf | 6 + .../fixture/confs-stdlocal/migrid-init.d-deb | 53 +- tests/fixture/confs-stdlocal/migrid-init.d-rh | 50 +- .../mig_shared_configuration--new.json | 1 + tests/test_mig_lib_accounting.py | 222 ++++++ 14 files changed, 1494 insertions(+), 9 deletions(-) create mode 100755 bin/showaccounting.py create mode 100644 mig/lib/accounting.py create mode 120000 mig/server/grid_accounting.py create mode 100755 sbin/grid_accounting.py create mode 100644 tests/test_mig_lib_accounting.py diff --git a/bin/showaccounting.py b/bin/showaccounting.py new file mode 100755 index 000000000..8d81703e1 --- /dev/null +++ b/bin/showaccounting.py @@ -0,0 +1,191 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# +# --- BEGIN_HEADER --- +# +# showaccounting - Display storage accounting +# Copyright (C) 2003-2026 The MiG Project by the Science HPC Center at UCPH +# +# This file is part of MiG. +# +# MiG is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# MiG is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# +# -- END_HEADER --- +# + +"""Create accounting information for users""" + +from __future__ import print_function +from __future__ import absolute_import + +import sys +import getopt +import re +import datetime + +from mig.shared.conf import get_configuration_object +from mig.lib.accounting import get_usage, human_readable_filesize + + +def usage(name='accounting.py'): + """Usage help""" + + print("""Create accounting information based on quota. +Usage: +%(name)s [ACCOUNTING_OPTIONS] +Where ACCOUNTING_OPTIONS may be one or more of: + -h Show this help + -v Verbose output + -c CONF_FILE Use CONF_FILE as server configuration + -f User filter Regex user (CERT_DN) filter + -m Minimum usage Only show accounts using more than + minimum usage (TB). + -t TIMESTAMP Use specific timestamp, latest if unset +""" % {'name': name}) + + +def show_accounting(configuration, + timestamp, + user_filter, + minimum_usage): + """Print user accointing report""" + user_filter_re = None + if user_filter: + try: + user_filter_re = re.compile(user_filter) + except Exception as err: + print("ERROR: Failed to compile user_filter: %r error: %s" + % (user_filter, err)) + return + + usage = get_usage(configuration, + timestamp=timestamp, + verbose=verbose) + + accounting = usage.get('accounting', {}) + accounting_timestamp = usage.get('timestamp', 0) + accounting_datestr \ + = datetime.datetime.fromtimestamp(accounting_timestamp) \ + .strftime('%d/%m/%Y-%H:%M:%S') + + # Sorted by total bytes and print usage for users + + report_total_users = 0 + report_shown_users = 0 + report_total_bytes = 0 + report_shown_bytes = 0 + total_bytes_map = {} + for username, values in accounting.items(): + # Do not show GDP project users + # projects are accounted for by the main user + if configuration.site_enable_gdp \ + and username.find("/GDP=") != -1: + continue + report_total_users += 1 + total_bytes = values.get('total_bytes', 0) + report_total_bytes += total_bytes + if total_bytes < minimum_usage \ + or user_filter_re and not user_filter_re.fullmatch(username): + continue + report_shown_users += 1 + report_shown_bytes += total_bytes + total_bytes_map_userlist = total_bytes_map.get(total_bytes, []) + total_bytes_map_userlist.append(username) + total_bytes_map[total_bytes] = total_bytes_map_userlist + sorted_total_bytes = sorted(list(total_bytes_map.keys()), reverse=True) + + print("\nAccounting (%d) %s for storage quota(s):" + % (accounting_timestamp, accounting_datestr)) + for quota_fs, values in usage.get('quota', {}).items(): + quota_mtime = values.get('mtime', 0) + quota_datestr = datetime.datetime.fromtimestamp(quota_mtime) \ + .strftime('%d/%m/%Y-%H:%M:%S') + print(" - %s (%d) %s" % (quota_fs, + quota_mtime, + quota_datestr)) + + print("Found a total of %s users using %s storage" + % (report_total_users, + human_readable_filesize(report_total_bytes))) + print("Showing details for %s users using %s storage " + % (report_shown_users, + human_readable_filesize(report_shown_bytes))) + print("User filter: %r" % user_filter) + print("Minumum usage: %s" % human_readable_filesize(minimum_usage)) + for total_bytes in sorted_total_bytes: + total_bytes_human = human_readable_filesize(total_bytes) + for username in total_bytes_map[total_bytes]: + report = accounting[username] + home_report = report.get('home_report', '') + freeze_report = report.get('freeze_report', '') + vgrid_report = report.get('vgrid_report', '') + ext_users_report = report.get('ext_users_report', '') + peers_report = report.get('peers_report', '') + print("\n%s:" % username) + print("Total usage: %s" % total_bytes_human) + if home_report: + print(home_report) + if freeze_report: + print(freeze_report) + if vgrid_report: + print(vgrid_report) + if ext_users_report: + print(ext_users_report) + if peers_report: + print(peers_report) + + +if '__main__' == __name__: + conf_path = None + user_filter = None + timestamp = 0 + minimum_usage = 0 + verbose = False + opt_args = 'hvc:f:m:t:' + try: + (opts, args) = getopt.getopt(sys.argv[1:], opt_args) + for (opt, val) in opts: + if opt == '-h': + usage() + sys.exit(0) + if opt == '-v': + verbose = True + elif opt == '-c': + conf_path = val + elif opt == '-f': + user_filter = val + elif opt == '-m': + minimum_usage = float(val)*(1024**4) + elif opt == '-t': + timestamp = int(val) + else: + print('Error: %s not supported!' % opt) + usage() + sys.exit(1) + except getopt.GetoptError as err: + print('Error: ', err.msg) + usage() + sys.exit(1) + + configuration = get_configuration_object(config_file=conf_path, + skip_log=True, + disable_auth_log=True) + + show_accounting(configuration, + timestamp, + user_filter, + minimum_usage) + + sys.exit(0) diff --git a/mig/install/MiGserver-template.conf b/mig/install/MiGserver-template.conf index 568cc737f..330804f82 100644 --- a/mig/install/MiGserver-template.conf +++ b/mig/install/MiGserver-template.conf @@ -127,6 +127,7 @@ workflows_home = %(state_path)s/workflows_home/ workflows_db_home = %(state_path)s/workflows_db_home/ notify_home = %(state_path)s/notify_home/ quota_home = %(state_path)s/quota_home/ +accounting_home = %(state_path)s/accounting_home/ # GDP data categories metadata and helpers json file gdp_data_categories = %(gdp_home)s/__GDP_DATA_CATEGORIES__ @@ -554,6 +555,9 @@ update_interval = __QUOTA_UPDATE_INTERVAL__ user_limit = __QUOTA_USER_LIMIT__ vgrid_limit = __QUOTA_VGRID_LIMIT__ +[ACCOUNTING] +update_interval = __ACCOUNTING_UPDATE_INTERVAL__ + [SITE] # Web site appearance # Whether to use Python 3 for all Python invocations @@ -677,6 +681,8 @@ enable_openid = __ENABLE_OPENID__ enable_sharelinks = __ENABLE_SHARELINKS__ # Enable storage quota enable_quota = __ENABLE_QUOTA__ +# Enable storage accounting +enable_accounting = __ENABLE_ACCOUNTING__ # Enable background data transfers daemon - requires lftp and rsync enable_transfers = __ENABLE_TRANSFERS__ # Explicit background transfer source addresses for use in pub key restrictions diff --git a/mig/install/migrid-init.d-deb-template b/mig/install/migrid-init.d-deb-template index b83ed781f..4d02c2701 100755 --- a/mig/install/migrid-init.d-deb-template +++ b/mig/install/migrid-init.d-deb-template @@ -70,13 +70,14 @@ MIG_NOTIFY=${MIG_CODE}/server/grid_notify.py MIG_IMNOTIFY=${MIG_CODE}/server/grid_imnotify.py MIG_VMPROXY=${MIG_CODE}/server/grid_vmproxy.py MIG_QUOTA=${MIG_CODE}/server/grid_quota.py +MIG_ACCOUNTING=${MIG_CODE}/server/grid_accounting.py MIG_CHKUSERROOT=${MIG_CODE}/server/chkuserroot.py MIG_CHKSIDROOT=${MIG_CODE}/server/chksidroot.py show_usage() { echo "Usage: migrid {start|stop|status|restart|reload}[daemon DAEMON]" echo "where daemon is left out for all or given along with DAEMON as one of the following" - echo "(script|monitor|sshmux|events|cron|janitor|transfers|openid|sftp|sftpsubsys|webdavs|ftps|notify|imnotify|vmproxy|quota|all)" + echo "(script|monitor|sshmux|events|cron|janitor|transfers|openid|sftp|sftpsubsys|webdavs|ftps|notify|imnotify|vmproxy|quota|accounting|all)" } check_enabled() { @@ -284,6 +285,18 @@ start_quota() { log_end_msg 1 || true fi } +start_accounting() { + check_enabled "accounting" || return 0 + DAEMON_PATH=${MIG_QUOTA} + SHORT_NAME=$(basename ${DAEMON_PATH}) + PID_FILE="$PID_DIR/${SHORT_NAME}.pid" + log_daemon_msg "Starting MiG accounting daemon" ${SHORT_NAME} || true + if start-stop-daemon --start --quiet --oknodo --pidfile ${PID_FILE} --make-pidfile --user root --chuid root --background --name ${SHORT_NAME} --startas ${DAEMON_PATH} ; then + log_end_msg 0 || true + else + log_end_msg 1 || true + fi +} start_sftpsubsys() { check_enabled "sftp_subsys" || return 0 DAEMON_PATH=${MIG_SFTPSUBSYS} @@ -313,6 +326,7 @@ start_all() { start_imnotify start_vmproxy start_quota + start_accounting return 0 } @@ -558,6 +572,19 @@ stop_quota() { log_end_msg 1 || true fi } +stop_accounting() { + check_enabled "accounting" || return 0 + DAEMON_PATH=${MIG_QUOTA} + SHORT_NAME=$(basename ${DAEMON_PATH}) + PID_FILE="$PID_DIR/${SHORT_NAME}.pid" + log_daemon_msg "Stopping MiG accounting" ${SHORT_NAME} || true + if start-stop-daemon --stop --quiet --oknodo --pidfile ${PID_FILE} ; then + rm -f ${PID_FILE} + log_end_msg 0 || true + else + log_end_msg 1 || true + fi +} stop_sftpsubsys_workers() { DAEMON_PATH=${MIG_SFTPSUBSYS_WORKER} SHORT_NAME=$(basename ${DAEMON_PATH}) @@ -598,6 +625,7 @@ stop_all() { stop_imnotify stop_vmproxy stop_quota + stop_accounting return 0 } @@ -782,6 +810,18 @@ reload_quota() { log_end_msg 1 || true fi } +reload_accounting() { + check_enabled "accounting" || return 0 + DAEMON_PATH=${MIG_QUOTA} + SHORT_NAME=$(basename ${DAEMON_PATH}) + PID_FILE="$PID_DIR/${SHORT_NAME}.pid" + log_daemon_msg "Reloading MiG accounting" ${SHORT_NAME} || true + if start-stop-daemon --stop --signal HUP --quiet --oknodo --pidfile ${PID_FILE} ; then + log_end_msg 0 || true + else + log_end_msg 1 || true + fi +} reload_sftpsubsys_workers() { DAEMON_PATH=${MIG_SFTPSUBSYS_WORKER} SHORT_NAME=$(basename ${DAEMON_PATH}) @@ -835,6 +875,7 @@ reload_all() { reload_imnotify reload_vmproxy reload_quota + reload_accounting # Apache helpers to verify proper chrooting reload_chkuserroot reload_chksidroot @@ -946,6 +987,13 @@ status_quota() { PID_FILE="$PID_DIR/${SHORT_NAME}.pid" status_of_proc -p ${PID_FILE} ${DAEMON_PATH} ${SHORT_NAME} } +status_accounting() { + check_enabled "accounting" || return 0 + DAEMON_PATH=${MIG_QUOTA} + SHORT_NAME=$(basename ${DAEMON_PATH}) + PID_FILE="$PID_DIR/${SHORT_NAME}.pid" + status_of_proc -p ${PID_FILE} ${DAEMON_PATH} ${SHORT_NAME} +} status_sftpsubsys_workers() { DAEMON_PATH=${MIG_SFTPSUBSYS_WORKER} SHORT_NAME=$(basename ${DAEMON_PATH}) @@ -985,6 +1033,7 @@ status_all() { status_imnotify status_vmproxy status_quota + status_accounting return 0 } @@ -996,7 +1045,7 @@ test -f ${MIG_SCRIPT} || exit 0 # Force valid target case "$2" in - script|monitor|sshmux|events|cron|janitor|transfers|openid|sftp|sftpsubsys|webdavs|ftps|notify|imnotify|vmproxy|quota|all) + script|monitor|sshmux|events|cron|janitor|transfers|openid|sftp|sftpsubsys|webdavs|ftps|notify|imnotify|vmproxy|quota|accounting|all) TARGET="$2" ;; '') diff --git a/mig/install/migrid-init.d-rh-template b/mig/install/migrid-init.d-rh-template index 9c838ef0e..cc5543400 100755 --- a/mig/install/migrid-init.d-rh-template +++ b/mig/install/migrid-init.d-rh-template @@ -36,6 +36,7 @@ # processname: grid_imnotify.py # processname: grid_vmproxy.py # processname: grid_quota.py +# processname: grid_accounting.py # processname: sshd # config: /etc/sysconfig/migrid # @@ -102,13 +103,14 @@ MIG_NOTIFY=${MIG_CODE}/server/grid_notify.py MIG_IMNOTIFY=${MIG_CODE}/server/grid_imnotify.py MIG_VMPROXY=${MIG_CODE}/server/grid_vmproxy.py MIG_QUOTA=${MIG_CODE}/server/grid_quota.py +MIG_ACCOUNTING=${MIG_CODE}/server/grid_accounting.py MIG_CHKUSERROOT=${MIG_CODE}/server/chkuserroot.py MIG_CHKSIDROOT=${MIG_CODE}/server/chksidroot.py show_usage() { echo "Usage: migrid {start|stop|status|restart|reload}[daemon DAEMON]" echo "where daemon is left out for all or given along with DAEMON as one of the following" - echo "(script|monitor|sshmux|events|cron|janitor|transfers|openid|sftp|sftpsubsys|webdavs|ftps|notify|imnotify|vmproxy|quota|all)" + echo "(script|monitor|sshmux|events|cron|janitor|transfers|openid|sftp|sftpsubsys|webdavs|ftps|notify|imnotify|vmproxy|quota|accounting|all)" } check_enabled() { @@ -377,6 +379,21 @@ start_quota() { [ $RET2 -ne 0 ] && echo "Warning: quota not started." echo } +start_accounting() { + check_enabled "accounting" || return + DAEMON_PATH=${MIG_ACCOUNTING} + SHORT_NAME=$(basename ${DAEMON_PATH}) + PID_FILE="$PID_DIR/${SHORT_NAME}.pid" + echo -n "Starting MiG accounting daemon: $SHORT_NAME" + daemon --user ${MIG_USER} --pidfile ${PID_FILE} \ + "${DAEMON_PATH} >> ${MIG_LOG}/accounting.out 2>&1 &" + fallback_save_pid "$DAEMON_PATH" "$PID_FILE" "$!" + RET2=$? + [ $RET2 -eq 0 ] && success + echo + [ $RET2 -ne 0 ] && echo "Warning: accounting not started." + echo +} start_sftpsubsys() { check_enabled "sftp_subsys" || return DAEMON_PATH=${MIG_SFTPSUBSYS} @@ -410,6 +427,7 @@ start_all() { start_imnotify start_vmproxy start_quota + start_accounting return 0 } @@ -579,6 +597,15 @@ stop_quota() { killproc ${DAEMON_PATH} echo } +stop_accounting() { + check_enabled "accounting" || return + DAEMON_PATH=${MIG_ACCOUNTING} + SHORT_NAME=$(basename ${DAEMON_PATH}) + PID_FILE="$PID_DIR/${SHORT_NAME}.pid" + echo -n "Shutting down MiG accounting: $SHORT_NAME " + killproc ${DAEMON_PATH} + echo +} stop_sftpsubsys_workers() { DAEMON_PATH=${MIG_SFTPSUBSYS_WORKER} SHORT_NAME=$(basename ${DAEMON_PATH}) @@ -623,6 +650,7 @@ stop_all() { stop_imnotify stop_vmproxy stop_quota + stop_accounting return 0 } @@ -761,6 +789,15 @@ reload_quota() { killproc ${DAEMON_PATH} -HUP echo } +reload_accounting() { + check_enabled "accounting" || return + DAEMON_PATH=${MIG_ACCOUNTING} + SHORT_NAME=$(basename ${DAEMON_PATH}) + PID_FILE="$PID_DIR/${SHORT_NAME}.pid" + echo -n "Reloading MiG accounting: $SHORT_NAME " + killproc ${DAEMON_PATH} -HUP + echo +} reload_sftpsubsys_workers() { DAEMON_PATH=${MIG_SFTPSUBSYS_WORKER} SHORT_NAME=$(basename ${DAEMON_PATH}) @@ -818,6 +855,7 @@ reload_all() { reload_imnotify reload_vmproxy reload_quota + reload_accounting # Apache helpers to verify proper chrooting reload_chkuserroot reload_chksidroot @@ -929,6 +967,13 @@ status_quota() { PID_FILE="$PID_DIR/${SHORT_NAME}.pid" status ${DAEMON_PATH} } +status_accounting() { + check_enabled "accounting" || return + DAEMON_PATH=${MIG_ACCOUNTING} + SHORT_NAME=$(basename ${DAEMON_PATH}) + PID_FILE="$PID_DIR/${SHORT_NAME}.pid" + status ${DAEMON_PATH} +} status_sftpsubsys_workers() { DAEMON_PATH=${MIG_SFTPSUBSYS_WORKER} SHORT_NAME=$(basename ${DAEMON_PATH}) @@ -969,6 +1014,7 @@ status_all() { status_imnotify status_vmproxy status_quota + status_accounting return 0 } @@ -980,7 +1026,7 @@ test -f ${MIG_SCRIPT} || exit 0 # Force valid target case "$2" in - script|monitor|sshmux|events|cron|janitor|transfers|openid|sftp|sftpsubsys|webdavs|ftps|notify|imnotify|vmproxy|quota|all) + script|monitor|sshmux|events|cron|janitor|transfers|openid|sftp|sftpsubsys|webdavs|ftps|notify|imnotify|vmproxy|quota|accounting|all) TARGET="$2" ;; '') diff --git a/mig/lib/accounting.py b/mig/lib/accounting.py new file mode 100644 index 000000000..692245084 --- /dev/null +++ b/mig/lib/accounting.py @@ -0,0 +1,709 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# +# --- BEGIN_HEADER --- +# +# accounting - helpers to support storage accounting +# Copyright (C) 2003-2026 The MiG Project by the Science HPC Center at UCPH +# +# This file is part of MiG. +# +# MiG is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# MiG is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, +# USA. +# +# --- END_HEADER --- +# + +"""helpers to support storage accounting""" + +from __future__ import print_function +from __future__ import absolute_import + +import os +import time +import datetime +import math + +from mig.shared.base import client_dir_id, force_native_str +from mig.shared.fileio import pickle, unpickle, load_json, make_symlink +from mig.shared.vgrid import vgrid_list_vgrids, vgrid_list +from mig.shared.useradm import get_accepted_peers + + +def __init_accounting_entry(user_bytes=0, + freeze_bytes=0, + vgrid_bytes=None, + peers=None, + ext_users=None): + """Return new user account dict entry""" + if vgrid_bytes is None: + vgrid_bytes = {} + if peers is None: + peers = {} + if ext_users is None: + ext_users = {} + + return {'user_bytes': user_bytes, + 'freeze_bytes': freeze_bytes, + 'vgrid_bytes': vgrid_bytes, + 'peers': peers, + 'ext_users': ext_users} + + +def __get_owned_vgrid(configuration, verbose=False): + """Find primary owner of vgrid and return a dictionary + with owner client_id as key and list of owned vgrid_names as values + NOTE: First owner of top-vgrid is primary owner""" + logger = configuration.logger + result = {} + (status, vgrids) = vgrid_list_vgrids(configuration) + if status: + for vgrid_name in vgrids: + #print("checkking vgrid: %s" % check_vgrid_name) + (owners_status, owners_list) = vgrid_list(vgrid_name, + 'owners', + configuration, + recursive=True) + # Find first non-zero owner + # NOTE: Some owner files contain empty owners) + owner = '' + if owners_status and owners_list: + owner = next(ent for ent in owners_list if ent) + if owner: + owned_vgrids = result.get(owner, []) + owned_vgrids.append(vgrid_name) + result[owner] = owned_vgrids + else: + msg = "Failed to find owner for vgrid: %s" \ + % vgrid_name + logger.warning(msg) + if verbose: + print("WARNING: %s" % msg) + + # for client_id, vgrids in result.items(): + # print("%s: %s" % (client_id, vgrids)) + + return result + + +def __get_peers_map(configuration, verbose=False): + """Create mapping between users and peers""" + result = {} + logger = configuration.logger + with os.scandir(configuration.user_settings) as it: + for user_entry in it: + client_id = client_dir_id(user_entry.name) + peer_result = result.get(client_id, {}) + accepted_peers = get_accepted_peers(configuration, client_id) + for ext_client_id, value in accepted_peers.items(): + if not isinstance(value, dict): + msg = "Invalid peers format: %s: %s: %s" \ + % (client_id, ext_client_id, value) + logger.warning(msg) + if verbose: + print("WARNING: %s" % msg) + continue + # Map external users to their peer + ext_users = peer_result.get('ext_users', {}) + ext_users[ext_client_id] = value + peer_result['ext_users'] = ext_users + # Map peers to their external user + ext_result = result.get(ext_client_id, {}) + peers = ext_result.get('peers', {}) + peers[client_id] = value + ext_result['peers'] = peers + result[ext_client_id] = ext_result + result[client_id] = peer_result + + return result + + +def update_accounting(configuration, + verbose=False): + """Update user accounting information""" + logger = configuration.logger + retval = True + result = {'accounting': {}, + 'quota': {}} + accounting = result['accounting'] + result['timestamp'] = int(time.time()) + + # Map vgrid to their primary owner + msg = "Creating vgrid owners map ..." + logger.info(msg) + if verbose: + print(msg) + t1 = time.time() + owned_vgrid = __get_owned_vgrid(configuration, verbose=verbose) + t2 = time.time() + msg = "Created vgrid owners map in %d secs" % (t2-t1) + logger.info(msg) + if verbose: + print(msg) + + # Map peers to their "owner" + + msg = "Creating peers map ..." + logger.info(msg) + if verbose: + print(msg) + t1 = time.time() + peers_map = __get_peers_map(configuration, verbose=verbose) + t2 = time.time() + msg = "Created peers map in %d secs" % (t2-t1) + logger.info(msg) + if verbose: + print(msg) + + # Create users, vgrid and freeze quota json file list + + user_quota_files = {} + vgrid_quota_files = {} + freeze_quota_files = {} + with os.scandir(configuration.quota_home) as it: + for entry in it: + # Load quota info if it exists + quota_info_pck = None + quota_info_json = None + quota_fs = None + if entry.name.endswith(".pck"): + quota_info_pck = entry.path + quota_fs = entry.name.replace(".pck", "") + elif entry.name.endswith(".json"): + quota_info_json = entry.path + quota_fs = entry.name.replace(".json", "") + else: + logger.debug("Skipping non quota info entry: %s" + % entry.name) + continue + quota_info = None + # Try .pck first then .json + if quota_info_pck: + quota_info = unpickle(quota_info_pck, configuration.logger) + elif quota_info_json: + quota_info = load_json(quota_info_json, + configuration.logger, + convert_utf8=False) + if not quota_info: + msg = "Failed to load quota info for FS entry: %s" \ + % entry.name + logger.error(msg) + if verbose: + print("ERROR: %s" % msg) + retval = False + continue + quota_basepath = os.path.join(configuration.quota_home, + quota_fs) + if not os.path.isdir(quota_basepath): + msg = "Missing quota_basepath: %r" % quota_basepath + logger.error(msg) + if verbose: + print("ERROR: %s" % msg) + retval = False + continue + quota_mtime = quota_info.get('mtime', 0) + quota_datestr = datetime.datetime.fromtimestamp(quota_mtime) \ + .strftime('%d/%m/%Y-%H:%M:%S') + result['quota'][quota_fs] = {'mtime': quota_mtime} + + # User quota + + user_path = os.path.join(quota_basepath, 'user') + if not os.path.isdir(user_path): + msg = "Missing quota user path: %r" % user_path + logger.error(msg) + if verbose: + print("ERROR: %s" % msg) + retval = False + continue + + msg = "Scanning %s user quota (%d) %s %r" \ + % (quota_fs, + quota_mtime, + quota_datestr, + user_path) + logger.info(msg) + if verbose: + print(msg) + t1 = time.time() + with os.scandir(user_path) as it2: + for user_entry in it2: + if user_entry.name.endswith(".pck"): + client_id = client_dir_id( + user_entry.name.replace('.pck', '')) + elif user_entry.name.endswith(".json"): + client_id = client_dir_id( + user_entry.name.replace('.json', '')) + else: + logger.debug("Skipping non-user entry: %s" + % user_entry.name) + continue + user_quota_files[client_id] = user_entry.path + + t2 = time.time() + msg = "Scanned %s user quota (%d) %s %r in %d secs" \ + % (quota_fs, + quota_mtime, + quota_datestr, + user_path, + (t2-t1)) + logger.info(msg) + if verbose: + print(msg) + + # Vgrid quota + + vgrid_path = os.path.join(quota_basepath, 'vgrid') + if not os.path.isdir(vgrid_path): + msg = "Missing quota vgrid path: %r" % vgrid_path + logger.error(msg) + if verbose: + print("ERROR: %s" % msg) + retval = False + continue + + msg = "Scanning %s vgrid quota (%d) %s %r" \ + % (quota_fs, + quota_mtime, + quota_datestr, + vgrid_path) + logger.info(msg) + if verbose: + print(msg) + t1 = time.time() + with os.scandir(vgrid_path) as it2: + for vgrid_entry in it2: + if vgrid_entry.name.endswith(".pck"): + vgrid_name = force_native_str( + vgrid_entry.name.replace('.pck', '')) + elif vgrid_entry.name.endswith(".json"): + vgrid_name = force_native_str( + vgrid_entry.name.replace('.json', '')) + else: + # logger.debug("Skipping non-vgrid entry: %s" + # % vgrid_entry.name) + continue + # NOTE: sub-vgrids uses ':' + # as delimiter in 'vgrid_files_writable' + vgrid_name = vgrid_name.replace(':', '/') + # print("%s: %s" % (vgrid_name, vgrid_entry.path)) + vgrid_quota_files[vgrid_name] = vgrid_entry.path + t2 = time.time() + msg = "Scanned %s vgrid quota (%d) %s %r in %d secs" \ + % (quota_fs, + quota_mtime, + quota_datestr, + vgrid_path, + (t2-t1)) + logger.info(msg) + if verbose: + print(msg) + + # Freeze quota + + if configuration.site_enable_freeze: + freeze_path = os.path.join(quota_basepath, 'freeze') + if not os.path.isdir(freeze_path): + msg = "Missing quota freeze path: %r" % freeze_path + logger.error(msg) + if verbose: + print("ERROR: %s" % msg) + retval = False + continue + + msg = "Scanning %s freeze quota (%d) %s %r" \ + % (quota_fs, + quota_mtime, + quota_datestr, + freeze_path) + logger.info(msg) + if verbose: + print(msg) + t1 = time.time() + with os.scandir(freeze_path) as it2: + for freeze_entry in it2: + if freeze_entry.name.endswith(".pck"): + freeze_client_id = client_dir_id( + freeze_entry.name.replace('.pck', '')) + elif freeze_entry.name.endswith(".json"): + freeze_client_id = client_dir_id( + freeze_entry.name.replace('.json', '')) + else: + logger.debug("Skipping non-freeze entry: %s" + % freeze_entry.name) + continue + freeze_quota_files[freeze_client_id] \ + = freeze_entry.path + t2 = time.time() + msg = "Scanned %s freeze quota (%d) %s %r in %d secs" \ + % (quota_fs, + quota_mtime, + quota_datestr, + freeze_path, + (t2-t1)) + logger.info(msg) + if verbose: + print(msg) + + # Generate accounting from user bytes, vgrid bytes and freeze bytes + + vgrids_accounted = [] + for client_id, user_quota_filepath in user_quota_files.items(): + # Init user accounting + peers = peers_map.get(client_id, {}).get('peers', {}) + ext_users = peers_map.get(client_id, {}).get('ext_users', {}) + accounting[client_id] = __init_accounting_entry(peers=peers, + ext_users=ext_users) + # Extract user bytes + if user_quota_filepath.endswith('.pck'): + user_quota = unpickle(user_quota_filepath, configuration) + elif user_quota_filepath.endswith('.json'): + user_quota = load_json(user_quota_filepath, + configuration.logger, + convert_utf8=False) + else: + msg = "Invalid user quota file: %r" % user_quota_filepath + logger.error(msg) + if verbose: + print("ERROR: %s" % msg) + retval = False + continue + try: + accounting[client_id]['user_bytes'] = user_quota['bytes'] + except Exception as err: + accounting[client_id]['user_bytes'] = 0 + msg = "Failed to load user quota: %r, error: %s" \ + % (user_quota_filepath, err) + logger.error(msg) + if verbose: + print("ERROR: %s" % msg) + retval = False + continue + + # Extract vgrid bytes for user 'client_id' + + for vgrid_name in owned_vgrid.get(client_id, []): + vgrid_quota_filepath = vgrid_quota_files.get(vgrid_name, '') + if not os.path.exists(vgrid_quota_filepath): + if verbose: + # NOTE: Legacy vgrids are accounted at by top-vgrid + vgrid_array = vgrid_name.split('/') + legacy_vgrid = os.path.join(configuration.vgrid_files_home, + vgrid_name) + if not os.path.isdir(legacy_vgrid) \ + or len(vgrid_array) == 1: + msg = "Missing quota for vgrid: %r" \ + % vgrid_name + logger.warning(msg) + if verbose: + print("WARNING: %s" % msg) + continue + if vgrid_quota_filepath.endswith('.pck'): + vgrid_quota = unpickle(vgrid_quota_filepath, configuration) + elif vgrid_quota_filepath.endswith('.json'): + vgrid_quota = load_json(vgrid_quota_filepath, + configuration.logger, + convert_utf8=False) + else: + msg = "Invalid vgrid quota file: %r" % vgrid_quota_filepath + logger.error(msg) + if verbose: + print("ERROR: %s" % msg) + retval = False + continue + try: + accounting[client_id]['vgrid_bytes'][vgrid_name] \ + = vgrid_quota['bytes'] + except Exception as err: + accounting[client_id]['vgrid_bytes'][vgrid_name] = 0 + msg = "Failed to load vgrid quota: %r, error: %s" \ + % (vgrid_quota_filepath, err) + logger.error(msg) + if verbose: + print("ERROR: %s" % msg) + retval = False + continue + # NOTE: Used for verification of + # that all vgrids was taken into account + vgrids_accounted.append(vgrid_name) + + # Check if all vgrids were accounted for + + for vgrid_name in vgrid_quota_files.keys(): + if not vgrid_name in vgrids_accounted: + vgridowner = '' + for owner, owned_vgrids in owned_vgrid.items(): + if vgrid_name in owned_vgrids: + vgridowner = owner + break + msg = "no accounting for vgrid: %r, missing owner?: %r" \ + % (vgrid_name, vgridowner) + logger.warning(msg) + if verbose: + print("WARNING: %s" % msg) + + # Extract freeze bytes + + for freeze_name, freeze_quota_filepath in freeze_quota_files.items(): + # Extract client_id from legacy freeze archive format + if freeze_name.startswith('archive-'): + legacy_freeze_meta_filepath \ + = os.path.join(configuration.freeze_home, + freeze_name, + 'meta.pck') + legacy_freeze_meta = unpickle(legacy_freeze_meta_filepath, + configuration.logger) + if not legacy_freeze_meta: + msg = "Missing metadata for archive: %r" \ + % freeze_name + logger.warning(msg) + if verbose: + print("WARNING: %s" % msg) + continue + client_id = legacy_freeze_meta.get('CREATOR', '') + if not client_id: + msg = "Failed to extract client_id from: %r" \ + % legacy_freeze_meta_filepath + logger.error(msg) + if verbose: + print("ERROR: %s" % msg) + retval = False + continue + else: + client_id = freeze_name + + # Load freeze quota + + freeze_bytes = 0 + if freeze_quota_filepath.endswith('.pck'): + freeze_quota = unpickle(freeze_quota_filepath, configuration) + elif freeze_quota_filepath.endswith('.json'): + freeze_quota = load_json(freeze_quota_filepath, + configuration.logger, + convert_utf8=False) + else: + msg = "Invalid freeze quota file: %r" % freeze_quota_filepath + logger.error(msg) + if verbose: + print("ERROR: %s" % msg) + retval = False + continue + try: + freeze_bytes = int(freeze_quota['bytes']) + except Exception as err: + freeze_bytes = 0 + msg = "Failed to fetch freeze quota: %r, error: %s" \ + % (freeze_quota_filepath, err) + logger.error(msg) + if verbose: + print("ERROR: %s" % msg) + retval = False + continue + + if freeze_bytes > 0: + freeze_accounting = accounting.get(client_id, '') + if not freeze_accounting: + msg = "added missing archive user: %r : %d" \ + % (client_id, freeze_bytes) + logger.warning(msg) + if verbose: + print("WARNING: %s" % msg) + accounting[client_id] = __init_accounting_entry() + freeze_accounting = accounting[client_id] + freeze_accounting['freeze_bytes'] += freeze_bytes + + # Save accounting result + + accounting_filepath = os.path.join(configuration.accounting_home, + "%s.pck" % result['timestamp']) + status = pickle(result, accounting_filepath, configuration.logger) + if status: + latest = os.path.join(configuration.accounting_home, 'latest') + status = make_symlink(accounting_filepath, latest, logger, force=True) + if not status: + retval = False + + return retval + + +def human_readable_filesize(filesize): + """Return human readable filesize""" + if filesize == 0: + return "0 B" + p = int(math.floor(math.log(filesize, 2)/10)) + return "%.3f %s" % (filesize/math.pow(1024, p), + ['B', + 'KiB', + 'MiB', + 'GiB', + 'TiB', + 'PiB', + 'EiB', + 'ZiB', + 'YiB'][p]) + + +def get_usage(configuration, + usernames=[], + timestamp=0, + verbose=False): + """Generate and return 'storage' usage""" + # Load accounting if it exists + logger = configuration.logger + if timestamp == 0: + accounting_filepath = os.path.join(configuration.accounting_home, + "latest") + else: + accounting_filepath = os.path.join(configuration.accounting_home, + "%s.pck" % timestamp) + data = unpickle(accounting_filepath, configuration.logger) + if not data: + msg = "Failed to load accounting data from: %r" % accounting_filepath + logger.error(msg) + if verbose: + print("ERROR: %s" % msg) + return None + + accounting = data.get('accounting', {}) + + # Do not show external users as a main account + # unless they act as both peer and external user + + ext_users = [] + peer_users = [] + skip_ext_users = [] + for values in accounting.values(): + ext_users.extend(list(values.get('ext_users', {}).keys())) + peer_users.extend(list(values.get('peers', {}).keys())) + skip_ext_users = [user for user in ext_users + if user not in peer_users] + + # Create accounting report + + account_usage = {} + for username, values in accounting.items(): + if usernames and username not in usernames: + continue + total_bytes = 0 + # Create home human readable bytes + home_report = "" + home_bytes = values.get('user_bytes', 0) + total_bytes += home_bytes + try: + home_bytes_human = human_readable_filesize(home_bytes) + except Exception: + home_bytes_human = "NaN" + home_report += "Home usage: %s" % home_bytes_human + + # Create freeze report + + freeze_report = "" + freeze_bytes = values.get('freeze_bytes', 0) + total_bytes += freeze_bytes + if freeze_bytes > 0: + try: + freeze_bytes_human = human_readable_filesize(freeze_bytes) + except Exception: + freeze_bytes_human = "NaN" + freeze_report += "Archive usage: %s" % freeze_bytes_human + + # Create vgrid report + + vgrid_report = "" + vgrid_total = 0 + for vgrid_name, vgrid_bytes in values.get('vgrid_bytes', {}).items(): + vgrid_total += vgrid_bytes + try: + vgrid_bytes_human = human_readable_filesize(vgrid_bytes) + except Exception as err: + vgrid_bytes_human = "NaN" + vgrid_report += "\n - %s: %s" % (vgrid_name, vgrid_bytes_human) + if vgrid_report: + vgrid_report = "%s usage (total: %s)%s" \ + % (configuration.site_vgrid_label, + human_readable_filesize(vgrid_total), + vgrid_report) + total_bytes += vgrid_total + account_usage[username] = {'total_bytes': total_bytes, + 'home_total': home_bytes, + 'vgrid_total': vgrid_total, + 'freeze_total': freeze_bytes, + 'ext_users_total': 0, + 'home_report': home_report, + 'freeze_report': freeze_report, + 'vgrid_report': vgrid_report, + 'ext_users_report': '', + 'peers_report': ''} + + # Create external users report + # NOTE: We need total bytes and therefore we need the above full report + # generated before external users report + + for username, values in accounting.items(): + # Create ext_users report + ext_users = values.get('ext_users', {}) + peers = values.get('peers', {}) + if not ext_users: + continue + if ext_users and peers: + msg = "User %r acts as both peer and external user" \ + % username + logger.warning(msg) + if verbose: + print("WARNING: %s" % msg) + + # Create external users report + + ext_users_report = "" + ext_users_total = 0 + for ext_user in ext_users.keys(): + ext_user_total_bytes = account_usage.get( + ext_user, {}).get('total_bytes', 0) + ext_users_total += ext_user_total_bytes + try: + ext_user_total_bytes_human = human_readable_filesize( + ext_user_total_bytes) + except Exception as err: + ext_user_total_bytes_human = "NaN" + ext_users_report += "\n - %s: %s" % (ext_user, + ext_user_total_bytes_human) + if ext_users_report: + ext_users_report = "External users usage (total: %s):%s" \ + % (human_readable_filesize(ext_users_total), + ext_users_report) + account_usage[username]['ext_users_total'] = ext_users_total + account_usage[username]['ext_users_report'] = ext_users_report + account_usage[username]['total_bytes'] += ext_users_total + + # Create peers report + + peers_report = "" + for peer in peers.keys(): + peers_report += "\n - %s" % peer + if peers_report: + peers_report = "Accepted by the following peer:%s" % peers_report + account_usage[username]['peers_report'] = peers_report + + # External users are accounted for by their peer + # unless the external user also act as a peer + + result = {} + result['timestamp'] = data.get('timestamp', 0) + result['quota'] = data.get('quota', {}) + result['accounting'] = {username: values for username, values + in account_usage.items() + if username not in skip_ext_users} + + return result diff --git a/mig/server/grid_accounting.py b/mig/server/grid_accounting.py new file mode 120000 index 000000000..6fa56546c --- /dev/null +++ b/mig/server/grid_accounting.py @@ -0,0 +1 @@ +../../sbin/grid_accounting.py \ No newline at end of file diff --git a/mig/shared/configuration.py b/mig/shared/configuration.py index 8dfaf4bce..c08eeafcf 100644 --- a/mig/shared/configuration.py +++ b/mig/shared/configuration.py @@ -262,6 +262,7 @@ def fix_missing(config_file, verbose=True): 'openid_store': '~/state/openid_store/', 'sitestats_home': '~/state/sitestats_home/', 'quota_home': '~/state/quota_home/', + 'accounting_home': '~/state/accounting_home/', 'public_key_file': '', 'javabin_home': '~/mig/java-bin', 'events_home': '~/state/events_home/', @@ -375,6 +376,7 @@ def fix_missing(config_file, verbose=True): 'user_auth_log': 'auth.log', 'user_shared_dhparams': '~/certs/dhparams.pem', 'user_quota_log': 'quota.log', + 'user_accounting_log': 'accounting.log', 'logfile': 'server.log', 'loglevel': 'info', 'sleep_period_for_empty_jobs': '80', @@ -412,6 +414,8 @@ def fix_missing(config_file, verbose=True): 'update_interval': 3600, 'user_limit': 1024**4, 'vgrid_limit': 1024**4} + accounting_section = {'update_interval': 3600} + defaults = { 'GLOBAL': global_section, 'SCHEDULER': scheduler_section, @@ -420,6 +424,7 @@ def fix_missing(config_file, verbose=True): 'FEASIBILITY': feasibility_section, 'WORKFLOWS': workflows_section, 'QUOTA': quota_section, + 'ACCOUNTING': accounting_section, } for section in defaults: if not section in config.sections(): @@ -662,6 +667,7 @@ def get(self, *args, **kwargs): 'user_auth_log': 'auth.log', 'user_shared_dhparams': '', 'user_quota_log': 'quota.log', + 'user_accounting_log': 'accounting.log', 'user_imnotify_address': '', 'user_imnotify_port': 6667, 'user_imnotify_channel': '', @@ -1188,6 +1194,8 @@ def reload_config(self, verbose, skip_log=False, disable_auth_log=False, self.sitestats_home = config.get('GLOBAL', 'sitestats_home') if config.has_option('GLOBAL', 'quota_home'): self.quota_home = config.get('GLOBAL', 'quota_home') + if config.has_option('GLOBAL', 'accounting_home'): + self.accounting_home = config.get('GLOBAL', 'accounting_home') if config.has_option('GLOBAL', 'jupyter_mount_files_dir'): self.jupyter_mount_files_dir = config.get( 'GLOBAL', 'jupyter_mount_files_dir') @@ -1778,6 +1786,9 @@ def reload_config(self, verbose, skip_log=False, disable_auth_log=False, if config.has_option('GLOBAL', 'user_quota_log'): self.user_quota_log = config.get('GLOBAL', 'user_quota_log') + if config.has_option('GLOBAL', 'user_accounting_log'): + self.user_quota_log = config.get('GLOBAL', + 'user_accounting_log') if config.has_option('GLOBAL', 'public_key_file'): self.public_key_file = config.get('GLOBAL', 'public_key_file') if config.has_option('GLOBAL', 'smtp_sender'): @@ -2053,6 +2064,10 @@ def reload_config(self, verbose, skip_log=False, disable_auth_log=False, self.quota_vgrid_limit = config.getint( 'QUOTA', 'vgrid_limit') + if config.has_option('ACCOUNTING', 'update_interval'): + self.accounting_update_interval = config.getint( + 'ACCOUNTING', 'update_interval') + if config.has_option('SITE', 'images'): self.site_images = config.get('SITE', 'images') else: @@ -2318,6 +2333,11 @@ def reload_config(self, verbose, skip_log=False, disable_auth_log=False, 'enable_quota') else: self.site_enable_quota = False + if config.has_option('SITE', 'enable_accounting'): + self.site_enable_accounting = config.getboolean('SITE', + 'enable_accounting') + else: + self.site_enable_accounting = False if config.has_option('SITE', 'enable_transfers'): self.site_enable_transfers = config.getboolean('SITE', 'enable_transfers') @@ -2750,7 +2770,8 @@ def reload_config(self, verbose, skip_log=False, disable_auth_log=False, 'user_janitor_log', 'user_transfers_log', 'user_notify_log', 'user_imnotify_log', 'user_auth_log', 'user_chkuserroot_log', - 'user_chksidroot_log', 'user_quota_log'): + 'user_chksidroot_log', 'user_quota_log', + 'user_accounting_log'): _log_path = getattr(self, _log_var) if not os.path.isabs(_log_path): setattr(self, _log_var, os.path.join(self.log_dir, _log_path)) diff --git a/mig/shared/install.py b/mig/shared/install.py index c9579fc60..500dd6a60 100644 --- a/mig/shared/install.py +++ b/mig/shared/install.py @@ -404,6 +404,7 @@ def generate_confs( enable_gravatars=False, enable_sitestatus=True, enable_quota=False, + enable_accounting=False, prefer_python3=False, io_account_expire=False, gdp_email_notify=False, @@ -536,6 +537,7 @@ def generate_confs( quota_update_interval=3600, quota_user_limit=(1024**4), quota_vgrid_limit=(1024**4), + accounting_update_interval=3600, ca_fqdn='', ca_user='mig-ca', ca_smtp='localhost', @@ -731,6 +733,7 @@ def _generate_confs_prepare( enable_gravatars, enable_sitestatus, enable_quota, + enable_accounting, prefer_python3, io_account_expire, gdp_email_notify, @@ -863,6 +866,7 @@ def _generate_confs_prepare( quota_update_interval, quota_user_limit, quota_vgrid_limit, + accounting_update_interval, ca_fqdn, ca_user, ca_smtp, @@ -988,6 +992,7 @@ def _generate_confs_prepare( user_dict['__ENABLE_GRAVATARS__'] = "%s" % enable_gravatars user_dict['__ENABLE_SITESTATUS__'] = "%s" % enable_sitestatus user_dict['__ENABLE_QUOTA__'] = "%s" % enable_quota + user_dict['__ENABLE_ACCOUNTING__'] = "%s" % enable_accounting user_dict['__PREFER_PYTHON3__'] = "%s" % prefer_python3 user_dict['__IO_ACCOUNT_EXPIRE__'] = "%s" % io_account_expire user_dict['__GDP_EMAIL_NOTIFY__'] = "%s" % gdp_email_notify @@ -1122,6 +1127,7 @@ def _generate_confs_prepare( user_dict['__QUOTA_UPDATE_INTERVAL__'] = "%s" % quota_update_interval user_dict['__QUOTA_USER_LIMIT__'] = "%s" % quota_user_limit user_dict['__QUOTA_VGRID_LIMIT__'] = "%s" % quota_vgrid_limit + user_dict['__ACCOUNTING_UPDATE_INTERVAL__'] = "%s" % accounting_update_interval user_dict['__CA_FQDN__'] = ca_fqdn user_dict['__CA_USER__'] = ca_user user_dict['__CA_SMTP__'] = ca_smtp diff --git a/sbin/grid_accounting.py b/sbin/grid_accounting.py new file mode 100755 index 000000000..b7fbdc325 --- /dev/null +++ b/sbin/grid_accounting.py @@ -0,0 +1,132 @@ +#!/usr/bin/python +# -*- coding: utf-8 -*- +# +# --- BEGIN_HEADER --- +# +# grid_accounting - daemon to create storage accounting +# Copyright (C) 2003-2026 The MiG Project by the Science HPC Center at UCPH +# +# This file is part of MiG. +# +# MiG is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# MiG is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# +# -- END_HEADER --- +# + +"""Daemon that create storage accounting""" + +from __future__ import absolute_import, print_function + +import os +import sys +import time +import traceback +import datetime + +from mig.lib.daemon import check_run, check_stop, interruptible_sleep, \ + register_run_handler, register_stop_handler, reset_run, stop_running +from mig.lib.accounting import update_accounting +from mig.shared.conf import get_configuration_object +from mig.shared.logger import daemon_logger, register_hangup_handler + + +if __name__ == "__main__": + print( + """This is the MiG lustre accounting daemon which collect storage accounting + information for users and vgrids. + +Set the MIG_CONF environment to the server configuration path +unless it is available in mig/server/MiGserver.conf +""" + ) + # Force no log init since we use separate logger + configuration = get_configuration_object(skip_log=True) + + log_level = configuration.loglevel + if sys.argv[1:] and sys.argv[1] in ["debug", "info", "warning", "error"]: + log_level = sys.argv[1] + + # Use separate logger + + logger = daemon_logger("accounting", + configuration.user_accounting_log, + log_level) + configuration.logger = logger + + # Check if accounting is enabled + + if not configuration.site_enable_accounting: + msg = "Accounting support is disabled in configuration!" + logger.error(msg) + print("%s ERROR: %s" + % (datetime.datetime.now(), msg), + file=sys.stderr) + sys.exit(1) + + # Allow e.g. logrotate to force log re-open after rotates + register_hangup_handler(configuration) + + # Allow trigger next run on SIGCONT to main process + register_run_handler(configuration) + + # Allow clean shutdown on SIGINT only to main process + register_stop_handler(configuration) + + throttle_secs = float(configuration.accounting_update_interval) + main_pid = os.getpid() + msg = "(%s) Starting accounting daemon with throttle: %d secs" \ + % (main_pid, throttle_secs) + logger.info(msg) + print("%s %s" % (datetime.datetime.now(), msg)) + + throttle = False + while not check_stop(): + try: + if throttle: + interruptible_sleep(configuration, throttle_secs, + (check_run, check_stop)) + reset_run() + if check_stop(): + break + t1 = time.time() + status = update_accounting(configuration, verbose=True) + t2 = time.time() + msg = "(%s) Updated accounting in %d secs with status: %s" \ + % (os.getpid(), int(t2-t1), status) + logger.info(msg) + print("%s %s" % (datetime.datetime.now(), msg)) + throttle = True + except KeyboardInterrupt: + stop_running() + # NOTE: we can't be sure if SIGINT was sent to only main process + # so we make sure to propagate to monitor child + msg = "(%s) Interrupt requested - shutdown" \ + % os.getpid() + logger.info(msg) + print("%s %s" % (datetime.datetime.now(), msg)) + except Exception as exc: + throttle = True + msg = "(%s) Caught unexpected exception:\n%s" \ + % (os.getpid(), traceback.format_exc()) + logger.error(msg) + print("%s ERROR: %s" + % (datetime.datetime.now(), msg), + file=sys.stderr) + + msg = "(%s) Accounting daemon shutting down" % main_pid + logger.info(msg) + print("%s %s" % (datetime.datetime.now(), msg)) + + sys.exit(0) diff --git a/tests/fixture/confs-stdlocal/MiGserver.conf b/tests/fixture/confs-stdlocal/MiGserver.conf index a9983263f..e8e761f6c 100644 --- a/tests/fixture/confs-stdlocal/MiGserver.conf +++ b/tests/fixture/confs-stdlocal/MiGserver.conf @@ -127,6 +127,7 @@ workflows_home = %(state_path)s/workflows_home/ workflows_db_home = %(state_path)s/workflows_db_home/ notify_home = %(state_path)s/notify_home/ quota_home = %(state_path)s/quota_home/ +accounting_home = %(state_path)s/accounting_home/ # GDP data categories metadata and helpers json file gdp_data_categories = %(gdp_home)s/data_categories.json @@ -554,6 +555,9 @@ update_interval = 3600 user_limit = 1099511627776 vgrid_limit = 1099511627776 +[ACCOUNTING] +update_interval = 3600 + [SITE] # Web site appearance # Whether to use Python 3 for all Python invocations @@ -677,6 +681,8 @@ enable_openid = False enable_sharelinks = True # Enable storage quota enable_quota = False +# Enable storage accounting +enable_accounting = False # Enable background data transfers daemon - requires lftp and rsync enable_transfers = False # Explicit background transfer source addresses for use in pub key restrictions diff --git a/tests/fixture/confs-stdlocal/migrid-init.d-deb b/tests/fixture/confs-stdlocal/migrid-init.d-deb index b83ed781f..4d02c2701 100755 --- a/tests/fixture/confs-stdlocal/migrid-init.d-deb +++ b/tests/fixture/confs-stdlocal/migrid-init.d-deb @@ -70,13 +70,14 @@ MIG_NOTIFY=${MIG_CODE}/server/grid_notify.py MIG_IMNOTIFY=${MIG_CODE}/server/grid_imnotify.py MIG_VMPROXY=${MIG_CODE}/server/grid_vmproxy.py MIG_QUOTA=${MIG_CODE}/server/grid_quota.py +MIG_ACCOUNTING=${MIG_CODE}/server/grid_accounting.py MIG_CHKUSERROOT=${MIG_CODE}/server/chkuserroot.py MIG_CHKSIDROOT=${MIG_CODE}/server/chksidroot.py show_usage() { echo "Usage: migrid {start|stop|status|restart|reload}[daemon DAEMON]" echo "where daemon is left out for all or given along with DAEMON as one of the following" - echo "(script|monitor|sshmux|events|cron|janitor|transfers|openid|sftp|sftpsubsys|webdavs|ftps|notify|imnotify|vmproxy|quota|all)" + echo "(script|monitor|sshmux|events|cron|janitor|transfers|openid|sftp|sftpsubsys|webdavs|ftps|notify|imnotify|vmproxy|quota|accounting|all)" } check_enabled() { @@ -284,6 +285,18 @@ start_quota() { log_end_msg 1 || true fi } +start_accounting() { + check_enabled "accounting" || return 0 + DAEMON_PATH=${MIG_QUOTA} + SHORT_NAME=$(basename ${DAEMON_PATH}) + PID_FILE="$PID_DIR/${SHORT_NAME}.pid" + log_daemon_msg "Starting MiG accounting daemon" ${SHORT_NAME} || true + if start-stop-daemon --start --quiet --oknodo --pidfile ${PID_FILE} --make-pidfile --user root --chuid root --background --name ${SHORT_NAME} --startas ${DAEMON_PATH} ; then + log_end_msg 0 || true + else + log_end_msg 1 || true + fi +} start_sftpsubsys() { check_enabled "sftp_subsys" || return 0 DAEMON_PATH=${MIG_SFTPSUBSYS} @@ -313,6 +326,7 @@ start_all() { start_imnotify start_vmproxy start_quota + start_accounting return 0 } @@ -558,6 +572,19 @@ stop_quota() { log_end_msg 1 || true fi } +stop_accounting() { + check_enabled "accounting" || return 0 + DAEMON_PATH=${MIG_QUOTA} + SHORT_NAME=$(basename ${DAEMON_PATH}) + PID_FILE="$PID_DIR/${SHORT_NAME}.pid" + log_daemon_msg "Stopping MiG accounting" ${SHORT_NAME} || true + if start-stop-daemon --stop --quiet --oknodo --pidfile ${PID_FILE} ; then + rm -f ${PID_FILE} + log_end_msg 0 || true + else + log_end_msg 1 || true + fi +} stop_sftpsubsys_workers() { DAEMON_PATH=${MIG_SFTPSUBSYS_WORKER} SHORT_NAME=$(basename ${DAEMON_PATH}) @@ -598,6 +625,7 @@ stop_all() { stop_imnotify stop_vmproxy stop_quota + stop_accounting return 0 } @@ -782,6 +810,18 @@ reload_quota() { log_end_msg 1 || true fi } +reload_accounting() { + check_enabled "accounting" || return 0 + DAEMON_PATH=${MIG_QUOTA} + SHORT_NAME=$(basename ${DAEMON_PATH}) + PID_FILE="$PID_DIR/${SHORT_NAME}.pid" + log_daemon_msg "Reloading MiG accounting" ${SHORT_NAME} || true + if start-stop-daemon --stop --signal HUP --quiet --oknodo --pidfile ${PID_FILE} ; then + log_end_msg 0 || true + else + log_end_msg 1 || true + fi +} reload_sftpsubsys_workers() { DAEMON_PATH=${MIG_SFTPSUBSYS_WORKER} SHORT_NAME=$(basename ${DAEMON_PATH}) @@ -835,6 +875,7 @@ reload_all() { reload_imnotify reload_vmproxy reload_quota + reload_accounting # Apache helpers to verify proper chrooting reload_chkuserroot reload_chksidroot @@ -946,6 +987,13 @@ status_quota() { PID_FILE="$PID_DIR/${SHORT_NAME}.pid" status_of_proc -p ${PID_FILE} ${DAEMON_PATH} ${SHORT_NAME} } +status_accounting() { + check_enabled "accounting" || return 0 + DAEMON_PATH=${MIG_QUOTA} + SHORT_NAME=$(basename ${DAEMON_PATH}) + PID_FILE="$PID_DIR/${SHORT_NAME}.pid" + status_of_proc -p ${PID_FILE} ${DAEMON_PATH} ${SHORT_NAME} +} status_sftpsubsys_workers() { DAEMON_PATH=${MIG_SFTPSUBSYS_WORKER} SHORT_NAME=$(basename ${DAEMON_PATH}) @@ -985,6 +1033,7 @@ status_all() { status_imnotify status_vmproxy status_quota + status_accounting return 0 } @@ -996,7 +1045,7 @@ test -f ${MIG_SCRIPT} || exit 0 # Force valid target case "$2" in - script|monitor|sshmux|events|cron|janitor|transfers|openid|sftp|sftpsubsys|webdavs|ftps|notify|imnotify|vmproxy|quota|all) + script|monitor|sshmux|events|cron|janitor|transfers|openid|sftp|sftpsubsys|webdavs|ftps|notify|imnotify|vmproxy|quota|accounting|all) TARGET="$2" ;; '') diff --git a/tests/fixture/confs-stdlocal/migrid-init.d-rh b/tests/fixture/confs-stdlocal/migrid-init.d-rh index 9c838ef0e..cc5543400 100755 --- a/tests/fixture/confs-stdlocal/migrid-init.d-rh +++ b/tests/fixture/confs-stdlocal/migrid-init.d-rh @@ -36,6 +36,7 @@ # processname: grid_imnotify.py # processname: grid_vmproxy.py # processname: grid_quota.py +# processname: grid_accounting.py # processname: sshd # config: /etc/sysconfig/migrid # @@ -102,13 +103,14 @@ MIG_NOTIFY=${MIG_CODE}/server/grid_notify.py MIG_IMNOTIFY=${MIG_CODE}/server/grid_imnotify.py MIG_VMPROXY=${MIG_CODE}/server/grid_vmproxy.py MIG_QUOTA=${MIG_CODE}/server/grid_quota.py +MIG_ACCOUNTING=${MIG_CODE}/server/grid_accounting.py MIG_CHKUSERROOT=${MIG_CODE}/server/chkuserroot.py MIG_CHKSIDROOT=${MIG_CODE}/server/chksidroot.py show_usage() { echo "Usage: migrid {start|stop|status|restart|reload}[daemon DAEMON]" echo "where daemon is left out for all or given along with DAEMON as one of the following" - echo "(script|monitor|sshmux|events|cron|janitor|transfers|openid|sftp|sftpsubsys|webdavs|ftps|notify|imnotify|vmproxy|quota|all)" + echo "(script|monitor|sshmux|events|cron|janitor|transfers|openid|sftp|sftpsubsys|webdavs|ftps|notify|imnotify|vmproxy|quota|accounting|all)" } check_enabled() { @@ -377,6 +379,21 @@ start_quota() { [ $RET2 -ne 0 ] && echo "Warning: quota not started." echo } +start_accounting() { + check_enabled "accounting" || return + DAEMON_PATH=${MIG_ACCOUNTING} + SHORT_NAME=$(basename ${DAEMON_PATH}) + PID_FILE="$PID_DIR/${SHORT_NAME}.pid" + echo -n "Starting MiG accounting daemon: $SHORT_NAME" + daemon --user ${MIG_USER} --pidfile ${PID_FILE} \ + "${DAEMON_PATH} >> ${MIG_LOG}/accounting.out 2>&1 &" + fallback_save_pid "$DAEMON_PATH" "$PID_FILE" "$!" + RET2=$? + [ $RET2 -eq 0 ] && success + echo + [ $RET2 -ne 0 ] && echo "Warning: accounting not started." + echo +} start_sftpsubsys() { check_enabled "sftp_subsys" || return DAEMON_PATH=${MIG_SFTPSUBSYS} @@ -410,6 +427,7 @@ start_all() { start_imnotify start_vmproxy start_quota + start_accounting return 0 } @@ -579,6 +597,15 @@ stop_quota() { killproc ${DAEMON_PATH} echo } +stop_accounting() { + check_enabled "accounting" || return + DAEMON_PATH=${MIG_ACCOUNTING} + SHORT_NAME=$(basename ${DAEMON_PATH}) + PID_FILE="$PID_DIR/${SHORT_NAME}.pid" + echo -n "Shutting down MiG accounting: $SHORT_NAME " + killproc ${DAEMON_PATH} + echo +} stop_sftpsubsys_workers() { DAEMON_PATH=${MIG_SFTPSUBSYS_WORKER} SHORT_NAME=$(basename ${DAEMON_PATH}) @@ -623,6 +650,7 @@ stop_all() { stop_imnotify stop_vmproxy stop_quota + stop_accounting return 0 } @@ -761,6 +789,15 @@ reload_quota() { killproc ${DAEMON_PATH} -HUP echo } +reload_accounting() { + check_enabled "accounting" || return + DAEMON_PATH=${MIG_ACCOUNTING} + SHORT_NAME=$(basename ${DAEMON_PATH}) + PID_FILE="$PID_DIR/${SHORT_NAME}.pid" + echo -n "Reloading MiG accounting: $SHORT_NAME " + killproc ${DAEMON_PATH} -HUP + echo +} reload_sftpsubsys_workers() { DAEMON_PATH=${MIG_SFTPSUBSYS_WORKER} SHORT_NAME=$(basename ${DAEMON_PATH}) @@ -818,6 +855,7 @@ reload_all() { reload_imnotify reload_vmproxy reload_quota + reload_accounting # Apache helpers to verify proper chrooting reload_chkuserroot reload_chksidroot @@ -929,6 +967,13 @@ status_quota() { PID_FILE="$PID_DIR/${SHORT_NAME}.pid" status ${DAEMON_PATH} } +status_accounting() { + check_enabled "accounting" || return + DAEMON_PATH=${MIG_ACCOUNTING} + SHORT_NAME=$(basename ${DAEMON_PATH}) + PID_FILE="$PID_DIR/${SHORT_NAME}.pid" + status ${DAEMON_PATH} +} status_sftpsubsys_workers() { DAEMON_PATH=${MIG_SFTPSUBSYS_WORKER} SHORT_NAME=$(basename ${DAEMON_PATH}) @@ -969,6 +1014,7 @@ status_all() { status_imnotify status_vmproxy status_quota + status_accounting return 0 } @@ -980,7 +1026,7 @@ test -f ${MIG_SCRIPT} || exit 0 # Force valid target case "$2" in - script|monitor|sshmux|events|cron|janitor|transfers|openid|sftp|sftpsubsys|webdavs|ftps|notify|imnotify|vmproxy|quota|all) + script|monitor|sshmux|events|cron|janitor|transfers|openid|sftp|sftpsubsys|webdavs|ftps|notify|imnotify|vmproxy|quota|accounting|all) TARGET="$2" ;; '') diff --git a/tests/fixture/mig_shared_configuration--new.json b/tests/fixture/mig_shared_configuration--new.json index bed1cc598..f98e271c3 100644 --- a/tests/fixture/mig_shared_configuration--new.json +++ b/tests/fixture/mig_shared_configuration--new.json @@ -243,6 +243,7 @@ "trac_ini_path": "", "twofactor_home": "", "usage_record_dir": null, + "user_accounting_log": "accounting.log", "user_auth_log": "auth.log", "user_cache": "", "user_chksidroot_log": "chkchroot.log", diff --git a/tests/test_mig_lib_accounting.py b/tests/test_mig_lib_accounting.py new file mode 100644 index 000000000..ba92d397f --- /dev/null +++ b/tests/test_mig_lib_accounting.py @@ -0,0 +1,222 @@ +# -*- coding: utf-8 -*- +# +# --- BEGIN_HEADER --- +# +# test_mig_lib_accounting - unit test of the corresponding mig lib module +# Copyright (C) 2003-2026 The MiG Project by the Science HPC Center at UCPH +# +# This file is part of MiG. +# +# MiG is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# MiG is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, +# USA. +# +# --- END_HEADER --- +# + +"""Unit tests for the migrid module pointed to in the filename""" + +import os +import pickle + +from mig.lib.accounting import update_accounting, get_usage +from mig.shared.base import client_id_dir +from mig.shared.defaults import peers_filename +from tests.support import MigTestCase, ensure_dirs_exist + + +class MigLibAccounting(MigTestCase): + """Unit tests for quota related helper functions""" + + def _provide_configuration(self): + """Prepare isolated test config""" + return 'testconfig' + + def before_each(self): + """Set up test configuration and reset state before each test""" + + # Define fake quota + + TEST_LUSTRE_QUOTA_INFO = {'next_pid': 192, 'mtime': 1768925307} + + TEST_CLIENT_DN = '/C=DK/ST=NA/L=NA/O=Test Org/OU=NA/CN=Test User/emailAddress=test@user.com' + TEST_VGRID_NAME1 = 'TestVgrid1' + TEST_VGRID_NAME2 = 'TestVgrid2' + TEST_VGRID_NAME3 = 'TestVgrid3' + TEST_EXT_DN = '/C=DK/ST=NA/L=NA/O=PEER Org/OU=NA/CN=Test Peer/emailAddress=peer@example.com' + + TEST_CLIENT_USAGE = {'lustre_pid': 42, + 'files': 11, + 'bytes': 206128256, + 'softlimit_bytes': 109951162777600, + 'hardlimit_bytes': 109951162777600, + 'mtime': 1768925307} + + TEST_VGRID_USAGE1 = {'lustre_pid': 43, + 'files': 111, + 'bytes': 406128256, + 'softlimit_bytes': 109951162777600, + 'hardlimit_bytes': 109951162777600, + 'mtime': 1768925307} + + TEST_VGRID_USAGE2 = {'lustre_pid': 44, + 'files': 222, + 'bytes': 606128256, + 'softlimit_bytes': 109951162777600, + 'hardlimit_bytes': 109951162777600, + 'mtime': 1768925307} + + TEST_VGRID_USAGE3 = {'lustre_pid': 45, + 'files': 333, + 'bytes': 806128256, + 'softlimit_bytes': 109951162777600, + 'hardlimit_bytes': 109951162777600, + 'mtime': 1768925307} + + TEST_EXT_USAGE = {'lustre_pid': 46, + 'files': 1, + 'bytes': 16806128256, + 'softlimit_bytes': 109951162777600, + 'hardlimit_bytes': 109951162777600, + 'mtime': 1768925307} + + TEST_FREEZE_USAGE = {'lustre_pid': 47, + 'files': 1, + 'bytes': 128256, + 'softlimit_bytes': 109951162777600, + 'hardlimit_bytes': 109951162777600, + 'mtime': 1768925307} + + TEST_PEERS = {TEST_EXT_DN: {'kind': 'collaboration', + 'distinguished_name': TEST_EXT_DN, + 'country': 'DK', + 'label': 'TEST', + 'state': '', + 'expire': '2222-12-31', + 'full_name': 'Test Peer', + 'organization': 'PEER Org', + 'email': 'peer@example.com' + }} + + # Create fake fs layout matching real systems + + self.configuration.site_enable_quota = True + self.configuration.site_enable_accounting = True + self.configuration.quota_backend = 'lustre' + + QUOTA_BASEPATH = os.path.join(self.configuration.quota_home, + self.configuration.quota_backend) + QUOTA_USER_PATH = os.path.join(QUOTA_BASEPATH, 'user') + QUOTA_VGRID_PATH = os.path.join(QUOTA_BASEPATH, 'vgrid') + QUOTA_FREEZE_PATH = os.path.join(QUOTA_BASEPATH, 'freeze') + TEST_CLIENT_PEERS_PATH = os.path.join(self.configuration.user_settings, + client_id_dir(TEST_CLIENT_DN)) + ensure_dirs_exist(self.configuration.vgrid_home) + ensure_dirs_exist(self.configuration.user_settings) + ensure_dirs_exist(self.configuration.accounting_home) + ensure_dirs_exist(self.configuration.quota_home) + ensure_dirs_exist(QUOTA_USER_PATH) + ensure_dirs_exist(QUOTA_VGRID_PATH) + ensure_dirs_exist(QUOTA_FREEZE_PATH) + ensure_dirs_exist(TEST_CLIENT_PEERS_PATH) + + # Ensure fake vgrid and write owner + + for vgrid_name in [TEST_VGRID_NAME1, + TEST_VGRID_NAME2, + TEST_VGRID_NAME3]: + vgrid_home_path = os.path.join( + self.configuration.vgrid_home, vgrid_name) + ensure_dirs_exist(vgrid_home_path) + vgrid_owners_filepath = os.path.join(vgrid_home_path, 'owners') + with open(vgrid_owners_filepath, 'wb') as fh: + fh.write(pickle.dumps([TEST_CLIENT_DN])) + + # Write fake quota + + TEST_LUSTRE_QUOTA_INFO_FILEPATH \ + = os.path.join(self.configuration.quota_home, + '%s.pck' % self.configuration.quota_backend) + with open(TEST_LUSTRE_QUOTA_INFO_FILEPATH, 'wb') as fh: + fh.write(pickle.dumps(TEST_LUSTRE_QUOTA_INFO)) + + QUOTA_TEST_CLIENT_PATH \ + = os.path.join(QUOTA_USER_PATH, + "%s.pck" % client_id_dir(TEST_CLIENT_DN)) + + with open(QUOTA_TEST_CLIENT_PATH, 'wb') as fh: + fh.write(pickle.dumps(TEST_CLIENT_USAGE)) + + QUOTA_TEST_VGRID_FILEPATH1 = os.path.join(QUOTA_VGRID_PATH, + "%s.pck" % TEST_VGRID_NAME1) + with open(QUOTA_TEST_VGRID_FILEPATH1, 'wb') as fh: + fh.write(pickle.dumps(TEST_VGRID_USAGE1)) + + QUOTA_TEST_VGRID_FILEPATH2 = os.path.join(QUOTA_VGRID_PATH, + "%s.pck" % TEST_VGRID_NAME2) + with open(QUOTA_TEST_VGRID_FILEPATH2, 'wb') as fh: + fh.write(pickle.dumps(TEST_VGRID_USAGE2)) + + QUOTA_TEST_VGRID_FILEPATH3 = os.path.join(QUOTA_VGRID_PATH, + "%s.pck" % TEST_VGRID_NAME3) + with open(QUOTA_TEST_VGRID_FILEPATH3, 'wb') as fh: + fh.write(pickle.dumps(TEST_VGRID_USAGE3)) + + TEST_CLIENT_PEERS_FILEPATH = os.path.join( + TEST_CLIENT_PEERS_PATH, peers_filename) + with open(TEST_CLIENT_PEERS_FILEPATH, 'wb') as fh: + fh.write(pickle.dumps(TEST_PEERS)) + + QUOTA_TEST_CLIENT_EXT_PATH \ + = os.path.join(QUOTA_USER_PATH, + "%s.pck" % client_id_dir(TEST_EXT_DN)) + with open(QUOTA_TEST_CLIENT_EXT_PATH, 'wb') as fh: + fh.write(pickle.dumps(TEST_EXT_USAGE)) + + QUOTA_TEST_FREEZE_PATH = os.path.join(QUOTA_FREEZE_PATH, + "%s.pck" + % client_id_dir(TEST_CLIENT_DN)) + with open(QUOTA_TEST_FREEZE_PATH, 'wb') as fh: + fh.write(pickle.dumps(TEST_FREEZE_USAGE)) + + def test_accounting(self): + """Test accounting update and usage""" + # Create accounting + retval = update_accounting(self.configuration) + self.assertTrue(retval) + + # Check accounting + + usage = get_usage(self.configuration) + self.assertNotEqual(usage, {}) + + accounting = usage.get('accounting', {}) + test_user_accounting = accounting.get( + '/C=DK/ST=NA/L=NA/O=Test Org/OU=NA/CN=Test User/emailAddress=test@user.com', {}) + self.assertNotEqual(test_user_accounting, {}) + + home_total = test_user_accounting.get('home_total', 0) + self.assertEqual(home_total, 206128256) + + vgrid_total = test_user_accounting.get('vgrid_total', 0) + self.assertEqual(vgrid_total, 1818384768) + + ext_users_total = test_user_accounting.get('ext_users_total', 0) + self.assertEqual(ext_users_total, 16806128256) + + freeze_total = test_user_accounting.get('freeze_total', 0) + self.assertEqual(freeze_total, 128256) + + total_bytes = test_user_accounting.get('total_bytes', 0) + self.assertEqual(total_bytes, 18830769536) From 4f537f19a4dce4c64113d47827934f244ec4ff20 Mon Sep 17 00:00:00 2001 From: Martin Rehr Date: Mon, 26 Jan 2026 14:44:27 +0100 Subject: [PATCH 02/14] Fixed 'quota' vs. 'accounting' copy paste error --- mig/shared/configuration.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mig/shared/configuration.py b/mig/shared/configuration.py index c08eeafcf..09a869905 100644 --- a/mig/shared/configuration.py +++ b/mig/shared/configuration.py @@ -1787,7 +1787,7 @@ def reload_config(self, verbose, skip_log=False, disable_auth_log=False, self.user_quota_log = config.get('GLOBAL', 'user_quota_log') if config.has_option('GLOBAL', 'user_accounting_log'): - self.user_quota_log = config.get('GLOBAL', + self.user_accounting_log = config.get('GLOBAL', 'user_accounting_log') if config.has_option('GLOBAL', 'public_key_file'): self.public_key_file = config.get('GLOBAL', 'public_key_file') From c84c47fe7f80abddca0ed411002be8579ab92f10 Mon Sep 17 00:00:00 2001 From: Martin Rehr Date: Mon, 26 Jan 2026 14:47:57 +0100 Subject: [PATCH 03/14] Fixed 'verbose' variable issue --- bin/showaccounting.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/bin/showaccounting.py b/bin/showaccounting.py index 8d81703e1..fa5c6c7ee 100755 --- a/bin/showaccounting.py +++ b/bin/showaccounting.py @@ -59,7 +59,8 @@ def usage(name='accounting.py'): def show_accounting(configuration, timestamp, user_filter, - minimum_usage): + minimum_usage, + verbose): """Print user accointing report""" user_filter_re = None if user_filter: @@ -186,6 +187,7 @@ def show_accounting(configuration, show_accounting(configuration, timestamp, user_filter, - minimum_usage) + minimum_usage, + verbose) sys.exit(0) From ec793467e0c4906f93e602175d4bd0019bf5334a Mon Sep 17 00:00:00 2001 From: Martin Rehr Date: Mon, 26 Jan 2026 17:14:07 +0100 Subject: [PATCH 04/14] Added accounting to MiGserver--customised tests --- tests/data/MiGserver--customised-include_sections.conf | 7 +++++++ tests/data/MiGserver--customised.conf | 7 +++++++ 2 files changed, 14 insertions(+) diff --git a/tests/data/MiGserver--customised-include_sections.conf b/tests/data/MiGserver--customised-include_sections.conf index 7608265b1..0cff0036a 100644 --- a/tests/data/MiGserver--customised-include_sections.conf +++ b/tests/data/MiGserver--customised-include_sections.conf @@ -123,6 +123,7 @@ workflows_db = %(workflows_db_home)sworkflows_db.pickle workflows_db_lock = %(workflows_db_home)sworkflows_db.lock notify_home = %(state_path)s/notify_home/ quota_home = %(state_path)s/quota_home/ +accounting_home = %(state_path)s/accounting_home/ # GDP data categories metadata and helpers json file gdp_data_categories = %(gdp_home)s/data_categories.json @@ -546,9 +547,13 @@ default_mount_re = SSHFS-2.X-1 [QUOTA] backend = lustre +update_interval = 3600 user_limit = 1099511627776 vgrid_limit = 1099511627776 +[ACCOUNTING] +update_interval = 3600 + [SITE] # Web site appearance # Whether to use Python 3 for all Python invocations @@ -670,6 +675,8 @@ enable_openid = False enable_sharelinks = True # Enable storage quota enable_quota = False +# Enable storage accounting +enable_accounting = False # Enable background data transfers daemon - requires lftp and rsync enable_transfers = False # Explicit background transfer source addresses for use in pub key restrictions diff --git a/tests/data/MiGserver--customised.conf b/tests/data/MiGserver--customised.conf index a39336d73..c993c499d 100644 --- a/tests/data/MiGserver--customised.conf +++ b/tests/data/MiGserver--customised.conf @@ -123,6 +123,7 @@ workflows_db = %(workflows_db_home)sworkflows_db.pickle workflows_db_lock = %(workflows_db_home)sworkflows_db.lock notify_home = %(state_path)s/notify_home/ quota_home = %(state_path)s/quota_home/ +accounting_home = %(state_path)s/accounting_home/ # GDP data categories metadata and helpers json file gdp_data_categories = %(gdp_home)s/data_categories.json @@ -546,9 +547,13 @@ default_mount_re = SSHFS-2.X-1 [QUOTA] backend = lustre +update_interval = 3600 user_limit = 1099511627776 vgrid_limit = 1099511627776 +[ACCOUNTING] +update_interval = 3600 + [SITE] # Web site appearance # Whether to use Python 3 for all Python invocations @@ -670,6 +675,8 @@ enable_openid = False enable_sharelinks = True # Enable storage quota enable_quota = False +# Enable storage accounting +enable_accounting = False # Enable background data transfers daemon - requires lftp and rsync enable_transfers = False # Explicit background transfer source addresses for use in pub key restrictions From cee7f77b382685489bd4bfb533223297b24d7116 Mon Sep 17 00:00:00 2001 From: Martin Rehr Date: Mon, 26 Jan 2026 17:14:29 +0100 Subject: [PATCH 05/14] Added accounting options to generateconfs --- mig/install/generateconfs.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/mig/install/generateconfs.py b/mig/install/generateconfs.py index 2299233cb..216fd9f48 100755 --- a/mig/install/generateconfs.py +++ b/mig/install/generateconfs.py @@ -255,6 +255,7 @@ def main(argv, _generate_confs=generate_confs, _print=print): 'quota_update_interval', 'quota_user_limit', 'quota_vgrid_limit', + 'accounting_update_interval', 'wwwserve_max_bytes', ] bool_names = [ @@ -273,6 +274,7 @@ def main(argv, _generate_confs=generate_confs, _print=print): 'enable_events', 'enable_sharelinks', 'enable_quota', + 'enable_accounting', 'enable_transfers', 'enable_freeze', 'enable_sandboxes', From 83d013371a67eba5adc9bbd319c4dcb22996bd7c Mon Sep 17 00:00:00 2001 From: Martin Rehr Date: Tue, 27 Jan 2026 14:46:42 +0100 Subject: [PATCH 06/14] Changed 'accounting' name to 'showaccounting' as suggested by @jonasbardino --- bin/showaccounting.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/showaccounting.py b/bin/showaccounting.py index fa5c6c7ee..55594c561 100755 --- a/bin/showaccounting.py +++ b/bin/showaccounting.py @@ -39,7 +39,7 @@ from mig.lib.accounting import get_usage, human_readable_filesize -def usage(name='accounting.py'): +def usage(name='showaccounting.py'): """Usage help""" print("""Create accounting information based on quota. From fecb7cf4d7bdf1ed9c192ad330d9e1e47ad5b78c Mon Sep 17 00:00:00 2001 From: Martin Rehr Date: Tue, 27 Jan 2026 14:48:54 +0100 Subject: [PATCH 07/14] Fixed typos thanks to @jonasbardino --- bin/showaccounting.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bin/showaccounting.py b/bin/showaccounting.py index 55594c561..0faa9ff8b 100755 --- a/bin/showaccounting.py +++ b/bin/showaccounting.py @@ -61,7 +61,7 @@ def show_accounting(configuration, user_filter, minimum_usage, verbose): - """Print user accointing report""" + """Print user accounting report""" user_filter_re = None if user_filter: try: @@ -124,7 +124,7 @@ def show_accounting(configuration, % (report_shown_users, human_readable_filesize(report_shown_bytes))) print("User filter: %r" % user_filter) - print("Minumum usage: %s" % human_readable_filesize(minimum_usage)) + print("Minimum usage: %s" % human_readable_filesize(minimum_usage)) for total_bytes in sorted_total_bytes: total_bytes_human = human_readable_filesize(total_bytes) for username in total_bytes_map[total_bytes]: From 8811c32dd39a8d7398703662abd7817c5696cb6e Mon Sep 17 00:00:00 2001 From: Martin Rehr Date: Tue, 27 Jan 2026 14:57:39 +0100 Subject: [PATCH 08/14] Replaced 'GDP' with mig.shared.gdp_distinguished_field as suggested by @jonasbardino --- bin/showaccounting.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/bin/showaccounting.py b/bin/showaccounting.py index 0faa9ff8b..4282f013b 100755 --- a/bin/showaccounting.py +++ b/bin/showaccounting.py @@ -36,6 +36,7 @@ import datetime from mig.shared.conf import get_configuration_object +from mig.shared.defaults import gdp_distinguished_field from mig.lib.accounting import get_usage, human_readable_filesize @@ -92,7 +93,7 @@ def show_accounting(configuration, # Do not show GDP project users # projects are accounted for by the main user if configuration.site_enable_gdp \ - and username.find("/GDP=") != -1: + and username.find("/%s=" % gdp_distinguished_field) != -1: continue report_total_users += 1 total_bytes = values.get('total_bytes', 0) From 5882a16bddf1ffaee032a54161f55d4415b91a9a Mon Sep 17 00:00:00 2001 From: Martin Rehr Date: Tue, 27 Jan 2026 15:06:29 +0100 Subject: [PATCH 09/14] Updated doc-string and startup text as suggested by @jonasbardino --- sbin/grid_accounting.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/sbin/grid_accounting.py b/sbin/grid_accounting.py index b7fbdc325..055f5f4e2 100755 --- a/sbin/grid_accounting.py +++ b/sbin/grid_accounting.py @@ -25,7 +25,8 @@ # -- END_HEADER --- # -"""Daemon that create storage accounting""" +"""Daemon that collect storage accounting +information for users and their associated vgrids, archives and peers""" from __future__ import absolute_import, print_function @@ -44,8 +45,8 @@ if __name__ == "__main__": print( - """This is the MiG lustre accounting daemon which collect storage accounting - information for users and vgrids. + """This is the MiG lustre accounting daemon that collect storage accounting + information for users and their associated vgrids, archives and peers. Set the MIG_CONF environment to the server configuration path unless it is available in mig/server/MiGserver.conf From 8e3918225932cc41e71069fc7c5e534f43158c6f Mon Sep 17 00:00:00 2001 From: Martin Rehr Date: Tue, 27 Jan 2026 15:07:10 +0100 Subject: [PATCH 10/14] Removed 'lustre' reference as suggested by @jonasbardino --- sbin/grid_accounting.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sbin/grid_accounting.py b/sbin/grid_accounting.py index 055f5f4e2..7e21e80db 100755 --- a/sbin/grid_accounting.py +++ b/sbin/grid_accounting.py @@ -45,7 +45,7 @@ if __name__ == "__main__": print( - """This is the MiG lustre accounting daemon that collect storage accounting + """This is the MiG accounting daemon that collect storage accounting information for users and their associated vgrids, archives and peers. Set the MIG_CONF environment to the server configuration path From 5be5964f78b90a6c7871d979b78cd3e934858c6c Mon Sep 17 00:00:00 2001 From: Martin Rehr Date: Tue, 27 Jan 2026 15:11:03 +0100 Subject: [PATCH 11/14] Adjusted import order as suggested by @jonasbardino --- bin/showaccounting.py | 9 ++++----- mig/lib/accounting.py | 11 +++++------ sbin/grid_accounting.py | 5 ++--- 3 files changed, 11 insertions(+), 14 deletions(-) diff --git a/bin/showaccounting.py b/bin/showaccounting.py index 4282f013b..20670f9d8 100755 --- a/bin/showaccounting.py +++ b/bin/showaccounting.py @@ -27,17 +27,16 @@ """Create accounting information for users""" -from __future__ import print_function -from __future__ import absolute_import +from __future__ import absolute_import, print_function -import sys +import datetime import getopt import re -import datetime +import sys +from mig.lib.accounting import get_usage, human_readable_filesize from mig.shared.conf import get_configuration_object from mig.shared.defaults import gdp_distinguished_field -from mig.lib.accounting import get_usage, human_readable_filesize def usage(name='showaccounting.py'): diff --git a/mig/lib/accounting.py b/mig/lib/accounting.py index 692245084..62472c768 100644 --- a/mig/lib/accounting.py +++ b/mig/lib/accounting.py @@ -28,18 +28,17 @@ """helpers to support storage accounting""" -from __future__ import print_function -from __future__ import absolute_import +from __future__ import absolute_import, print_function -import os -import time import datetime import math +import os +import time from mig.shared.base import client_dir_id, force_native_str -from mig.shared.fileio import pickle, unpickle, load_json, make_symlink -from mig.shared.vgrid import vgrid_list_vgrids, vgrid_list +from mig.shared.fileio import load_json, make_symlink, pickle, unpickle from mig.shared.useradm import get_accepted_peers +from mig.shared.vgrid import vgrid_list, vgrid_list_vgrids def __init_accounting_entry(user_bytes=0, diff --git a/sbin/grid_accounting.py b/sbin/grid_accounting.py index 7e21e80db..c0cf7a84d 100755 --- a/sbin/grid_accounting.py +++ b/sbin/grid_accounting.py @@ -30,19 +30,18 @@ from __future__ import absolute_import, print_function +import datetime import os import sys import time import traceback -import datetime +from mig.lib.accounting import update_accounting from mig.lib.daemon import check_run, check_stop, interruptible_sleep, \ register_run_handler, register_stop_handler, reset_run, stop_running -from mig.lib.accounting import update_accounting from mig.shared.conf import get_configuration_object from mig.shared.logger import daemon_logger, register_hangup_handler - if __name__ == "__main__": print( """This is the MiG accounting daemon that collect storage accounting From 1601adf53208fd08ed2204a820e02ddde0bd47a6 Mon Sep 17 00:00:00 2001 From: Martin Rehr Date: Tue, 27 Jan 2026 15:12:35 +0100 Subject: [PATCH 12/14] Fixed copy/paste quota -> accounting leftover as suggested by @jonasbardino --- tests/test_mig_lib_accounting.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_mig_lib_accounting.py b/tests/test_mig_lib_accounting.py index ba92d397f..b5ffb7e7c 100644 --- a/tests/test_mig_lib_accounting.py +++ b/tests/test_mig_lib_accounting.py @@ -37,7 +37,7 @@ class MigLibAccounting(MigTestCase): - """Unit tests for quota related helper functions""" + """Unit tests for accounting related helper functions""" def _provide_configuration(self): """Prepare isolated test config""" From 6a03b1dd794abe3a00c954add4390a59ec4170e2 Mon Sep 17 00:00:00 2001 From: Martin Rehr Date: Wed, 28 Jan 2026 12:48:02 +0100 Subject: [PATCH 13/14] Removed explicit 'keys' extraction from dict when creating list of dict keys (suggested by @jonasbardino) --- bin/showaccounting.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/showaccounting.py b/bin/showaccounting.py index 20670f9d8..1c5af61ef 100755 --- a/bin/showaccounting.py +++ b/bin/showaccounting.py @@ -105,7 +105,7 @@ def show_accounting(configuration, total_bytes_map_userlist = total_bytes_map.get(total_bytes, []) total_bytes_map_userlist.append(username) total_bytes_map[total_bytes] = total_bytes_map_userlist - sorted_total_bytes = sorted(list(total_bytes_map.keys()), reverse=True) + sorted_total_bytes = sorted(list(total_bytes_map), reverse=True) print("\nAccounting (%d) %s for storage quota(s):" % (accounting_timestamp, accounting_datestr)) From 08d6799b4345c5a9f6953537ce8c405746a12336 Mon Sep 17 00:00:00 2001 From: Martin Rehr Date: Wed, 28 Jan 2026 13:56:21 +0100 Subject: [PATCH 14/14] Cleaned up unittest as suggested by @jonasbardino --- tests/test_mig_lib_accounting.py | 214 ++++++++++++++++--------------- 1 file changed, 111 insertions(+), 103 deletions(-) diff --git a/tests/test_mig_lib_accounting.py b/tests/test_mig_lib_accounting.py index b5ffb7e7c..6dc9c0393 100644 --- a/tests/test_mig_lib_accounting.py +++ b/tests/test_mig_lib_accounting.py @@ -35,6 +35,75 @@ from mig.shared.defaults import peers_filename from tests.support import MigTestCase, ensure_dirs_exist +TEST_MTIME = 1768925307 +TEST_SOFTLIMIT_BYTES = 109951162777600 +TEST_HARDLIMIT_BYTES = 109951162777600 +TEST_CLIENT_DN = '/C=DK/ST=NA/L=NA/O=Test Org/OU=NA/CN=Test User/emailAddress=test@user.com' +TEST_CLIENT_BYTES = 206128256 +TEST_EXT_DN = '/C=DK/ST=NA/L=NA/O=PEER Org/OU=NA/CN=Test Peer/emailAddress=peer@example.com' +TEST_EXT_BYTES = 16806128256 +TEST_FREEZE_BYTES = 128256 +TEST_VGRID_NAME1 = 'TestVgrid1' +TEST_VGRID_BYTES1 = 406128256 +TEST_VGRID_NAME2 = 'TestVgrid2' +TEST_VGRID_BYTES2 = 606128256 +TEST_VGRID_NAME3 = 'TestVgrid3' +TEST_VGRID_BYTES3 = 806128256 +TEST_VGRID_TOTAL_BYTES = TEST_VGRID_BYTES1 \ + + TEST_VGRID_BYTES2 \ + + TEST_VGRID_BYTES3 +TEST_TOTAL_BYTES = TEST_CLIENT_BYTES \ + + TEST_EXT_BYTES \ + + TEST_FREEZE_BYTES \ + + TEST_VGRID_TOTAL_BYTES +TEST_LUSTRE_QUOTA_INFO = {'next_pid': 192, 'mtime': TEST_MTIME} +TEST_CLIENT_USAGE = {'lustre_pid': 42, + 'files': 11, + 'bytes': TEST_CLIENT_BYTES, + 'softlimit_bytes': TEST_SOFTLIMIT_BYTES, + 'hardlimit_bytes': TEST_HARDLIMIT_BYTES, + 'mtime': TEST_MTIME} +TEST_VGRID_USAGE1 = {'lustre_pid': 43, + 'files': 111, + 'bytes': TEST_VGRID_BYTES1, + 'softlimit_bytes': TEST_SOFTLIMIT_BYTES, + 'hardlimit_bytes': TEST_HARDLIMIT_BYTES, + 'mtime': TEST_MTIME} +TEST_VGRID_USAGE2 = {'lustre_pid': 44, + 'files': 222, + 'bytes': TEST_VGRID_BYTES2, + 'softlimit_bytes': TEST_SOFTLIMIT_BYTES, + 'hardlimit_bytes': TEST_HARDLIMIT_BYTES, + 'mtime': TEST_MTIME} +TEST_VGRID_USAGE3 = {'lustre_pid': 45, + 'files': 333, + 'bytes': TEST_VGRID_BYTES3, + 'softlimit_bytes': TEST_SOFTLIMIT_BYTES, + 'hardlimit_bytes': TEST_HARDLIMIT_BYTES, + 'mtime': TEST_MTIME} +TEST_EXT_USAGE = {'lustre_pid': 46, + 'files': 1, + 'bytes': TEST_EXT_BYTES, + 'softlimit_bytes': TEST_SOFTLIMIT_BYTES, + 'hardlimit_bytes': TEST_HARDLIMIT_BYTES, + 'mtime': TEST_MTIME} +TEST_FREEZE_USAGE = {'lustre_pid': 47, + 'files': 1, + 'bytes': TEST_FREEZE_BYTES, + 'softlimit_bytes': TEST_SOFTLIMIT_BYTES, + 'hardlimit_bytes': TEST_HARDLIMIT_BYTES, + 'mtime': TEST_MTIME} +TEST_PEERS = {TEST_EXT_DN: {'kind': 'collaboration', + 'distinguished_name': TEST_EXT_DN, + 'country': 'DK', + 'label': 'TEST', + 'state': '', + 'expire': '2222-12-31', + 'full_name': 'Test Peer', + 'organization': 'PEER Org', + 'email': 'peer@example.com' + }} + class MigLibAccounting(MigTestCase): """Unit tests for accounting related helper functions""" @@ -46,90 +115,28 @@ def _provide_configuration(self): def before_each(self): """Set up test configuration and reset state before each test""" - # Define fake quota - - TEST_LUSTRE_QUOTA_INFO = {'next_pid': 192, 'mtime': 1768925307} - - TEST_CLIENT_DN = '/C=DK/ST=NA/L=NA/O=Test Org/OU=NA/CN=Test User/emailAddress=test@user.com' - TEST_VGRID_NAME1 = 'TestVgrid1' - TEST_VGRID_NAME2 = 'TestVgrid2' - TEST_VGRID_NAME3 = 'TestVgrid3' - TEST_EXT_DN = '/C=DK/ST=NA/L=NA/O=PEER Org/OU=NA/CN=Test Peer/emailAddress=peer@example.com' - - TEST_CLIENT_USAGE = {'lustre_pid': 42, - 'files': 11, - 'bytes': 206128256, - 'softlimit_bytes': 109951162777600, - 'hardlimit_bytes': 109951162777600, - 'mtime': 1768925307} - - TEST_VGRID_USAGE1 = {'lustre_pid': 43, - 'files': 111, - 'bytes': 406128256, - 'softlimit_bytes': 109951162777600, - 'hardlimit_bytes': 109951162777600, - 'mtime': 1768925307} - - TEST_VGRID_USAGE2 = {'lustre_pid': 44, - 'files': 222, - 'bytes': 606128256, - 'softlimit_bytes': 109951162777600, - 'hardlimit_bytes': 109951162777600, - 'mtime': 1768925307} - - TEST_VGRID_USAGE3 = {'lustre_pid': 45, - 'files': 333, - 'bytes': 806128256, - 'softlimit_bytes': 109951162777600, - 'hardlimit_bytes': 109951162777600, - 'mtime': 1768925307} - - TEST_EXT_USAGE = {'lustre_pid': 46, - 'files': 1, - 'bytes': 16806128256, - 'softlimit_bytes': 109951162777600, - 'hardlimit_bytes': 109951162777600, - 'mtime': 1768925307} - - TEST_FREEZE_USAGE = {'lustre_pid': 47, - 'files': 1, - 'bytes': 128256, - 'softlimit_bytes': 109951162777600, - 'hardlimit_bytes': 109951162777600, - 'mtime': 1768925307} - - TEST_PEERS = {TEST_EXT_DN: {'kind': 'collaboration', - 'distinguished_name': TEST_EXT_DN, - 'country': 'DK', - 'label': 'TEST', - 'state': '', - 'expire': '2222-12-31', - 'full_name': 'Test Peer', - 'organization': 'PEER Org', - 'email': 'peer@example.com' - }} - # Create fake fs layout matching real systems self.configuration.site_enable_quota = True self.configuration.site_enable_accounting = True self.configuration.quota_backend = 'lustre' - QUOTA_BASEPATH = os.path.join(self.configuration.quota_home, + quota_basepath = os.path.join(self.configuration.quota_home, self.configuration.quota_backend) - QUOTA_USER_PATH = os.path.join(QUOTA_BASEPATH, 'user') - QUOTA_VGRID_PATH = os.path.join(QUOTA_BASEPATH, 'vgrid') - QUOTA_FREEZE_PATH = os.path.join(QUOTA_BASEPATH, 'freeze') - TEST_CLIENT_PEERS_PATH = os.path.join(self.configuration.user_settings, + quota_user_path = os.path.join(quota_basepath, 'user') + quota_vgrid_path = os.path.join(quota_basepath, 'vgrid') + quota_freeze_path = os.path.join(quota_basepath, 'freeze') + test_client_peers_path = os.path.join(self.configuration.user_settings, client_id_dir(TEST_CLIENT_DN)) + ensure_dirs_exist(self.configuration.vgrid_home) ensure_dirs_exist(self.configuration.user_settings) ensure_dirs_exist(self.configuration.accounting_home) ensure_dirs_exist(self.configuration.quota_home) - ensure_dirs_exist(QUOTA_USER_PATH) - ensure_dirs_exist(QUOTA_VGRID_PATH) - ensure_dirs_exist(QUOTA_FREEZE_PATH) - ensure_dirs_exist(TEST_CLIENT_PEERS_PATH) + ensure_dirs_exist(quota_user_path) + ensure_dirs_exist(quota_vgrid_path) + ensure_dirs_exist(quota_freeze_path) + ensure_dirs_exist(test_client_peers_path) # Ensure fake vgrid and write owner @@ -145,78 +152,79 @@ def before_each(self): # Write fake quota - TEST_LUSTRE_QUOTA_INFO_FILEPATH \ + test_lustre_quota_info_filepath \ = os.path.join(self.configuration.quota_home, '%s.pck' % self.configuration.quota_backend) - with open(TEST_LUSTRE_QUOTA_INFO_FILEPATH, 'wb') as fh: + with open(test_lustre_quota_info_filepath, 'wb') as fh: fh.write(pickle.dumps(TEST_LUSTRE_QUOTA_INFO)) - QUOTA_TEST_CLIENT_PATH \ - = os.path.join(QUOTA_USER_PATH, + quota_test_client_path \ + = os.path.join(quota_user_path, "%s.pck" % client_id_dir(TEST_CLIENT_DN)) - with open(QUOTA_TEST_CLIENT_PATH, 'wb') as fh: + with open(quota_test_client_path, 'wb') as fh: fh.write(pickle.dumps(TEST_CLIENT_USAGE)) - QUOTA_TEST_VGRID_FILEPATH1 = os.path.join(QUOTA_VGRID_PATH, - "%s.pck" % TEST_VGRID_NAME1) - with open(QUOTA_TEST_VGRID_FILEPATH1, 'wb') as fh: + quot_test_vgrid_filepath1 = os.path.join(quota_vgrid_path, + "%s.pck" % TEST_VGRID_NAME1) + with open(quot_test_vgrid_filepath1, 'wb') as fh: fh.write(pickle.dumps(TEST_VGRID_USAGE1)) - QUOTA_TEST_VGRID_FILEPATH2 = os.path.join(QUOTA_VGRID_PATH, - "%s.pck" % TEST_VGRID_NAME2) - with open(QUOTA_TEST_VGRID_FILEPATH2, 'wb') as fh: + quot_test_vgrid_filepath2 = os.path.join(quota_vgrid_path, + "%s.pck" % TEST_VGRID_NAME2) + with open(quot_test_vgrid_filepath2, 'wb') as fh: fh.write(pickle.dumps(TEST_VGRID_USAGE2)) - QUOTA_TEST_VGRID_FILEPATH3 = os.path.join(QUOTA_VGRID_PATH, - "%s.pck" % TEST_VGRID_NAME3) - with open(QUOTA_TEST_VGRID_FILEPATH3, 'wb') as fh: + quot_test_vgrid_filepath3 = os.path.join(quota_vgrid_path, + "%s.pck" % TEST_VGRID_NAME3) + with open(quot_test_vgrid_filepath3, 'wb') as fh: fh.write(pickle.dumps(TEST_VGRID_USAGE3)) - TEST_CLIENT_PEERS_FILEPATH = os.path.join( - TEST_CLIENT_PEERS_PATH, peers_filename) - with open(TEST_CLIENT_PEERS_FILEPATH, 'wb') as fh: + test_client_peers_filepath = os.path.join( + test_client_peers_path, peers_filename) + with open(test_client_peers_filepath, 'wb') as fh: fh.write(pickle.dumps(TEST_PEERS)) - QUOTA_TEST_CLIENT_EXT_PATH \ - = os.path.join(QUOTA_USER_PATH, + quota_test_client_ext_path \ + = os.path.join(quota_user_path, "%s.pck" % client_id_dir(TEST_EXT_DN)) - with open(QUOTA_TEST_CLIENT_EXT_PATH, 'wb') as fh: + with open(quota_test_client_ext_path, 'wb') as fh: fh.write(pickle.dumps(TEST_EXT_USAGE)) - QUOTA_TEST_FREEZE_PATH = os.path.join(QUOTA_FREEZE_PATH, + quota_test_freeze_path = os.path.join(quota_freeze_path, "%s.pck" % client_id_dir(TEST_CLIENT_DN)) - with open(QUOTA_TEST_FREEZE_PATH, 'wb') as fh: + with open(quota_test_freeze_path, 'wb') as fh: fh.write(pickle.dumps(TEST_FREEZE_USAGE)) def test_accounting(self): """Test accounting update and usage""" - # Create accounting + + # Update accounting information based on quote + retval = update_accounting(self.configuration) self.assertTrue(retval) - # Check accounting + # Check updated accounting data usage = get_usage(self.configuration) self.assertNotEqual(usage, {}) accounting = usage.get('accounting', {}) - test_user_accounting = accounting.get( - '/C=DK/ST=NA/L=NA/O=Test Org/OU=NA/CN=Test User/emailAddress=test@user.com', {}) + test_user_accounting = accounting.get(TEST_CLIENT_DN, {}) self.assertNotEqual(test_user_accounting, {}) home_total = test_user_accounting.get('home_total', 0) - self.assertEqual(home_total, 206128256) + self.assertEqual(home_total, TEST_CLIENT_BYTES) vgrid_total = test_user_accounting.get('vgrid_total', 0) - self.assertEqual(vgrid_total, 1818384768) + self.assertEqual(vgrid_total, TEST_VGRID_TOTAL_BYTES) ext_users_total = test_user_accounting.get('ext_users_total', 0) - self.assertEqual(ext_users_total, 16806128256) + self.assertEqual(ext_users_total, TEST_EXT_BYTES) freeze_total = test_user_accounting.get('freeze_total', 0) - self.assertEqual(freeze_total, 128256) + self.assertEqual(freeze_total, TEST_FREEZE_BYTES) total_bytes = test_user_accounting.get('total_bytes', 0) - self.assertEqual(total_bytes, 18830769536) + self.assertEqual(total_bytes, TEST_TOTAL_BYTES)