From dfe542dbfc2b16a8dbdb8f9ae85dad718a146950 Mon Sep 17 00:00:00 2001 From: "Peter B. Robinson" Date: Wed, 20 May 2026 14:02:31 -0700 Subject: [PATCH 01/21] AI pass at branch 3 --- ats/atsMachines/fluxScheduled.py | 4 + ats/machines.py | 765 +++++++++++++++++++++++++------ 2 files changed, 622 insertions(+), 147 deletions(-) diff --git a/ats/atsMachines/fluxScheduled.py b/ats/atsMachines/fluxScheduled.py index 48c4cab..6e8a018 100755 --- a/ats/atsMachines/fluxScheduled.py +++ b/ats/atsMachines/fluxScheduled.py @@ -175,6 +175,10 @@ def get_physical_node(self, rel_index): raise IndexError(f"Relative index {rel_index} out of range (0-{len(nodes)-1})") return nodes[rel_index] + def getNumberOfProcessors(self): + """Return total schedulable processor slots in the Flux allocation.""" + return self.maxCores + def kill(self, test): """ diff --git a/ats/machines.py b/ats/machines.py index 2915f49..365322b 100644 --- a/ats/machines.py +++ b/ats/machines.py @@ -1,6 +1,7 @@ """Definition of class Machine for overriding. """ -import subprocess, sys, os, time, shlex +from collections import deque +import selectors, subprocess, sys, os, threading, time, shlex from ats.atsut import RUNNING, TIMEDOUT, PASSED, FAILED, LSFERROR, \ SKIPPED, HALTED, AtsError from ats.log import log, terminal @@ -85,26 +86,57 @@ def checkForTimeOut(self, test): def checkRunning(self): """Find those tests still running. getStatus checks for timeout. """ - # print("DEBUG checkRunning 100\n") - from ats import configuration - time.sleep(self.naptime) - stillRunning = [] - for test in self.running: - done = self.getStatus(test) - if not done: - stillRunning.append(test) - else: # test has finished - if test.status is not PASSED: - if configuration.options.oneFailure: - raise AtsError("Test failed in oneFailure mode.") - self.running = stillRunning + completion_limit = self._completionFastPathDrainLimit() + if self._useLegacyCompletionPolling(): + if self._pollRunningTests( + allow_running_checks=True, + completion_limit=completion_limit, + ): + return + time.sleep(self.naptime) + self._pollRunningTests( + allow_running_checks=True, + completion_limit=completion_limit, + ) + return + if self._useQueuedCompletionDetection(): + self._incrementCompletionStat("check_running_completion_queue_mode") + if self._pollQueuedCompletionTests(completion_limit=completion_limit): + self._incrementCompletionStat("check_running_queue_pre_drain_completed") + return + self._incrementCompletionStat("check_running_queue_pre_drain_empty") + self._incrementCompletionStat("check_running_wait_for_completion_signal") + self._waitForCompletionSignal() + if self._pollQueuedCompletionTests(completion_limit=completion_limit): + self._incrementCompletionStat("check_running_queue_post_wait_completed") + return + self._incrementCompletionStat("check_running_queue_post_wait_empty") + self._incrementCompletionStat("check_running_queue_fallback_poll_running") + self._pollRunningTests( + allow_running_checks=True, + completion_limit=completion_limit, + ) + return + if self._pollRunningTests( + allow_running_checks=False, + completion_limit=completion_limit, + ): + return + completion_hints = self._waitForCompletionSignal() + if completion_hints and self._pollRunningTests( + allow_running_checks=False, + prioritized=completion_hints, + completion_limit=completion_limit, + ): + return + self._pollRunningTests(allow_running_checks=True) def remainingCapacity(self): """How many processors are free? Could be overriden to answer the real question, what is the largest job you could start at this time?""" return self.numberTestsRunningMax - self.numberTestsRunning - def getStatus(self, test): + def getStatus(self, test, allow_running_checks=True): """ Override this only if not using subprocess (unusual). Obtains the exit code of the test object process and then sets @@ -116,162 +148,591 @@ def getStatus(self, test): testEnded will call your bookkeeping method noteEnd. """ from ats import configuration - test.child.poll() - # print(f"This is the return code for the test:{test.child.returncode}") - if test.child.returncode is None: - overtime, fraction = self.checkForTimeOut(test) - #print "DEBUG getStatus 100" - #print overtime - #print fraction - #print "DEBUG getStatus 200" - if fraction > .9 or overtime != 0: - # If a process produces a lot of output, it may fill its output - # buffer and then block until something is read from it. - - # How should testStdout handle this? ??? - # - # 2017-08-15 SAD putting back in poll, to see if it fixes hang. - if configuration.SYS_TYPE.startswith('somesystemxxx'): - stdoutdata, stderrdata = test.child.communicate() - - # Now, poll it again. - test.child.poll() - - - if test.child.returncode is None: #still running, but too long? - overtime, fraction = self.checkForTimeOut(test) - #print "DEBUG getStatus 300" - #print overtime - #print fraction - #print "DEBUG getStatus 400" - if overtime != 0: - self.kill(test) - test.statusCode = 2 - test.setEndDateTime() - if overtime > 0: - status = TIMEDOUT - else: - status = HALTED #one minute mode - else: - #print "DEBUG getStatus 320" - # SAD - # Coding to detect SLURM deficiencies, and abort job. - # Implemented 2016-Aug-30 - slurm_error = False - f = open(test.errname, 'r', errors='replace') - lines = f.readlines() - f.close - for line in lines: - if slurm_error == False: - if "Slurmd could not set up environment for batch job" in line: - print("ATS Halting test %s. Detected slurm launch failure : %s " % (test.name, line)) - slurm_error = True - elif "srun: error: Unable to create job step" in line: - print("ATS Halting test %s. Detected slurm error : %s " % (test.name, line)) - slurm_error = True - elif "Error opening remote shared memory object in shm_open" in line: - print("ATS Halting test %s. Detected MPI shared memory failure : %s " % (test.name, line)) - slurm_error = True - elif "PSM could not set up shared memory segment" in line: - print("ATS Halting test %s. Detected MPI shared memory failure : %s " % (test.name, line)) - slurm_error = True - elif "Attempting to use an MPI routine before initializing MPICH" in line: - print("ATS Halting test %s. Detected MPI Error : %s " % (test.name, line)) - slurm_error = True - elif "Bus error)" in line: - print("ATS Halting test %s. Detected Bus Error (perhaps MPI related) : %s " % (test.name, line)) - slurm_error = True - - if slurm_error: - self.kill(test) - test.statusCode = 2 - test.setEndDateTime() - status = HALTED + self._pollChild(test) + if test.child.returncode is not None: + return self._finishCompletedTest(test) - else: - return False - else: - # print "DEBUG getStatus 400" + if not allow_running_checks: + return False + + overtime, fraction = self.checkForTimeOut(test) + if fraction > .9 or overtime != 0: + if configuration.SYS_TYPE.startswith('somesystemxxx'): + stdoutdata, stderrdata = test.child.communicate() + + self._pollChild(test) + if test.child.returncode is not None: + return self._finishCompletedTest(test) + + overtime, fraction = self.checkForTimeOut(test) + if overtime != 0: + self.kill(test) + test.statusCode = 2 test.setEndDateTime() - test.statusCode = test.child.returncode - # If the user set ignoreReturnCode to True then set statusCode to 0. - ignoreReturnCode = test.options.get('ignoreReturnCode', False) - if ignoreReturnCode: - test.statusCode = 0 - if test.statusCode == 0: # process is done - status = PASSED - # This checks for flux timeouts since ATS' method for determining timeouts doesnt work with flux - elif "flux" in configuration.MACHINE_TYPE and test.statusCode == 142: # 142 is the return code for a timeout from flux + if overtime > 0: status = TIMEDOUT else: - # Coding to detect LSF deficiencies - # Implemented 2018-12-12 - lsf_error = False - f = open(test.errname, 'r', errors='replace') + status = HALTED + return self._completeTest(test, status) + + if self._detectRunningSlurmError(test): + self.kill(test) + test.statusCode = 2 + test.setEndDateTime() + return self._completeTest(test, HALTED) + + return False + + def _pollRunningTests( + self, + allow_running_checks, + prioritized=None, + stop_after_completion=False, + completion_limit=None, + ): + """Poll running tests, optionally prioritizing likely completions.""" + from ats import configuration + + start_us = time.time_ns() // 1000 + self._incrementCompletionStat("_pollRunningTests_called") + if allow_running_checks: + self._incrementCompletionStat("_pollRunningTests_allow_running_checks_true") + else: + self._incrementCompletionStat("_pollRunningTests_allow_running_checks_false") + + prioritized = list(prioritized or []) + prioritized_count = len(prioritized) + ordered_count = 0 + completed = 0 + result_kind = "completed_none" + try: + ordered = [] + seen_ids = set() + for test in prioritized: + test_id = id(test) + if test_id in seen_ids: + continue + ordered.append(test) + seen_ids.add(test_id) + for test in self.running: + test_id = id(test) + if test_id in seen_ids: + continue + ordered.append(test) + seen_ids.add(test_id) + + ordered_count = len(ordered) + self._incrementCompletionStat("_pollRunningTests_total_ordered", ordered_count) + + remaining = [] + for index, test in enumerate(ordered): + done = self.getStatus(test, allow_running_checks=allow_running_checks) + if not done: + remaining.append(test) + continue + completed += 1 + if test.status is not PASSED and configuration.options.oneFailure: + raise AtsError("Test failed in oneFailure mode.") + if stop_after_completion or ( + completion_limit is not None and completed >= completion_limit + ): + remaining.extend(ordered[index + 1:]) + self._preserve_new_running_tests(remaining, seen_ids) + self.running = remaining + result_kind = "stopped_after_completion" + self._incrementCompletionStat("_pollRunningTests_stopped_after_completion") + self._incrementCompletionStat("_pollRunningTests_total_completed", completed) + return completed + + self._preserve_new_running_tests(remaining, seen_ids) + self.running = remaining + self._incrementCompletionStat("_pollRunningTests_total_completed", completed) + if completed: + result_kind = "completed" + self._incrementCompletionStat("_pollRunningTests_completed") + else: + self._incrementCompletionStat("_pollRunningTests_completed_none") + return completed + finally: + self._recordCompletionInternalSpan( + "_pollRunningTests", + start_us, + time.time_ns() // 1000, + metadata={ + "mode": getattr(self, "completion_detection_mode", ""), + "allow_running_checks": bool(allow_running_checks), + "prioritized_count": prioritized_count, + "ordered_count": ordered_count, + "stop_after_completion": bool(stop_after_completion), + "completion_limit": completion_limit, + "completed_count": completed, + "result": result_kind, + }, + ) + + def _preserve_new_running_tests(self, remaining, seen_ids): + """Keep tests appended to ``self.running`` during completion callbacks.""" + remaining_ids = {id(test) for test in remaining} + for test in self.running: + test_id = id(test) + if test_id in seen_ids or test_id in remaining_ids: + continue + remaining.append(test) + remaining_ids.add(test_id) + + def _waitForCompletionSignal(self): + """Wait for a local child exit, using pidfds when available.""" + start_us = time.time_ns() // 1000 + self._incrementCompletionStat("_waitForCompletionSignal_called") + registered = False + registered_count = 0 + ready = [] + selector = None + used_queue_event_wait = False + result_kind = "sleep_fallback" + try: + selector = selectors.DefaultSelector() + except Exception: + selector = None + + try: + if selector is not None: + try: + for test in self.running: + pidfd = self._ensurePidfd(test) + if pidfd is None: + continue + try: + selector.register(pidfd, selectors.EVENT_READ, test) + registered = True + registered_count += 1 + except Exception: + self._closePidfd(test) + if registered: + self._incrementCompletionStat("_waitForCompletionSignal_pidfd_registered") + ready = [key.data for key, _mask in selector.select(self.naptime)] + if ready: + result_kind = "pidfd_ready" + self._incrementCompletionStat("_waitForCompletionSignal_pidfd_ready") + self._incrementCompletionStat("_waitForCompletionSignal_total_ready", len(ready)) + for test in ready: + self._recordCompletionSignal(test) + else: + result_kind = "pidfd_timeout" + self._incrementCompletionStat("_waitForCompletionSignal_pidfd_timeout") + finally: + selector.close() + + if registered: + return ready + + if self._useQueuedCompletionDetection(): + used_queue_event_wait = True + result_kind = "queue_event_wait" + self._incrementCompletionStat("_waitForCompletionSignal_queue_event_wait") + self._completionEvent.wait(self.naptime) + return [] + + self._incrementCompletionStat("_waitForCompletionSignal_sleep_fallback") + time.sleep(self.naptime) + return [] + finally: + self._recordCompletionInternalSpan( + "_waitForCompletionSignal", + start_us, + time.time_ns() // 1000, + metadata={ + "mode": getattr(self, "completion_detection_mode", ""), + "running_count": len(self.running), + "registered": bool(registered), + "registered_count": registered_count, + "ready_count": len(ready), + "used_queue_event_wait": bool(used_queue_event_wait), + "result": result_kind, + }, + ) + + def _useLegacyCompletionPolling(self): + mode = str(getattr(self, "completion_detection_mode", "") or "").strip().lower() + return mode == "legacy_poll" + + def _useQueuedCompletionDetection(self): + mode = str(getattr(self, "completion_detection_mode", "") or "").strip().lower() + return mode == "completion_queue" + + def _completionStatsEnabled(self): + return bool(getattr(self, "completion_detection_stats", False)) + + def _completionSpansEnabled(self): + return bool(getattr(self, "completion_detection_spans", False)) + + def _incrementCompletionStat(self, name, amount=1): + if not self._completionStatsEnabled(): + return + with self._completionStatsLock: + self._completionStats[name] = self._completionStats.get(name, 0) + amount + + def _completionStatsSnapshot(self): + with self._completionStatsLock: + return dict(self._completionStats) + + def _addMachineHook(self, hook_attr, callback, description): + if not callable(callback): + raise AtsError("%s hook must be callable" % description) + hooks = getattr(self, hook_attr, None) + if hooks is None: + hooks = [] + setattr(self, hook_attr, hooks) + hooks.append(callback) + return callback + + def _removeMachineHook(self, hook_attr, callback): + hooks = getattr(self, hook_attr, None) + if hooks is None: + return + try: + hooks.remove(callback) + except ValueError: + pass + + def add_completion_span_hook(self, callback): + """Register a callback for internal completion-detection timing spans.""" + return self._addMachineHook( + "_completion_span_hooks", + callback, + "completion span", + ) + + def remove_completion_span_hook(self, callback): + """Unregister a completion span callback.""" + self._removeMachineHook("_completion_span_hooks", callback) + + def add_completion_queue_snapshot_hook(self, callback): + """Register a callback for completion queue depth snapshots.""" + return self._addMachineHook( + "_completion_queue_snapshot_hooks", + callback, + "completion queue snapshot", + ) + + def remove_completion_queue_snapshot_hook(self, callback): + """Unregister a completion queue snapshot callback.""" + self._removeMachineHook("_completion_queue_snapshot_hooks", callback) + + def _recordCompletionInternalSpan(self, name, start_us, end_us, metadata=None): + if not self._completionSpansEnabled(): + return + for callback in list(getattr(self, "_completion_span_hooks", [])): + callback(name, start_us, end_us, metadata or {}) + + def _recordCompletionQueueSnapshot(self, depth, reason, timestamp_us=None, metadata=None): + if timestamp_us is None: + timestamp_us = time.time_ns() // 1000 + depth = max(0, int(depth)) + if self._completionStatsEnabled(): + with self._completionStatsLock: + self._completionStats["completion_queue_depth_latest"] = depth + peak = int(self._completionStats.get("completion_queue_depth_peak", 0)) + if depth > peak: + self._completionStats["completion_queue_depth_peak"] = depth + payload = { + "completion_queue_depth": depth, + "reason": reason, + } + if metadata: + payload.update(metadata) + for callback in list(getattr(self, "_completion_queue_snapshot_hooks", [])): + callback(timestamp_us, payload) + + def _completionFastPathDrainLimit(self): + limit = getattr(self, "completion_fast_path_drain_limit", 128) + try: + limit = int(limit) + except (TypeError, ValueError): + limit = 128 + return max(1, limit) + + def _pollChild(self, test): + test.child.poll() + return test.child.returncode + + def _recordCompletionSignal(self, test, observed_us=None): + if observed_us is None: + observed_us = time.time_ns() // 1000 + if getattr(test, "ats_completion_signal_us", None) is None: + test.ats_completion_signal_us = observed_us + self._incrementCompletionStat("completion_signal_recorded") + if not self._useQueuedCompletionDetection(): + return + with self._completionQueueLock: + test_id = id(test) + if test_id in self._completionQueueIds: + self._incrementCompletionStat("completion_queue_duplicate_signal") + return + self._completionQueue.append(test) + self._completionQueueIds.add(test_id) + self._completionEvent.set() + self._incrementCompletionStat("completion_queue_enqueued") + depth = len(self._completionQueue) + self._recordCompletionQueueSnapshot( + depth, + "completion_queue_enqueue", + timestamp_us=observed_us, + ) + + def _drainCompletionQueue(self, completion_limit=None): + queued = [] + with self._completionQueueLock: + while self._completionQueue: + if completion_limit is not None and len(queued) >= completion_limit: + break + test = self._completionQueue.popleft() + self._completionQueueIds.discard(id(test)) + queued.append(test) + remaining_depth = len(self._completionQueue) + if not self._completionQueue: + self._completionEvent.clear() + if queued: + self._recordCompletionQueueSnapshot( + remaining_depth, + "completion_queue_drain", + metadata={ + "drained_count": len(queued), + "completion_limit": completion_limit, + }, + ) + return queued + + def _pollQueuedCompletionTests(self, completion_limit=None): + from ats import configuration + + start_us = time.time_ns() // 1000 + self._incrementCompletionStat("_pollQueuedCompletionTests_called") + queued_count = 0 + selected_count = 0 + stale_count = 0 + completed = 0 + result_kind = "empty" + try: + queued = self._drainCompletionQueue(completion_limit=completion_limit) + queued_count = len(queued) + self._incrementCompletionStat("_pollQueuedCompletionTests_total_queued", queued_count) + if not queued: + self._incrementCompletionStat("_pollQueuedCompletionTests_empty") + return 0 + + selected = [] + selected_ids = set() + running_ids = {id(test) for test in self.running} + for test in queued: + test_id = id(test) + if test_id in selected_ids: + continue + if test_id not in running_ids: + stale_count += 1 + continue + selected.append(test) + selected_ids.add(test_id) + + selected_count = len(selected) + self._incrementCompletionStat("_pollQueuedCompletionTests_total_selected", selected_count) + self._incrementCompletionStat("_pollQueuedCompletionTests_total_stale", stale_count) + if stale_count: + self._incrementCompletionStat("_pollQueuedCompletionTests_saw_stale_entries") + if not selected: + result_kind = "stale_only" + self._incrementCompletionStat("_pollQueuedCompletionTests_selected_none") + return 0 + + completed_ids = set() + for test in selected: + done = self.getStatus(test, allow_running_checks=False) + if not done: + continue + completed_ids.add(id(test)) + completed += 1 + if test.status is not PASSED and configuration.options.oneFailure: + raise AtsError("Test failed in oneFailure mode.") + + self._incrementCompletionStat("_pollQueuedCompletionTests_total_completed", completed) + if completed_ids: + self.running = [ + test for test in self.running if id(test) not in completed_ids + ] + result_kind = "completed" + self._incrementCompletionStat("_pollQueuedCompletionTests_completed") + else: + result_kind = "selected_none_completed" + self._incrementCompletionStat("_pollQueuedCompletionTests_selected_none_completed") + return completed + finally: + self._recordCompletionInternalSpan( + "_pollQueuedCompletionTests", + start_us, + time.time_ns() // 1000, + metadata={ + "mode": getattr(self, "completion_detection_mode", ""), + "completion_limit": completion_limit, + "queued_count": queued_count, + "selected_count": selected_count, + "stale_count": stale_count, + "completed_count": completed, + "result": result_kind, + }, + ) + + def _finishCompletedTest(self, test): + from ats import configuration + + if getattr(test, "ats_returncode_observed_us", None) is None: + test.ats_returncode_observed_us = time.time_ns() // 1000 + test.setEndDateTime() + test.statusCode = test.child.returncode + ignoreReturnCode = test.options.get('ignoreReturnCode', False) + if ignoreReturnCode: + test.statusCode = 0 + if test.statusCode == 0: + status = PASSED + elif "flux" in configuration.MACHINE_TYPE and test.statusCode == 142: + status = TIMEDOUT + else: + lsf_error = False + with open(test.errname, 'r', errors='replace') as f: lines = f.readlines() - f.close + for line in lines: + if lsf_error == False: + if "Terminated while pending" in line: + print("ATS ERROR: Detected LSF Job Start Error %s. Detected LSF launch failure : %s " % (test.name, line)) + lsf_error = True + elif "JSM daemon timed" in line: + print("ATS ERROR: Detected LSF Job Start Error %s. Detected LSF launch failure : %s " % (test.name, line)) + lsf_error = True + elif "Error initializing RM" in line: + print("ATS ERROR: Detected LSF Job Start Error %s. Detected LSF launch failure : %s " % (test.name, line)) + lsf_error = True + elif "Bus error)" in line: + print("ATS ERROR: Halting test %s. Detected Bus Error (perhaps MPI related) : %s " % (test.name, line)) + lsf_error = True + + if not lsf_error: + with open(test.outname, 'r', errors='replace') as f: + lines = f.readlines() for line in lines: if lsf_error == False: - if "Terminated while pending" in line: + if "ATS Error: Locate pipe file" in line: print("ATS ERROR: Detected LSF Job Start Error %s. Detected LSF launch failure : %s " % (test.name, line)) lsf_error = True - elif "JSM daemon timed" in line: - print("ATS ERROR: Detected LSF Job Start Error %s. Detected LSF launch failure : %s " % (test.name, line)) + elif "Could not read jskill" in line: + print("ATS ERROR: Detected LSF Job Scheduler Error %s. : %s " % (test.name, line)) lsf_error = True - #time.sleep(10) # See if sleeiping helps the JSM daemon recover - elif "Error initializing RM" in line: + elif "AST Error: initializing RM" in line: print("ATS ERROR: Detected LSF Job Start Error %s. Detected LSF launch failure : %s " % (test.name, line)) lsf_error = True - #time.sleep(10) # See if sleeiping helps the JSM daemon recover - elif "Bus error)" in line: - print("ATS ERROR: Halting test %s. Detected Bus Error (perhaps MPI related) : %s " % (test.name, line)) - lsf_error = True - - if not lsf_error: - f = open(test.outname, 'r', errors='replace') - lines = f.readlines() - f.close - for line in lines: - if lsf_error == False: - if "ATS Error: Locate pipe file" in line: - print("ATS ERROR: Detected LSF Job Start Error %s. Detected LSF launch failure : %s " % (test.name, line)) - lsf_error = True - elif "Could not read jskill" in line: - print("ATS ERROR: Detected LSF Job Scheduler Error %s. : %s " % (test.name, line)) - lsf_error = True - elif "AST Error: initializing RM" in line: - print("ATS ERROR: Detected LSF Job Start Error %s. Detected LSF launch failure : %s " % (test.name, line)) - lsf_error = True - - - #sys.exit(-1) SAD ambyr - #print "DEBUG getStatus 420 statusCode is %d " % test.statusCode - if lsf_error: - print("ATS LSF Development: LSFE Detected statusCode is %d " % test.statusCode) - test.statusCode = 2 - test.setEndDateTime() - status = LSFERROR - else: - status = FAILED + if lsf_error: + print("ATS LSF Development: LSFE Detected statusCode is %d " % test.statusCode) + test.statusCode = 2 + test.setEndDateTime() + status = LSFERROR + else: + status = FAILED + return self._completeTest(test, status) - # Send test's stdout/stderr to file and to terminal + def _completeTest(self, test, status): if test.stdOutLocGet() == 'both': outhandle, errhandle = test.fileHandleGet() for line in test.child.stdout: print(line) print(line, file=outhandle) + self._closePidfd(test) self.testEnded(test, status) - - #if hasattr(test, 'runningWithinSalloc'): - # if test.runningWithinSalloc == True: - # print "DEBUG Sleeping 1 sec after job end %s" % test.name - # time.sleep(1) - return True + def _detectRunningSlurmError(self, test): + with open(test.errname, 'r', errors='replace') as f: + lines = f.readlines() + for line in lines: + if "Slurmd could not set up environment for batch job" in line: + print("ATS Halting test %s. Detected slurm launch failure : %s " % (test.name, line)) + return True + elif "srun: error: Unable to create job step" in line: + print("ATS Halting test %s. Detected slurm error : %s " % (test.name, line)) + return True + elif "Error opening remote shared memory object in shm_open" in line: + print("ATS Halting test %s. Detected MPI shared memory failure : %s " % (test.name, line)) + return True + elif "PSM could not set up shared memory segment" in line: + print("ATS Halting test %s. Detected MPI shared memory failure : %s " % (test.name, line)) + return True + elif "Attempting to use an MPI routine before initializing MPICH" in line: + print("ATS Halting test %s. Detected MPI Error : %s " % (test.name, line)) + return True + elif "Bus error)" in line: + print("ATS Halting test %s. Detected Bus Error (perhaps MPI related) : %s " % (test.name, line)) + return True + return False + + def _ensurePidfd(self, test): + if getattr(self, "_pidfdUnavailable", False): + self._ensureCompletionWatcher(test) + return None + pidfd = getattr(test, "_pidfd", None) + if pidfd is not None: + return pidfd + if not hasattr(os, "pidfd_open"): + self._pidfdUnavailable = True + self._ensureCompletionWatcher(test) + return None + child = getattr(test, "child", None) + if child is None or getattr(child, "pid", None) is None: + return None + try: + pidfd = os.pidfd_open(child.pid) + except OSError: + self._ensureCompletionWatcher(test) + return None + except AttributeError: + self._pidfdUnavailable = True + self._ensureCompletionWatcher(test) + return None + test._pidfd = pidfd + return pidfd + + def _ensureCompletionWatcher(self, test): + child = getattr(test, "child", None) + if child is None: + return + watcher = getattr(test, "_completionWatcher", None) + if watcher is not None: + return + + def _watch_for_completion(): + try: + child.wait() + except Exception: + return + self._recordCompletionSignal(test) + + watcher = threading.Thread( + target=_watch_for_completion, + name=f"ats-completion-{getattr(child, 'pid', 'unknown')}", + daemon=True, + ) + test._completionWatcher = watcher + watcher.start() + + def _closePidfd(self, test): + pidfd = getattr(test, "_pidfd", None) + if pidfd is None: + return + try: + os.close(pidfd) + except OSError: + pass + test._pidfd = None + def testEnded(self, test, status): """Do book-keeping when a job has exited; call noteEnd for machine-specific part. @@ -336,6 +797,7 @@ def kill(self, test): # override if not using subprocess "Kill the job running test." if test.child: test.child.kill() + self._closePidfd(test) if test.stdOutLocGet() != 'terminal': test.fileHandleClose() @@ -660,6 +1122,7 @@ def _launch(self, test): else: test.child = subprocess.Popen(test.commandList, cwd=test.directory, stdout = subprocess.PIPE, stderr=subprocess.STDOUT, env=E, stdin=testStdin) + self._ensurePidfd(test) test.set(RUNNING, test.commandLine) self.running.append(test) @@ -816,6 +1279,14 @@ def __init__(self, name, npMaxH): self.hardLimit = (npMaxH > 0) self.naptime = 0.2 #number of seconds to sleep between checks on running tests. self.running = [] + self._completionEvent = threading.Event() + self._completionQueue = deque() + self._completionQueueIds = set() + self._completionQueueLock = threading.Lock() + self._completionStats = {} + self._completionStatsLock = threading.Lock() + self._completion_span_hooks = [] + self._completion_queue_snapshot_hooks = [] self.runOrder = 0 from ats import schedulers self.scheduler = schedulers.StandardScheduler() From 30e498ec0a07e2907c3531a112168f8b9ef8979f Mon Sep 17 00:00:00 2001 From: "Peter B. Robinson" Date: Wed, 20 May 2026 14:26:03 -0700 Subject: [PATCH 02/21] add documentation --- ats/atsMachines/fluxScheduled.py | 7 +- ats/machines.py | 259 ++++++++++++++++++++++++++++++- 2 files changed, 257 insertions(+), 9 deletions(-) diff --git a/ats/atsMachines/fluxScheduled.py b/ats/atsMachines/fluxScheduled.py index 6e8a018..4542a78 100755 --- a/ats/atsMachines/fluxScheduled.py +++ b/ats/atsMachines/fluxScheduled.py @@ -176,7 +176,12 @@ def get_physical_node(self, rel_index): return nodes[rel_index] def getNumberOfProcessors(self): - """Return total schedulable processor slots in the Flux allocation.""" + """Return total schedulable processor slots in the Flux allocation. + + Returns: + int: Total processor capacity ATS may schedule inside the current + Flux allocation. + """ return self.maxCores diff --git a/ats/machines.py b/ats/machines.py index 365322b..8c23cbc 100644 --- a/ats/machines.py +++ b/ats/machines.py @@ -84,7 +84,11 @@ def checkForTimeOut(self, test): return 0, fraction def checkRunning(self): - """Find those tests still running. getStatus checks for timeout. + """Update ``self.running`` after checking for finished child processes. + + Returns: + None: ``self.running`` is rewritten in place and completion + callbacks may run for newly finished tests. """ completion_limit = self._completionFastPathDrainLimit() if self._useLegacyCompletionPolling(): @@ -142,10 +146,19 @@ def getStatus(self, test, allow_running_checks=True): Obtains the exit code of the test object process and then sets the status of the test object accordingly. Returns True if test done. + Args: + test: ATS test object whose child status should be checked. + allow_running_checks (bool): When ``False``, skip timeout and + running-error detection for children that have not yet exited. + When a test has completed you must set test.statusCode and call self.testEnded(test, status). You may add a message as a third arg, which will be shown in the test's final report. testEnded will call your bookkeeping method noteEnd. + + Returns: + bool: ``True`` when completion handling ran for ``test``, else + ``False`` while the child remains running. """ from ats import configuration self._pollChild(test) @@ -190,7 +203,21 @@ def _pollRunningTests( stop_after_completion=False, completion_limit=None, ): - """Poll running tests, optionally prioritizing likely completions.""" + """Poll running tests, optionally prioritizing likely completions. + + Args: + allow_running_checks (bool): When ``False``, skip timeout and + runtime error checks for children that have not yet exited. + prioritized (iterable|None): Optional running-test candidates to + check before the rest of ``self.running``. + stop_after_completion (bool): If ``True``, stop after the first + completed test is handled. + completion_limit (int|None): Maximum number of completions to + process before returning control to the scheduler. + + Returns: + int: Number of completed tests processed in this polling pass. + """ from ats import configuration start_us = time.time_ns() // 1000 @@ -271,7 +298,16 @@ def _pollRunningTests( ) def _preserve_new_running_tests(self, remaining, seen_ids): - """Keep tests appended to ``self.running`` during completion callbacks.""" + """Keep tests appended to ``self.running`` during completion callbacks. + + Args: + remaining (list): Running tests that should remain after the current + polling pass. + seen_ids (set): Object ids already considered in the polling pass. + + Returns: + None: ``remaining`` is updated in place. + """ remaining_ids = {id(test) for test in remaining} for test in self.running: test_id = id(test) @@ -281,7 +317,13 @@ def _preserve_new_running_tests(self, remaining, seen_ids): remaining_ids.add(test_id) def _waitForCompletionSignal(self): - """Wait for a local child exit, using pidfds when available.""" + """Wait for a local child exit, using pidfds when available. + + Returns: + list: Running tests that were signaled as likely completed during + this wait interval. Queue-wait and sleep-fallback paths return an + empty list. + """ start_us = time.time_ns() // 1000 self._incrementCompletionStat("_waitForCompletionSignal_called") registered = False @@ -353,30 +395,77 @@ def _waitForCompletionSignal(self): ) def _useLegacyCompletionPolling(self): + """Return whether legacy double-poll completion detection is enabled. + + Returns: + bool: ``True`` when ``completion_detection_mode`` is + ``"legacy_poll"``. + """ mode = str(getattr(self, "completion_detection_mode", "") or "").strip().lower() return mode == "legacy_poll" def _useQueuedCompletionDetection(self): + """Return whether queued completion detection is enabled. + + Returns: + bool: ``True`` when ``completion_detection_mode`` is + ``"completion_queue"``. + """ mode = str(getattr(self, "completion_detection_mode", "") or "").strip().lower() return mode == "completion_queue" def _completionStatsEnabled(self): + """Return whether aggregated completion counters should be tracked. + + Returns: + bool: ``True`` when completion statistics are enabled. + """ return bool(getattr(self, "completion_detection_stats", False)) def _completionSpansEnabled(self): + """Return whether internal completion spans should be emitted. + + Returns: + bool: ``True`` when completion span hooks are enabled. + """ return bool(getattr(self, "completion_detection_spans", False)) def _incrementCompletionStat(self, name, amount=1): + """Increment one aggregated completion counter. + + Args: + name (str): Counter key to increment. + amount (int): Value added to the counter. + + Returns: + None: The in-memory stats dictionary is updated when enabled. + """ if not self._completionStatsEnabled(): return with self._completionStatsLock: self._completionStats[name] = self._completionStats.get(name, 0) + amount def _completionStatsSnapshot(self): + """Return a copy of the current completion statistics. + + Returns: + dict: Snapshot of aggregated completion counters. + """ with self._completionStatsLock: return dict(self._completionStats) def _addMachineHook(self, hook_attr, callback, description): + """Register a machine hook callback on one hook list. + + Args: + hook_attr (str): Attribute name holding the callback list. + callback (callable): Hook function to register. + description (str): Human-readable hook name used in validation + errors. + + Returns: + callable: The registered callback. + """ if not callable(callback): raise AtsError("%s hook must be callable" % description) hooks = getattr(self, hook_attr, None) @@ -387,6 +476,15 @@ def _addMachineHook(self, hook_attr, callback, description): return callback def _removeMachineHook(self, hook_attr, callback): + """Remove a callback from one machine hook list if present. + + Args: + hook_attr (str): Attribute name holding the callback list. + callback (callable): Hook function to remove. + + Returns: + None: Missing callbacks are ignored. + """ hooks = getattr(self, hook_attr, None) if hooks is None: return @@ -396,7 +494,15 @@ def _removeMachineHook(self, hook_attr, callback): pass def add_completion_span_hook(self, callback): - """Register a callback for internal completion-detection timing spans.""" + """Register a callback for internal completion-detection timing spans. + + Args: + callback (callable): Function called as + ``callback(name, start_us, end_us, metadata)``. + + Returns: + callable: The registered callback. + """ return self._addMachineHook( "_completion_span_hooks", callback, @@ -404,11 +510,26 @@ def add_completion_span_hook(self, callback): ) def remove_completion_span_hook(self, callback): - """Unregister a completion span callback.""" + """Unregister a completion span callback. + + Args: + callback (callable): Previously registered completion span hook. + + Returns: + None: Missing callbacks are ignored. + """ self._removeMachineHook("_completion_span_hooks", callback) def add_completion_queue_snapshot_hook(self, callback): - """Register a callback for completion queue depth snapshots.""" + """Register a callback for completion queue depth snapshots. + + Args: + callback (callable): Function called as + ``callback(timestamp_us, metadata)``. + + Returns: + callable: The registered callback. + """ return self._addMachineHook( "_completion_queue_snapshot_hooks", callback, @@ -416,16 +537,46 @@ def add_completion_queue_snapshot_hook(self, callback): ) def remove_completion_queue_snapshot_hook(self, callback): - """Unregister a completion queue snapshot callback.""" + """Unregister a completion queue snapshot callback. + + Args: + callback (callable): Previously registered queue snapshot hook. + + Returns: + None: Missing callbacks are ignored. + """ self._removeMachineHook("_completion_queue_snapshot_hooks", callback) def _recordCompletionInternalSpan(self, name, start_us, end_us, metadata=None): + """Emit one internal completion-detection timing span. + + Args: + name (str): Span name. + start_us (int): Inclusive start timestamp in microseconds. + end_us (int): End timestamp in microseconds. + metadata (dict|None): Optional structured span metadata. + + Returns: + None: Registered hooks are called when enabled. + """ if not self._completionSpansEnabled(): return for callback in list(getattr(self, "_completion_span_hooks", [])): callback(name, start_us, end_us, metadata or {}) def _recordCompletionQueueSnapshot(self, depth, reason, timestamp_us=None, metadata=None): + """Emit one completion-queue depth snapshot and update queue stats. + + Args: + depth (int): Queue depth after the observed event. + reason (str): Short reason label for the snapshot. + timestamp_us (int|None): Event timestamp in microseconds. Uses the + current time when omitted. + metadata (dict|None): Optional extra snapshot metadata. + + Returns: + None: Registered hooks are called when present. + """ if timestamp_us is None: timestamp_us = time.time_ns() // 1000 depth = max(0, int(depth)) @@ -445,6 +596,11 @@ def _recordCompletionQueueSnapshot(self, depth, reason, timestamp_us=None, metad callback(timestamp_us, payload) def _completionFastPathDrainLimit(self): + """Return the configured maximum completions drained per wakeup. + + Returns: + int: Positive completion drain limit. + """ limit = getattr(self, "completion_fast_path_drain_limit", 128) try: limit = int(limit) @@ -453,10 +609,28 @@ def _completionFastPathDrainLimit(self): return max(1, limit) def _pollChild(self, test): + """Poll one child process and return its current return code. + + Args: + test: ATS test object whose ``child`` process should be polled. + + Returns: + int|None: Child return code, or ``None`` while still running. + """ test.child.poll() return test.child.returncode def _recordCompletionSignal(self, test, observed_us=None): + """Record a likely completion signal for one running test. + + Args: + test: ATS test object associated with the completion signal. + observed_us (int|None): Signal timestamp in microseconds. Uses the + current time when omitted. + + Returns: + None: Internal timestamps, queue state, and statistics are updated. + """ if observed_us is None: observed_us = time.time_ns() // 1000 if getattr(test, "ats_completion_signal_us", None) is None: @@ -481,6 +655,15 @@ def _recordCompletionSignal(self, test, observed_us=None): ) def _drainCompletionQueue(self, completion_limit=None): + """Remove queued completion candidates up to the configured limit. + + Args: + completion_limit (int|None): Maximum number of queued tests to + return. ``None`` drains the entire queue. + + Returns: + list: Queued tests selected for completion re-checking. + """ queued = [] with self._completionQueueLock: while self._completionQueue: @@ -504,6 +687,15 @@ def _drainCompletionQueue(self, completion_limit=None): return queued def _pollQueuedCompletionTests(self, completion_limit=None): + """Handle completion candidates from the queued completion path. + + Args: + completion_limit (int|None): Maximum number of queued candidates to + process in this pass. + + Returns: + int: Number of running tests confirmed completed in this pass. + """ from ats import configuration start_us = time.time_ns() // 1000 @@ -582,6 +774,14 @@ def _pollQueuedCompletionTests(self, completion_limit=None): ) def _finishCompletedTest(self, test): + """Finalize status selection for a child that has already exited. + + Args: + test: ATS test object whose child return code is available. + + Returns: + bool: ``True`` after the completion has been fully handled. + """ from ats import configuration if getattr(test, "ats_returncode_observed_us", None) is None: @@ -640,6 +840,15 @@ def _finishCompletedTest(self, test): return self._completeTest(test, status) def _completeTest(self, test, status): + """Run completion bookkeeping for a finished test. + + Args: + test: ATS test object that has finished. + status: ATS status object chosen for the finished test. + + Returns: + bool: Always ``True`` after completion handling runs. + """ if test.stdOutLocGet() == 'both': outhandle, errhandle = test.fileHandleGet() for line in test.child.stdout: @@ -651,6 +860,14 @@ def _completeTest(self, test, status): return True def _detectRunningSlurmError(self, test): + """Check a still-running test for known SLURM launch/runtime failures. + + Args: + test: ATS test object whose stderr should be inspected. + + Returns: + bool: ``True`` when a known SLURM-related fatal error was found. + """ with open(test.errname, 'r', errors='replace') as f: lines = f.readlines() for line in lines: @@ -675,6 +892,15 @@ def _detectRunningSlurmError(self, test): return False def _ensurePidfd(self, test): + """Return or create a pidfd for one running child when supported. + + Args: + test: ATS test object whose child process should be observed. + + Returns: + int|None: Open pidfd file descriptor, or ``None`` when pidfds are + unavailable and ATS must use the watcher fallback. + """ if getattr(self, "_pidfdUnavailable", False): self._ensureCompletionWatcher(test) return None @@ -701,6 +927,15 @@ def _ensurePidfd(self, test): return pidfd def _ensureCompletionWatcher(self, test): + """Start the watcher-thread fallback for completion signaling. + + Args: + test: ATS test object whose child should be watched with + ``child.wait()``. + + Returns: + None: A daemon watcher thread is created at most once per test. + """ child = getattr(test, "child", None) if child is None: return @@ -724,6 +959,14 @@ def _watch_for_completion(): watcher.start() def _closePidfd(self, test): + """Close a pidfd associated with one test if it exists. + + Args: + test: ATS test object that may own ``_pidfd``. + + Returns: + None: Missing or already-closed pidfds are ignored. + """ pidfd = getattr(test, "_pidfd", None) if pidfd is None: return From 621f4a52fe3e834b510c53dd0727c1fb4eacbd08 Mon Sep 17 00:00:00 2001 From: "Peter B. Robinson" Date: Wed, 20 May 2026 16:44:17 -0700 Subject: [PATCH 03/21] refactor completion detection to be a bit more object oriented. --- ats/.management.py.swp | Bin 0 -> 90112 bytes ats/completion_detector.py | 126 +++++++++++++++++++ ats/completion_fast_path.py | 32 +++++ ats/completion_legacy_poll.py | 40 +++++++ ats/completion_queue.py | 44 +++++++ ats/machines.py | 140 ++++++++++------------ docs/source/scheduler_extensions.rst | 58 +++++++++ test/test_completion_detector_examples.py | 56 +++++++++ test/test_streaming_core_helpers.py | 117 ++++++++++++++++++ 9 files changed, 536 insertions(+), 77 deletions(-) create mode 100644 ats/.management.py.swp create mode 100644 ats/completion_detector.py create mode 100644 ats/completion_fast_path.py create mode 100644 ats/completion_legacy_poll.py create mode 100644 ats/completion_queue.py create mode 100644 test/test_completion_detector_examples.py create mode 100644 test/test_streaming_core_helpers.py diff --git a/ats/.management.py.swp b/ats/.management.py.swp new file mode 100644 index 0000000000000000000000000000000000000000..36a440a848239e5a060d0a807eb685ba3a65d76d GIT binary patch literal 90112 zcmeI537lM6UGGbFkY(5xfw^=OyE2pXBr^jHnaPZqq%#Dvk?vs_GMP?wcXg*G)m3d( zcamn<6+ux^L0LpLQQ1U6HWxO1fC8c*iwJ@ug7_3gcx+nZpL71(IcH|eWA^OJE*sxc<8xlEcIuz*bkn6@eCsV<|K$hOmU`WpR;S<9 ztF=D8$DcgbT3Tu@KjgBtg4Hfs>Gv+WZgT5IA>oS_TFVzT+RaPmuIu)W_m>*8&HhEr zQ%l|6@^ov_z80Ikh2}-e-EO;o(T=IfiyF)QixwN5#zJ$k*;yW6T3s{8b1%=ez_}K% z7Fb!H-+bAfYL{HR)rR`$$b+)G-|gCSjdQLA&b7d~7C6@e=UU)g3!H0#b1iVL1?0AEIU{s?#{cosMb7Qth| zbznRACIa|3z+Zt6g13X;2PeS}@Kpr(_kw4G$APPDXC;x4};X$*5O?dxC#L-u(jjbMOK1TCfPN z1rGz?MV|iO;Ge-?f_H#dfS&?afsNoi@I3_KPl7)Lt6&mr2j_!3fG?m|cpLa5@N3|) z;7V{0@ILy;`pIY^^Pl-&TMmhY-65MYd-6AdDRau>k7I_=}z%uSXwh%pS{J>H67xx919Tvej-= zHcMo#Iqw4yx885I=Qmj(a?dj%_pIGrsE_TKn`_Sb;M>MFWxeK7uRgyqR!Je1#<o@Z}6(MT5 z(+cGmEzLvKfw)T4(rYfS^g4yEG6LFd10`#>7!u9JrRCLesHi!6th?Q)hQiNu`^{p3 zQ+lFghmYjRhIX<<`?0k~pwmz=wSz{6vK-HLwwsNf%ibdAUskj1Ahc>C+t>%iF`!>vnu2@YgwU*Zipvh`i zbDk{b)(F+@9PG8az1DK;@#}=!*=@I*vrKSnq&Cx-GukPTfmKPR8S={aU`0F3x-6$! zi_M9SrZxy)vzK*ibFSF1{vl!Jn1=alajD+!b{3{JjWm|GPxV$3${jh>Y;~x0qfN!L znMS`gt9T8Ktd&_tM>8ck{xV8Z?kS~NX~Ii4EJUHx(@J}K+Gy0T6IvClz!zi-z3$3V zKij-Hn`_OATpGRAtUK=}e|wpAXKtdc`;6eZ<|!&Zvzm2Q7H670pF{--E4k);1ClP& z@&%cKd$KdK90(hz`GTyHj0ryyrowRs%1FlWJPFk-H{C39K{iO&M6w>IW>nbGy5^-k zq{jYiquus(DpB^8vCa`;*5= zT(zWwY#b`iNphX*cHHVjFN;PY7J_RCWXp^j(N#|9HaGc_g`rxfF-W7mx>4~$hbgB% z6LUlPt+eNi6M|jpcUO9|WPuCoQnS}y^@%CCMO`a3;^5%ghePtQ!oW}R4UeDJ1tLO|2mS14(;S?*%5{v6>YpKzuX;~ZL_IPXZcz8mV z%($CNs2GXaZx>;t(StZnvhZ3jFF`dT6KdPEAH8PVTEJdkHB!@*MwF|2T~_K|zrMT` z9`xdhYVHrTSXs+9OVQznY;*hp1=QbeD(|0cEg!3ojN;+EZ6wT>lQ86SFpaeLf+pwY zGBcvs8qn&%Qo|CcWKg|@gSqT=`%463Xw;RsN;u!^E@sSFTmgZW$_=fuT z8P)zJ-MODcJJ9cHF0QS*QRXep@|(+`Z2X<-^8v1*G_H9 z9C4?gExccgA-s5 zOn^&427U|r0xG%UR_#(Q2zW{#-UIu;wbio9;5ZoSo1%1IM z!8^dKz_Y1pw{)6*j`J>oqkqN;IxWl1bXJuLoCP>m|nk%i{6mkvj^W$}+qHOCob*+{oDi3rJC z_oMJe;+VbBkHVWogt7q1--U3wjAWtmzmhQfcI;LZQ4Sy9B*8->E`o&M$aUm7=w%XT^7=`iK`DHAk%TR9W7M+ncM=Zn)A%=ngrCV< z&*t$mFnmc5_0@R&Bs5GBkHi|qP;~BrB_knT#Xph|v&WjV$FFIxsI&9m$@>)qB75n6 zByaq4eI-wkU`+HNx$rZ28(!IW@_YqF?(A{mxF5+I|6E_m(-rjhEg!*tCGWzU^7E4o zqg#C?p`8SZV4=o~e+-1k-zIO0gOm`(2T4epp4a@FJX=B5le&%<;b-zTf9HNAZ^*=D zl{|BDI*;9t`Bn6mWE3v4!3 z!KGKL4DM}3>i=y;TaZ<Dpnm7M2{A zwC&aF4R1Ep;&N3A6<=)1)6xo63Kp_)i&t2hwhvvhn3Fs&KtY45D?Y3_1|WIo9`M}M zvN4W(!1H9)FGNe86DnWz3t{r7H3()`B#k6c@kNcICNKG`7ADi=ad>3%{~M5*uSUL> z{2%=D--C?*ZtzAh3myfo0uKP6LbiVmcrJJjm;v_#_XR%;{v4V9<=|!D7r>K14>Z9R za8Gb6a7*wp@_jN;{D`RF7PVw z6mSi=JNP)dlUIOa;342H;GNj{UkIKC_Je)k0pRo4_df{U1YQMR2wnhw0%*U$<=~Fs zGuZk+4c-A>1fBvO2d)SA1n^DMivWRno(1F*G+n|pqXd8Q`5#dl8MK~%o9S>Xqxcicx+=?Z2yw?X9|}7r1}C zwIpry4SDdn?hf1)u)0aZSl?KDIkFc;X+uVP4z-{Ij&)C}shbV_9#p5qtn0!AlUbg8 z$yA+8@S5zE{IYcTy_Jreur&+Qqu2I1BO@fH>W^mo8mkoK1hY6iMkbab>u6nB(5$W# zHcY?HL_fbGvT@0cW@aMr1}x_gvbfZ?<2Pg^z!0VErF)FTNd7ouOigAJ`*&prc4w$o zv&pF)htQW|Pftylk=X2MUhw!lPiyaiYowZ;AO)U{OZ1DOR6~UvQLZ}=?4R1R|Jof> zdk*ZkDA2nlPyfrCp zz!fGCJqLaw+gelY^%YD5H?+{>tJa_UnsjmW9|5_P!960$T(helR%k>*BT`;K^(E?b~tadOUu3fX~kw zZ9j~BMGHl03A40jut>uys6I)X7Hgqmjcq>D&sda_l)Ba2PjJQL|LgNs4%VuNCMK`l zOY5F(t12Pur0r+=xuFw%Wx~{=*KIXmr4AP zEk6S!ZoM+#YZK&5Kjp{nf}dcaTOtc*$ksv@)&-Y}R-uj_t8%ITNS=w60h<*Iwl`3! zRrRsL6kQtAcW z<&C@IgVHd*hvOsHg{9W5(q}vd$JS9U?(kpc^es_y_)zkx#*HlA&avg#GINQEI zdbG0ka_UgB;S;E_E(fCq4woCZWg5S-2dAy83ae$iwGeYhZy}{Q%6Z7@1HDxDO72gz zdC;H86-s^K8fCEChL1eI0{fcefhuY8&}bCXIhH=<<)jFoK=j_8Qb8hn<-`Mg)`OTswakSA%hVZ!@a3F+r=fK+uJvqegS&xmAnU&YG=XgV zw*+5Awtoe92Dky-349v4{_WtU;3eQE!2&n{9tQ3VK8jrbGSCDUgRdde{|WdtpgjO} za31&+^86>kYrry)|Nl<#Fi;0$Kr;X5k?sEsybvscHrNL)0CxsqKftfkZU=z1pKrT! zU*}rjTnqev~i|9FvZ_FZQMC+oVS^r)5e{pj#JC* zIc?lIZCq{>|8Jm;%N1RwKbUQFYJr*5cS&`X<=bsCZLwyT4tg9}4-}6552(zf_tDOl z@HFv+^I9JH|6`i2{V&;)k^ftsZT~Ft{rkYH!85=LSOy2d?ZEqx@!t+!1fB+F!Db-& z|0~G$p9OCLeegK23w#^-{!`#ZKsNvD!A>CA|5o5HkoDgTB>(?9=z)zuw*D`HcYt33 zj|NwO9|f}ae+uX)5c~mn88`wS z4K{-s_#U$UTftkvOTddk4~&3MA@~0>=z>`wJ-~gyy}{qns80k>01Z$FW8mM>wY(X; z2rPmh0~dpPgWH2oV&9kD|Jh&`>;;#D2ZGyyFVIgP2G(D`uL6w}J&(fEf<{vhz0=g5 z1E$ifaF|k{88(O-|D6-S!_6f~P5hST5G!k^*WChZ+W+cqY%ursTK{PZ23QvSn?jpYl+w2Kp~rhYB0 zj>SFOgN;~-8v|0j#J5d>FvY2T`s3P)G~Q?Hp4_(TI8-(DI<{DD7v3a)DSKItSWjI1 z>%UF%9ZCNy%;Sz>%MT597#FG#1XU>3%bo!cSf9`b3|gkJgT_D*3{@Egdwy-nGP z?rdXbrHyG*+sM+!GzDx(V4bLWMD>uu%Ev(8sfsyGeDJrXv(%Xzm!|qyEGzEd^doA{ z!A)?9**J!$9NUx1jaMaCtgjKQYcJ91ST;6(Q@7QzzUXfpW`G@Vmn`UrV|nC`A?Fb4 zVS80mn+|Roc#LTKsWyY+m(j7!ec5v3xHQCAi*oP2Os+X@#OYO|7iu$r$(oN-1@wuDSQubxnHsdsQ}E|IAQ9#$^R zC8Z&$;?FhP<>DMZ73V7jq2Q;?$!A!pHA*c622;PESn+}dNCcO6{p#Y3e1;UowxinY zlNx=4lVSY=rIr@C)M{aW24 z-M8eLE$W?8g6E1&)q2K+FXx|FFrOIpD1`No2ZE=AGsIw3@|I8)EvqY^yp%B~qV6@H zyJj}K(&J2})H2RKb&iEl>tC+1*3o!zUej{?%dqjq0Zk@G*)~V^{n)<1B8z%`JiFt^XOoKLFyZ#-6QuwIHgtZsUz&*&1TCM;)TlRI{0 zyC$x_7MH{IGOAg`F&E-6p(=R#%rNKXHDUZ@_tT zB7)6mZd9sTq_m6hgrOeZ`yVU3FD*|n&RbexGX(23K1jIMEaP}`m6e&bg9x3u>6MO! zZ_fGAty)zK==pfM7H#eKDwd|=Sl42Z@i0&1t)*IoO)c}u=+2uz*^05v?Ee=dN50w1 z|Bv%B;Dulj>;RJU z-v(X^ejdoL|4OhC$lm{IFbBrLoxw+t-~S4{5&R5T0z1LI!CxZ3KNI{Y_$V^_N5EUa z^MQQ&XMz0sF9JUd?f~T5|6AY$SOEvXdEjr6**^td2abYi@Mv%wpf-P#+Wm|_>f`it z^r*-sctha|lx6ZS2~H(Jh-jSXGFL8lSDb@MpY4rvEA6H`Qz@iZkn=z6 zuj64D+QJ=sG@BeeBx-3VHM~$F1XM0lP}I2ghT^F` zX{R2QI7jV3tzU3~j@Hu8)SFvLxQ(l&IJO1VN9r%H%pQ;Pnv3HC6rYB|;Qe$BJ+f3v zQxRbaB2!<}lk&%yhST?AhGi53SUNNG1LWEAP6Esra*-wrj+~srqx82K2Kbf-mOc5<%-;lgmc2-pt5GyjB}?b*R$l zK>^3~$FNEau#(Dx79NxYL*CZ>o3G4lGC_v+9bi;v4kE>ts^+4VL*-WIG#5{8>r%9p zC=Luxqs3g3c$Ay(q&iayr2j4>#rIQv?7#6yhzFR}HI9me6(}VivS&)=Apwz(j#=|` zRzbrdq{g~JYK&udkXfK8m?_{|6OC%Yh>Aox|CxevF zMGjqTMyTXuWM5A@FWLO>XVp~TjXT!-%OD6wy>+aQ4??0kK-Gld#6q$$E`d7|K5l>W zZm&#!1%9K(+0Y|opAR$dryij3=m9^QC5zeUnoe(uFMZAq_KPDfF&Mq z)DCLZAsB3QV%@{M;SL;?G_K6G%Teo8O+BIyIab>S17W2ciZtg8RtoWhGSVex(9}wo zW~x#Sw^C>ghJIyNvHGz$4daBYA+fiW!s#$y8r`6$ot;`?*~r;SAz4w1HreD^L5tW`+@xap8zff_XYO> zvi)nX|IYx){hvj)*SQ08;8O4($n__H_V9lYdHxib1N*_Zkl|keejPjqJPB+7w*#`_ ze-U~9ufgww7lNM!yTK^9HMkY{H{^Pqzo&Ek=fNb{0B#RHkDUJ@@G|gpFa!312Y?#* z81ntwz#oAYH~_8ylKa1ejQ@||z2H6ImEcK0Hht~k|0S>+e1rD=Tkt8Mwtg@8L4HId z`Y@TbI(F`5BLBj)L-Q#K)hsaoKy+?vl?>yIIonnpPvU32a_8ETu1zzVBsoA+<1ieP zxiQ+9~V z#lNs=Iq!Af`tgkA=db!&_&PXSPf~}FRAn-HX-S1cU$?12 zd{WI*I&v57N1;K&)7T_Rq}VSsf^&@5tK_qG8Ny+)X&Ku6`KE6AD3&2S{lS;P9fDhI z(qj!~l2R+UXFvG-bsR5eE>M6CbK1^LSJXvpI)!2m)gFQ?gUqwa`NLxCgvZG|sXkup zFSrG=a$uZB2a*6$M|Rt3zXK(7W`{!g{G+(u4@$n}G`AGClUvn&Dmn^rOTxMOm@)i7 z4K(_TLrJ7Klm_*ZQ;T;#O1`3m046q zHN6cV3*}F0&Mwv%Q2)mQ_zmcVLWE~LRJDfa^hSNEKd3ZEt19G+5=TUsC%%sfF-%Iy z+7w3q_mOItW|CChX#PIZbkjzXqHhi*nR`_o9vdCI5TT>zm*8fPJ1X|u-47c#@vJ7?v zB?HQ77n0m5OX!LN=DQGSkkebY5yL6yNh!P;b!Z!`o2Hv5lN5iTeAQlDH9Qj~ z305X6Bum|TlkimGL>85GfTL2UF_MQX9W~4kFOPGAeom*6AvAiG5#?GA9%|#-V-rw? zrUNR)&!er7)rTO2-9~|{Qz7stBA5N0Dthh%&o9iO?rG%{NqqaD( zKIs3tnU1tx@OYy*2Ch2ydFEfL72{)&D;r#r6~_m+6*N!7|45oa!6r>?LeGLG&|a0~ z0yg>2<)b_<4@yj=VMmOGsIbs@D1^VPsnd3a1=y;$apBIsDJ{IZNC~B)@ezdwX99zr#o_P_BBmrVcbE)RmXa`}^5_Qq$rm z$SK~>Vutnl$PFFZL8dwvT6#^Ua*JcGT*T3wUQ7hyBbI%9GW_1ZbJ9c?Q)+k_UVL37 zg;9~3Bo7xg!z74>p+3T&Yv>Y+@f~kil|(LPP=TOB$R(6KOO;_jScv5aJb$VL2%%hpS_0Ge`2B*B-y79A#`2$UBaaHeUvmzp-D;Id_|gPU(x|>K$v-kmvpGfCd_Y138Q^#SX&NF zZ7l=U87CH)?5IofbB!ABY$i7Io6My136ZU2{9sV*@u)6_r(ZP+Ifo#S+_Hzv8S zwZ26`pP6o8d2^{3MDvRD18Gz)eT57H{l!J=^m15hRV-vMuqd8h`ahWUZj<@{-y3H9 zHD3O|k>C2f4Ow6N{%-)=!4*Jz0PYGT|GyaA5quFj|DV8b0?GZl&+lK6_x}al3?2^- z0r~mg8GJ>ye(fS&+2f*s(#;CA3M=mkCv-V5FXUJVw&1A)%=e?NFR z_<8U+a1Hn#x_}RXSAivP6}T<bKEnv~=t3*S9TusD- z;#PXnjQYkMVENbvp9k z!;Q_!+6}xk-6g@c)29c+$uZ^VKg?n1 zX6ta3=85CUB|VBb8;?)W{-G({Lu7)MgYUxdiroUaKVO*Gd9`Jd~lds)ou~Qk+?Wm z{WS(6QLAZ;40h^bF)1!=|5^0P;*O+K(o7|C0R7I-OB-KvU_pl$8IQvR-YeSQv?ncpzfbm=LTc0_cs);aL@&7@gSveB&vXO}^<-`<4~5i5g%nD)Xft z4`wN6@v|EEduAfBX<7*3gL!LYbi<`{BWN;R?F)8AXZ(_;+w5PlsRHHbJ4Rft2R@fA z9=5C16zCib-=ANF^)$BAUlHXJqQ zFczx$>x-+?9nmpFF`nC1Ru=awUZN)o*@PC+^$dBd*TP$U(=wK3c4c*<#nw{AwGn+J zZbWynx2vd?rP{t>j(aB1X#28P2HH5-fFa$@(FI-Qt!5-b;85R!C8*OWt?;5C9%{Uo zA|#_pnT=)_Z`u0r&08M2`Qpp6NtjkOJGrSWlEmc7pY%-RATn z^=shZSSkl=es-keDYDxWsg$TD)UvZQNIJ}#Qadc`?U#ph$HlQ{L*)&*U?Qup@Oiso_7e%aVQezEzfWu$|ko^BH zpu7H-fzJBd0CcwhN5Lz>3Gid!DsUe705bm*fc*Zyij4mi@HgOD;IZIxa31(~Wc&|+ z_k%Zp8^IB97x3rE_#XgI2A6?{fl(lt|D{0p_+15RKxg=Aum6X^hrsi|vp^Hv6?_Ak z{_Eg(!0W)Xf#m%?U;}tC_yF?#NpKuo4Rn^@t-(i;>t79?4VFP4YytNHcK}~Tw*PnV zG4Mw4EU*J?0rvs_M!UZosQq88J}7=1DY$=6e?zSJioza9mfWJauN2Q{pKi}-@@lRq z?%#nepeE)Zwfm3VH)59Be*RGFQ*S(6++FFAsVQ4WO1V`ozd0!EjgRZ3yN_rfK*HI@ z(@`rGk$(Owy88^Sdp=mf&4-zRt%bejv~wpji3f9dxQ{4(?417nlT=u-1sX1>(e_u6 z#d*M9Y1h;t=dG4(perPo@KTNTA$QADOr@ego|#VGBuS?g@&WZI3E_*3a;3V+I=-v! zIA^Wk93$8~)mX&!tQSj_%BS$@TB}cHTtCajMO?To2}$fb(Sv(}HLg#GHOt)h#05Cs zLmayftcyZspp?w6%4znF9<>yX9t~wVdX(c){TXy% zUyssj)M2gV^zNNK^9q9J2)QBx&$JsIZYI($eOx3?9NxO^h#uQ*yX=dX5S+VCjK`cb zz&3L~4L3^i-EN*}wzI95#rJ6z4LQqdm}yU9gYwu(j`S5S8je-- z!!{}Ol*%kcD2Mpi?-*FscDwW*li&7KA9Dy&u;Q=rq_YnYDO6ZAON#c~=m&mf9Ae2HQ!LDG}bhKTT8;u`zt zC_*bI^;e^tvLYPhYOYq&72&h&9nUm0L{LV=l}nol(YQtJg)xHDKXDSyiOptom8q!d z542WIAF6#V1PR}X|8jIa^>#kZO%-(JH6oGDl$LQ3frp6S*&b7)htq;`jv|Jr9$F+f z&b8-rnie&E#9oO7cd99O+>Lo^6EY~Q(x(?fm62#ee9QoHvZjEX_u>5EATSYvJ4toK zUrdJAu}QL!ry9E!;_F-ek>{vMpzdT>TauRXKKDgLcZT#A*!np;pvOv_C=&C&t)K3A z414P0d^4KuU;%cTN+Z?$#$rxGlS8{|P+G9k`iP>W9qJ#x$$UCZW5ornjHiU1NMq=? z-+p@G_N zH>s0)jSiPAw=QH)wQWBOvC~;ML~@yz^LA&BrTTqgrTZT;DF%MlK9W?(oJHnUroY}zcgv9xkHEtI2B-O!jrjeN1 zv_^Kfbk?lRt$tdh61}F>wsx*WYX3{DBWh9lN&~EVTFI@3#pct*G8TqvDOWSxz*-Dh zn=R@EXZWd3j*_c$jA*@q)piBwVZoG|iVAAF%42U-FmtQ$|?s!{1(Ac9B=QLj8*~h>vE0VNub5Flj27%R)T6RyO0* zmliC$c#xn}w^4=&Aa2!(0W)?Pb~L2>Gj3}kDK8cz5(I^WWp3#lE!cH;7x_}R-I`@$ zuov zA7`&N>bSFnolEWW4E^c}tH`q_jaw`O_OK;?BFcPA4i56T!wh>`dEn2#+XauaqNWwk zq1k-o`LSXf6pK99owY*`ccu(ovZN2KqYkJOqn8`D2OFBXSuJjNxUXnaSE*vdXMAE^ zs|XcMH?lwQ+4GIr7I!kN(&zYr43WeJ)jy(vk*keU&r{F>r_)sYERL+?Nq?NXcAIS- zqn1pv>P3BMn}G;Pi7g{f^fZ&(8G=~0EekoFP9{oa%ml3Isp}6;um}}9Fusb?fW`5b zl2yk_UzXybkG44SF)C&i-jpI}w%L2&nrzpeLlZlv4jj5(F)6}$JVB~v?ZrPK@aeNXBEVi~Gwzj$% zRQecLiBMW5L`jLb=G1#*H`Wi|c*Dq%@e4NIF!JC;W1}Keh8?r8V`*5GJnf-IZ1{D` z=ejhO&{BcKumK+Aj3)|!FBN1wdy=CUYoUcK6cvVobSz<85&d*ULgGO3KtqYu ztDTZ5u_~=%q0)@<94uPQy%@tkF9tSGBsH@()fdCZSH&oS3oD2B&-1|L)YjLKFE?;CH_c=rf;oK^v+ATzm0?KFwE&Goy;S9|YGsvT z>*G@{TBh79pL{qM*YTRtwZpVSGny;WPUGgp?pS<%(N|)Bp|#f3J^#agi~J3|6g~1w0f>j1d{*ngY5Yf$)L#plU@#eAF}?_yv+Z6DQ8}Nt7_D=%3$FB(<3Ov4`K3Q!$p-qS}pJl|+ znQpheaa(rXF*srjsoI5(qb9mJ3#F4hQGt_~HgzM(c9eWA#R1WG+aZoJVT&uau{vRlrSb1+pJMP75GgvyYTAsje>KiDWyBokv zv?U{CF>6%@HJaSN#PDZmYfBMpjOvGz`wBj)<|`(ZHg@Gg)`=uP zs zoK%={G?czGpCI zNeXrZWL#ou7|pGzS?QB_MlRl(s1R1L3^)M_5~yw^f%Vw_GS6eGtvuOE?bJ0gSBvOH0LlmdKnOwSdaI|jSh7so@d%0n*;0y@sQ96=ahvZ#k=g=jQtCTd-7*wlh27H=^^{RE=q^!UyN3O@r=C}`q!ff6w z8)+gU4;6Y8uGk<|SrT2^uYtCbMolkbg=1RfvEt`l(;|jBK2Vvd5}Axht~A{jr&pC$ zC%oeD@8ul3GSae#Dx8_s&KSg%`f^-hjD^J)#)wy#UnxvIaifLY*hNOMbOs9Ia}22NaiHEBRk#NpJ#jYsCP?9-FXIQ z?Y&ge7siv`F)k%paJNFaLI%l`PSb)*U7XWQGMj{YT&*E)RPLwjSnQOotRe|j(mx-R z6V7meDlMaE8=sOP`IhLSv(qW=`Av0MkeWlL2S3&)!Yy%;9@i}}$J5Mo z8@;*d;^v8>>!MFT%$v3bi!mwYqrB$b6kXHr^kWs>iOyXkx1wT*X7gniMH_ft@ll+Q zw3Nx?$&JkNO-_Yj?+ylurccn%c_CojI(*N!Es`4U{p40 zp>?W9lHM98;rTza8XY}4&;#kXQJivj;?lg_gKgU9PHfXONmsa;Aunc}Ns|#iI0{*b zA^!Zn^rJ_UhG!-T$)yW8EWm+C#i4inuq&4}9ek>bKJKa=)HjUC5-tu7C8Wj^gp zPotmCB*^gQ9m*sO?M;sRwaGuuegkO)D~yf?9$D`8`_q?BpStX!(+|CL#GU{5AY{|u zkc^7_|2QwFz8Ts77zpwoCL?gwO|Xl2K@E-71a1*#M_#k?LCxZiEKiCfb4V}Qtz|+AhI0YUJ4ub9AOXve$4&)PXIk+SE zXY>KO2k0?F!&I78h9A^S9Ab>30?!V-~S2VA}|i_1LOC_+WZx2 z_w_#M5F1St0{v`hWd^B9*IB`kNuG6fVYi{(ZT)91w2kBFK^gGcF0eo7GI)g zcYZ!Q*6kh-CVMk6Xsv{4@oELHYegSY@yUZtv+o!gjxBM;B{RgaMK$c_MckWU`{?Of zGC~^WKi^)B^dDa6JC?+}8}jS^szsN@qP=N3~oR^@=o|z2dxT$??zp$k5Tn{B~i1kTyo!?c-Un+mG z@)FQPwH-5EwSI>XtJCcITxpJT>56GD+`XBmVjJwVVf|B7Jgi^o5Tdb+C3R+HIdOOo zufn&96(&~EEBKD$OXI8^CwuZFMMAUNC$&`pQ=*+~d-P~6#7@7JQc}CD&fZ}zD}V>v z;hcUHKw2j5$#q&S?Weql^TU+@e!962*w*19_OCaFp{*< z!Gx*V5b7?|V3GJnxq8`Dt!90}_1rGYoI~|End=>~#V8eZrkkhxvgYkSGa< z1sly8E4W(pTAX4mZJL=xwfjT2x2~t1474&zrJ~dNMrsrcl13xiGozYIX-eBo9pf&0 z*&W$aVpBk=p?D9MS61I@OZQT{p)EvG?zY>T8_S!!Y$+DJ-WH7z%hi@ zFv!PEJFJ4^6DOdV(ux*?+tfmMH^tcCD1{w`64;1CkHtxih73vr?IXZs*>*Sr6!EzsWw;KnH6@~)9TU;XK>h57j<(; z=pz3&UCxo|Npi59PQ{+6U~EAg=F7J+c1L-B)iq?Dl_jr_DpucgNPb)(6=SfahfQWy zvp9`8Jz;879I(P_X;Z23355ozl9?qn#A!@TDd8yInbBTNq~|=f;Y%GWQ&_M7S0awQD zR;YCO@hUedU2|hfUOCSO#wk%QB{+-bb|Rb36Y^JA`4k5iFca@_U({mg_u36nx-kM( zpv0u(Mm^6GMY_~ThmWL{vGV$vXZ*)k&~0!NU_KLh!xYCw`%xK(t;+3&_<&s`@=%m2 zZ=%Scmq&cCBjze5+3jk|Abip{$@p`cBh1ELmAnzXsKL)h- z|9;^8$oWqI+rZty=aAFd%^&bPz1INI9z-_>% zk>}qG-UNOboCLZHunG16`3C$h=z-nfZs0@6_P+og4;~KA1AmKb|101rU>~>zcs=s` z9Jm~O0r~x1;6>oaf$aYo_z<%Dv%rsmFC(x2JNQ+w7n~11g1r7$;HSVt!B=UkR|2)! zQ`BzX&nI+a`J1s5do~W9axncJCbpgqjM`J0Ybqw6sx-ngm=jGl5ak{=OL9GE#o{wa z%^|kFxx+JOdvmtPSjA5BWP5cpOt)FvlS#3b#xMW&8#+_YCa-V?Cx(;5QkTRsw0=^9 z6z8aNYLyHx7pSvyZ|)45irz`eKcyxc-bb+?;S1Akt!`A3R%$EvMyMh?npUHn@#!R5 z+#{Q&7Jj?QJny;*SiByt0&YW>d{6r(ld*j|5v$@wxpFSma z!jx+rFzHNY$6BaQ`Fk`AlA7_(#$8fBw>L6t&T%lBt-!gkX3bV{d{sy|UAoc(O~Gxd7q6g;2ZGHmUicS*a;mKVhI2B+tVn_DASov$a&78;CRK1vGuRhz*@N2#J;d){Nn2Cm8^y{oQ`#TyZtLS z@#%b;jwchREur6e5T`q<4(KYyd{i3!yEs{)!(8H&$4(}$m2=SF!)tCbEcBe$906K> zyqqvH;;xgS*6_BK$tr0RZc2x_mCl^9yAy{d;23^i2dCL?^{0m`w=0^LmZ?{!&zf0Q za~{X>N#tfHh9ZG4Fg_{-MpbPzn_;ZY!#;}G@}7e34=|W|CuQcFQx}-xP+Y8)4JLC) zVI;3Hd+4aq<5CMazR`7o%_w^4g*+THL`?%JIcH><=AKT8l2{xQKgfi^u!9N^ zyQ4#>t#r(wD2zn~%G1cB(AKPuX)9AHhz;|ToWJ@=`p`}t1z=(Eg?+Qdx5&S#J=l+h zi;g?ph#ni(P9`m1t=i?vho}SURXIjgm=GG4rO=Wz6x#qxFf=JIvo4`548q+F=oDO5LROB${IS0A& zY?pJz;~X6+T`lWfeP|#W(wZO$|9R{}D@kWtAykw-J)j?!w8bU%R+`cZdzE~_Y|?di zZLo&((6Jj{DOSqJ77uifT4*JIOorNuDp<5x1U{i>4#tKIex)Oa$-eQzJZ?qjaPo5S1H#MMv@V0}x^nMU979#~35gT}}i zU)SU&9m)R>M1FmYUC8_I1WRB9dUeCJ&*kzu7|vyMMRf&LNjQ%vNRG;gj$-6Eksp@ z|CFC7Y}5_0ZJ`L>Yfg@rXPeGjqWx7*>R@XKN1gULDSL?0Rq36+H{$6POA2e}e0?^h zQHfN$YX64HJ?Tj$s@<~{Abe(+a0gjEy)96=xqSrY*!ncmsI!aNT7j-Pu=OkOH`>0? zoGUUfA!C>rc+qO<9{a0S9=BPd(?n4VCB``)|B2kTCNpfzsuogDzy0+RPFau_TdYp7 z&DdrEGY!j8#_GfC;(C#$7&9!dx$Zz?o7h?LjpDA%wp12N2^Q9C*0kBxpR9*%h)H6a$pDJ~G>Ho^RVV2F6VcqX|ER zF(zRE2z@74+F)4?B9}46;^0g~s#z|je-O_?-7_a_nq}IFMXOEZDP*c(H3>o!nzMF$ zNOC*oPS`n@K3PpUrvA^JB@$B~tk53ma)uL_N-L(gz;io3nfpm9i=Zg_7_r`4Uqe@8AR=C-47M!3WK zGbgt35Z%c>Lx!BHVEq{GF=r*#mfv>u2vu-3Z-rE~7PW*#>mypC4ow_9aA=D4KAYIP zzWbNfOGb-IHEg zxmw6O{Z?F2>iPtx87DX2I(g4F3PLS5b(jfa7;#|F%cb31jE-E?oD|Lyp^fe^gyP6k zv{Z&NBT9Nwf^SpXXwG0!GRFs6(ln`w26V33ZZ0sQtB$=7X|A)Hhb^3`PH8v;8xqy` zx~xtJj43+&toj}qWYSP($jvG!@M#1aOr^bXLsNzP)Z8{)rdcAYHUtcfT;X(9$zQ4} zWRx=xMP*^m^OTdTZVN z)I!Xv?@@2858LRZ){yhtaW(-}>7x*x)^10wkDSiq%}8HWq4v)tp?gd!>#?-sj0p#7_3xiAn*1X21%cY+B#QK@_28`gW-lpIMSACnlCw z25i=P4o}MCJ3W8?XGqp9iiOAGS)}mOK2fb6C^xeD!5x#ZJ)5#{5AdceT>C>c^XSrU zScHU!n4(n@%O5MdPHM?6Wz$wsZLYokg9H2rsl8C8*PLjFt2b()!NT{N=_rsD{Qpn= zhS3bG;Q;LwS?EFAmFhLreGj^^y%Gm>v_88e1{e6kYv{9*oj41H&R}zkXmhiGct>NK z)>mw;vF+uw3a^q>oP?;y`V1jv&*cpkxWwlb&CtV%)n!)vcYdADPD5J_j7TcC0EeJf zo1Lt>PM|-Sn_==j(@03k*5X=-7``=KqZEY1S9(?u)oM$`IZUrSGJGauA0;syIkF#mGix7jO2(zxY!Wp-l$ zORB=_n?^8ZE4@HWJ?-9DU6tAWU6; z){GN1JyJ=K*@gBzVSdfuBa%zf{YGW!c{t=^osjHbNm^j8b)v=AViM>Uh$v_O%N?pZ zRgzOlJo)|w-~xR@9F+oZUX`KcBALQ;A6w0u#f#%o{G84n&O&~Lbcd~hLuRVAmydOn z1aa{gzHk&udNx&6i-ym-HC1)a&~#%hkTX?7v?F1xlnw*+)vh-C6e=r@HPJq53a3)2 z#o=pguaJqp;S2>g24qs(l!e5-qKXSVhqXpg=O1Rst(qZyN7}?<6H0N~h+{vb9S8aL zk_;R>J%mjhjJ7Ag2LtBI$>1wxl1v{?+%5+5!`+&di=@-Dx{9??x$>-rjRyr%*BeO< zwd*OASPBb8uV|@pK6tqvPxk+NAzQvsGG?;>|Lw^6C%|TKYj7(dAApwwo%??b>;mK9 zOUU-`2fqbQ0PO?#8nXO{z$?Jd1Nj9!7F-E7gC7C+10O-Ie;Rly=z|SFcLKfwJO*q7 zlJo1}Uf>Sk&yeXQOPmU!oJ!K&J|#d)XW4V7gZ_P`A$yYSBY3h%v~^+v%n6z&!dXdZfWjnZ7r@W z!ULdK;EG_@d9|7ItUG$t5@$av3cF<-9%J&M6I^50!k_xKIOz^JijA(oC#+GceWzB% z;CN;RN|W7=4SH3Wn#Me}*(rwSwoR6Kck`E2y2X;^Z=FwXR(HjGv^I28Y5Rsvk(Y>a z9aIGmRW30fX8z^~>|U{hG_}9?o}#R6ih89guVGD}(g%fU%u4KCP~BENBt%Df z+qaBwNi|NP@qM}M<{RAwwAM{SkgL14r}8QuzqkCbHbm%>BMWL3$@9|YbKWFdCUn}j z-S2~w_C|Qp;#EWnsFb8H;I4Z88E)sv6Bcdw0GiWnL5$<6JY4LYf-Dn9>$V`6{o&}* zq)6IZj<$Zq44HP}lx?Ao#8N##dc%VX3>mQ)T+QJ$s0q3(8t^%&{8%Qv{N9jYFkvFd+hUG4QBVlu ziepWclr?DFOUFpquZ>II`H3{dZn;422su@T&D_o>aV8GU!Qr7D1*?j`xxc)M87to% zQrgPzg*j&^BuAHJuaXM+l z#!239&nL>k7HIg>g_tijJYyDV;}%-8l!kIw%$Rd|c9DvVj$>%(=oSu>;$(`jl|VHt z4n94B_s`c~_U-EYZ3e6{3+X8v#tPdOlWRN9QO|ACi$r^kiXZ_c@mF)K@Q^Py_oRJalxDvGBz)zUP|Oq^8XE982uOI{ZE3IfG2|nI10`O4+PQ&$PeJ7;Jx6b;70Ipa6Wh-xE1&W zx`6k9cLV7LR=`z2=LGyA^8YV^8^Arlr;+`C8|bcpIWPc zaA)vsOkm6N17y3fG?Rg1P6|1HuNnHrx z)byxocb+L=*V=U!N@9F(}5kOw?6hO4+Jkod$YUG3n`3QBaaJ_W5W@{tsew8++Q)<(C}HdNEf zDQ*eL%Afp`q%x&qu>R(?nV%h>ZNpZg4bPut1E2bdBD=jgJ`LqkPYZk=+GrYI(Oe#jkAsX5X{!5PPPFPq6GCn32vi>5=S@QUa)4vr2+r}6 zGs5gV!g18W_(`fOJF8~!w&UY$8(PEd*w{qtGQv$%LM}x2V(f98Nk)mHt6>=-nsB@T zNUA5|r<~10+$-YLV150P@!dovW;JaRX&?u;Frg#pYuMl~ zf2@T7=QiXaCN`29PEm}7E;?-I+ilC?VuKy|61|LtGp^a@$WD_7QVx)Ym3BO~m7|#v zf36G-30ptz|B^pS_4G3m9NT<^@aGMkk-UsPW#C5?9QqV=4?(~b40|}mfqR!;1$5ED#-fw~JN!)h>4SLs zPU(Ub+Nlh!yBSOL3(+LoIKoiJ;++wuVxLP$89vRmllzy8{LpDJ^RHF>v{FsgN;7?& zN>IKa zvGk$%nnF}dqoIl=O2Mib0=>y&l5V9dBRPbXC~ytyc+x7_@W~g8;p@rZ!Z@@{l!XLm z6Wv4_H1QV`bC~v}D18X0$ZV=4slLTe8OKlwujFuMuUdGj@Bd0`Zb7@|I1zP_JOBS4 zOi|DD^8aJJSSkDe&0q_-2uR-lTjY720igZ=C&6XlZs0GF<#i6gjo=8l0bC8fg-rju zU=cLHBp3&`2Y-cJ{~~Y_90Hete?+GLc`yOKhAjUt;I-g2;A!AWupQhT$fy6K;Eh0X z{-ePqK<5Cy51IZ~K^Ht2+yQ(OdHx&VQ{d;p&w(b`2lj#=0>6t)e+=9ooCiLKO#cty zPr*~bH9&H{{QCbl^8D+-D!3j<#vcX$hD`q@Ab4g(w!^2w0#U#&3x2OLmyD#S+)tWT0}V*M^3Qkl3F8N(QrND0TKZh6w*r4fO)1oXPTOY0fi!&J!fYFBF`r9* zp&2a|V^(}otWuPIK8eMj#;t_Pm6(&XDMJ^TI_+^$f|LaIR-|@$ z6et1Dq4`dZPJzlou?&M!%1r13JzN!K1ED8H?KEz z!=N+>v7TI})y9E@Pz`%!yr%{SER*)&TuwW&os8XgF4z7v+XQ_U@U2!)H98kraxs^` z7XXUsjiYE6@n{z_|2vL#85Y}$6RQA+lBS&}I7OlDsEw1l1hKz@3x{^J@tad*bUnbL z2_TGbvAZ%wb2b^hzLEXI&cTm(JiAWiL*xk((McScdkdURo}aC!xuV+gZtXHwl~2~DT4;te|6UPVVN4@%;4wEU}98}SSox=jHUA!h1aYieUlXn zby3<-rtY%&)~XePMwQhQ^D3&`BLg$y$hPco+@)g4rJGVp^XHMGpDW3wmujz*RvL6L zp@VS)O_;6MNv|TtV6r>iS@_Ml(@bf1}hZ%V)V-T!v9$cG}GYj z0{YSi@_&{Bc^~L0%n;^cA3~R(m!gTsnNuXf>zoRv+{LAO8=+Sxe=atbolkfvmOsca zHxz^J&|j*m(WedR?A6w=Obd3%S%M|J=hj|Zkx)D|IA2=Q zpVLX(V(OMHuaK~Gw8|tyP<^cdo997=^N+NHI}>#ct)xf8&fYM(84y3q67aCYS(sG9 zp)7WKa+>jJ9?9Yh=Zjfo0wWU&ct(b{THYM|SjYSXJ5+VrP*@|PV!?*pa2 z(aZlwz5M@r-2JR0&gIxbE&;*;nUBLU1>z@N8 z=idu_2HE~y;F;hVKxh143GN4O4Q>TKj(o5E{POR=Kln#v`nQ0mgKNNTz?YEeKL!36 z1e<=7-xDAMHSkGfdfo5$WN-sG40eLAAlv^3_#5z(KtBIZ1CIl`58!-oOYj-w{C9#k zgATYSc>68gJ%4ZD`&Hlt;Q8P1G|%^itj8Chi;DKwi#sLjWwA@ zTNocm#+@Ha1J-QXIE*Q-wR|nBT6;{)>|tA<4v(of#aE0SIqU?;&0CKcGv~!g$*)OV zFJ+E-+a|A~(0CT<#>l0FeVL!Y%hpmmi)i+;KbBjKOYsi3IEY-ywj_FCnqO$zt@Hx{Hx{@A zFF}kb;cRrn9OhML&NtEZ@R;j2=fOL6#7U=z$E5#=e<_QV{uueIoyC_-PwRk>>FEJ= zh|do1yO4f}D;o~~U1&^e$f5KQm-jx}g)!F|E+dkfp5~Tk&;Fa5yjs$g! z1*+NmauC|qx0J=9HitV=dQp_Y*Kx$!6jJX*R+0YKY36AchjgR3s#AKCa)w5(DK~wj zpFpISOy`?&6E`I{c4_NE%>r0cyZ5dMP;1!hZ8bX+DAJ44Qpb;GyIVcF8Vj@8Z(%>Z zpPp%POtILlc)#Tvw&MQXv|i<-=mw@lmo_Um)u+f*h2peI8Po9c#8Z(BC%7_PR5eBsnE3{N3y? zp%B%Iis~O10nHeRgU-l7c>y`9S0J^TnVOK_%pmF1b1rRK&spd-CauOn%_I8uy%tsE zsbYaueNf4556#L@AGzWp5ttqR-oG+&R7=80IhG~iu1wlr1h0*+C_tQ+R8|L7#;s(b zy)Cmzn~XFS%Fa<2$3fwhJ+9e%CwJQ+xk(Xh78mt1+E3Qa01=Mx{?$ zYM17jmU++q$L!dM@M8gTm&=MGlC{G*YDb7C~THs9AAKB@PKy6nXc6 zrL1n0WrS6jq2dCPHiqsdE}!IMjaanH#YC%uNS#dqckkG 0) self.naptime = 0.2 #number of seconds to sleep between checks on running tests. self.running = [] - self._completionEvent = threading.Event() - self._completionQueue = deque() - self._completionQueueIds = set() - self._completionQueueLock = threading.Lock() - self._completionStats = {} - self._completionStatsLock = threading.Lock() - self._completion_span_hooks = [] - self._completion_queue_snapshot_hooks = [] + super(Machine, self).__init__( + completion_detection_mode=completion_detection_mode, + ) self.runOrder = 0 from ats import schedulers self.scheduler = schedulers.StandardScheduler() diff --git a/docs/source/scheduler_extensions.rst b/docs/source/scheduler_extensions.rst index 5aa7d15..6dece5c 100644 --- a/docs/source/scheduler_extensions.rst +++ b/docs/source/scheduler_extensions.rst @@ -49,6 +49,64 @@ A custom scheduler should preserve two invariants: readiness, but it should still call ``machine.canRunNow(test)`` or ``machine.startRun(test)`` before consuming resources. +Completion Detectors +==================== + +ATS machines delegate running-test completion policy to a completion detector. +The detector keeps strategy choice out of ``MachineCore.checkRunning()`` while +reusing the same machine-owned helpers for pidfds, watcher threads, completion +queues, and aggregated completion statistics. + +ATS ships three detector types: + +* ``ats.completion_fast_path.FastPathCompletionDetector`` waits for likely + completions first and then falls back to a full running-test check; +* ``ats.completion_queue.CompletionQueueCompletionDetector`` records signaled + completions into a queue and drains only those tests before falling back; +* ``ats.completion_legacy_poll.LegacyPollCompletionDetector`` preserves the + historical double-poll-with-sleep behavior. + +The normal machine constructor accepts ``completion_detection_mode`` and +instantiates the matching detector: + +:: + + from ats.machines import Machine + + machine = Machine( + "generic", + -1, + completion_detection_mode="completion_queue", + ) + +Wrappers that need to choose the detector before ATS configuration creates the +machine can set ``ATS_COMPLETION_DETECTION_MODE`` in the environment: + +:: + + import os + + os.environ["ATS_COMPLETION_DETECTION_MODE"] = "legacy_poll" + + import ats + + ats.manager.init(clas="...") + +Custom machine subclasses should pass the mode through to ``Machine`` so the +selection stays explicit at construction time: + +:: + + from ats import machines + + class MyMachine(machines.Machine): + def __init__(self, name, npMaxH, completion_detection_mode="fast_path"): + super(MyMachine, self).__init__( + name, + npMaxH, + completion_detection_mode=completion_detection_mode, + ) + ReadyWorkSet ============ diff --git a/test/test_completion_detector_examples.py b/test/test_completion_detector_examples.py new file mode 100644 index 0000000..ae3050b --- /dev/null +++ b/test/test_completion_detector_examples.py @@ -0,0 +1,56 @@ +import os +import unittest +from unittest import mock + +from ats.completion_queue import CompletionQueueCompletionDetector +from ats.completion_fast_path import FastPathCompletionDetector +from ats.completion_legacy_poll import LegacyPollCompletionDetector +from ats.machines import Machine + + +class CompletionDetectorExamplesTest(unittest.TestCase): + """Keep the scheduler-extension completion-detector examples executable.""" + + def test_constructor_argument_selects_requested_completion_detector(self): + """Constructor selection example should instantiate the requested detector.""" + machine = Machine( + "example", + 1, + completion_detection_mode="completion_queue", + ) + + self.assertEqual(machine.completion_detection_mode, "completion_queue") + self.assertIsInstance( + machine._completionDetector, + CompletionQueueCompletionDetector, + ) + + def test_environment_variable_selects_default_completion_detector(self): + """Environment-selection example should influence default machine init.""" + with mock.patch.dict( + os.environ, + {"ATS_COMPLETION_DETECTION_MODE": "legacy_poll"}, + clear=False, + ): + machine = Machine("example", 1) + + self.assertEqual(machine.completion_detection_mode, "legacy_poll") + self.assertIsInstance( + machine._completionDetector, + LegacyPollCompletionDetector, + ) + + def test_fast_path_is_the_default_when_no_mode_is_requested(self): + """Default machine construction should preserve the fast-path detector.""" + with mock.patch.dict(os.environ, {}, clear=True): + machine = Machine("example", 1) + + self.assertEqual(machine.completion_detection_mode, "fast_path") + self.assertIsInstance( + machine._completionDetector, + FastPathCompletionDetector, + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/test_streaming_core_helpers.py b/test/test_streaming_core_helpers.py new file mode 100644 index 0000000..f388bf1 --- /dev/null +++ b/test/test_streaming_core_helpers.py @@ -0,0 +1,117 @@ +import types +import unittest + +from ats.atsut import CREATED, PASSED +from ats.management import AtsManager +from ats.schedulers import StandardScheduler + + +class _Group(list): + def __init__(self, number): + """Create a sortable fake ATS group. + + Args: + number (int): Stable group number used for scheduler ordering. + """ + super().__init__() + self.number = number + self.totalPriority = 0 + + def __lt__(self, other): + """Compare fake groups by group number. + + Args: + other (_Group): Group to compare against. + + Returns: + bool: ``True`` when this group should sort before ``other``. + """ + return self.number < other.number + + +class StreamingCoreHelperTest(unittest.TestCase): + def test_streaming_definition_accepts_wrapped_group(self): + """Verify streamed definitions normalize wrapper and list inputs.""" + manager = AtsManager() + tests = [types.SimpleNamespace(serialNumber=1, group=object())] + testcase = types.SimpleNamespace(atsGroup=tests) + + self.assertEqual(manager._streamingTestsFromDefinition(testcase), tests) + self.assertEqual(manager._streamingTestsFromDefinition(tests), tests) + self.assertEqual(manager._streamingTestsFromDefinition("not a test"), []) + + def test_streaming_finalize_waits_adds_missing_parent_once(self): + """Verify streamed wait-finalization adds each live parent once.""" + manager = AtsManager() + child = types.SimpleNamespace(serialNumber=2, waitUntil=[]) + parent = types.SimpleNamespace( + serialNumber=1, + status=CREATED, + dependents=[child], + ) + + manager._streamingFinalizeWaits([child], [parent]) + manager._streamingFinalizeWaits([child], [parent]) + + self.assertEqual(child.waitUntil, [parent]) + + done_parent = types.SimpleNamespace( + serialNumber=3, + status=PASSED, + dependents=[child], + ) + manager._streamingFinalizeWaits([child], [done_parent]) + + self.assertEqual(child.waitUntil, [parent]) + + def test_streaming_distinct_names_match_ats_suffix_style(self): + """Verify streamed duplicate names use the same suffixes as ATS collect.""" + manager = AtsManager() + tests = [ + types.SimpleNamespace(name="sample"), + types.SimpleNamespace(name="SAMPLE"), + types.SimpleNamespace(name="sample"), + ] + + manager._streamingEnsureDistinctNames(tests, {}) + + self.assertEqual([test.name for test in tests], ["sample", "SAMPLE#2", "sample#3"]) + + def test_standard_scheduler_accepts_incremental_interactive_tests(self): + """Verify the default scheduler can load tests after initial load.""" + scheduler = StandardScheduler() + scheduler.groups = [] + scheduled = [] + + def record_schedule(*args): + """Record one scheduler log message. + + Args: + *args: Positional values passed by ``StandardScheduler``. + + Returns: + None. + """ + scheduled.append(args) + + scheduler.schedule = record_schedule + group = _Group(7) + test = types.SimpleNamespace( + group=group, + name="streamed", + priority=3, + serialNumber=11, + totalPriority=5, + waitUntil=[], + ) + group.append(test) + + self.assertTrue(scheduler.addInteractiveTests([test])) + + self.assertEqual(scheduler.groups, [group]) + self.assertEqual(group.totalPriority, 5) + self.assertEqual(len(scheduled), 1) + + +if __name__ == "__main__": + unittest.main() From 3bf3d0dd1690e248f5583b0bc2306cd2802de2ad Mon Sep 17 00:00:00 2001 From: "Peter B. Robinson" Date: Wed, 20 May 2026 16:44:44 -0700 Subject: [PATCH 04/21] remove swp file --- ats/.management.py.swp | Bin 90112 -> 0 bytes 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 ats/.management.py.swp diff --git a/ats/.management.py.swp b/ats/.management.py.swp deleted file mode 100644 index 36a440a848239e5a060d0a807eb685ba3a65d76d..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 90112 zcmeI537lM6UGGbFkY(5xfw^=OyE2pXBr^jHnaPZqq%#Dvk?vs_GMP?wcXg*G)m3d( zcamn<6+ux^L0LpLQQ1U6HWxO1fC8c*iwJ@ug7_3gcx+nZpL71(IcH|eWA^OJE*sxc<8xlEcIuz*bkn6@eCsV<|K$hOmU`WpR;S<9 ztF=D8$DcgbT3Tu@KjgBtg4Hfs>Gv+WZgT5IA>oS_TFVzT+RaPmuIu)W_m>*8&HhEr zQ%l|6@^ov_z80Ikh2}-e-EO;o(T=IfiyF)QixwN5#zJ$k*;yW6T3s{8b1%=ez_}K% z7Fb!H-+bAfYL{HR)rR`$$b+)G-|gCSjdQLA&b7d~7C6@e=UU)g3!H0#b1iVL1?0AEIU{s?#{cosMb7Qth| zbznRACIa|3z+Zt6g13X;2PeS}@Kpr(_kw4G$APPDXC;x4};X$*5O?dxC#L-u(jjbMOK1TCfPN z1rGz?MV|iO;Ge-?f_H#dfS&?afsNoi@I3_KPl7)Lt6&mr2j_!3fG?m|cpLa5@N3|) z;7V{0@ILy;`pIY^^Pl-&TMmhY-65MYd-6AdDRau>k7I_=}z%uSXwh%pS{J>H67xx919Tvej-= zHcMo#Iqw4yx885I=Qmj(a?dj%_pIGrsE_TKn`_Sb;M>MFWxeK7uRgyqR!Je1#<o@Z}6(MT5 z(+cGmEzLvKfw)T4(rYfS^g4yEG6LFd10`#>7!u9JrRCLesHi!6th?Q)hQiNu`^{p3 zQ+lFghmYjRhIX<<`?0k~pwmz=wSz{6vK-HLwwsNf%ibdAUskj1Ahc>C+t>%iF`!>vnu2@YgwU*Zipvh`i zbDk{b)(F+@9PG8az1DK;@#}=!*=@I*vrKSnq&Cx-GukPTfmKPR8S={aU`0F3x-6$! zi_M9SrZxy)vzK*ibFSF1{vl!Jn1=alajD+!b{3{JjWm|GPxV$3${jh>Y;~x0qfN!L znMS`gt9T8Ktd&_tM>8ck{xV8Z?kS~NX~Ii4EJUHx(@J}K+Gy0T6IvClz!zi-z3$3V zKij-Hn`_OATpGRAtUK=}e|wpAXKtdc`;6eZ<|!&Zvzm2Q7H670pF{--E4k);1ClP& z@&%cKd$KdK90(hz`GTyHj0ryyrowRs%1FlWJPFk-H{C39K{iO&M6w>IW>nbGy5^-k zq{jYiquus(DpB^8vCa`;*5= zT(zWwY#b`iNphX*cHHVjFN;PY7J_RCWXp^j(N#|9HaGc_g`rxfF-W7mx>4~$hbgB% z6LUlPt+eNi6M|jpcUO9|WPuCoQnS}y^@%CCMO`a3;^5%ghePtQ!oW}R4UeDJ1tLO|2mS14(;S?*%5{v6>YpKzuX;~ZL_IPXZcz8mV z%($CNs2GXaZx>;t(StZnvhZ3jFF`dT6KdPEAH8PVTEJdkHB!@*MwF|2T~_K|zrMT` z9`xdhYVHrTSXs+9OVQznY;*hp1=QbeD(|0cEg!3ojN;+EZ6wT>lQ86SFpaeLf+pwY zGBcvs8qn&%Qo|CcWKg|@gSqT=`%463Xw;RsN;u!^E@sSFTmgZW$_=fuT z8P)zJ-MODcJJ9cHF0QS*QRXep@|(+`Z2X<-^8v1*G_H9 z9C4?gExccgA-s5 zOn^&427U|r0xG%UR_#(Q2zW{#-UIu;wbio9;5ZoSo1%1IM z!8^dKz_Y1pw{)6*j`J>oqkqN;IxWl1bXJuLoCP>m|nk%i{6mkvj^W$}+qHOCob*+{oDi3rJC z_oMJe;+VbBkHVWogt7q1--U3wjAWtmzmhQfcI;LZQ4Sy9B*8->E`o&M$aUm7=w%XT^7=`iK`DHAk%TR9W7M+ncM=Zn)A%=ngrCV< z&*t$mFnmc5_0@R&Bs5GBkHi|qP;~BrB_knT#Xph|v&WjV$FFIxsI&9m$@>)qB75n6 zByaq4eI-wkU`+HNx$rZ28(!IW@_YqF?(A{mxF5+I|6E_m(-rjhEg!*tCGWzU^7E4o zqg#C?p`8SZV4=o~e+-1k-zIO0gOm`(2T4epp4a@FJX=B5le&%<;b-zTf9HNAZ^*=D zl{|BDI*;9t`Bn6mWE3v4!3 z!KGKL4DM}3>i=y;TaZ<Dpnm7M2{A zwC&aF4R1Ep;&N3A6<=)1)6xo63Kp_)i&t2hwhvvhn3Fs&KtY45D?Y3_1|WIo9`M}M zvN4W(!1H9)FGNe86DnWz3t{r7H3()`B#k6c@kNcICNKG`7ADi=ad>3%{~M5*uSUL> z{2%=D--C?*ZtzAh3myfo0uKP6LbiVmcrJJjm;v_#_XR%;{v4V9<=|!D7r>K14>Z9R za8Gb6a7*wp@_jN;{D`RF7PVw z6mSi=JNP)dlUIOa;342H;GNj{UkIKC_Je)k0pRo4_df{U1YQMR2wnhw0%*U$<=~Fs zGuZk+4c-A>1fBvO2d)SA1n^DMivWRno(1F*G+n|pqXd8Q`5#dl8MK~%o9S>Xqxcicx+=?Z2yw?X9|}7r1}C zwIpry4SDdn?hf1)u)0aZSl?KDIkFc;X+uVP4z-{Ij&)C}shbV_9#p5qtn0!AlUbg8 z$yA+8@S5zE{IYcTy_Jreur&+Qqu2I1BO@fH>W^mo8mkoK1hY6iMkbab>u6nB(5$W# zHcY?HL_fbGvT@0cW@aMr1}x_gvbfZ?<2Pg^z!0VErF)FTNd7ouOigAJ`*&prc4w$o zv&pF)htQW|Pftylk=X2MUhw!lPiyaiYowZ;AO)U{OZ1DOR6~UvQLZ}=?4R1R|Jof> zdk*ZkDA2nlPyfrCp zz!fGCJqLaw+gelY^%YD5H?+{>tJa_UnsjmW9|5_P!960$T(helR%k>*BT`;K^(E?b~tadOUu3fX~kw zZ9j~BMGHl03A40jut>uys6I)X7Hgqmjcq>D&sda_l)Ba2PjJQL|LgNs4%VuNCMK`l zOY5F(t12Pur0r+=xuFw%Wx~{=*KIXmr4AP zEk6S!ZoM+#YZK&5Kjp{nf}dcaTOtc*$ksv@)&-Y}R-uj_t8%ITNS=w60h<*Iwl`3! zRrRsL6kQtAcW z<&C@IgVHd*hvOsHg{9W5(q}vd$JS9U?(kpc^es_y_)zkx#*HlA&avg#GINQEI zdbG0ka_UgB;S;E_E(fCq4woCZWg5S-2dAy83ae$iwGeYhZy}{Q%6Z7@1HDxDO72gz zdC;H86-s^K8fCEChL1eI0{fcefhuY8&}bCXIhH=<<)jFoK=j_8Qb8hn<-`Mg)`OTswakSA%hVZ!@a3F+r=fK+uJvqegS&xmAnU&YG=XgV zw*+5Awtoe92Dky-349v4{_WtU;3eQE!2&n{9tQ3VK8jrbGSCDUgRdde{|WdtpgjO} za31&+^86>kYrry)|Nl<#Fi;0$Kr;X5k?sEsybvscHrNL)0CxsqKftfkZU=z1pKrT! zU*}rjTnqev~i|9FvZ_FZQMC+oVS^r)5e{pj#JC* zIc?lIZCq{>|8Jm;%N1RwKbUQFYJr*5cS&`X<=bsCZLwyT4tg9}4-}6552(zf_tDOl z@HFv+^I9JH|6`i2{V&;)k^ftsZT~Ft{rkYH!85=LSOy2d?ZEqx@!t+!1fB+F!Db-& z|0~G$p9OCLeegK23w#^-{!`#ZKsNvD!A>CA|5o5HkoDgTB>(?9=z)zuw*D`HcYt33 zj|NwO9|f}ae+uX)5c~mn88`wS z4K{-s_#U$UTftkvOTddk4~&3MA@~0>=z>`wJ-~gyy}{qns80k>01Z$FW8mM>wY(X; z2rPmh0~dpPgWH2oV&9kD|Jh&`>;;#D2ZGyyFVIgP2G(D`uL6w}J&(fEf<{vhz0=g5 z1E$ifaF|k{88(O-|D6-S!_6f~P5hST5G!k^*WChZ+W+cqY%ursTK{PZ23QvSn?jpYl+w2Kp~rhYB0 zj>SFOgN;~-8v|0j#J5d>FvY2T`s3P)G~Q?Hp4_(TI8-(DI<{DD7v3a)DSKItSWjI1 z>%UF%9ZCNy%;Sz>%MT597#FG#1XU>3%bo!cSf9`b3|gkJgT_D*3{@Egdwy-nGP z?rdXbrHyG*+sM+!GzDx(V4bLWMD>uu%Ev(8sfsyGeDJrXv(%Xzm!|qyEGzEd^doA{ z!A)?9**J!$9NUx1jaMaCtgjKQYcJ91ST;6(Q@7QzzUXfpW`G@Vmn`UrV|nC`A?Fb4 zVS80mn+|Roc#LTKsWyY+m(j7!ec5v3xHQCAi*oP2Os+X@#OYO|7iu$r$(oN-1@wuDSQubxnHsdsQ}E|IAQ9#$^R zC8Z&$;?FhP<>DMZ73V7jq2Q;?$!A!pHA*c622;PESn+}dNCcO6{p#Y3e1;UowxinY zlNx=4lVSY=rIr@C)M{aW24 z-M8eLE$W?8g6E1&)q2K+FXx|FFrOIpD1`No2ZE=AGsIw3@|I8)EvqY^yp%B~qV6@H zyJj}K(&J2})H2RKb&iEl>tC+1*3o!zUej{?%dqjq0Zk@G*)~V^{n)<1B8z%`JiFt^XOoKLFyZ#-6QuwIHgtZsUz&*&1TCM;)TlRI{0 zyC$x_7MH{IGOAg`F&E-6p(=R#%rNKXHDUZ@_tT zB7)6mZd9sTq_m6hgrOeZ`yVU3FD*|n&RbexGX(23K1jIMEaP}`m6e&bg9x3u>6MO! zZ_fGAty)zK==pfM7H#eKDwd|=Sl42Z@i0&1t)*IoO)c}u=+2uz*^05v?Ee=dN50w1 z|Bv%B;Dulj>;RJU z-v(X^ejdoL|4OhC$lm{IFbBrLoxw+t-~S4{5&R5T0z1LI!CxZ3KNI{Y_$V^_N5EUa z^MQQ&XMz0sF9JUd?f~T5|6AY$SOEvXdEjr6**^td2abYi@Mv%wpf-P#+Wm|_>f`it z^r*-sctha|lx6ZS2~H(Jh-jSXGFL8lSDb@MpY4rvEA6H`Qz@iZkn=z6 zuj64D+QJ=sG@BeeBx-3VHM~$F1XM0lP}I2ghT^F` zX{R2QI7jV3tzU3~j@Hu8)SFvLxQ(l&IJO1VN9r%H%pQ;Pnv3HC6rYB|;Qe$BJ+f3v zQxRbaB2!<}lk&%yhST?AhGi53SUNNG1LWEAP6Esra*-wrj+~srqx82K2Kbf-mOc5<%-;lgmc2-pt5GyjB}?b*R$l zK>^3~$FNEau#(Dx79NxYL*CZ>o3G4lGC_v+9bi;v4kE>ts^+4VL*-WIG#5{8>r%9p zC=Luxqs3g3c$Ay(q&iayr2j4>#rIQv?7#6yhzFR}HI9me6(}VivS&)=Apwz(j#=|` zRzbrdq{g~JYK&udkXfK8m?_{|6OC%Yh>Aox|CxevF zMGjqTMyTXuWM5A@FWLO>XVp~TjXT!-%OD6wy>+aQ4??0kK-Gld#6q$$E`d7|K5l>W zZm&#!1%9K(+0Y|opAR$dryij3=m9^QC5zeUnoe(uFMZAq_KPDfF&Mq z)DCLZAsB3QV%@{M;SL;?G_K6G%Teo8O+BIyIab>S17W2ciZtg8RtoWhGSVex(9}wo zW~x#Sw^C>ghJIyNvHGz$4daBYA+fiW!s#$y8r`6$ot;`?*~r;SAz4w1HreD^L5tW`+@xap8zff_XYO> zvi)nX|IYx){hvj)*SQ08;8O4($n__H_V9lYdHxib1N*_Zkl|keejPjqJPB+7w*#`_ ze-U~9ufgww7lNM!yTK^9HMkY{H{^Pqzo&Ek=fNb{0B#RHkDUJ@@G|gpFa!312Y?#* z81ntwz#oAYH~_8ylKa1ejQ@||z2H6ImEcK0Hht~k|0S>+e1rD=Tkt8Mwtg@8L4HId z`Y@TbI(F`5BLBj)L-Q#K)hsaoKy+?vl?>yIIonnpPvU32a_8ETu1zzVBsoA+<1ieP zxiQ+9~V z#lNs=Iq!Af`tgkA=db!&_&PXSPf~}FRAn-HX-S1cU$?12 zd{WI*I&v57N1;K&)7T_Rq}VSsf^&@5tK_qG8Ny+)X&Ku6`KE6AD3&2S{lS;P9fDhI z(qj!~l2R+UXFvG-bsR5eE>M6CbK1^LSJXvpI)!2m)gFQ?gUqwa`NLxCgvZG|sXkup zFSrG=a$uZB2a*6$M|Rt3zXK(7W`{!g{G+(u4@$n}G`AGClUvn&Dmn^rOTxMOm@)i7 z4K(_TLrJ7Klm_*ZQ;T;#O1`3m046q zHN6cV3*}F0&Mwv%Q2)mQ_zmcVLWE~LRJDfa^hSNEKd3ZEt19G+5=TUsC%%sfF-%Iy z+7w3q_mOItW|CChX#PIZbkjzXqHhi*nR`_o9vdCI5TT>zm*8fPJ1X|u-47c#@vJ7?v zB?HQ77n0m5OX!LN=DQGSkkebY5yL6yNh!P;b!Z!`o2Hv5lN5iTeAQlDH9Qj~ z305X6Bum|TlkimGL>85GfTL2UF_MQX9W~4kFOPGAeom*6AvAiG5#?GA9%|#-V-rw? zrUNR)&!er7)rTO2-9~|{Qz7stBA5N0Dthh%&o9iO?rG%{NqqaD( zKIs3tnU1tx@OYy*2Ch2ydFEfL72{)&D;r#r6~_m+6*N!7|45oa!6r>?LeGLG&|a0~ z0yg>2<)b_<4@yj=VMmOGsIbs@D1^VPsnd3a1=y;$apBIsDJ{IZNC~B)@ezdwX99zr#o_P_BBmrVcbE)RmXa`}^5_Qq$rm z$SK~>Vutnl$PFFZL8dwvT6#^Ua*JcGT*T3wUQ7hyBbI%9GW_1ZbJ9c?Q)+k_UVL37 zg;9~3Bo7xg!z74>p+3T&Yv>Y+@f~kil|(LPP=TOB$R(6KOO;_jScv5aJb$VL2%%hpS_0Ge`2B*B-y79A#`2$UBaaHeUvmzp-D;Id_|gPU(x|>K$v-kmvpGfCd_Y138Q^#SX&NF zZ7l=U87CH)?5IofbB!ABY$i7Io6My136ZU2{9sV*@u)6_r(ZP+Ifo#S+_Hzv8S zwZ26`pP6o8d2^{3MDvRD18Gz)eT57H{l!J=^m15hRV-vMuqd8h`ahWUZj<@{-y3H9 zHD3O|k>C2f4Ow6N{%-)=!4*Jz0PYGT|GyaA5quFj|DV8b0?GZl&+lK6_x}al3?2^- z0r~mg8GJ>ye(fS&+2f*s(#;CA3M=mkCv-V5FXUJVw&1A)%=e?NFR z_<8U+a1Hn#x_}RXSAivP6}T<bKEnv~=t3*S9TusD- z;#PXnjQYkMVENbvp9k z!;Q_!+6}xk-6g@c)29c+$uZ^VKg?n1 zX6ta3=85CUB|VBb8;?)W{-G({Lu7)MgYUxdiroUaKVO*Gd9`Jd~lds)ou~Qk+?Wm z{WS(6QLAZ;40h^bF)1!=|5^0P;*O+K(o7|C0R7I-OB-KvU_pl$8IQvR-YeSQv?ncpzfbm=LTc0_cs);aL@&7@gSveB&vXO}^<-`<4~5i5g%nD)Xft z4`wN6@v|EEduAfBX<7*3gL!LYbi<`{BWN;R?F)8AXZ(_;+w5PlsRHHbJ4Rft2R@fA z9=5C16zCib-=ANF^)$BAUlHXJqQ zFczx$>x-+?9nmpFF`nC1Ru=awUZN)o*@PC+^$dBd*TP$U(=wK3c4c*<#nw{AwGn+J zZbWynx2vd?rP{t>j(aB1X#28P2HH5-fFa$@(FI-Qt!5-b;85R!C8*OWt?;5C9%{Uo zA|#_pnT=)_Z`u0r&08M2`Qpp6NtjkOJGrSWlEmc7pY%-RATn z^=shZSSkl=es-keDYDxWsg$TD)UvZQNIJ}#Qadc`?U#ph$HlQ{L*)&*U?Qup@Oiso_7e%aVQezEzfWu$|ko^BH zpu7H-fzJBd0CcwhN5Lz>3Gid!DsUe705bm*fc*Zyij4mi@HgOD;IZIxa31(~Wc&|+ z_k%Zp8^IB97x3rE_#XgI2A6?{fl(lt|D{0p_+15RKxg=Aum6X^hrsi|vp^Hv6?_Ak z{_Eg(!0W)Xf#m%?U;}tC_yF?#NpKuo4Rn^@t-(i;>t79?4VFP4YytNHcK}~Tw*PnV zG4Mw4EU*J?0rvs_M!UZosQq88J}7=1DY$=6e?zSJioza9mfWJauN2Q{pKi}-@@lRq z?%#nepeE)Zwfm3VH)59Be*RGFQ*S(6++FFAsVQ4WO1V`ozd0!EjgRZ3yN_rfK*HI@ z(@`rGk$(Owy88^Sdp=mf&4-zRt%bejv~wpji3f9dxQ{4(?417nlT=u-1sX1>(e_u6 z#d*M9Y1h;t=dG4(perPo@KTNTA$QADOr@ego|#VGBuS?g@&WZI3E_*3a;3V+I=-v! zIA^Wk93$8~)mX&!tQSj_%BS$@TB}cHTtCajMO?To2}$fb(Sv(}HLg#GHOt)h#05Cs zLmayftcyZspp?w6%4znF9<>yX9t~wVdX(c){TXy% zUyssj)M2gV^zNNK^9q9J2)QBx&$JsIZYI($eOx3?9NxO^h#uQ*yX=dX5S+VCjK`cb zz&3L~4L3^i-EN*}wzI95#rJ6z4LQqdm}yU9gYwu(j`S5S8je-- z!!{}Ol*%kcD2Mpi?-*FscDwW*li&7KA9Dy&u;Q=rq_YnYDO6ZAON#c~=m&mf9Ae2HQ!LDG}bhKTT8;u`zt zC_*bI^;e^tvLYPhYOYq&72&h&9nUm0L{LV=l}nol(YQtJg)xHDKXDSyiOptom8q!d z542WIAF6#V1PR}X|8jIa^>#kZO%-(JH6oGDl$LQ3frp6S*&b7)htq;`jv|Jr9$F+f z&b8-rnie&E#9oO7cd99O+>Lo^6EY~Q(x(?fm62#ee9QoHvZjEX_u>5EATSYvJ4toK zUrdJAu}QL!ry9E!;_F-ek>{vMpzdT>TauRXKKDgLcZT#A*!np;pvOv_C=&C&t)K3A z414P0d^4KuU;%cTN+Z?$#$rxGlS8{|P+G9k`iP>W9qJ#x$$UCZW5ornjHiU1NMq=? z-+p@G_N zH>s0)jSiPAw=QH)wQWBOvC~;ML~@yz^LA&BrTTqgrTZT;DF%MlK9W?(oJHnUroY}zcgv9xkHEtI2B-O!jrjeN1 zv_^Kfbk?lRt$tdh61}F>wsx*WYX3{DBWh9lN&~EVTFI@3#pct*G8TqvDOWSxz*-Dh zn=R@EXZWd3j*_c$jA*@q)piBwVZoG|iVAAF%42U-FmtQ$|?s!{1(Ac9B=QLj8*~h>vE0VNub5Flj27%R)T6RyO0* zmliC$c#xn}w^4=&Aa2!(0W)?Pb~L2>Gj3}kDK8cz5(I^WWp3#lE!cH;7x_}R-I`@$ zuov zA7`&N>bSFnolEWW4E^c}tH`q_jaw`O_OK;?BFcPA4i56T!wh>`dEn2#+XauaqNWwk zq1k-o`LSXf6pK99owY*`ccu(ovZN2KqYkJOqn8`D2OFBXSuJjNxUXnaSE*vdXMAE^ zs|XcMH?lwQ+4GIr7I!kN(&zYr43WeJ)jy(vk*keU&r{F>r_)sYERL+?Nq?NXcAIS- zqn1pv>P3BMn}G;Pi7g{f^fZ&(8G=~0EekoFP9{oa%ml3Isp}6;um}}9Fusb?fW`5b zl2yk_UzXybkG44SF)C&i-jpI}w%L2&nrzpeLlZlv4jj5(F)6}$JVB~v?ZrPK@aeNXBEVi~Gwzj$% zRQecLiBMW5L`jLb=G1#*H`Wi|c*Dq%@e4NIF!JC;W1}Keh8?r8V`*5GJnf-IZ1{D` z=ejhO&{BcKumK+Aj3)|!FBN1wdy=CUYoUcK6cvVobSz<85&d*ULgGO3KtqYu ztDTZ5u_~=%q0)@<94uPQy%@tkF9tSGBsH@()fdCZSH&oS3oD2B&-1|L)YjLKFE?;CH_c=rf;oK^v+ATzm0?KFwE&Goy;S9|YGsvT z>*G@{TBh79pL{qM*YTRtwZpVSGny;WPUGgp?pS<%(N|)Bp|#f3J^#agi~J3|6g~1w0f>j1d{*ngY5Yf$)L#plU@#eAF}?_yv+Z6DQ8}Nt7_D=%3$FB(<3Ov4`K3Q!$p-qS}pJl|+ znQpheaa(rXF*srjsoI5(qb9mJ3#F4hQGt_~HgzM(c9eWA#R1WG+aZoJVT&uau{vRlrSb1+pJMP75GgvyYTAsje>KiDWyBokv zv?U{CF>6%@HJaSN#PDZmYfBMpjOvGz`wBj)<|`(ZHg@Gg)`=uP zs zoK%={G?czGpCI zNeXrZWL#ou7|pGzS?QB_MlRl(s1R1L3^)M_5~yw^f%Vw_GS6eGtvuOE?bJ0gSBvOH0LlmdKnOwSdaI|jSh7so@d%0n*;0y@sQ96=ahvZ#k=g=jQtCTd-7*wlh27H=^^{RE=q^!UyN3O@r=C}`q!ff6w z8)+gU4;6Y8uGk<|SrT2^uYtCbMolkbg=1RfvEt`l(;|jBK2Vvd5}Axht~A{jr&pC$ zC%oeD@8ul3GSae#Dx8_s&KSg%`f^-hjD^J)#)wy#UnxvIaifLY*hNOMbOs9Ia}22NaiHEBRk#NpJ#jYsCP?9-FXIQ z?Y&ge7siv`F)k%paJNFaLI%l`PSb)*U7XWQGMj{YT&*E)RPLwjSnQOotRe|j(mx-R z6V7meDlMaE8=sOP`IhLSv(qW=`Av0MkeWlL2S3&)!Yy%;9@i}}$J5Mo z8@;*d;^v8>>!MFT%$v3bi!mwYqrB$b6kXHr^kWs>iOyXkx1wT*X7gniMH_ft@ll+Q zw3Nx?$&JkNO-_Yj?+ylurccn%c_CojI(*N!Es`4U{p40 zp>?W9lHM98;rTza8XY}4&;#kXQJivj;?lg_gKgU9PHfXONmsa;Aunc}Ns|#iI0{*b zA^!Zn^rJ_UhG!-T$)yW8EWm+C#i4inuq&4}9ek>bKJKa=)HjUC5-tu7C8Wj^gp zPotmCB*^gQ9m*sO?M;sRwaGuuegkO)D~yf?9$D`8`_q?BpStX!(+|CL#GU{5AY{|u zkc^7_|2QwFz8Ts77zpwoCL?gwO|Xl2K@E-71a1*#M_#k?LCxZiEKiCfb4V}Qtz|+AhI0YUJ4ub9AOXve$4&)PXIk+SE zXY>KO2k0?F!&I78h9A^S9Ab>30?!V-~S2VA}|i_1LOC_+WZx2 z_w_#M5F1St0{v`hWd^B9*IB`kNuG6fVYi{(ZT)91w2kBFK^gGcF0eo7GI)g zcYZ!Q*6kh-CVMk6Xsv{4@oELHYegSY@yUZtv+o!gjxBM;B{RgaMK$c_MckWU`{?Of zGC~^WKi^)B^dDa6JC?+}8}jS^szsN@qP=N3~oR^@=o|z2dxT$??zp$k5Tn{B~i1kTyo!?c-Un+mG z@)FQPwH-5EwSI>XtJCcITxpJT>56GD+`XBmVjJwVVf|B7Jgi^o5Tdb+C3R+HIdOOo zufn&96(&~EEBKD$OXI8^CwuZFMMAUNC$&`pQ=*+~d-P~6#7@7JQc}CD&fZ}zD}V>v z;hcUHKw2j5$#q&S?Weql^TU+@e!962*w*19_OCaFp{*< z!Gx*V5b7?|V3GJnxq8`Dt!90}_1rGYoI~|End=>~#V8eZrkkhxvgYkSGa< z1sly8E4W(pTAX4mZJL=xwfjT2x2~t1474&zrJ~dNMrsrcl13xiGozYIX-eBo9pf&0 z*&W$aVpBk=p?D9MS61I@OZQT{p)EvG?zY>T8_S!!Y$+DJ-WH7z%hi@ zFv!PEJFJ4^6DOdV(ux*?+tfmMH^tcCD1{w`64;1CkHtxih73vr?IXZs*>*Sr6!EzsWw;KnH6@~)9TU;XK>h57j<(; z=pz3&UCxo|Npi59PQ{+6U~EAg=F7J+c1L-B)iq?Dl_jr_DpucgNPb)(6=SfahfQWy zvp9`8Jz;879I(P_X;Z23355ozl9?qn#A!@TDd8yInbBTNq~|=f;Y%GWQ&_M7S0awQD zR;YCO@hUedU2|hfUOCSO#wk%QB{+-bb|Rb36Y^JA`4k5iFca@_U({mg_u36nx-kM( zpv0u(Mm^6GMY_~ThmWL{vGV$vXZ*)k&~0!NU_KLh!xYCw`%xK(t;+3&_<&s`@=%m2 zZ=%Scmq&cCBjze5+3jk|Abip{$@p`cBh1ELmAnzXsKL)h- z|9;^8$oWqI+rZty=aAFd%^&bPz1INI9z-_>% zk>}qG-UNOboCLZHunG16`3C$h=z-nfZs0@6_P+og4;~KA1AmKb|101rU>~>zcs=s` z9Jm~O0r~x1;6>oaf$aYo_z<%Dv%rsmFC(x2JNQ+w7n~11g1r7$;HSVt!B=UkR|2)! zQ`BzX&nI+a`J1s5do~W9axncJCbpgqjM`J0Ybqw6sx-ngm=jGl5ak{=OL9GE#o{wa z%^|kFxx+JOdvmtPSjA5BWP5cpOt)FvlS#3b#xMW&8#+_YCa-V?Cx(;5QkTRsw0=^9 z6z8aNYLyHx7pSvyZ|)45irz`eKcyxc-bb+?;S1Akt!`A3R%$EvMyMh?npUHn@#!R5 z+#{Q&7Jj?QJny;*SiByt0&YW>d{6r(ld*j|5v$@wxpFSma z!jx+rFzHNY$6BaQ`Fk`AlA7_(#$8fBw>L6t&T%lBt-!gkX3bV{d{sy|UAoc(O~Gxd7q6g;2ZGHmUicS*a;mKVhI2B+tVn_DASov$a&78;CRK1vGuRhz*@N2#J;d){Nn2Cm8^y{oQ`#TyZtLS z@#%b;jwchREur6e5T`q<4(KYyd{i3!yEs{)!(8H&$4(}$m2=SF!)tCbEcBe$906K> zyqqvH;;xgS*6_BK$tr0RZc2x_mCl^9yAy{d;23^i2dCL?^{0m`w=0^LmZ?{!&zf0Q za~{X>N#tfHh9ZG4Fg_{-MpbPzn_;ZY!#;}G@}7e34=|W|CuQcFQx}-xP+Y8)4JLC) zVI;3Hd+4aq<5CMazR`7o%_w^4g*+THL`?%JIcH><=AKT8l2{xQKgfi^u!9N^ zyQ4#>t#r(wD2zn~%G1cB(AKPuX)9AHhz;|ToWJ@=`p`}t1z=(Eg?+Qdx5&S#J=l+h zi;g?ph#ni(P9`m1t=i?vho}SURXIjgm=GG4rO=Wz6x#qxFf=JIvo4`548q+F=oDO5LROB${IS0A& zY?pJz;~X6+T`lWfeP|#W(wZO$|9R{}D@kWtAykw-J)j?!w8bU%R+`cZdzE~_Y|?di zZLo&((6Jj{DOSqJ77uifT4*JIOorNuDp<5x1U{i>4#tKIex)Oa$-eQzJZ?qjaPo5S1H#MMv@V0}x^nMU979#~35gT}}i zU)SU&9m)R>M1FmYUC8_I1WRB9dUeCJ&*kzu7|vyMMRf&LNjQ%vNRG;gj$-6Eksp@ z|CFC7Y}5_0ZJ`L>Yfg@rXPeGjqWx7*>R@XKN1gULDSL?0Rq36+H{$6POA2e}e0?^h zQHfN$YX64HJ?Tj$s@<~{Abe(+a0gjEy)96=xqSrY*!ncmsI!aNT7j-Pu=OkOH`>0? zoGUUfA!C>rc+qO<9{a0S9=BPd(?n4VCB``)|B2kTCNpfzsuogDzy0+RPFau_TdYp7 z&DdrEGY!j8#_GfC;(C#$7&9!dx$Zz?o7h?LjpDA%wp12N2^Q9C*0kBxpR9*%h)H6a$pDJ~G>Ho^RVV2F6VcqX|ER zF(zRE2z@74+F)4?B9}46;^0g~s#z|je-O_?-7_a_nq}IFMXOEZDP*c(H3>o!nzMF$ zNOC*oPS`n@K3PpUrvA^JB@$B~tk53ma)uL_N-L(gz;io3nfpm9i=Zg_7_r`4Uqe@8AR=C-47M!3WK zGbgt35Z%c>Lx!BHVEq{GF=r*#mfv>u2vu-3Z-rE~7PW*#>mypC4ow_9aA=D4KAYIP zzWbNfOGb-IHEg zxmw6O{Z?F2>iPtx87DX2I(g4F3PLS5b(jfa7;#|F%cb31jE-E?oD|Lyp^fe^gyP6k zv{Z&NBT9Nwf^SpXXwG0!GRFs6(ln`w26V33ZZ0sQtB$=7X|A)Hhb^3`PH8v;8xqy` zx~xtJj43+&toj}qWYSP($jvG!@M#1aOr^bXLsNzP)Z8{)rdcAYHUtcfT;X(9$zQ4} zWRx=xMP*^m^OTdTZVN z)I!Xv?@@2858LRZ){yhtaW(-}>7x*x)^10wkDSiq%}8HWq4v)tp?gd!>#?-sj0p#7_3xiAn*1X21%cY+B#QK@_28`gW-lpIMSACnlCw z25i=P4o}MCJ3W8?XGqp9iiOAGS)}mOK2fb6C^xeD!5x#ZJ)5#{5AdceT>C>c^XSrU zScHU!n4(n@%O5MdPHM?6Wz$wsZLYokg9H2rsl8C8*PLjFt2b()!NT{N=_rsD{Qpn= zhS3bG;Q;LwS?EFAmFhLreGj^^y%Gm>v_88e1{e6kYv{9*oj41H&R}zkXmhiGct>NK z)>mw;vF+uw3a^q>oP?;y`V1jv&*cpkxWwlb&CtV%)n!)vcYdADPD5J_j7TcC0EeJf zo1Lt>PM|-Sn_==j(@03k*5X=-7``=KqZEY1S9(?u)oM$`IZUrSGJGauA0;syIkF#mGix7jO2(zxY!Wp-l$ zORB=_n?^8ZE4@HWJ?-9DU6tAWU6; z){GN1JyJ=K*@gBzVSdfuBa%zf{YGW!c{t=^osjHbNm^j8b)v=AViM>Uh$v_O%N?pZ zRgzOlJo)|w-~xR@9F+oZUX`KcBALQ;A6w0u#f#%o{G84n&O&~Lbcd~hLuRVAmydOn z1aa{gzHk&udNx&6i-ym-HC1)a&~#%hkTX?7v?F1xlnw*+)vh-C6e=r@HPJq53a3)2 z#o=pguaJqp;S2>g24qs(l!e5-qKXSVhqXpg=O1Rst(qZyN7}?<6H0N~h+{vb9S8aL zk_;R>J%mjhjJ7Ag2LtBI$>1wxl1v{?+%5+5!`+&di=@-Dx{9??x$>-rjRyr%*BeO< zwd*OASPBb8uV|@pK6tqvPxk+NAzQvsGG?;>|Lw^6C%|TKYj7(dAApwwo%??b>;mK9 zOUU-`2fqbQ0PO?#8nXO{z$?Jd1Nj9!7F-E7gC7C+10O-Ie;Rly=z|SFcLKfwJO*q7 zlJo1}Uf>Sk&yeXQOPmU!oJ!K&J|#d)XW4V7gZ_P`A$yYSBY3h%v~^+v%n6z&!dXdZfWjnZ7r@W z!ULdK;EG_@d9|7ItUG$t5@$av3cF<-9%J&M6I^50!k_xKIOz^JijA(oC#+GceWzB% z;CN;RN|W7=4SH3Wn#Me}*(rwSwoR6Kck`E2y2X;^Z=FwXR(HjGv^I28Y5Rsvk(Y>a z9aIGmRW30fX8z^~>|U{hG_}9?o}#R6ih89guVGD}(g%fU%u4KCP~BENBt%Df z+qaBwNi|NP@qM}M<{RAwwAM{SkgL14r}8QuzqkCbHbm%>BMWL3$@9|YbKWFdCUn}j z-S2~w_C|Qp;#EWnsFb8H;I4Z88E)sv6Bcdw0GiWnL5$<6JY4LYf-Dn9>$V`6{o&}* zq)6IZj<$Zq44HP}lx?Ao#8N##dc%VX3>mQ)T+QJ$s0q3(8t^%&{8%Qv{N9jYFkvFd+hUG4QBVlu ziepWclr?DFOUFpquZ>II`H3{dZn;422su@T&D_o>aV8GU!Qr7D1*?j`xxc)M87to% zQrgPzg*j&^BuAHJuaXM+l z#!239&nL>k7HIg>g_tijJYyDV;}%-8l!kIw%$Rd|c9DvVj$>%(=oSu>;$(`jl|VHt z4n94B_s`c~_U-EYZ3e6{3+X8v#tPdOlWRN9QO|ACi$r^kiXZ_c@mF)K@Q^Py_oRJalxDvGBz)zUP|Oq^8XE982uOI{ZE3IfG2|nI10`O4+PQ&$PeJ7;Jx6b;70Ipa6Wh-xE1&W zx`6k9cLV7LR=`z2=LGyA^8YV^8^Arlr;+`C8|bcpIWPc zaA)vsOkm6N17y3fG?Rg1P6|1HuNnHrx z)byxocb+L=*V=U!N@9F(}5kOw?6hO4+Jkod$YUG3n`3QBaaJ_W5W@{tsew8++Q)<(C}HdNEf zDQ*eL%Afp`q%x&qu>R(?nV%h>ZNpZg4bPut1E2bdBD=jgJ`LqkPYZk=+GrYI(Oe#jkAsX5X{!5PPPFPq6GCn32vi>5=S@QUa)4vr2+r}6 zGs5gV!g18W_(`fOJF8~!w&UY$8(PEd*w{qtGQv$%LM}x2V(f98Nk)mHt6>=-nsB@T zNUA5|r<~10+$-YLV150P@!dovW;JaRX&?u;Frg#pYuMl~ zf2@T7=QiXaCN`29PEm}7E;?-I+ilC?VuKy|61|LtGp^a@$WD_7QVx)Ym3BO~m7|#v zf36G-30ptz|B^pS_4G3m9NT<^@aGMkk-UsPW#C5?9QqV=4?(~b40|}mfqR!;1$5ED#-fw~JN!)h>4SLs zPU(Ub+Nlh!yBSOL3(+LoIKoiJ;++wuVxLP$89vRmllzy8{LpDJ^RHF>v{FsgN;7?& zN>IKa zvGk$%nnF}dqoIl=O2Mib0=>y&l5V9dBRPbXC~ytyc+x7_@W~g8;p@rZ!Z@@{l!XLm z6Wv4_H1QV`bC~v}D18X0$ZV=4slLTe8OKlwujFuMuUdGj@Bd0`Zb7@|I1zP_JOBS4 zOi|DD^8aJJSSkDe&0q_-2uR-lTjY720igZ=C&6XlZs0GF<#i6gjo=8l0bC8fg-rju zU=cLHBp3&`2Y-cJ{~~Y_90Hete?+GLc`yOKhAjUt;I-g2;A!AWupQhT$fy6K;Eh0X z{-ePqK<5Cy51IZ~K^Ht2+yQ(OdHx&VQ{d;p&w(b`2lj#=0>6t)e+=9ooCiLKO#cty zPr*~bH9&H{{QCbl^8D+-D!3j<#vcX$hD`q@Ab4g(w!^2w0#U#&3x2OLmyD#S+)tWT0}V*M^3Qkl3F8N(QrND0TKZh6w*r4fO)1oXPTOY0fi!&J!fYFBF`r9* zp&2a|V^(}otWuPIK8eMj#;t_Pm6(&XDMJ^TI_+^$f|LaIR-|@$ z6et1Dq4`dZPJzlou?&M!%1r13JzN!K1ED8H?KEz z!=N+>v7TI})y9E@Pz`%!yr%{SER*)&TuwW&os8XgF4z7v+XQ_U@U2!)H98kraxs^` z7XXUsjiYE6@n{z_|2vL#85Y}$6RQA+lBS&}I7OlDsEw1l1hKz@3x{^J@tad*bUnbL z2_TGbvAZ%wb2b^hzLEXI&cTm(JiAWiL*xk((McScdkdURo}aC!xuV+gZtXHwl~2~DT4;te|6UPVVN4@%;4wEU}98}SSox=jHUA!h1aYieUlXn zby3<-rtY%&)~XePMwQhQ^D3&`BLg$y$hPco+@)g4rJGVp^XHMGpDW3wmujz*RvL6L zp@VS)O_;6MNv|TtV6r>iS@_Ml(@bf1}hZ%V)V-T!v9$cG}GYj z0{YSi@_&{Bc^~L0%n;^cA3~R(m!gTsnNuXf>zoRv+{LAO8=+Sxe=atbolkfvmOsca zHxz^J&|j*m(WedR?A6w=Obd3%S%M|J=hj|Zkx)D|IA2=Q zpVLX(V(OMHuaK~Gw8|tyP<^cdo997=^N+NHI}>#ct)xf8&fYM(84y3q67aCYS(sG9 zp)7WKa+>jJ9?9Yh=Zjfo0wWU&ct(b{THYM|SjYSXJ5+VrP*@|PV!?*pa2 z(aZlwz5M@r-2JR0&gIxbE&;*;nUBLU1>z@N8 z=idu_2HE~y;F;hVKxh143GN4O4Q>TKj(o5E{POR=Kln#v`nQ0mgKNNTz?YEeKL!36 z1e<=7-xDAMHSkGfdfo5$WN-sG40eLAAlv^3_#5z(KtBIZ1CIl`58!-oOYj-w{C9#k zgATYSc>68gJ%4ZD`&Hlt;Q8P1G|%^itj8Chi;DKwi#sLjWwA@ zTNocm#+@Ha1J-QXIE*Q-wR|nBT6;{)>|tA<4v(of#aE0SIqU?;&0CKcGv~!g$*)OV zFJ+E-+a|A~(0CT<#>l0FeVL!Y%hpmmi)i+;KbBjKOYsi3IEY-ywj_FCnqO$zt@Hx{Hx{@A zFF}kb;cRrn9OhML&NtEZ@R;j2=fOL6#7U=z$E5#=e<_QV{uueIoyC_-PwRk>>FEJ= zh|do1yO4f}D;o~~U1&^e$f5KQm-jx}g)!F|E+dkfp5~Tk&;Fa5yjs$g! z1*+NmauC|qx0J=9HitV=dQp_Y*Kx$!6jJX*R+0YKY36AchjgR3s#AKCa)w5(DK~wj zpFpISOy`?&6E`I{c4_NE%>r0cyZ5dMP;1!hZ8bX+DAJ44Qpb;GyIVcF8Vj@8Z(%>Z zpPp%POtILlc)#Tvw&MQXv|i<-=mw@lmo_Um)u+f*h2peI8Po9c#8Z(BC%7_PR5eBsnE3{N3y? zp%B%Iis~O10nHeRgU-l7c>y`9S0J^TnVOK_%pmF1b1rRK&spd-CauOn%_I8uy%tsE zsbYaueNf4556#L@AGzWp5ttqR-oG+&R7=80IhG~iu1wlr1h0*+C_tQ+R8|L7#;s(b zy)Cmzn~XFS%Fa<2$3fwhJ+9e%CwJQ+xk(Xh78mt1+E3Qa01=Mx{?$ zYM17jmU++q$L!dM@M8gTm&=MGlC{G*YDb7C~THs9AAKB@PKy6nXc6 zrL1n0WrS6jq2dCPHiqsdE}!IMjaanH#YC%uNS#dqckkG Date: Wed, 20 May 2026 17:01:12 -0700 Subject: [PATCH 05/21] move helper methods --- ats/completion_detector.py | 327 +++++++++++++++++++++++- ats/completion_fast_path.py | 9 +- ats/completion_legacy_poll.py | 6 +- ats/completion_queue.py | 163 +++++++++++- ats/machines.py | 460 +--------------------------------- 5 files changed, 492 insertions(+), 473 deletions(-) diff --git a/ats/completion_detector.py b/ats/completion_detector.py index d817d31..1261691 100644 --- a/ats/completion_detector.py +++ b/ats/completion_detector.py @@ -2,8 +2,11 @@ from abc import ABC, abstractmethod import os +import selectors +import threading +import time -from ats.atsut import AtsError +from ats.atsut import AtsError, PASSED def completion_detection_mode_from_env(default="fast_path"): @@ -44,6 +47,19 @@ def __init__(self, machine): """ self.machine = machine + def completion_fast_path_drain_limit(self): + """Return the configured maximum completions drained per wakeup. + + Returns: + int: Positive completion drain limit. + """ + limit = getattr(self.machine, "completion_fast_path_drain_limit", 128) + try: + limit = int(limit) + except (TypeError, ValueError): + limit = 128 + return max(1, limit) + @property def uses_completion_queue(self): """Return whether completion signals should be queued. @@ -73,7 +89,18 @@ def prepare_for_launch(self, test): None: Completion wait primitives are installed when needed. """ if self.uses_signal_wait: - self.machine._ensurePidfd(test) + self.ensure_pidfd(test) + + def close_for_test(self, test): + """Release completion-detector resources associated with one test. + + Args: + test: ATS test object that may own detector-specific wait state. + + Returns: + None: Detector-owned wait state is cleaned up when present. + """ + self.close_pidfd(test) def wait_for_completion_signal(self): """Wait for likely completions according to detector policy. @@ -81,9 +108,301 @@ def wait_for_completion_signal(self): Returns: list: Tests that were signaled as likely completed during the wait. """ - return self.machine._waitForCompletionSignal( - use_queue_event_wait=self.uses_completion_queue, + start_us = time.time_ns() // 1000 + machine = self.machine + machine._incrementCompletionStat("_waitForCompletionSignal_called") + registered = False + registered_count = 0 + ready = [] + selector = None + used_queue_event_wait = False + result_kind = "sleep_fallback" + try: + try: + selector = selectors.DefaultSelector() + except Exception: + selector = None + + if selector is not None: + try: + for test in machine.running: + pidfd = self.ensure_pidfd(test) + if pidfd is None: + continue + try: + selector.register(pidfd, selectors.EVENT_READ, test) + registered = True + registered_count += 1 + except Exception: + self.close_pidfd(test) + if registered: + machine._incrementCompletionStat("_waitForCompletionSignal_pidfd_registered") + ready = [key.data for key, _mask in selector.select(machine.naptime)] + if ready: + result_kind = "pidfd_ready" + machine._incrementCompletionStat("_waitForCompletionSignal_pidfd_ready") + machine._incrementCompletionStat("_waitForCompletionSignal_total_ready", len(ready)) + for test in ready: + self.record_completion_signal(test) + else: + result_kind = "pidfd_timeout" + machine._incrementCompletionStat("_waitForCompletionSignal_pidfd_timeout") + finally: + selector.close() + + if registered: + return ready + + if self.uses_completion_queue: + used_queue_event_wait = True + result_kind = "queue_event_wait" + machine._incrementCompletionStat("_waitForCompletionSignal_queue_event_wait") + machine._completionEvent.wait(machine.naptime) + return [] + + machine._incrementCompletionStat("_waitForCompletionSignal_sleep_fallback") + time.sleep(machine.naptime) + return [] + finally: + machine._recordCompletionInternalSpan( + "_waitForCompletionSignal", + start_us, + time.time_ns() // 1000, + metadata={ + "mode": getattr(machine, "completion_detection_mode", ""), + "running_count": len(machine.running), + "registered": bool(registered), + "registered_count": registered_count, + "ready_count": len(ready), + "used_queue_event_wait": bool(used_queue_event_wait), + "result": result_kind, + }, + ) + + def poll_running_tests( + self, + allow_running_checks, + prioritized=None, + stop_after_completion=False, + completion_limit=None, + ): + """Poll running tests, optionally prioritizing likely completions. + + Args: + allow_running_checks (bool): When ``False``, skip timeout and + runtime error checks for children that have not yet exited. + prioritized (iterable|None): Optional running-test candidates to + check before the rest of ``machine.running``. + stop_after_completion (bool): If ``True``, stop after the first + completed test is handled. + completion_limit (int|None): Maximum number of completions to + process before returning control to the scheduler. + + Returns: + int: Number of completed tests processed in this polling pass. + """ + from ats import configuration + + machine = self.machine + start_us = time.time_ns() // 1000 + machine._incrementCompletionStat("_pollRunningTests_called") + if allow_running_checks: + machine._incrementCompletionStat("_pollRunningTests_allow_running_checks_true") + else: + machine._incrementCompletionStat("_pollRunningTests_allow_running_checks_false") + + prioritized = list(prioritized or []) + prioritized_count = len(prioritized) + ordered_count = 0 + completed = 0 + result_kind = "completed_none" + try: + ordered = [] + seen_ids = set() + for test in prioritized: + test_id = id(test) + if test_id in seen_ids: + continue + ordered.append(test) + seen_ids.add(test_id) + for test in machine.running: + test_id = id(test) + if test_id in seen_ids: + continue + ordered.append(test) + seen_ids.add(test_id) + + ordered_count = len(ordered) + machine._incrementCompletionStat("_pollRunningTests_total_ordered", ordered_count) + + remaining = [] + for index, test in enumerate(ordered): + done = machine.getStatus(test, allow_running_checks=allow_running_checks) + if not done: + remaining.append(test) + continue + completed += 1 + if test.status is not PASSED and configuration.options.oneFailure: + raise AtsError("Test failed in oneFailure mode.") + if stop_after_completion or ( + completion_limit is not None and completed >= completion_limit + ): + remaining.extend(ordered[index + 1:]) + self.preserve_new_running_tests(remaining, seen_ids) + machine.running = remaining + result_kind = "stopped_after_completion" + machine._incrementCompletionStat("_pollRunningTests_stopped_after_completion") + machine._incrementCompletionStat("_pollRunningTests_total_completed", completed) + return completed + + self.preserve_new_running_tests(remaining, seen_ids) + machine.running = remaining + machine._incrementCompletionStat("_pollRunningTests_total_completed", completed) + if completed: + result_kind = "completed" + machine._incrementCompletionStat("_pollRunningTests_completed") + else: + machine._incrementCompletionStat("_pollRunningTests_completed_none") + return completed + finally: + machine._recordCompletionInternalSpan( + "_pollRunningTests", + start_us, + time.time_ns() // 1000, + metadata={ + "mode": getattr(machine, "completion_detection_mode", ""), + "allow_running_checks": bool(allow_running_checks), + "prioritized_count": prioritized_count, + "ordered_count": ordered_count, + "stop_after_completion": bool(stop_after_completion), + "completion_limit": completion_limit, + "completed_count": completed, + "result": result_kind, + }, + ) + + def preserve_new_running_tests(self, remaining, seen_ids): + """Keep tests appended to ``machine.running`` during completion callbacks. + + Args: + remaining (list): Running tests that should remain after the current + polling pass. + seen_ids (set): Object ids already considered in the polling pass. + + Returns: + None: ``remaining`` is updated in place. + """ + remaining_ids = {id(test) for test in remaining} + for test in self.machine.running: + test_id = id(test) + if test_id in seen_ids or test_id in remaining_ids: + continue + remaining.append(test) + remaining_ids.add(test_id) + + def record_completion_signal(self, test, observed_us=None): + """Record a likely completion signal for one running test. + + Args: + test: ATS test object associated with the completion signal. + observed_us (int|None): Signal timestamp in microseconds. Uses the + current time when omitted. + + Returns: + None: Internal timestamps, queue state, and statistics are updated. + """ + machine = self.machine + if observed_us is None: + observed_us = time.time_ns() // 1000 + if getattr(test, "ats_completion_signal_us", None) is None: + test.ats_completion_signal_us = observed_us + machine._incrementCompletionStat("completion_signal_recorded") + + def ensure_pidfd(self, test): + """Return or create a pidfd for one running child when supported. + + Args: + test: ATS test object whose child process should be observed. + + Returns: + int|None: Open pidfd file descriptor, or ``None`` when pidfds are + unavailable and ATS must use the watcher fallback. + """ + machine = self.machine + if getattr(machine, "_pidfdUnavailable", False): + self.ensure_completion_watcher(test) + return None + pidfd = getattr(test, "_pidfd", None) + if pidfd is not None: + return pidfd + if not hasattr(os, "pidfd_open"): + machine._pidfdUnavailable = True + self.ensure_completion_watcher(test) + return None + child = getattr(test, "child", None) + if child is None or getattr(child, "pid", None) is None: + return None + try: + pidfd = os.pidfd_open(child.pid) + except OSError: + self.ensure_completion_watcher(test) + return None + except AttributeError: + machine._pidfdUnavailable = True + self.ensure_completion_watcher(test) + return None + test._pidfd = pidfd + return pidfd + + def ensure_completion_watcher(self, test): + """Start the watcher-thread fallback for completion signaling. + + Args: + test: ATS test object whose child should be watched with + ``child.wait()``. + + Returns: + None: A daemon watcher thread is created at most once per test. + """ + child = getattr(test, "child", None) + if child is None: + return + watcher = getattr(test, "_completionWatcher", None) + if watcher is not None: + return + + def _watch_for_completion(): + try: + child.wait() + except Exception: + return + self.record_completion_signal(test) + + watcher = threading.Thread( + target=_watch_for_completion, + name=f"ats-completion-{getattr(child, 'pid', 'unknown')}", + daemon=True, ) + test._completionWatcher = watcher + watcher.start() + + def close_pidfd(self, test): + """Close a pidfd associated with one test if it exists. + + Args: + test: ATS test object that may own ``_pidfd``. + + Returns: + None: Missing or already-closed pidfds are ignored. + """ + pidfd = getattr(test, "_pidfd", None) + if pidfd is None: + return + try: + os.close(pidfd) + except OSError: + pass + test._pidfd = None @abstractmethod def check_running(self): diff --git a/ats/completion_fast_path.py b/ats/completion_fast_path.py index 3efaaff..43cc44a 100644 --- a/ats/completion_fast_path.py +++ b/ats/completion_fast_path.py @@ -15,18 +15,17 @@ def check_running(self): None: Completed tests may be finalized and removed from ``machine.running``. """ - machine = self.machine - completion_limit = machine._completionFastPathDrainLimit() - if machine._pollRunningTests( + completion_limit = self.completion_fast_path_drain_limit() + if self.poll_running_tests( allow_running_checks=False, completion_limit=completion_limit, ): return completion_hints = self.wait_for_completion_signal() - if completion_hints and machine._pollRunningTests( + if completion_hints and self.poll_running_tests( allow_running_checks=False, prioritized=completion_hints, completion_limit=completion_limit, ): return - machine._pollRunningTests(allow_running_checks=True) + self.poll_running_tests(allow_running_checks=True) diff --git a/ats/completion_legacy_poll.py b/ats/completion_legacy_poll.py index 4c63560..288eea7 100644 --- a/ats/completion_legacy_poll.py +++ b/ats/completion_legacy_poll.py @@ -27,14 +27,14 @@ def check_running(self): ``machine.running``. """ machine = self.machine - completion_limit = machine._completionFastPathDrainLimit() - if machine._pollRunningTests( + completion_limit = self.completion_fast_path_drain_limit() + if self.poll_running_tests( allow_running_checks=True, completion_limit=completion_limit, ): return time.sleep(machine.naptime) - machine._pollRunningTests( + self.poll_running_tests( allow_running_checks=True, completion_limit=completion_limit, ) diff --git a/ats/completion_queue.py b/ats/completion_queue.py index 9d6afd7..66f34a8 100644 --- a/ats/completion_queue.py +++ b/ats/completion_queue.py @@ -1,5 +1,8 @@ """Queued completion detector for ATS machines.""" +import time + +from ats.atsut import AtsError, PASSED from ats.completion_detector import CompletionDetector @@ -25,20 +28,172 @@ def check_running(self): ``machine.running``. """ machine = self.machine - completion_limit = machine._completionFastPathDrainLimit() + completion_limit = self.completion_fast_path_drain_limit() machine._incrementCompletionStat("check_running_completion_queue_mode") - if machine._pollQueuedCompletionTests(completion_limit=completion_limit): + if self.poll_queued_completion_tests(completion_limit=completion_limit): machine._incrementCompletionStat("check_running_queue_pre_drain_completed") return machine._incrementCompletionStat("check_running_queue_pre_drain_empty") machine._incrementCompletionStat("check_running_wait_for_completion_signal") self.wait_for_completion_signal() - if machine._pollQueuedCompletionTests(completion_limit=completion_limit): + if self.poll_queued_completion_tests(completion_limit=completion_limit): machine._incrementCompletionStat("check_running_queue_post_wait_completed") return machine._incrementCompletionStat("check_running_queue_post_wait_empty") machine._incrementCompletionStat("check_running_queue_fallback_poll_running") - machine._pollRunningTests( + self.poll_running_tests( allow_running_checks=True, completion_limit=completion_limit, ) + + def record_completion_signal(self, test, observed_us=None): + """Record a likely completion signal and enqueue it for later draining. + + Args: + test: ATS test object associated with the completion signal. + observed_us (int|None): Signal timestamp in microseconds. Uses the + current time when omitted. + + Returns: + None: Internal timestamps, queue state, and statistics are updated. + """ + machine = self.machine + super().record_completion_signal(test, observed_us=observed_us) + if observed_us is None: + observed_us = getattr(test, "ats_completion_signal_us", None) + with machine._completionQueueLock: + test_id = id(test) + if test_id in machine._completionQueueIds: + machine._incrementCompletionStat("completion_queue_duplicate_signal") + return + machine._completionQueue.append(test) + machine._completionQueueIds.add(test_id) + machine._completionEvent.set() + machine._incrementCompletionStat("completion_queue_enqueued") + depth = len(machine._completionQueue) + machine._recordCompletionQueueSnapshot( + depth, + "completion_queue_enqueue", + timestamp_us=observed_us, + ) + + def drain_completion_queue(self, completion_limit=None): + """Remove queued completion candidates up to the configured limit. + + Args: + completion_limit (int|None): Maximum number of queued tests to + return. ``None`` drains the entire queue. + + Returns: + list: Queued tests selected for completion re-checking. + """ + machine = self.machine + queued = [] + with machine._completionQueueLock: + while machine._completionQueue: + if completion_limit is not None and len(queued) >= completion_limit: + break + test = machine._completionQueue.popleft() + machine._completionQueueIds.discard(id(test)) + queued.append(test) + remaining_depth = len(machine._completionQueue) + if not machine._completionQueue: + machine._completionEvent.clear() + if queued: + machine._recordCompletionQueueSnapshot( + remaining_depth, + "completion_queue_drain", + metadata={ + "drained_count": len(queued), + "completion_limit": completion_limit, + }, + ) + return queued + + def poll_queued_completion_tests(self, completion_limit=None): + """Handle completion candidates from the queued completion path. + + Args: + completion_limit (int|None): Maximum number of queued candidates to + process in this pass. + + Returns: + int: Number of running tests confirmed completed in this pass. + """ + from ats import configuration + + machine = self.machine + start_us = time.time_ns() // 1000 + machine._incrementCompletionStat("_pollQueuedCompletionTests_called") + queued_count = 0 + selected_count = 0 + stale_count = 0 + completed = 0 + result_kind = "empty" + try: + queued = self.drain_completion_queue(completion_limit=completion_limit) + queued_count = len(queued) + machine._incrementCompletionStat("_pollQueuedCompletionTests_total_queued", queued_count) + if not queued: + machine._incrementCompletionStat("_pollQueuedCompletionTests_empty") + return 0 + + selected = [] + selected_ids = set() + running_ids = {id(test) for test in machine.running} + for test in queued: + test_id = id(test) + if test_id in selected_ids: + continue + if test_id not in running_ids: + stale_count += 1 + continue + selected.append(test) + selected_ids.add(test_id) + + selected_count = len(selected) + machine._incrementCompletionStat("_pollQueuedCompletionTests_total_selected", selected_count) + machine._incrementCompletionStat("_pollQueuedCompletionTests_total_stale", stale_count) + if stale_count: + machine._incrementCompletionStat("_pollQueuedCompletionTests_saw_stale_entries") + if not selected: + result_kind = "stale_only" + machine._incrementCompletionStat("_pollQueuedCompletionTests_selected_none") + return 0 + + completed_ids = set() + for test in selected: + done = machine.getStatus(test, allow_running_checks=False) + if not done: + continue + completed_ids.add(id(test)) + completed += 1 + if test.status is not PASSED and configuration.options.oneFailure: + raise AtsError("Test failed in oneFailure mode.") + + machine._incrementCompletionStat("_pollQueuedCompletionTests_total_completed", completed) + if completed_ids: + machine.running = [ + test for test in machine.running if id(test) not in completed_ids + ] + result_kind = "completed" + machine._incrementCompletionStat("_pollQueuedCompletionTests_completed") + else: + result_kind = "selected_none_completed" + machine._incrementCompletionStat("_pollQueuedCompletionTests_selected_none_completed") + return completed + finally: + machine._recordCompletionInternalSpan( + "_pollQueuedCompletionTests", + start_us, + time.time_ns() // 1000, + metadata={ + "mode": getattr(machine, "completion_detection_mode", ""), + "completion_limit": completion_limit, + "queued_count": queued_count, + "selected_count": selected_count, + "stale_count": stale_count, + "completed_count": completed, + "result": result_kind, + }, + ) diff --git a/ats/machines.py b/ats/machines.py index dbf7244..5ca0eac 100644 --- a/ats/machines.py +++ b/ats/machines.py @@ -1,7 +1,7 @@ """Definition of class Machine for overriding. """ from collections import deque -import selectors, subprocess, sys, os, threading, time, shlex +import subprocess, sys, os, threading, time, shlex from ats.completion_detector import ( completion_detection_mode_from_env, create_completion_detector, @@ -197,209 +197,6 @@ def getStatus(self, test, allow_running_checks=True): return False - def _pollRunningTests( - self, - allow_running_checks, - prioritized=None, - stop_after_completion=False, - completion_limit=None, - ): - """Poll running tests, optionally prioritizing likely completions. - - Args: - allow_running_checks (bool): When ``False``, skip timeout and - runtime error checks for children that have not yet exited. - prioritized (iterable|None): Optional running-test candidates to - check before the rest of ``self.running``. - stop_after_completion (bool): If ``True``, stop after the first - completed test is handled. - completion_limit (int|None): Maximum number of completions to - process before returning control to the scheduler. - - Returns: - int: Number of completed tests processed in this polling pass. - """ - from ats import configuration - - start_us = time.time_ns() // 1000 - self._incrementCompletionStat("_pollRunningTests_called") - if allow_running_checks: - self._incrementCompletionStat("_pollRunningTests_allow_running_checks_true") - else: - self._incrementCompletionStat("_pollRunningTests_allow_running_checks_false") - - prioritized = list(prioritized or []) - prioritized_count = len(prioritized) - ordered_count = 0 - completed = 0 - result_kind = "completed_none" - try: - ordered = [] - seen_ids = set() - for test in prioritized: - test_id = id(test) - if test_id in seen_ids: - continue - ordered.append(test) - seen_ids.add(test_id) - for test in self.running: - test_id = id(test) - if test_id in seen_ids: - continue - ordered.append(test) - seen_ids.add(test_id) - - ordered_count = len(ordered) - self._incrementCompletionStat("_pollRunningTests_total_ordered", ordered_count) - - remaining = [] - for index, test in enumerate(ordered): - done = self.getStatus(test, allow_running_checks=allow_running_checks) - if not done: - remaining.append(test) - continue - completed += 1 - if test.status is not PASSED and configuration.options.oneFailure: - raise AtsError("Test failed in oneFailure mode.") - if stop_after_completion or ( - completion_limit is not None and completed >= completion_limit - ): - remaining.extend(ordered[index + 1:]) - self._preserve_new_running_tests(remaining, seen_ids) - self.running = remaining - result_kind = "stopped_after_completion" - self._incrementCompletionStat("_pollRunningTests_stopped_after_completion") - self._incrementCompletionStat("_pollRunningTests_total_completed", completed) - return completed - - self._preserve_new_running_tests(remaining, seen_ids) - self.running = remaining - self._incrementCompletionStat("_pollRunningTests_total_completed", completed) - if completed: - result_kind = "completed" - self._incrementCompletionStat("_pollRunningTests_completed") - else: - self._incrementCompletionStat("_pollRunningTests_completed_none") - return completed - finally: - self._recordCompletionInternalSpan( - "_pollRunningTests", - start_us, - time.time_ns() // 1000, - metadata={ - "mode": getattr(self, "completion_detection_mode", ""), - "allow_running_checks": bool(allow_running_checks), - "prioritized_count": prioritized_count, - "ordered_count": ordered_count, - "stop_after_completion": bool(stop_after_completion), - "completion_limit": completion_limit, - "completed_count": completed, - "result": result_kind, - }, - ) - - def _preserve_new_running_tests(self, remaining, seen_ids): - """Keep tests appended to ``self.running`` during completion callbacks. - - Args: - remaining (list): Running tests that should remain after the current - polling pass. - seen_ids (set): Object ids already considered in the polling pass. - - Returns: - None: ``remaining`` is updated in place. - """ - remaining_ids = {id(test) for test in remaining} - for test in self.running: - test_id = id(test) - if test_id in seen_ids or test_id in remaining_ids: - continue - remaining.append(test) - remaining_ids.add(test_id) - - def _waitForCompletionSignal(self, use_queue_event_wait=False): - """Wait for a local child exit, using pidfds when available. - - Args: - use_queue_event_wait (bool): When ``True``, fall back to waiting on - the machine completion event instead of sleeping if pidfds are - unavailable. - - Returns: - list: Running tests that were signaled as likely completed during - this wait interval. Queue-wait and sleep-fallback paths return an - empty list. - """ - start_us = time.time_ns() // 1000 - self._incrementCompletionStat("_waitForCompletionSignal_called") - registered = False - registered_count = 0 - ready = [] - selector = None - used_queue_event_wait = False - result_kind = "sleep_fallback" - try: - selector = selectors.DefaultSelector() - except Exception: - selector = None - - try: - if selector is not None: - try: - for test in self.running: - pidfd = self._ensurePidfd(test) - if pidfd is None: - continue - try: - selector.register(pidfd, selectors.EVENT_READ, test) - registered = True - registered_count += 1 - except Exception: - self._closePidfd(test) - if registered: - self._incrementCompletionStat("_waitForCompletionSignal_pidfd_registered") - ready = [key.data for key, _mask in selector.select(self.naptime)] - if ready: - result_kind = "pidfd_ready" - self._incrementCompletionStat("_waitForCompletionSignal_pidfd_ready") - self._incrementCompletionStat("_waitForCompletionSignal_total_ready", len(ready)) - for test in ready: - self._recordCompletionSignal(test) - else: - result_kind = "pidfd_timeout" - self._incrementCompletionStat("_waitForCompletionSignal_pidfd_timeout") - finally: - selector.close() - - if registered: - return ready - - if use_queue_event_wait: - used_queue_event_wait = True - result_kind = "queue_event_wait" - self._incrementCompletionStat("_waitForCompletionSignal_queue_event_wait") - self._completionEvent.wait(self.naptime) - return [] - - self._incrementCompletionStat("_waitForCompletionSignal_sleep_fallback") - time.sleep(self.naptime) - return [] - finally: - self._recordCompletionInternalSpan( - "_waitForCompletionSignal", - start_us, - time.time_ns() // 1000, - metadata={ - "mode": getattr(self, "completion_detection_mode", ""), - "running_count": len(self.running), - "registered": bool(registered), - "registered_count": registered_count, - "ready_count": len(ready), - "used_queue_event_wait": bool(used_queue_event_wait), - "result": result_kind, - }, - ) - def _completionStatsEnabled(self): """Return whether aggregated completion counters should be tracked. @@ -581,19 +378,6 @@ def _recordCompletionQueueSnapshot(self, depth, reason, timestamp_us=None, metad for callback in list(getattr(self, "_completion_queue_snapshot_hooks", [])): callback(timestamp_us, payload) - def _completionFastPathDrainLimit(self): - """Return the configured maximum completions drained per wakeup. - - Returns: - int: Positive completion drain limit. - """ - limit = getattr(self, "completion_fast_path_drain_limit", 128) - try: - limit = int(limit) - except (TypeError, ValueError): - limit = 128 - return max(1, limit) - def _pollChild(self, test): """Poll one child process and return its current return code. @@ -606,159 +390,6 @@ def _pollChild(self, test): test.child.poll() return test.child.returncode - def _recordCompletionSignal(self, test, observed_us=None): - """Record a likely completion signal for one running test. - - Args: - test: ATS test object associated with the completion signal. - observed_us (int|None): Signal timestamp in microseconds. Uses the - current time when omitted. - - Returns: - None: Internal timestamps, queue state, and statistics are updated. - """ - if observed_us is None: - observed_us = time.time_ns() // 1000 - if getattr(test, "ats_completion_signal_us", None) is None: - test.ats_completion_signal_us = observed_us - self._incrementCompletionStat("completion_signal_recorded") - if not getattr(self._completionDetector, "uses_completion_queue", False): - return - with self._completionQueueLock: - test_id = id(test) - if test_id in self._completionQueueIds: - self._incrementCompletionStat("completion_queue_duplicate_signal") - return - self._completionQueue.append(test) - self._completionQueueIds.add(test_id) - self._completionEvent.set() - self._incrementCompletionStat("completion_queue_enqueued") - depth = len(self._completionQueue) - self._recordCompletionQueueSnapshot( - depth, - "completion_queue_enqueue", - timestamp_us=observed_us, - ) - - def _drainCompletionQueue(self, completion_limit=None): - """Remove queued completion candidates up to the configured limit. - - Args: - completion_limit (int|None): Maximum number of queued tests to - return. ``None`` drains the entire queue. - - Returns: - list: Queued tests selected for completion re-checking. - """ - queued = [] - with self._completionQueueLock: - while self._completionQueue: - if completion_limit is not None and len(queued) >= completion_limit: - break - test = self._completionQueue.popleft() - self._completionQueueIds.discard(id(test)) - queued.append(test) - remaining_depth = len(self._completionQueue) - if not self._completionQueue: - self._completionEvent.clear() - if queued: - self._recordCompletionQueueSnapshot( - remaining_depth, - "completion_queue_drain", - metadata={ - "drained_count": len(queued), - "completion_limit": completion_limit, - }, - ) - return queued - - def _pollQueuedCompletionTests(self, completion_limit=None): - """Handle completion candidates from the queued completion path. - - Args: - completion_limit (int|None): Maximum number of queued candidates to - process in this pass. - - Returns: - int: Number of running tests confirmed completed in this pass. - """ - from ats import configuration - - start_us = time.time_ns() // 1000 - self._incrementCompletionStat("_pollQueuedCompletionTests_called") - queued_count = 0 - selected_count = 0 - stale_count = 0 - completed = 0 - result_kind = "empty" - try: - queued = self._drainCompletionQueue(completion_limit=completion_limit) - queued_count = len(queued) - self._incrementCompletionStat("_pollQueuedCompletionTests_total_queued", queued_count) - if not queued: - self._incrementCompletionStat("_pollQueuedCompletionTests_empty") - return 0 - - selected = [] - selected_ids = set() - running_ids = {id(test) for test in self.running} - for test in queued: - test_id = id(test) - if test_id in selected_ids: - continue - if test_id not in running_ids: - stale_count += 1 - continue - selected.append(test) - selected_ids.add(test_id) - - selected_count = len(selected) - self._incrementCompletionStat("_pollQueuedCompletionTests_total_selected", selected_count) - self._incrementCompletionStat("_pollQueuedCompletionTests_total_stale", stale_count) - if stale_count: - self._incrementCompletionStat("_pollQueuedCompletionTests_saw_stale_entries") - if not selected: - result_kind = "stale_only" - self._incrementCompletionStat("_pollQueuedCompletionTests_selected_none") - return 0 - - completed_ids = set() - for test in selected: - done = self.getStatus(test, allow_running_checks=False) - if not done: - continue - completed_ids.add(id(test)) - completed += 1 - if test.status is not PASSED and configuration.options.oneFailure: - raise AtsError("Test failed in oneFailure mode.") - - self._incrementCompletionStat("_pollQueuedCompletionTests_total_completed", completed) - if completed_ids: - self.running = [ - test for test in self.running if id(test) not in completed_ids - ] - result_kind = "completed" - self._incrementCompletionStat("_pollQueuedCompletionTests_completed") - else: - result_kind = "selected_none_completed" - self._incrementCompletionStat("_pollQueuedCompletionTests_selected_none_completed") - return completed - finally: - self._recordCompletionInternalSpan( - "_pollQueuedCompletionTests", - start_us, - time.time_ns() // 1000, - metadata={ - "mode": getattr(self, "completion_detection_mode", ""), - "completion_limit": completion_limit, - "queued_count": queued_count, - "selected_count": selected_count, - "stale_count": stale_count, - "completed_count": completed, - "result": result_kind, - }, - ) - def _finishCompletedTest(self, test): """Finalize status selection for a child that has already exited. @@ -841,7 +472,7 @@ def _completeTest(self, test, status): print(line) print(line, file=outhandle) - self._closePidfd(test) + self._completionDetector.close_for_test(test) self.testEnded(test, status) return True @@ -877,91 +508,6 @@ def _detectRunningSlurmError(self, test): return True return False - def _ensurePidfd(self, test): - """Return or create a pidfd for one running child when supported. - - Args: - test: ATS test object whose child process should be observed. - - Returns: - int|None: Open pidfd file descriptor, or ``None`` when pidfds are - unavailable and ATS must use the watcher fallback. - """ - if getattr(self, "_pidfdUnavailable", False): - self._ensureCompletionWatcher(test) - return None - pidfd = getattr(test, "_pidfd", None) - if pidfd is not None: - return pidfd - if not hasattr(os, "pidfd_open"): - self._pidfdUnavailable = True - self._ensureCompletionWatcher(test) - return None - child = getattr(test, "child", None) - if child is None or getattr(child, "pid", None) is None: - return None - try: - pidfd = os.pidfd_open(child.pid) - except OSError: - self._ensureCompletionWatcher(test) - return None - except AttributeError: - self._pidfdUnavailable = True - self._ensureCompletionWatcher(test) - return None - test._pidfd = pidfd - return pidfd - - def _ensureCompletionWatcher(self, test): - """Start the watcher-thread fallback for completion signaling. - - Args: - test: ATS test object whose child should be watched with - ``child.wait()``. - - Returns: - None: A daemon watcher thread is created at most once per test. - """ - child = getattr(test, "child", None) - if child is None: - return - watcher = getattr(test, "_completionWatcher", None) - if watcher is not None: - return - - def _watch_for_completion(): - try: - child.wait() - except Exception: - return - self._recordCompletionSignal(test) - - watcher = threading.Thread( - target=_watch_for_completion, - name=f"ats-completion-{getattr(child, 'pid', 'unknown')}", - daemon=True, - ) - test._completionWatcher = watcher - watcher.start() - - def _closePidfd(self, test): - """Close a pidfd associated with one test if it exists. - - Args: - test: ATS test object that may own ``_pidfd``. - - Returns: - None: Missing or already-closed pidfds are ignored. - """ - pidfd = getattr(test, "_pidfd", None) - if pidfd is None: - return - try: - os.close(pidfd) - except OSError: - pass - test._pidfd = None - def testEnded(self, test, status): """Do book-keeping when a job has exited; call noteEnd for machine-specific part. @@ -1026,7 +572,7 @@ def kill(self, test): # override if not using subprocess "Kill the job running test." if test.child: test.child.kill() - self._closePidfd(test) + self._completionDetector.close_for_test(test) if test.stdOutLocGet() != 'terminal': test.fileHandleClose() From 368bd1921817a43bb02b99a5d15cc795db514954 Mon Sep 17 00:00:00 2001 From: "Peter B. Robinson" Date: Thu, 21 May 2026 06:06:40 -0700 Subject: [PATCH 06/21] only do completion_queue in addition to legacy --- ats/completion_detector.py | 370 +-------------------- ats/completion_fast_path.py | 31 -- ats/completion_legacy_poll.py | 371 +++++++++++++++++++++- ats/completion_queue.py | 6 +- ats/configuration.py | 143 ++++++++- ats/machines.py | 21 +- ats/management.py | 32 +- docs/source/scheduler_extensions.rst | 33 +- test/test_completion_detector_examples.py | 29 +- 9 files changed, 568 insertions(+), 468 deletions(-) delete mode 100644 ats/completion_fast_path.py diff --git a/ats/completion_detector.py b/ats/completion_detector.py index 1261691..78eba86 100644 --- a/ats/completion_detector.py +++ b/ats/completion_detector.py @@ -1,24 +1,8 @@ """Completion-detection strategy interface and factory helpers.""" from abc import ABC, abstractmethod -import os -import selectors -import threading -import time -from ats.atsut import AtsError, PASSED - - -def completion_detection_mode_from_env(default="fast_path"): - """Return the configured completion-detection mode from the environment. - - Args: - default (str): Mode used when the environment variable is unset. - - Returns: - str: Requested completion-detection mode. - """ - return os.environ.get("ATS_COMPLETION_DETECTION_MODE", default) +from ats.atsut import AtsError def normalize_completion_detection_mode(mode): @@ -28,9 +12,10 @@ def normalize_completion_detection_mode(mode): mode (str|None): Requested mode name. Returns: - str: Normalized lowercase mode name, defaulting to ``"fast_path"``. + str: Normalized lowercase mode name, defaulting to + ``"completion_queue"``. """ - return str(mode or "fast_path").strip().lower() or "fast_path" + return str(mode or "completion_queue").strip().lower() or "completion_queue" class CompletionDetector(ABC): @@ -47,38 +32,6 @@ def __init__(self, machine): """ self.machine = machine - def completion_fast_path_drain_limit(self): - """Return the configured maximum completions drained per wakeup. - - Returns: - int: Positive completion drain limit. - """ - limit = getattr(self.machine, "completion_fast_path_drain_limit", 128) - try: - limit = int(limit) - except (TypeError, ValueError): - limit = 128 - return max(1, limit) - - @property - def uses_completion_queue(self): - """Return whether completion signals should be queued. - - Returns: - bool: ``True`` when signals should be enqueued for later draining. - """ - return False - - @property - def uses_signal_wait(self): - """Return whether child-completion wait primitives should be prepared. - - Returns: - bool: ``True`` when pidfds or watcher threads should be set up at - launch time. - """ - return True - def prepare_for_launch(self, test): """Prepare one launched test for completion signaling. @@ -86,10 +39,8 @@ def prepare_for_launch(self, test): test: ATS test object whose child process has just been launched. Returns: - None: Completion wait primitives are installed when needed. + None: Default detector preparation is a no-op. """ - if self.uses_signal_wait: - self.ensure_pidfd(test) def close_for_test(self, test): """Release completion-detector resources associated with one test. @@ -98,311 +49,8 @@ def close_for_test(self, test): test: ATS test object that may own detector-specific wait state. Returns: - None: Detector-owned wait state is cleaned up when present. - """ - self.close_pidfd(test) - - def wait_for_completion_signal(self): - """Wait for likely completions according to detector policy. - - Returns: - list: Tests that were signaled as likely completed during the wait. - """ - start_us = time.time_ns() // 1000 - machine = self.machine - machine._incrementCompletionStat("_waitForCompletionSignal_called") - registered = False - registered_count = 0 - ready = [] - selector = None - used_queue_event_wait = False - result_kind = "sleep_fallback" - try: - try: - selector = selectors.DefaultSelector() - except Exception: - selector = None - - if selector is not None: - try: - for test in machine.running: - pidfd = self.ensure_pidfd(test) - if pidfd is None: - continue - try: - selector.register(pidfd, selectors.EVENT_READ, test) - registered = True - registered_count += 1 - except Exception: - self.close_pidfd(test) - if registered: - machine._incrementCompletionStat("_waitForCompletionSignal_pidfd_registered") - ready = [key.data for key, _mask in selector.select(machine.naptime)] - if ready: - result_kind = "pidfd_ready" - machine._incrementCompletionStat("_waitForCompletionSignal_pidfd_ready") - machine._incrementCompletionStat("_waitForCompletionSignal_total_ready", len(ready)) - for test in ready: - self.record_completion_signal(test) - else: - result_kind = "pidfd_timeout" - machine._incrementCompletionStat("_waitForCompletionSignal_pidfd_timeout") - finally: - selector.close() - - if registered: - return ready - - if self.uses_completion_queue: - used_queue_event_wait = True - result_kind = "queue_event_wait" - machine._incrementCompletionStat("_waitForCompletionSignal_queue_event_wait") - machine._completionEvent.wait(machine.naptime) - return [] - - machine._incrementCompletionStat("_waitForCompletionSignal_sleep_fallback") - time.sleep(machine.naptime) - return [] - finally: - machine._recordCompletionInternalSpan( - "_waitForCompletionSignal", - start_us, - time.time_ns() // 1000, - metadata={ - "mode": getattr(machine, "completion_detection_mode", ""), - "running_count": len(machine.running), - "registered": bool(registered), - "registered_count": registered_count, - "ready_count": len(ready), - "used_queue_event_wait": bool(used_queue_event_wait), - "result": result_kind, - }, - ) - - def poll_running_tests( - self, - allow_running_checks, - prioritized=None, - stop_after_completion=False, - completion_limit=None, - ): - """Poll running tests, optionally prioritizing likely completions. - - Args: - allow_running_checks (bool): When ``False``, skip timeout and - runtime error checks for children that have not yet exited. - prioritized (iterable|None): Optional running-test candidates to - check before the rest of ``machine.running``. - stop_after_completion (bool): If ``True``, stop after the first - completed test is handled. - completion_limit (int|None): Maximum number of completions to - process before returning control to the scheduler. - - Returns: - int: Number of completed tests processed in this polling pass. + None: Default detector cleanup is a no-op. """ - from ats import configuration - - machine = self.machine - start_us = time.time_ns() // 1000 - machine._incrementCompletionStat("_pollRunningTests_called") - if allow_running_checks: - machine._incrementCompletionStat("_pollRunningTests_allow_running_checks_true") - else: - machine._incrementCompletionStat("_pollRunningTests_allow_running_checks_false") - - prioritized = list(prioritized or []) - prioritized_count = len(prioritized) - ordered_count = 0 - completed = 0 - result_kind = "completed_none" - try: - ordered = [] - seen_ids = set() - for test in prioritized: - test_id = id(test) - if test_id in seen_ids: - continue - ordered.append(test) - seen_ids.add(test_id) - for test in machine.running: - test_id = id(test) - if test_id in seen_ids: - continue - ordered.append(test) - seen_ids.add(test_id) - - ordered_count = len(ordered) - machine._incrementCompletionStat("_pollRunningTests_total_ordered", ordered_count) - - remaining = [] - for index, test in enumerate(ordered): - done = machine.getStatus(test, allow_running_checks=allow_running_checks) - if not done: - remaining.append(test) - continue - completed += 1 - if test.status is not PASSED and configuration.options.oneFailure: - raise AtsError("Test failed in oneFailure mode.") - if stop_after_completion or ( - completion_limit is not None and completed >= completion_limit - ): - remaining.extend(ordered[index + 1:]) - self.preserve_new_running_tests(remaining, seen_ids) - machine.running = remaining - result_kind = "stopped_after_completion" - machine._incrementCompletionStat("_pollRunningTests_stopped_after_completion") - machine._incrementCompletionStat("_pollRunningTests_total_completed", completed) - return completed - - self.preserve_new_running_tests(remaining, seen_ids) - machine.running = remaining - machine._incrementCompletionStat("_pollRunningTests_total_completed", completed) - if completed: - result_kind = "completed" - machine._incrementCompletionStat("_pollRunningTests_completed") - else: - machine._incrementCompletionStat("_pollRunningTests_completed_none") - return completed - finally: - machine._recordCompletionInternalSpan( - "_pollRunningTests", - start_us, - time.time_ns() // 1000, - metadata={ - "mode": getattr(machine, "completion_detection_mode", ""), - "allow_running_checks": bool(allow_running_checks), - "prioritized_count": prioritized_count, - "ordered_count": ordered_count, - "stop_after_completion": bool(stop_after_completion), - "completion_limit": completion_limit, - "completed_count": completed, - "result": result_kind, - }, - ) - - def preserve_new_running_tests(self, remaining, seen_ids): - """Keep tests appended to ``machine.running`` during completion callbacks. - - Args: - remaining (list): Running tests that should remain after the current - polling pass. - seen_ids (set): Object ids already considered in the polling pass. - - Returns: - None: ``remaining`` is updated in place. - """ - remaining_ids = {id(test) for test in remaining} - for test in self.machine.running: - test_id = id(test) - if test_id in seen_ids or test_id in remaining_ids: - continue - remaining.append(test) - remaining_ids.add(test_id) - - def record_completion_signal(self, test, observed_us=None): - """Record a likely completion signal for one running test. - - Args: - test: ATS test object associated with the completion signal. - observed_us (int|None): Signal timestamp in microseconds. Uses the - current time when omitted. - - Returns: - None: Internal timestamps, queue state, and statistics are updated. - """ - machine = self.machine - if observed_us is None: - observed_us = time.time_ns() // 1000 - if getattr(test, "ats_completion_signal_us", None) is None: - test.ats_completion_signal_us = observed_us - machine._incrementCompletionStat("completion_signal_recorded") - - def ensure_pidfd(self, test): - """Return or create a pidfd for one running child when supported. - - Args: - test: ATS test object whose child process should be observed. - - Returns: - int|None: Open pidfd file descriptor, or ``None`` when pidfds are - unavailable and ATS must use the watcher fallback. - """ - machine = self.machine - if getattr(machine, "_pidfdUnavailable", False): - self.ensure_completion_watcher(test) - return None - pidfd = getattr(test, "_pidfd", None) - if pidfd is not None: - return pidfd - if not hasattr(os, "pidfd_open"): - machine._pidfdUnavailable = True - self.ensure_completion_watcher(test) - return None - child = getattr(test, "child", None) - if child is None or getattr(child, "pid", None) is None: - return None - try: - pidfd = os.pidfd_open(child.pid) - except OSError: - self.ensure_completion_watcher(test) - return None - except AttributeError: - machine._pidfdUnavailable = True - self.ensure_completion_watcher(test) - return None - test._pidfd = pidfd - return pidfd - - def ensure_completion_watcher(self, test): - """Start the watcher-thread fallback for completion signaling. - - Args: - test: ATS test object whose child should be watched with - ``child.wait()``. - - Returns: - None: A daemon watcher thread is created at most once per test. - """ - child = getattr(test, "child", None) - if child is None: - return - watcher = getattr(test, "_completionWatcher", None) - if watcher is not None: - return - - def _watch_for_completion(): - try: - child.wait() - except Exception: - return - self.record_completion_signal(test) - - watcher = threading.Thread( - target=_watch_for_completion, - name=f"ats-completion-{getattr(child, 'pid', 'unknown')}", - daemon=True, - ) - test._completionWatcher = watcher - watcher.start() - - def close_pidfd(self, test): - """Close a pidfd associated with one test if it exists. - - Args: - test: ATS test object that may own ``_pidfd``. - - Returns: - None: Missing or already-closed pidfds are ignored. - """ - pidfd = getattr(test, "_pidfd", None) - if pidfd is None: - return - try: - os.close(pidfd) - except OSError: - pass - test._pidfd = None @abstractmethod def check_running(self): @@ -427,10 +75,6 @@ def create_completion_detector(machine, mode): AtsError: If ``mode`` is not one of the supported detector modes. """ normalized_mode = normalize_completion_detection_mode(mode) - if normalized_mode == "fast_path": - from ats.completion_fast_path import FastPathCompletionDetector - - return FastPathCompletionDetector(machine) if normalized_mode == "completion_queue": from ats.completion_queue import CompletionQueueCompletionDetector @@ -441,5 +85,5 @@ def create_completion_detector(machine, mode): return LegacyPollCompletionDetector(machine) raise AtsError( "Unknown completion detection mode %r. Expected one of: " - "'fast_path', 'completion_queue', 'legacy_poll'." % mode + "'completion_queue', 'legacy_poll'." % mode ) diff --git a/ats/completion_fast_path.py b/ats/completion_fast_path.py deleted file mode 100644 index 43cc44a..0000000 --- a/ats/completion_fast_path.py +++ /dev/null @@ -1,31 +0,0 @@ -"""Fast-path completion detector for ATS machines.""" - -from ats.completion_detector import CompletionDetector - - -class FastPathCompletionDetector(CompletionDetector): - """Prefer signaled completions before slower running-state checks.""" - - mode_name = "fast_path" - - def check_running(self): - """Advance machine state using the fast completion-detection path. - - Returns: - None: Completed tests may be finalized and removed from - ``machine.running``. - """ - completion_limit = self.completion_fast_path_drain_limit() - if self.poll_running_tests( - allow_running_checks=False, - completion_limit=completion_limit, - ): - return - completion_hints = self.wait_for_completion_signal() - if completion_hints and self.poll_running_tests( - allow_running_checks=False, - prioritized=completion_hints, - completion_limit=completion_limit, - ): - return - self.poll_running_tests(allow_running_checks=True) diff --git a/ats/completion_legacy_poll.py b/ats/completion_legacy_poll.py index 288eea7..0c5ed38 100644 --- a/ats/completion_legacy_poll.py +++ b/ats/completion_legacy_poll.py @@ -1,11 +1,376 @@ -"""Legacy polling completion detector for ATS machines.""" +"""Legacy polling detector and shared polling helpers for ATS machines.""" +import os +import selectors +import threading import time +from ats.atsut import AtsError, PASSED from ats.completion_detector import CompletionDetector -class LegacyPollCompletionDetector(CompletionDetector): +class PollingCompletionDetector(CompletionDetector): + """Shared polling and signal-wait helpers used by ATS detectors.""" + + @property + def uses_completion_queue(self): + """Return whether completion signals should be queued. + + Returns: + bool: ``True`` when signals should be enqueued for later draining. + """ + return False + + @property + def uses_signal_wait(self): + """Return whether child-completion wait primitives should be prepared. + + Returns: + bool: ``True`` when pidfds or watcher threads should be set up at + launch time. + """ + return True + + def completion_drain_limit(self): + """Return the configured maximum completions drained per wakeup. + + Returns: + int: Positive completion drain limit. + """ + limit = getattr(self.machine, "completion_fast_path_drain_limit", 128) + try: + limit = int(limit) + except (TypeError, ValueError): + limit = 128 + return max(1, limit) + + def prepare_for_launch(self, test): + """Prepare one launched test for completion signaling. + + Args: + test: ATS test object whose child process has just been launched. + + Returns: + None: Completion wait primitives are installed when needed. + """ + if self.uses_signal_wait: + self.ensure_pidfd(test) + + def close_for_test(self, test): + """Release completion-detector resources associated with one test. + + Args: + test: ATS test object that may own detector-specific wait state. + + Returns: + None: Detector-owned wait state is cleaned up when present. + """ + self.close_pidfd(test) + + def wait_for_completion_signal(self): + """Wait for likely completions according to detector policy. + + Returns: + list: Tests that were signaled as likely completed during the wait. + """ + start_us = time.time_ns() // 1000 + machine = self.machine + machine._incrementCompletionStat("_waitForCompletionSignal_called") + registered = False + registered_count = 0 + ready = [] + selector = None + used_queue_event_wait = False + result_kind = "sleep_fallback" + try: + try: + selector = selectors.DefaultSelector() + except Exception: + selector = None + + if selector is not None: + try: + for test in machine.running: + pidfd = self.ensure_pidfd(test) + if pidfd is None: + continue + try: + selector.register(pidfd, selectors.EVENT_READ, test) + registered = True + registered_count += 1 + except Exception: + self.close_pidfd(test) + if registered: + machine._incrementCompletionStat("_waitForCompletionSignal_pidfd_registered") + ready = [key.data for key, _mask in selector.select(machine.naptime)] + if ready: + result_kind = "pidfd_ready" + machine._incrementCompletionStat("_waitForCompletionSignal_pidfd_ready") + machine._incrementCompletionStat("_waitForCompletionSignal_total_ready", len(ready)) + for test in ready: + self.record_completion_signal(test) + else: + result_kind = "pidfd_timeout" + machine._incrementCompletionStat("_waitForCompletionSignal_pidfd_timeout") + finally: + selector.close() + + if registered: + return ready + + if self.uses_completion_queue: + used_queue_event_wait = True + result_kind = "queue_event_wait" + machine._incrementCompletionStat("_waitForCompletionSignal_queue_event_wait") + machine._completionEvent.wait(machine.naptime) + return [] + + machine._incrementCompletionStat("_waitForCompletionSignal_sleep_fallback") + time.sleep(machine.naptime) + return [] + finally: + machine._recordCompletionInternalSpan( + "_waitForCompletionSignal", + start_us, + time.time_ns() // 1000, + metadata={ + "mode": getattr(machine, "completion_detection_mode", ""), + "running_count": len(machine.running), + "registered": bool(registered), + "registered_count": registered_count, + "ready_count": len(ready), + "used_queue_event_wait": bool(used_queue_event_wait), + "result": result_kind, + }, + ) + + def poll_running_tests( + self, + allow_running_checks, + prioritized=None, + stop_after_completion=False, + completion_limit=None, + ): + """Poll running tests, optionally prioritizing likely completions. + + Args: + allow_running_checks (bool): When ``False``, skip timeout and + runtime error checks for children that have not yet exited. + prioritized (iterable|None): Optional running-test candidates to + check before the rest of ``machine.running``. + stop_after_completion (bool): If ``True``, stop after the first + completed test is handled. + completion_limit (int|None): Maximum number of completions to + process before returning control to the scheduler. + + Returns: + int: Number of completed tests processed in this polling pass. + """ + from ats import configuration + + machine = self.machine + start_us = time.time_ns() // 1000 + machine._incrementCompletionStat("_pollRunningTests_called") + if allow_running_checks: + machine._incrementCompletionStat("_pollRunningTests_allow_running_checks_true") + else: + machine._incrementCompletionStat("_pollRunningTests_allow_running_checks_false") + + prioritized = list(prioritized or []) + prioritized_count = len(prioritized) + ordered_count = 0 + completed = 0 + result_kind = "completed_none" + try: + ordered = [] + seen_ids = set() + for test in prioritized: + test_id = id(test) + if test_id in seen_ids: + continue + ordered.append(test) + seen_ids.add(test_id) + for test in machine.running: + test_id = id(test) + if test_id in seen_ids: + continue + ordered.append(test) + seen_ids.add(test_id) + + ordered_count = len(ordered) + machine._incrementCompletionStat("_pollRunningTests_total_ordered", ordered_count) + + remaining = [] + for index, test in enumerate(ordered): + done = machine.getStatus(test, allow_running_checks=allow_running_checks) + if not done: + remaining.append(test) + continue + completed += 1 + if test.status is not PASSED and configuration.options.oneFailure: + raise AtsError("Test failed in oneFailure mode.") + if stop_after_completion or ( + completion_limit is not None and completed >= completion_limit + ): + remaining.extend(ordered[index + 1:]) + self.preserve_new_running_tests(remaining, seen_ids) + machine.running = remaining + result_kind = "stopped_after_completion" + machine._incrementCompletionStat("_pollRunningTests_stopped_after_completion") + machine._incrementCompletionStat("_pollRunningTests_total_completed", completed) + return completed + + self.preserve_new_running_tests(remaining, seen_ids) + machine.running = remaining + machine._incrementCompletionStat("_pollRunningTests_total_completed", completed) + if completed: + result_kind = "completed" + machine._incrementCompletionStat("_pollRunningTests_completed") + else: + machine._incrementCompletionStat("_pollRunningTests_completed_none") + return completed + finally: + machine._recordCompletionInternalSpan( + "_pollRunningTests", + start_us, + time.time_ns() // 1000, + metadata={ + "mode": getattr(machine, "completion_detection_mode", ""), + "allow_running_checks": bool(allow_running_checks), + "prioritized_count": prioritized_count, + "ordered_count": ordered_count, + "stop_after_completion": bool(stop_after_completion), + "completion_limit": completion_limit, + "completed_count": completed, + "result": result_kind, + }, + ) + + def preserve_new_running_tests(self, remaining, seen_ids): + """Keep tests appended to ``machine.running`` during completion callbacks. + + Args: + remaining (list): Running tests that should remain after the current + polling pass. + seen_ids (set): Object ids already considered in the polling pass. + + Returns: + None: ``remaining`` is updated in place. + """ + remaining_ids = {id(test) for test in remaining} + for test in self.machine.running: + test_id = id(test) + if test_id in seen_ids or test_id in remaining_ids: + continue + remaining.append(test) + remaining_ids.add(test_id) + + def record_completion_signal(self, test, observed_us=None): + """Record a likely completion signal for one running test. + + Args: + test: ATS test object associated with the completion signal. + observed_us (int|None): Signal timestamp in microseconds. Uses the + current time when omitted. + + Returns: + None: Internal timestamps and statistics are updated. + """ + machine = self.machine + if observed_us is None: + observed_us = time.time_ns() // 1000 + if getattr(test, "ats_completion_signal_us", None) is None: + test.ats_completion_signal_us = observed_us + machine._incrementCompletionStat("completion_signal_recorded") + + def ensure_pidfd(self, test): + """Return or create a pidfd for one running child when supported. + + Args: + test: ATS test object whose child process should be observed. + + Returns: + int|None: Open pidfd file descriptor, or ``None`` when pidfds are + unavailable and ATS must use the watcher fallback. + """ + machine = self.machine + if getattr(machine, "_pidfdUnavailable", False): + self.ensure_completion_watcher(test) + return None + pidfd = getattr(test, "_pidfd", None) + if pidfd is not None: + return pidfd + if not hasattr(os, "pidfd_open"): + machine._pidfdUnavailable = True + self.ensure_completion_watcher(test) + return None + child = getattr(test, "child", None) + if child is None or getattr(child, "pid", None) is None: + return None + try: + pidfd = os.pidfd_open(child.pid) + except OSError: + self.ensure_completion_watcher(test) + return None + except AttributeError: + machine._pidfdUnavailable = True + self.ensure_completion_watcher(test) + return None + test._pidfd = pidfd + return pidfd + + def ensure_completion_watcher(self, test): + """Start the watcher-thread fallback for completion signaling. + + Args: + test: ATS test object whose child should be watched with + ``child.wait()``. + + Returns: + None: A daemon watcher thread is created at most once per test. + """ + child = getattr(test, "child", None) + if child is None: + return + watcher = getattr(test, "_completionWatcher", None) + if watcher is not None: + return + + def _watch_for_completion(): + try: + child.wait() + except Exception: + return + self.record_completion_signal(test) + + watcher = threading.Thread( + target=_watch_for_completion, + name=f"ats-completion-{getattr(child, 'pid', 'unknown')}", + daemon=True, + ) + test._completionWatcher = watcher + watcher.start() + + def close_pidfd(self, test): + """Close a pidfd associated with one test if it exists. + + Args: + test: ATS test object that may own ``_pidfd``. + + Returns: + None: Missing or already-closed pidfds are ignored. + """ + pidfd = getattr(test, "_pidfd", None) + if pidfd is None: + return + try: + os.close(pidfd) + except OSError: + pass + test._pidfd = None + + +class LegacyPollCompletionDetector(PollingCompletionDetector): """Preserve the historical ATS double-poll-with-sleep behavior.""" mode_name = "legacy_poll" @@ -27,7 +392,7 @@ def check_running(self): ``machine.running``. """ machine = self.machine - completion_limit = self.completion_fast_path_drain_limit() + completion_limit = self.completion_drain_limit() if self.poll_running_tests( allow_running_checks=True, completion_limit=completion_limit, diff --git a/ats/completion_queue.py b/ats/completion_queue.py index 66f34a8..b848fb6 100644 --- a/ats/completion_queue.py +++ b/ats/completion_queue.py @@ -3,10 +3,10 @@ import time from ats.atsut import AtsError, PASSED -from ats.completion_detector import CompletionDetector +from ats.completion_legacy_poll import PollingCompletionDetector -class CompletionQueueCompletionDetector(CompletionDetector): +class CompletionQueueCompletionDetector(PollingCompletionDetector): """Drain explicitly signaled completions from a machine-owned queue.""" mode_name = "completion_queue" @@ -28,7 +28,7 @@ def check_running(self): ``machine.running``. """ machine = self.machine - completion_limit = self.completion_fast_path_drain_limit() + completion_limit = self.completion_drain_limit() machine._incrementCompletionStat("check_running_completion_queue_mode") if self.poll_queued_completion_tests(completion_limit=completion_limit): machine._incrementCompletionStat("check_running_queue_pre_drain_completed") diff --git a/ats/configuration.py b/ats/configuration.py index 25a2d4f..f5ca89e 100644 --- a/ats/configuration.py +++ b/ats/configuration.py @@ -11,6 +11,7 @@ from argparse import ArgumentParser from glob import glob import importlib +import inspect import os import re import sys @@ -474,7 +475,69 @@ def get_machine_factory(module_name, machine_class, log(f"Importing {module_name} from {machine_package} caused the following error:\n{e}", echo=True) return None -def get_machine(file_text, file_name, is_batch=False): +def _instantiate_machine(machine_factory, + machine_name, + npMaxH, + completion_detection_mode=None): + """Instantiate a machine factory with optional detector selection. + + Args: + machine_factory (callable): Factory or class used to build the machine. + machine_name (str): Machine name passed to the constructor. + npMaxH (int|str): Hardware processor-slot limit for the machine. + completion_detection_mode (str|None): Requested completion detector + mode. When the factory supports it, the mode is passed as a keyword + argument. Older two-argument constructors remain supported. + + Returns: + object: Instantiated machine object. + """ + npMaxH = int(npMaxH) + if completion_detection_mode is None: + return machine_factory(machine_name, npMaxH) + + try: + signature = inspect.signature(machine_factory) + except (TypeError, ValueError): + signature = None + + if signature is not None: + parameters = signature.parameters.values() + supports_mode_kwarg = any( + parameter.kind == inspect.Parameter.VAR_KEYWORD + or parameter.name == "completion_detection_mode" + for parameter in parameters + ) + if not supports_mode_kwarg: + return machine_factory(machine_name, npMaxH) + + try: + return machine_factory( + machine_name, + npMaxH, + completion_detection_mode=completion_detection_mode, + ) + except TypeError: + return machine_factory(machine_name, npMaxH) + +def get_machine(file_text, + file_name, + is_batch=False, + completion_detection_mode=None): + """Create the machine declared in one machine-specification file. + + Args: + file_text (str): Full text of the candidate machine module. + file_name (str): Basename of the candidate machine module. + is_batch (bool): When ``True``, search ``#BATS:`` declarations instead + of ``#ATS:`` declarations. + completion_detection_mode (str|None): Requested completion detector + mode to pass through machine construction when supported. + + Returns: + object|None: Matching machine instance, or ``None`` when the file does + not define the active machine type. + """ header = '#BATS:' if is_batch else '#ATS:' machine_type = BATCH_TYPE if is_batch else MACHINE_TYPE ats_lines = (ats_line for ats_line in file_text.splitlines() @@ -503,7 +566,12 @@ def get_machine(file_text, file_name, is_batch=False): f"import {machine_class} as Machine") if machine_factory: - machine = machine_factory(machine_name, int(npMaxH)) + machine = _instantiate_machine( + machine_factory, + machine_name, + npMaxH, + completion_detection_mode=completion_detection_mode, + ) break else: @@ -511,7 +579,7 @@ def get_machine(file_text, file_name, is_batch=False): return machine -def get_machine_entry_points(machine_class): +def get_machine_entry_points(machine_class, completion_detection_mode=None): """ Looks for custom machine type via entry_points plugins installed by ats wrappers. @@ -521,6 +589,16 @@ def get_machine_entry_points(machine_class): Batch mode not really supported this way? -> would want to tag batch vs ats headers as an instance/class variable instead in this mode rather than rely on the header comments + + Args: + machine_class (str): Machine type to resolve through installed entry + points. + completion_detection_mode (str|None): Requested completion detector + mode to pass through machine construction when supported. + + Returns: + object|None: Machine instance loaded from an entry-point plugin, or + ``None`` when no plugin matches. """ log("Machine Factory: looping over available machine plugins:", echo=False) @@ -531,7 +609,12 @@ def get_machine_entry_points(machine_class): for machine_factory in ats_machines: if machine_class in machine_factory.value: log(f"Machine Factory: Found machine {machine_factory.name} of class {machine_factory.value}: {machine_factory}") - return machine_factory.load()(machine_class, -1) + return _instantiate_machine( + machine_factory.load(), + machine_class, + -1, + completion_detection_mode=completion_detection_mode, + ) else: ats_machines = {machine.name: machine for group, machines in entry_points().items() @@ -541,17 +624,33 @@ def get_machine_entry_points(machine_class): for name, machine_factory in ats_machines.items(): if machine_class in machine_factory.value: log(f"Machine Factory: Found machine {name} of class {machine_class}: {machine_factory}") - return machine_factory.load()(machine_class, -1) + return _instantiate_machine( + machine_factory.load(), + machine_class, + -1, + completion_detection_mode=completion_detection_mode, + ) # Downstream needs to be able to detect if machine isn't found return None -def init(clas = '', adder = None, examiner=None): - """Called by manager.init(class, adder, examiner) - Initialize configuration and process command-line options; create log, - options, inputFiles, timelimit, machine, and batchmatchine. - Call backs to machine and to adder/examiner for options. +def init(clas = '', adder = None, examiner=None, + completion_detection_mode=None): + """Initialize ATS configuration, options, and machine instances. + + Args: + clas (str): ATS command-line string to parse instead of + ``sys.argv[1:]``. + adder (callable|None): Optional callback that adds parser options + before ATS parses the command line. + examiner (callable|None): Optional callback that inspects parsed + options after initialization. + completion_detection_mode (str|None): Requested completion detector + mode to pass through machine construction when supported. + + Returns: + None: Module-level ATS configuration state is updated in place. """ global log, options, inputFiles, timelimit, machine, batchmachine,\ defaultExecutable, ATSROOT, cuttime @@ -599,25 +698,41 @@ def init(clas = '', adder = None, examiner=None): file_name = os.path.basename(full_path) if not machine and re.search(ATS_PATTERN, file_text): - machine = get_machine(file_text, file_name) + machine = get_machine( + file_text, + file_name, + completion_detection_mode=completion_detection_mode, + ) specFoundIn = full_path if not batchmachine and re.search(BATS_PATTERN, file_text): - batchmachine = get_machine(file_text, file_name, is_batch=True) + batchmachine = get_machine( + file_text, + file_name, + is_batch=True, + completion_detection_mode=completion_detection_mode, + ) bspecFoundIn = full_path if machine and batchmachine: break # Check entry_points plugins to override built-in machines - machine_plugin = get_machine_entry_points(MACHINE_TYPE) + machine_plugin = get_machine_entry_points( + MACHINE_TYPE, + completion_detection_mode=completion_detection_mode, + ) if machine_plugin: machine = machine_plugin if machine is None: terminal("No machine specifications for", SYS_TYPE, "found, using generic.") - machine = machines.Machine('generic', -1) + machine = machines.Machine( + 'generic', + -1, + completion_detection_mode=completion_detection_mode, + ) # create the option set usage = "usage: %(prog)s [options] [input files]" diff --git a/ats/machines.py b/ats/machines.py index 5ca0eac..b0e284a 100644 --- a/ats/machines.py +++ b/ats/machines.py @@ -2,10 +2,7 @@ """ from collections import deque import subprocess, sys, os, threading, time, shlex -from ats.completion_detector import ( - completion_detection_mode_from_env, - create_completion_detector, -) +from ats.completion_detector import create_completion_detector from ats.atsut import RUNNING, TIMEDOUT, PASSED, FAILED, LSFERROR, \ SKIPPED, HALTED, AtsError from ats.log import log, terminal @@ -31,8 +28,7 @@ def __init__(self, completion_detection_mode=None): Args: completion_detection_mode (str|None): Requested completion-detector - mode. When omitted, ATS uses ``ATS_COMPLETION_DETECTION_MODE`` - from the environment and falls back to ``"fast_path"``. + mode. When omitted, ATS falls back to ``"completion_queue"``. Returns: None: Completion detector state and hooks are initialized. @@ -114,17 +110,16 @@ def configureCompletionDetector(self, completion_detection_mode=None): Args: completion_detection_mode (str|None): Requested detector mode. When - omitted, ATS uses ``ATS_COMPLETION_DETECTION_MODE`` from the - environment. + omitted, ATS falls back to ``"completion_queue"``. Returns: object: Newly created completion detector strategy instance. """ - mode = completion_detection_mode - if mode is None: - mode = completion_detection_mode_from_env() - self.completion_detection_mode = mode - self._completionDetector = create_completion_detector(self, mode) + self._completionDetector = create_completion_detector( + self, + completion_detection_mode, + ) + self.completion_detection_mode = self._completionDetector.mode_name return self._completionDetector def checkRunning(self): diff --git a/ats/management.py b/ats/management.py index 0734845..e27bbe5 100644 --- a/ats/management.py +++ b/ats/management.py @@ -734,14 +734,34 @@ def postprocess(self): log("-------------------------------", echo=True) return True - def init(self, clas = '', adder=None, examiner=None): - """This initialization is separate so that unit tests can be done on this module. - For this reason we delay any logging until main is called. - adder and examiner are called in configuration if given to allow user - a chance to add options and see results of option parsing. + def init(self, clas = '', adder=None, examiner=None, + completion_detection_mode=None): + """Initialize ATS configuration and machine state. + + This initialization is separate so that unit tests can exercise this + module without running ``main``. Logging is therefore delayed until + after initialization completes. + + Args: + clas (str): ATS command-line string to parse instead of + ``sys.argv[1:]``. + adder (callable|None): Optional callback that adds parser options + before ATS parses the command line. + examiner (callable|None): Optional callback that inspects parsed + options after initialization. + completion_detection_mode (str|None): Requested completion + detector mode to pass through ATS machine construction. + + Returns: + None: Manager state is updated in place. """ tempfile.tempdir = os.getcwd() - configuration.init(clas, adder, examiner) + configuration.init( + clas, + adder, + examiner, + completion_detection_mode=completion_detection_mode, + ) self.options = configuration.options self.inputFiles = configuration.inputFiles self.machine = configuration.machine diff --git a/docs/source/scheduler_extensions.rst b/docs/source/scheduler_extensions.rst index 6dece5c..39cca88 100644 --- a/docs/source/scheduler_extensions.rst +++ b/docs/source/scheduler_extensions.rst @@ -57,40 +57,37 @@ The detector keeps strategy choice out of ``MachineCore.checkRunning()`` while reusing the same machine-owned helpers for pidfds, watcher threads, completion queues, and aggregated completion statistics. -ATS ships three detector types: +ATS ships two detector types: -* ``ats.completion_fast_path.FastPathCompletionDetector`` waits for likely - completions first and then falls back to a full running-test check; * ``ats.completion_queue.CompletionQueueCompletionDetector`` records signaled completions into a queue and drains only those tests before falling back; * ``ats.completion_legacy_poll.LegacyPollCompletionDetector`` preserves the historical double-poll-with-sleep behavior. -The normal machine constructor accepts ``completion_detection_mode`` and -instantiates the matching detector: +The ATS initialization path accepts ``completion_detection_mode`` and passes it +through machine construction: :: - from ats.machines import Machine + import ats - machine = Machine( - "generic", - -1, + ats.manager.init( + clas="...", completion_detection_mode="completion_queue", ) -Wrappers that need to choose the detector before ATS configuration creates the -machine can set ``ATS_COMPLETION_DETECTION_MODE`` in the environment: +Machine constructors also accept the same argument directly and instantiate the +matching detector: :: - import os - - os.environ["ATS_COMPLETION_DETECTION_MODE"] = "legacy_poll" - - import ats + from ats.machines import Machine - ats.manager.init(clas="...") + machine = Machine( + "generic", + -1, + completion_detection_mode="completion_queue", + ) Custom machine subclasses should pass the mode through to ``Machine`` so the selection stays explicit at construction time: @@ -100,7 +97,7 @@ selection stays explicit at construction time: from ats import machines class MyMachine(machines.Machine): - def __init__(self, name, npMaxH, completion_detection_mode="fast_path"): + def __init__(self, name, npMaxH, completion_detection_mode="completion_queue"): super(MyMachine, self).__init__( name, npMaxH, diff --git a/test/test_completion_detector_examples.py b/test/test_completion_detector_examples.py index ae3050b..89399c3 100644 --- a/test/test_completion_detector_examples.py +++ b/test/test_completion_detector_examples.py @@ -1,9 +1,6 @@ -import os import unittest -from unittest import mock from ats.completion_queue import CompletionQueueCompletionDetector -from ats.completion_fast_path import FastPathCompletionDetector from ats.completion_legacy_poll import LegacyPollCompletionDetector from ats.machines import Machine @@ -25,14 +22,13 @@ def test_constructor_argument_selects_requested_completion_detector(self): CompletionQueueCompletionDetector, ) - def test_environment_variable_selects_default_completion_detector(self): - """Environment-selection example should influence default machine init.""" - with mock.patch.dict( - os.environ, - {"ATS_COMPLETION_DETECTION_MODE": "legacy_poll"}, - clear=False, - ): - machine = Machine("example", 1) + def test_constructor_argument_selects_legacy_completion_detector(self): + """Constructor selection should also support the legacy detector.""" + machine = Machine( + "example", + 1, + completion_detection_mode="legacy_poll", + ) self.assertEqual(machine.completion_detection_mode, "legacy_poll") self.assertIsInstance( @@ -40,15 +36,14 @@ def test_environment_variable_selects_default_completion_detector(self): LegacyPollCompletionDetector, ) - def test_fast_path_is_the_default_when_no_mode_is_requested(self): - """Default machine construction should preserve the fast-path detector.""" - with mock.patch.dict(os.environ, {}, clear=True): - machine = Machine("example", 1) + def test_completion_queue_is_the_default_when_no_mode_is_requested(self): + """Default machine construction should preserve the queue detector.""" + machine = Machine("example", 1) - self.assertEqual(machine.completion_detection_mode, "fast_path") + self.assertEqual(machine.completion_detection_mode, "completion_queue") self.assertIsInstance( machine._completionDetector, - FastPathCompletionDetector, + CompletionQueueCompletionDetector, ) From 447aaa411c7f84303ed4ce4be4196b0a8980a6b6 Mon Sep 17 00:00:00 2001 From: "Peter B. Robinson" Date: Thu, 21 May 2026 08:56:38 -0700 Subject: [PATCH 07/21] bring legacy poll back to original behavior without psfd stuff --- ats/completion_legacy_poll.py | 179 ++------------------------- ats/machines.py | 6 + docs/source/scheduler_extensions.rst | 6 +- 3 files changed, 20 insertions(+), 171 deletions(-) diff --git a/ats/completion_legacy_poll.py b/ats/completion_legacy_poll.py index 0c5ed38..747beab 100644 --- a/ats/completion_legacy_poll.py +++ b/ats/completion_legacy_poll.py @@ -1,8 +1,5 @@ """Legacy polling detector and shared polling helpers for ATS machines.""" -import os -import selectors -import threading import time from ats.atsut import AtsError, PASSED @@ -10,7 +7,7 @@ class PollingCompletionDetector(CompletionDetector): - """Shared polling and signal-wait helpers used by ATS detectors.""" + """Shared polling helpers used by ATS completion detectors.""" @property def uses_completion_queue(self): @@ -21,16 +18,6 @@ def uses_completion_queue(self): """ return False - @property - def uses_signal_wait(self): - """Return whether child-completion wait primitives should be prepared. - - Returns: - bool: ``True`` when pidfds or watcher threads should be set up at - launch time. - """ - return True - def completion_drain_limit(self): """Return the configured maximum completions drained per wakeup. @@ -45,16 +32,14 @@ def completion_drain_limit(self): return max(1, limit) def prepare_for_launch(self, test): - """Prepare one launched test for completion signaling. + """Prepare one launched test for completion detection. Args: test: ATS test object whose child process has just been launched. Returns: - None: Completion wait primitives are installed when needed. + None: Legacy polling does not need launch-time setup. """ - if self.uses_signal_wait: - self.ensure_pidfd(test) def close_for_test(self, test): """Release completion-detector resources associated with one test. @@ -63,61 +48,21 @@ def close_for_test(self, test): test: ATS test object that may own detector-specific wait state. Returns: - None: Detector-owned wait state is cleaned up when present. + None: Legacy polling does not need detector-specific cleanup. """ - self.close_pidfd(test) def wait_for_completion_signal(self): - """Wait for likely completions according to detector policy. + """Wait for one scheduler polling interval. Returns: - list: Tests that were signaled as likely completed during the wait. + list: Always returns an empty list for polling-only waits. """ start_us = time.time_ns() // 1000 machine = self.machine machine._incrementCompletionStat("_waitForCompletionSignal_called") - registered = False - registered_count = 0 - ready = [] - selector = None used_queue_event_wait = False result_kind = "sleep_fallback" try: - try: - selector = selectors.DefaultSelector() - except Exception: - selector = None - - if selector is not None: - try: - for test in machine.running: - pidfd = self.ensure_pidfd(test) - if pidfd is None: - continue - try: - selector.register(pidfd, selectors.EVENT_READ, test) - registered = True - registered_count += 1 - except Exception: - self.close_pidfd(test) - if registered: - machine._incrementCompletionStat("_waitForCompletionSignal_pidfd_registered") - ready = [key.data for key, _mask in selector.select(machine.naptime)] - if ready: - result_kind = "pidfd_ready" - machine._incrementCompletionStat("_waitForCompletionSignal_pidfd_ready") - machine._incrementCompletionStat("_waitForCompletionSignal_total_ready", len(ready)) - for test in ready: - self.record_completion_signal(test) - else: - result_kind = "pidfd_timeout" - machine._incrementCompletionStat("_waitForCompletionSignal_pidfd_timeout") - finally: - selector.close() - - if registered: - return ready - if self.uses_completion_queue: used_queue_event_wait = True result_kind = "queue_event_wait" @@ -136,9 +81,9 @@ def wait_for_completion_signal(self): metadata={ "mode": getattr(machine, "completion_detection_mode", ""), "running_count": len(machine.running), - "registered": bool(registered), - "registered_count": registered_count, - "ready_count": len(ready), + "registered": False, + "registered_count": 0, + "ready_count": 0, "used_queue_event_wait": bool(used_queue_event_wait), "result": result_kind, }, @@ -283,123 +228,21 @@ def record_completion_signal(self, test, observed_us=None): test.ats_completion_signal_us = observed_us machine._incrementCompletionStat("completion_signal_recorded") - def ensure_pidfd(self, test): - """Return or create a pidfd for one running child when supported. - - Args: - test: ATS test object whose child process should be observed. - - Returns: - int|None: Open pidfd file descriptor, or ``None`` when pidfds are - unavailable and ATS must use the watcher fallback. - """ - machine = self.machine - if getattr(machine, "_pidfdUnavailable", False): - self.ensure_completion_watcher(test) - return None - pidfd = getattr(test, "_pidfd", None) - if pidfd is not None: - return pidfd - if not hasattr(os, "pidfd_open"): - machine._pidfdUnavailable = True - self.ensure_completion_watcher(test) - return None - child = getattr(test, "child", None) - if child is None or getattr(child, "pid", None) is None: - return None - try: - pidfd = os.pidfd_open(child.pid) - except OSError: - self.ensure_completion_watcher(test) - return None - except AttributeError: - machine._pidfdUnavailable = True - self.ensure_completion_watcher(test) - return None - test._pidfd = pidfd - return pidfd - - def ensure_completion_watcher(self, test): - """Start the watcher-thread fallback for completion signaling. - - Args: - test: ATS test object whose child should be watched with - ``child.wait()``. - - Returns: - None: A daemon watcher thread is created at most once per test. - """ - child = getattr(test, "child", None) - if child is None: - return - watcher = getattr(test, "_completionWatcher", None) - if watcher is not None: - return - - def _watch_for_completion(): - try: - child.wait() - except Exception: - return - self.record_completion_signal(test) - - watcher = threading.Thread( - target=_watch_for_completion, - name=f"ats-completion-{getattr(child, 'pid', 'unknown')}", - daemon=True, - ) - test._completionWatcher = watcher - watcher.start() - - def close_pidfd(self, test): - """Close a pidfd associated with one test if it exists. - - Args: - test: ATS test object that may own ``_pidfd``. - - Returns: - None: Missing or already-closed pidfds are ignored. - """ - pidfd = getattr(test, "_pidfd", None) - if pidfd is None: - return - try: - os.close(pidfd) - except OSError: - pass - test._pidfd = None - class LegacyPollCompletionDetector(PollingCompletionDetector): - """Preserve the historical ATS double-poll-with-sleep behavior.""" + """Preserve the plain ATS polling behavior from the ``ale3d`` branch.""" mode_name = "legacy_poll" - @property - def uses_signal_wait(self): - """Return whether launch-time signal wait setup is needed. - - Returns: - bool: Always ``False`` for the legacy polling detector. - """ - return False - def check_running(self): - """Advance machine state using the historical polling behavior. + """Advance machine state using plain sleep-then-poll behavior. Returns: None: Completed tests may be finalized and removed from ``machine.running``. """ machine = self.machine - completion_limit = self.completion_drain_limit() - if self.poll_running_tests( - allow_running_checks=True, - completion_limit=completion_limit, - ): - return time.sleep(machine.naptime) self.poll_running_tests( allow_running_checks=True, - completion_limit=completion_limit, ) diff --git a/ats/machines.py b/ats/machines.py index b0e284a..3ba7606 100644 --- a/ats/machines.py +++ b/ats/machines.py @@ -166,9 +166,13 @@ def getStatus(self, test, allow_running_checks=True): overtime, fraction = self.checkForTimeOut(test) if fraction > .9 or overtime != 0: + # If a process produces a lot of output, it may fill its output + # buffer and then block until something is read from it. if configuration.SYS_TYPE.startswith('somesystemxxx'): stdoutdata, stderrdata = test.child.communicate() + # Poll again after the optional readback because the child may have + # exited while ATS was handling timeout-adjacent bookkeeping. self._pollChild(test) if test.child.returncode is not None: return self._finishCompletedTest(test) @@ -406,8 +410,10 @@ def _finishCompletedTest(self, test): if test.statusCode == 0: status = PASSED elif "flux" in configuration.MACHINE_TYPE and test.statusCode == 142: + # Flux reports scheduler-enforced timeouts as return code 142. status = TIMEDOUT else: + # Preserve ATS' historical LSF launch/runtime deficiency checks. lsf_error = False with open(test.errname, 'r', errors='replace') as f: lines = f.readlines() diff --git a/docs/source/scheduler_extensions.rst b/docs/source/scheduler_extensions.rst index 39cca88..4ee4c82 100644 --- a/docs/source/scheduler_extensions.rst +++ b/docs/source/scheduler_extensions.rst @@ -54,15 +54,15 @@ Completion Detectors ATS machines delegate running-test completion policy to a completion detector. The detector keeps strategy choice out of ``MachineCore.checkRunning()`` while -reusing the same machine-owned helpers for pidfds, watcher threads, completion -queues, and aggregated completion statistics. +reusing the same machine-owned helpers for completion queues, polling, and +aggregated completion statistics. ATS ships two detector types: * ``ats.completion_queue.CompletionQueueCompletionDetector`` records signaled completions into a queue and drains only those tests before falling back; * ``ats.completion_legacy_poll.LegacyPollCompletionDetector`` preserves the - historical double-poll-with-sleep behavior. + plain sleep-then-poll behavior from the ``ale3d`` ATS branch. The ATS initialization path accepts ``completion_detection_mode`` and passes it through machine construction: From aa0464cbfd91e9b781c6524599c4f1c6e3d69961 Mon Sep 17 00:00:00 2001 From: "Peter B. Robinson" Date: Fri, 22 May 2026 10:33:25 -0700 Subject: [PATCH 08/21] move compeletion queue only code to completion queue --- ats/completion_legacy_poll.py | 47 ----------------------------------- ats/completion_queue.py | 32 ++++++++++++++++++++---- 2 files changed, 27 insertions(+), 52 deletions(-) diff --git a/ats/completion_legacy_poll.py b/ats/completion_legacy_poll.py index 747beab..d75fbd1 100644 --- a/ats/completion_legacy_poll.py +++ b/ats/completion_legacy_poll.py @@ -9,15 +9,6 @@ class PollingCompletionDetector(CompletionDetector): """Shared polling helpers used by ATS completion detectors.""" - @property - def uses_completion_queue(self): - """Return whether completion signals should be queued. - - Returns: - bool: ``True`` when signals should be enqueued for later draining. - """ - return False - def completion_drain_limit(self): """Return the configured maximum completions drained per wakeup. @@ -51,44 +42,6 @@ def close_for_test(self, test): None: Legacy polling does not need detector-specific cleanup. """ - def wait_for_completion_signal(self): - """Wait for one scheduler polling interval. - - Returns: - list: Always returns an empty list for polling-only waits. - """ - start_us = time.time_ns() // 1000 - machine = self.machine - machine._incrementCompletionStat("_waitForCompletionSignal_called") - used_queue_event_wait = False - result_kind = "sleep_fallback" - try: - if self.uses_completion_queue: - used_queue_event_wait = True - result_kind = "queue_event_wait" - machine._incrementCompletionStat("_waitForCompletionSignal_queue_event_wait") - machine._completionEvent.wait(machine.naptime) - return [] - - machine._incrementCompletionStat("_waitForCompletionSignal_sleep_fallback") - time.sleep(machine.naptime) - return [] - finally: - machine._recordCompletionInternalSpan( - "_waitForCompletionSignal", - start_us, - time.time_ns() // 1000, - metadata={ - "mode": getattr(machine, "completion_detection_mode", ""), - "running_count": len(machine.running), - "registered": False, - "registered_count": 0, - "ready_count": 0, - "used_queue_event_wait": bool(used_queue_event_wait), - "result": result_kind, - }, - ) - def poll_running_tests( self, allow_running_checks, diff --git a/ats/completion_queue.py b/ats/completion_queue.py index b848fb6..ee05c97 100644 --- a/ats/completion_queue.py +++ b/ats/completion_queue.py @@ -11,14 +11,36 @@ class CompletionQueueCompletionDetector(PollingCompletionDetector): mode_name = "completion_queue" - @property - def uses_completion_queue(self): - """Return whether completion signals should be queued. + def wait_for_completion_signal(self): + """Wait one polling interval for queued completion signals. Returns: - bool: Always ``True`` for the queued completion detector. + list: Always returns an empty list because queue-mode wakeups only + signal that queued completions may be available. """ - return True + start_us = time.time_ns() // 1000 + machine = self.machine + machine._incrementCompletionStat("_waitForCompletionSignal_called") + result_kind = "queue_event_wait" + try: + machine._incrementCompletionStat("_waitForCompletionSignal_queue_event_wait") + machine._completionEvent.wait(machine.naptime) + return [] + finally: + machine._recordCompletionInternalSpan( + "_waitForCompletionSignal", + start_us, + time.time_ns() // 1000, + metadata={ + "mode": getattr(machine, "completion_detection_mode", ""), + "running_count": len(machine.running), + "registered": False, + "registered_count": 0, + "ready_count": 0, + "used_queue_event_wait": True, + "result": result_kind, + }, + ) def check_running(self): """Advance machine state by draining the queued completion set first. From 406856a4e85bb3f225e054b136838a261a21de12 Mon Sep 17 00:00:00 2001 From: "Peter B. Robinson" Date: Fri, 22 May 2026 12:36:30 -0700 Subject: [PATCH 09/21] some cleanup before reaper thread refactor. --- ats/completion_detector.py | 46 ++++++++++-- ats/completion_legacy_poll.py | 91 +++++------------------ ats/completion_queue.py | 78 ++++++++++++++++--- ats/machines.py | 6 +- docs/source/scheduler_extensions.rst | 2 +- test/test_completion_detector_examples.py | 11 +++ 6 files changed, 139 insertions(+), 95 deletions(-) diff --git a/ats/completion_detector.py b/ats/completion_detector.py index 78eba86..5fb7d0c 100644 --- a/ats/completion_detector.py +++ b/ats/completion_detector.py @@ -1,8 +1,9 @@ -"""Completion-detection strategy interface and factory helpers.""" +"""Completion-detection strategy base class and factory helpers.""" from abc import ABC, abstractmethod +import time -from ats.atsut import AtsError +from ats.atsut import AtsError, PASSED def normalize_completion_detection_mode(mode): @@ -18,8 +19,34 @@ def normalize_completion_detection_mode(mode): return str(mode or "completion_queue").strip().lower() or "completion_queue" +def _validate_completion_detection_mode_for_machine(machine, normalized_mode): + """Reject detector modes unsupported by one machine implementation. + + Args: + machine: Machine instance that would own the detector. + normalized_mode (str): Normalized detector mode name. + + Returns: + None: Validation succeeds without modifying ``machine``. + + Raises: + AtsError: If ``normalized_mode`` is unsupported for ``machine``. + """ + machine_class = machine.__class__.__name__ + machine_module = machine.__class__.__module__ + if ( + normalized_mode == "completion_queue" + and machine_class == "FluxDirect" + and machine_module.endswith("flux_direct") + ): + raise AtsError( + "completion_detection_mode='completion_queue' is unsupported for " + "FluxDirect. Use 'legacy_poll' for this experimental machine." + ) + + class CompletionDetector(ABC): - """Abstract policy object for machine completion detection.""" + """Abstract policy object with shared ATS completion-detection helpers.""" mode_name = "" @@ -29,27 +56,31 @@ def __init__(self, machine): Args: machine: Machine instance that owns completion helpers and running test state. + + Returns: + None: The detector stores a reference to ``machine``. """ self.machine = machine def prepare_for_launch(self, test): - """Prepare one launched test for completion signaling. + """Prepare one launched test for detector-specific completion work. Args: test: ATS test object whose child process has just been launched. Returns: - None: Default detector preparation is a no-op. + None: The default detector implementation needs no launch-time + setup. """ def close_for_test(self, test): - """Release completion-detector resources associated with one test. + """Release detector-owned state associated with one finished test. Args: test: ATS test object that may own detector-specific wait state. Returns: - None: Default detector cleanup is a no-op. + None: The default detector implementation needs no cleanup. """ @abstractmethod @@ -75,6 +106,7 @@ def create_completion_detector(machine, mode): AtsError: If ``mode`` is not one of the supported detector modes. """ normalized_mode = normalize_completion_detection_mode(mode) + _validate_completion_detection_mode_for_machine(machine, normalized_mode) if normalized_mode == "completion_queue": from ats.completion_queue import CompletionQueueCompletionDetector diff --git a/ats/completion_legacy_poll.py b/ats/completion_legacy_poll.py index d75fbd1..267a871 100644 --- a/ats/completion_legacy_poll.py +++ b/ats/completion_legacy_poll.py @@ -1,46 +1,33 @@ -"""Legacy polling detector and shared polling helpers for ATS machines.""" +"""Legacy polling detector for ATS machines.""" import time -from ats.atsut import AtsError, PASSED from ats.completion_detector import CompletionDetector -class PollingCompletionDetector(CompletionDetector): - """Shared polling helpers used by ATS completion detectors.""" - - def completion_drain_limit(self): - """Return the configured maximum completions drained per wakeup. - - Returns: - int: Positive completion drain limit. - """ - limit = getattr(self.machine, "completion_fast_path_drain_limit", 128) - try: - limit = int(limit) - except (TypeError, ValueError): - limit = 128 - return max(1, limit) - - def prepare_for_launch(self, test): - """Prepare one launched test for completion detection. - - Args: - test: ATS test object whose child process has just been launched. +class LegacyPollCompletionDetector(CompletionDetector): + """Preserve the plain ATS polling behavior from the ``ale3d`` branch.""" - Returns: - None: Legacy polling does not need launch-time setup. - """ + mode_name = "legacy_poll" - def close_for_test(self, test): - """Release completion-detector resources associated with one test. + def preserve_new_running_tests(self, remaining, seen_ids): + """Keep tests appended to ``machine.running`` during completion callbacks. Args: - test: ATS test object that may own detector-specific wait state. + remaining (list): Running tests that should remain after the current + polling pass. + seen_ids (set): Object ids already considered in the polling pass. Returns: - None: Legacy polling does not need detector-specific cleanup. + None: ``remaining`` is updated in place. """ + remaining_ids = {id(test) for test in remaining} + for test in self.machine.running: + test_id = id(test) + if test_id in seen_ids or test_id in remaining_ids: + continue + remaining.append(test) + remaining_ids.add(test_id) def poll_running_tests( self, @@ -56,7 +43,7 @@ def poll_running_tests( runtime error checks for children that have not yet exited. prioritized (iterable|None): Optional running-test candidates to check before the rest of ``machine.running``. - stop_after_completion (bool): If ``True``, stop after the first + stop_after_completion (bool): When ``True``, stop after the first completed test is handled. completion_limit (int|None): Maximum number of completions to process before returning control to the scheduler. @@ -144,48 +131,6 @@ def poll_running_tests( }, ) - def preserve_new_running_tests(self, remaining, seen_ids): - """Keep tests appended to ``machine.running`` during completion callbacks. - - Args: - remaining (list): Running tests that should remain after the current - polling pass. - seen_ids (set): Object ids already considered in the polling pass. - - Returns: - None: ``remaining`` is updated in place. - """ - remaining_ids = {id(test) for test in remaining} - for test in self.machine.running: - test_id = id(test) - if test_id in seen_ids or test_id in remaining_ids: - continue - remaining.append(test) - remaining_ids.add(test_id) - - def record_completion_signal(self, test, observed_us=None): - """Record a likely completion signal for one running test. - - Args: - test: ATS test object associated with the completion signal. - observed_us (int|None): Signal timestamp in microseconds. Uses the - current time when omitted. - - Returns: - None: Internal timestamps and statistics are updated. - """ - machine = self.machine - if observed_us is None: - observed_us = time.time_ns() // 1000 - if getattr(test, "ats_completion_signal_us", None) is None: - test.ats_completion_signal_us = observed_us - machine._incrementCompletionStat("completion_signal_recorded") - - -class LegacyPollCompletionDetector(PollingCompletionDetector): - """Preserve the plain ATS polling behavior from the ``ale3d`` branch.""" - - mode_name = "legacy_poll" def check_running(self): """Advance machine state using plain sleep-then-poll behavior. diff --git a/ats/completion_queue.py b/ats/completion_queue.py index ee05c97..c9aebe6 100644 --- a/ats/completion_queue.py +++ b/ats/completion_queue.py @@ -1,16 +1,30 @@ """Queued completion detector for ATS machines.""" +import threading import time from ats.atsut import AtsError, PASSED -from ats.completion_legacy_poll import PollingCompletionDetector +from ats.completion_detector import CompletionDetector -class CompletionQueueCompletionDetector(PollingCompletionDetector): +class CompletionQueueCompletionDetector(CompletionDetector): """Drain explicitly signaled completions from a machine-owned queue.""" mode_name = "completion_queue" + def completion_drain_limit(self): + """Return the configured maximum completions drained per wakeup. + + Returns: + int: Positive completion drain limit. + """ + limit = getattr(self.machine, "completion_fast_path_drain_limit", 128) + try: + limit = int(limit) + except (TypeError, ValueError): + limit = 128 + return max(1, limit) + def wait_for_completion_signal(self): """Wait one polling interval for queued completion signals. @@ -42,6 +56,50 @@ def wait_for_completion_signal(self): }, ) + def prepare_for_launch(self, test): + """Start the queued completion watcher for one launched test. + + Args: + test: ATS test object whose child process has just been launched. + + Returns: + None: A daemon watcher thread is created at most once per test. + """ + child = getattr(test, "child", None) + if child is None: + return + watcher = getattr(test, "_completionWatcher", None) + if watcher is not None: + return + + def watch_for_completion(): + """Wait for one child to exit and then enqueue its completion.""" + try: + child.wait() + except Exception: + return + self.record_completion_signal(test) + + watcher = threading.Thread( + target=watch_for_completion, + name=f"ats-completion-{getattr(child, 'pid', 'unknown')}", + daemon=True, + ) + test._completionWatcher = watcher + watcher.start() + + def close_for_test(self, test): + """Clear watcher bookkeeping for one finished test. + + Args: + test: ATS test object that may own a completion watcher thread. + + Returns: + None: Detector bookkeeping attributes are removed when present. + """ + if hasattr(test, "_completionWatcher"): + test._completionWatcher = None + def check_running(self): """Advance machine state by draining the queued completion set first. @@ -62,13 +120,8 @@ def check_running(self): machine._incrementCompletionStat("check_running_queue_post_wait_completed") return machine._incrementCompletionStat("check_running_queue_post_wait_empty") - machine._incrementCompletionStat("check_running_queue_fallback_poll_running") - self.poll_running_tests( - allow_running_checks=True, - completion_limit=completion_limit, - ) - def record_completion_signal(self, test, observed_us=None): + def record_completion_signal(self, test): """Record a likely completion signal and enqueue it for later draining. Args: @@ -80,9 +133,10 @@ def record_completion_signal(self, test, observed_us=None): None: Internal timestamps, queue state, and statistics are updated. """ machine = self.machine - super().record_completion_signal(test, observed_us=observed_us) - if observed_us is None: - observed_us = getattr(test, "ats_completion_signal_us", None) + observed_us = time.time_ns() // 1000 + if getattr(test, "ats_completion_signal_us", None) is None: + test.ats_completion_signal_us = observed_us + machine._incrementCompletionStat("completion_signal_recorded") with machine._completionQueueLock: test_id = id(test) if test_id in machine._completionQueueIds: @@ -202,7 +256,7 @@ def poll_queued_completion_tests(self, completion_limit=None): machine._incrementCompletionStat("_pollQueuedCompletionTests_completed") else: result_kind = "selected_none_completed" - machine._incrementCompletionStat("_pollQueuedCompletionTests_selected_none_completed") + machine._incrementCompletionStat("_pollQueuedCompletionTests_selected_none_completed") return completed finally: machine._recordCompletionInternalSpan( diff --git a/ats/machines.py b/ats/machines.py index 3ba7606..8afc5ca 100644 --- a/ats/machines.py +++ b/ats/machines.py @@ -573,7 +573,6 @@ def kill(self, test): # override if not using subprocess "Kill the job running test." if test.child: test.child.kill() - self._completionDetector.close_for_test(test) if test.stdOutLocGet() != 'terminal': test.fileHandleClose() @@ -898,11 +897,14 @@ def _launch(self, test): else: test.child = subprocess.Popen(test.commandList, cwd=test.directory, stdout = subprocess.PIPE, stderr=subprocess.STDOUT, env=E, stdin=testStdin) - self._completionDetector.prepare_for_launch(test) test.set(RUNNING, test.commandLine) self.running.append(test) self.numberTestsRunning += 1 + # Detector-specific completion wait state is armed only after the + # test is visible in ``self.running`` so early exits still map back + # to a live ATS test object. + self._completionDetector.prepare_for_launch(test) if MachineCore.debugClass or MachineCore.canRunNow_debugClass: print("DEBUG MachineCore.testEnded increased self.numberTestsRunning by 1 to %d " % self.numberTestsRunning) diff --git a/docs/source/scheduler_extensions.rst b/docs/source/scheduler_extensions.rst index 4ee4c82..675f85f 100644 --- a/docs/source/scheduler_extensions.rst +++ b/docs/source/scheduler_extensions.rst @@ -60,7 +60,7 @@ aggregated completion statistics. ATS ships two detector types: * ``ats.completion_queue.CompletionQueueCompletionDetector`` records signaled - completions into a queue and drains only those tests before falling back; + completions into a queue and drains only those tests; * ``ats.completion_legacy_poll.LegacyPollCompletionDetector`` preserves the plain sleep-then-poll behavior from the ``ale3d`` ATS branch. diff --git a/test/test_completion_detector_examples.py b/test/test_completion_detector_examples.py index 89399c3..0dbed78 100644 --- a/test/test_completion_detector_examples.py +++ b/test/test_completion_detector_examples.py @@ -1,6 +1,8 @@ import unittest +from ats.atsut import AtsError from ats.completion_queue import CompletionQueueCompletionDetector +from ats.completion_detector import create_completion_detector from ats.completion_legacy_poll import LegacyPollCompletionDetector from ats.machines import Machine @@ -46,6 +48,15 @@ def test_completion_queue_is_the_default_when_no_mode_is_requested(self): CompletionQueueCompletionDetector, ) + def test_flux_direct_rejects_completion_queue_mode(self): + """FluxDirect should reject the queued completion detector mode.""" + + class FluxDirect: + __module__ = "ats.atsMachines.FutureMachines.flux_direct" + + with self.assertRaisesRegex(AtsError, "unsupported for FluxDirect"): + create_completion_detector(FluxDirect(), "completion_queue") + if __name__ == "__main__": unittest.main() From d52da18a0e16a642f379f10362f2a0f8a77c1b54 Mon Sep 17 00:00:00 2001 From: "Peter B. Robinson" Date: Fri, 22 May 2026 13:12:32 -0700 Subject: [PATCH 10/21] reaper thread ai pass --- ats/completion_detector.py | 8 ++ ats/completion_legacy_poll.py | 1 + ats/completion_queue.py | 143 ++++++++++++++++------ ats/machines.py | 87 +++++++++++-- test/test_completion_detector_examples.py | 132 +++++++++++++++++++- 5 files changed, 322 insertions(+), 49 deletions(-) diff --git a/ats/completion_detector.py b/ats/completion_detector.py index 5fb7d0c..c0c341f 100644 --- a/ats/completion_detector.py +++ b/ats/completion_detector.py @@ -83,6 +83,14 @@ def close_for_test(self, test): None: The default detector implementation needs no cleanup. """ + def owns_child_reaping(self): + """Return whether this detector is responsible for child reaping. + + Returns: + bool: ``False`` for the default detector behavior. + """ + return False + @abstractmethod def check_running(self): """Update machine running state according to one detector strategy. diff --git a/ats/completion_legacy_poll.py b/ats/completion_legacy_poll.py index 267a871..52e4896 100644 --- a/ats/completion_legacy_poll.py +++ b/ats/completion_legacy_poll.py @@ -3,6 +3,7 @@ import time from ats.completion_detector import CompletionDetector +from ats.atsut import AtsError, PASSED class LegacyPollCompletionDetector(CompletionDetector): diff --git a/ats/completion_queue.py b/ats/completion_queue.py index c9aebe6..532ef44 100644 --- a/ats/completion_queue.py +++ b/ats/completion_queue.py @@ -1,5 +1,6 @@ """Queued completion detector for ATS machines.""" +import os import threading import time @@ -12,6 +13,82 @@ class CompletionQueueCompletionDetector(CompletionDetector): mode_name = "completion_queue" + def __init__(self, machine): + """Initialize queue-mode reaper state for one ATS machine. + + Args: + machine: Machine instance that owns queue-mode running tests. + """ + super().__init__(machine) + self._reaper_thread = None + self._reaper_stop = False + self._reaper_lock = threading.Lock() + self._reaper_condition = threading.Condition(self._reaper_lock) + self._registered_tests_by_pid = {} + + def owns_child_reaping(self): + """Queue mode owns child reaping through the detector reaper.""" + return True + + def _ensure_reaper_started(self): + """Start the queue-mode reaper thread on first child registration.""" + if self._reaper_thread is not None: + return + self._reaper_thread = threading.Thread( + target=self._reaper_loop, + name="ats-completion-reaper", + daemon=True, + ) + self._reaper_thread.start() + + def _wait_status_to_returncode(self, wait_status): + """Convert one raw wait status to subprocess-style return codes.""" + if os.WIFEXITED(wait_status): + return os.WEXITSTATUS(wait_status) + if os.WIFSIGNALED(wait_status): + return -os.WTERMSIG(wait_status) + return wait_status + + def _reaper_loop(self): + """Reap registered queue-mode children and enqueue their completions.""" + machine = self.machine + while True: + with self._reaper_condition: + while not self._reaper_stop and not self._registered_tests_by_pid: + self._reaper_condition.wait() + if self._reaper_stop and not self._registered_tests_by_pid: + return + + try: + pid, wait_status = os.waitpid(-1, 0) + except ChildProcessError: + machine._incrementCompletionStat("completion_queue_reaper_child_process_error") + with self._reaper_condition: + if not self._registered_tests_by_pid: + continue + time.sleep(0.01) + continue + except OSError: + machine._incrementCompletionStat("completion_queue_reaper_waitpid_error") + time.sleep(0.01) + continue + + with self._reaper_condition: + test = self._registered_tests_by_pid.pop(pid, None) + + if test is None: + machine._incrementCompletionStat("completion_queue_reaper_unknown_pid") + continue + + child = getattr(test, "child", None) + if child is None: + machine._incrementCompletionStat("completion_queue_reaper_missing_child") + continue + + child.returncode = self._wait_status_to_returncode(wait_status) + machine._incrementCompletionStat("completion_queue_reaper_reaped") + self.record_completion_signal(test) + def completion_drain_limit(self): """Return the configured maximum completions drained per wakeup. @@ -57,48 +134,43 @@ def wait_for_completion_signal(self): ) def prepare_for_launch(self, test): - """Start the queued completion watcher for one launched test. + """Register one launched child with the queued completion reaper. Args: test: ATS test object whose child process has just been launched. Returns: - None: A daemon watcher thread is created at most once per test. + None: The queue-mode reaper is started lazily and the child pid is + registered for reaping. """ child = getattr(test, "child", None) - if child is None: - return - watcher = getattr(test, "_completionWatcher", None) - if watcher is not None: + pid = getattr(child, "pid", None) + if child is None or pid is None: return - - def watch_for_completion(): - """Wait for one child to exit and then enqueue its completion.""" - try: - child.wait() - except Exception: - return - self.record_completion_signal(test) - - watcher = threading.Thread( - target=watch_for_completion, - name=f"ats-completion-{getattr(child, 'pid', 'unknown')}", - daemon=True, - ) - test._completionWatcher = watcher - watcher.start() + with self._reaper_condition: + self._ensure_reaper_started() + self._registered_tests_by_pid[pid] = test + self._reaper_condition.notify() + self.machine._incrementCompletionStat("completion_queue_reaper_registered") def close_for_test(self, test): - """Clear watcher bookkeeping for one finished test. + """Clear reaper bookkeeping for one finished test. Args: - test: ATS test object that may own a completion watcher thread. + test: ATS test object that may still be registered with the queue + reaper. Returns: - None: Detector bookkeeping attributes are removed when present. + None: Any stale pid registration is removed when present. """ - if hasattr(test, "_completionWatcher"): - test._completionWatcher = None + child = getattr(test, "child", None) + pid = getattr(child, "pid", None) + if pid is None: + return + with self._reaper_condition: + registered = self._registered_tests_by_pid.get(pid) + if registered is test: + self._registered_tests_by_pid.pop(pid, None) def check_running(self): """Advance machine state by draining the queued completion set first. @@ -112,14 +184,15 @@ def check_running(self): machine._incrementCompletionStat("check_running_completion_queue_mode") if self.poll_queued_completion_tests(completion_limit=completion_limit): machine._incrementCompletionStat("check_running_queue_pre_drain_completed") - return - machine._incrementCompletionStat("check_running_queue_pre_drain_empty") - machine._incrementCompletionStat("check_running_wait_for_completion_signal") - self.wait_for_completion_signal() - if self.poll_queued_completion_tests(completion_limit=completion_limit): - machine._incrementCompletionStat("check_running_queue_post_wait_completed") - return - machine._incrementCompletionStat("check_running_queue_post_wait_empty") + else: + machine._incrementCompletionStat("check_running_queue_pre_drain_empty") + machine._incrementCompletionStat("check_running_wait_for_completion_signal") + self.wait_for_completion_signal() + if self.poll_queued_completion_tests(completion_limit=completion_limit): + machine._incrementCompletionStat("check_running_queue_post_wait_completed") + else: + machine._incrementCompletionStat("check_running_queue_post_wait_empty") + machine.scan_running_tests_for_health() def record_completion_signal(self, test): """Record a likely completion signal and enqueue it for later draining. diff --git a/ats/machines.py b/ats/machines.py index 8afc5ca..642b336 100644 --- a/ats/machines.py +++ b/ats/machines.py @@ -157,13 +157,39 @@ def getStatus(self, test, allow_running_checks=True): ``False`` while the child remains running. """ from ats import configuration - self._pollChild(test) - if test.child.returncode is not None: - return self._finishCompletedTest(test) + if self._observeCompletedChild(test): + return True if not allow_running_checks: return False + return self._checkRunningHealth(test) + + def _observeCompletedChild(self, test): + """Finalize a child that has already exited if its return code is known. + + Args: + test: ATS test object whose child may already be complete. + + Returns: + bool: ``True`` when completion finalization ran. + """ + self._pollChild(test) + if test.child.returncode is None: + return False + return self._finishCompletedTest(test) + + def _checkRunningHealth(self, test): + """Apply timeout and runtime-error checks to one still-running test. + + Args: + test: ATS test object whose child is still expected to be running. + + Returns: + bool: ``True`` when the health check finishes the test. + """ + from ats import configuration + overtime, fraction = self.checkForTimeOut(test) if fraction > .9 or overtime != 0: # If a process produces a lot of output, it may fill its output @@ -171,11 +197,8 @@ def getStatus(self, test, allow_running_checks=True): if configuration.SYS_TYPE.startswith('somesystemxxx'): stdoutdata, stderrdata = test.child.communicate() - # Poll again after the optional readback because the child may have - # exited while ATS was handling timeout-adjacent bookkeeping. - self._pollChild(test) - if test.child.returncode is not None: - return self._finishCompletedTest(test) + if self._observeCompletedChild(test): + return True overtime, fraction = self.checkForTimeOut(test) if overtime != 0: @@ -386,9 +409,51 @@ def _pollChild(self, test): Returns: int|None: Child return code, or ``None`` while still running. """ + if self._completionDetector.owns_child_reaping(): + return test.child.returncode test.child.poll() return test.child.returncode + def _preserve_new_running_tests(self, remaining, seen_ids): + """Keep tests appended to ``self.running`` during a running-state scan. + + Args: + remaining (list): Running tests that should remain after the scan. + seen_ids (set): Object ids already considered in the current pass. + + Returns: + None: ``remaining`` is updated in place. + """ + remaining_ids = {id(test) for test in remaining} + for test in self.running: + test_id = id(test) + if test_id in seen_ids or test_id in remaining_ids: + continue + remaining.append(test) + remaining_ids.add(test_id) + + def scan_running_tests_for_health(self): + """Check still-running tests for timeout and runtime launch failures. + + Returns: + int: Number of running tests completed by health checks. + """ + ordered = list(self.running) + seen_ids = {id(test) for test in ordered} + remaining = [] + completed = 0 + for test in ordered: + if getattr(test.child, "returncode", None) is not None: + remaining.append(test) + continue + if self._checkRunningHealth(test): + completed += 1 + continue + remaining.append(test) + self._preserve_new_running_tests(remaining, seen_ids) + self.running = remaining + return completed + def _finishCompletedTest(self, test): """Finalize status selection for a child that has already exited. @@ -897,14 +962,10 @@ def _launch(self, test): else: test.child = subprocess.Popen(test.commandList, cwd=test.directory, stdout = subprocess.PIPE, stderr=subprocess.STDOUT, env=E, stdin=testStdin) + self._completionDetector.prepare_for_launch(test) test.set(RUNNING, test.commandLine) - self.running.append(test) self.numberTestsRunning += 1 - # Detector-specific completion wait state is armed only after the - # test is visible in ``self.running`` so early exits still map back - # to a live ATS test object. - self._completionDetector.prepare_for_launch(test) if MachineCore.debugClass or MachineCore.canRunNow_debugClass: print("DEBUG MachineCore.testEnded increased self.numberTestsRunning by 1 to %d " % self.numberTestsRunning) diff --git a/test/test_completion_detector_examples.py b/test/test_completion_detector_examples.py index 0dbed78..5f0c21e 100644 --- a/test/test_completion_detector_examples.py +++ b/test/test_completion_detector_examples.py @@ -1,11 +1,67 @@ +from collections import deque +import subprocess +import sys +import threading +import time +from types import SimpleNamespace import unittest +from unittest.mock import Mock -from ats.atsut import AtsError +from ats import configuration +from ats.atsut import AtsError, PASSED from ats.completion_queue import CompletionQueueCompletionDetector from ats.completion_detector import create_completion_detector from ats.completion_legacy_poll import LegacyPollCompletionDetector from ats.machines import Machine +if not hasattr(configuration, "options"): + configuration.options = SimpleNamespace( + oneFailure=False, + verbose=False, + skip=False, + logUsage=False, + removeStartNote=False, + removeEndNote=False, + debug=False, + ) + + +class _DetectorMachineStub: + """Minimal machine stub for queue-detector focused unit tests.""" + + def __init__(self, naptime=0.01): + self.naptime = naptime + self.running = [] + self.completion_detection_mode = "completion_queue" + self.completion_fast_path_drain_limit = 128 + self._completionEvent = threading.Event() + self._completionQueue = deque() + self._completionQueueIds = set() + self._completionQueueLock = threading.Lock() + self.stats = {} + self.get_status_calls = [] + self.health_scan_calls = 0 + + def _incrementCompletionStat(self, name, amount=1): + self.stats[name] = self.stats.get(name, 0) + amount + + def _recordCompletionInternalSpan(self, name, start_us, end_us, metadata=None): + pass + + def _recordCompletionQueueSnapshot(self, depth, reason, timestamp_us=None, metadata=None): + pass + + def getStatus(self, test, allow_running_checks=True): + self.get_status_calls.append((test, allow_running_checks)) + if test.child.returncode is None: + return False + test.status = PASSED + return True + + def scan_running_tests_for_health(self): + self.health_scan_calls += 1 + return 0 + class CompletionDetectorExamplesTest(unittest.TestCase): """Keep the scheduler-extension completion-detector examples executable.""" @@ -57,6 +113,80 @@ class FluxDirect: with self.assertRaisesRegex(AtsError, "unsupported for FluxDirect"): create_completion_detector(FluxDirect(), "completion_queue") + def test_queue_mode_does_not_call_child_poll(self): + """Queue mode should trust reaper-populated return codes.""" + machine = Machine( + "example", + 1, + completion_detection_mode="completion_queue", + ) + child = Mock() + child.returncode = 17 + test = SimpleNamespace(child=child) + + self.assertEqual(machine._pollChild(test), 17) + child.poll.assert_not_called() + + def test_legacy_mode_still_calls_child_poll(self): + """Legacy polling mode should continue to use Popen.poll().""" + machine = Machine( + "example", + 1, + completion_detection_mode="legacy_poll", + ) + child = Mock() + child.returncode = 0 + test = SimpleNamespace(child=child) + + self.assertEqual(machine._pollChild(test), 0) + child.poll.assert_called_once_with() + + def test_completion_queue_reaper_sets_child_returncode(self): + """Queue-mode reaper should publish subprocess-style return codes.""" + machine = _DetectorMachineStub() + detector = CompletionQueueCompletionDetector(machine) + child = subprocess.Popen( + [sys.executable, "-c", "import sys; sys.exit(7)"], + ) + test = SimpleNamespace( + child=child, + ats_completion_signal_us=None, + status=PASSED, + ) + + detector.prepare_for_launch(test) + self.assertTrue(machine._completionEvent.wait(5.0)) + + deadline = time.time() + 5.0 + while child.returncode is None and time.time() < deadline: + time.sleep(0.01) + + self.assertEqual(child.returncode, 7) + self.assertEqual(detector.drain_completion_queue(), [test]) + + def test_completion_queue_drain_avoids_fallback_completion_rescan(self): + """Queue mode should only finalize queued completions and still scan health.""" + machine = _DetectorMachineStub() + detector = CompletionQueueCompletionDetector(machine) + queued_test = SimpleNamespace( + child=SimpleNamespace(returncode=0), + ats_completion_signal_us=None, + status=PASSED, + ) + still_running = SimpleNamespace( + child=SimpleNamespace(returncode=None), + ats_completion_signal_us=None, + status=PASSED, + ) + machine.running = [queued_test, still_running] + + detector.record_completion_signal(queued_test) + detector.check_running() + + self.assertEqual(machine.get_status_calls, [(queued_test, False)]) + self.assertEqual(machine.health_scan_calls, 1) + self.assertEqual(machine.running, [still_running]) + if __name__ == "__main__": unittest.main() From 7485ca67775ed484565ea7a0dda62d7b2acf22ba Mon Sep 17 00:00:00 2001 From: "Peter B. Robinson" Date: Fri, 22 May 2026 13:45:07 -0700 Subject: [PATCH 11/21] bring other threading strategy in as a spearate option --- ats/completion_detector.py | 26 +++++++++---- ats/completion_queue_simple.py | 45 ++++++++++++++++++++++ docs/source/scheduler_extensions.rst | 14 +++++-- test/test_completion_detector_examples.py | 46 +++++++++++++++++++++++ 4 files changed, 121 insertions(+), 10 deletions(-) create mode 100644 ats/completion_queue_simple.py diff --git a/ats/completion_detector.py b/ats/completion_detector.py index c0c341f..5da4c97 100644 --- a/ats/completion_detector.py +++ b/ats/completion_detector.py @@ -1,9 +1,15 @@ """Completion-detection strategy base class and factory helpers.""" from abc import ABC, abstractmethod -import time -from ats.atsut import AtsError, PASSED +from ats.atsut import AtsError + + +_COMPLETION_DETECTION_MODE_ALIASES = { + "poll": "legacy_poll", + "reap": "completion_queue", + "queue": "completion_queue_simple", +} def normalize_completion_detection_mode(mode): @@ -16,7 +22,8 @@ def normalize_completion_detection_mode(mode): str: Normalized lowercase mode name, defaulting to ``"completion_queue"``. """ - return str(mode or "completion_queue").strip().lower() or "completion_queue" + normalized_mode = str(mode or "completion_queue").strip().lower() or "completion_queue" + return _COMPLETION_DETECTION_MODE_ALIASES.get(normalized_mode, normalized_mode) def _validate_completion_detection_mode_for_machine(machine, normalized_mode): @@ -35,13 +42,13 @@ def _validate_completion_detection_mode_for_machine(machine, normalized_mode): machine_class = machine.__class__.__name__ machine_module = machine.__class__.__module__ if ( - normalized_mode == "completion_queue" + normalized_mode in ("completion_queue", "completion_queue_simple") and machine_class == "FluxDirect" and machine_module.endswith("flux_direct") ): raise AtsError( - "completion_detection_mode='completion_queue' is unsupported for " - "FluxDirect. Use 'legacy_poll' for this experimental machine." + "queue-based completion detection is unsupported for FluxDirect. " + "Use 'legacy_poll' or its alias 'poll' for this experimental machine." ) @@ -119,11 +126,16 @@ def create_completion_detector(machine, mode): from ats.completion_queue import CompletionQueueCompletionDetector return CompletionQueueCompletionDetector(machine) + if normalized_mode == "completion_queue_simple": + from ats.completion_queue_simple import CompletionQueueSimpleCompletionDetector + + return CompletionQueueSimpleCompletionDetector(machine) if normalized_mode == "legacy_poll": from ats.completion_legacy_poll import LegacyPollCompletionDetector return LegacyPollCompletionDetector(machine) raise AtsError( "Unknown completion detection mode %r. Expected one of: " - "'completion_queue', 'legacy_poll'." % mode + "'completion_queue', 'completion_queue_simple', 'legacy_poll' " + "(aliases: 'reap', 'queue', 'poll')." % mode ) diff --git a/ats/completion_queue_simple.py b/ats/completion_queue_simple.py new file mode 100644 index 0000000..6887106 --- /dev/null +++ b/ats/completion_queue_simple.py @@ -0,0 +1,45 @@ +"""Queued completion detector using one watcher thread per child.""" + +import threading + +from ats.completion_queue import CompletionQueueCompletionDetector + + +class CompletionQueueSimpleCompletionDetector(CompletionQueueCompletionDetector): + """Preserve the original queued completion watcher-thread strategy.""" + + mode_name = "completion_queue_simple" + + def owns_child_reaping(self): + """Simple queue mode leaves child reaping to ``subprocess.Popen``.""" + return False + + def prepare_for_launch(self, test): + """Start one watcher thread that waits for this child to exit.""" + child = getattr(test, "child", None) + if child is None: + return + watcher = getattr(test, "_completionWatcher", None) + if watcher is not None: + return + + def watch_for_completion(): + """Wait for one child to exit and then enqueue its completion.""" + try: + child.wait() + except Exception: + return + self.record_completion_signal(test) + + watcher = threading.Thread( + target=watch_for_completion, + name=f"ats-completion-{getattr(child, 'pid', 'unknown')}", + daemon=True, + ) + test._completionWatcher = watcher + watcher.start() + + def close_for_test(self, test): + """Clear watcher bookkeeping for one finished test.""" + if hasattr(test, "_completionWatcher"): + test._completionWatcher = None diff --git a/docs/source/scheduler_extensions.rst b/docs/source/scheduler_extensions.rst index 675f85f..901a089 100644 --- a/docs/source/scheduler_extensions.rst +++ b/docs/source/scheduler_extensions.rst @@ -57,13 +57,21 @@ The detector keeps strategy choice out of ``MachineCore.checkRunning()`` while reusing the same machine-owned helpers for completion queues, polling, and aggregated completion statistics. -ATS ships two detector types: +ATS ships three detector types: -* ``ats.completion_queue.CompletionQueueCompletionDetector`` records signaled - completions into a queue and drains only those tests; +* ``ats.completion_queue.CompletionQueueCompletionDetector`` owns child reaping + with a dedicated ``waitpid`` reaper and records completed tests into a queue; +* ``ats.completion_queue_simple.CompletionQueueSimpleCompletionDetector`` + preserves the original watcher-thread-per-child queue strategy; * ``ats.completion_legacy_poll.LegacyPollCompletionDetector`` preserves the plain sleep-then-poll behavior from the ``ale3d`` ATS branch. +The normalizer also accepts short aliases: + +* ``reap`` -> ``completion_queue`` +* ``queue`` -> ``completion_queue_simple`` +* ``poll`` -> ``legacy_poll`` + The ATS initialization path accepts ``completion_detection_mode`` and passes it through machine construction: diff --git a/test/test_completion_detector_examples.py b/test/test_completion_detector_examples.py index 5f0c21e..c45517a 100644 --- a/test/test_completion_detector_examples.py +++ b/test/test_completion_detector_examples.py @@ -12,6 +12,7 @@ from ats.completion_queue import CompletionQueueCompletionDetector from ats.completion_detector import create_completion_detector from ats.completion_legacy_poll import LegacyPollCompletionDetector +from ats.completion_queue_simple import CompletionQueueSimpleCompletionDetector from ats.machines import Machine if not hasattr(configuration, "options"): @@ -94,6 +95,20 @@ def test_constructor_argument_selects_legacy_completion_detector(self): LegacyPollCompletionDetector, ) + def test_constructor_argument_selects_simple_queue_completion_detector(self): + """Constructor selection should support the simple queue detector.""" + machine = Machine( + "example", + 1, + completion_detection_mode="completion_queue_simple", + ) + + self.assertEqual(machine.completion_detection_mode, "completion_queue_simple") + self.assertIsInstance( + machine._completionDetector, + CompletionQueueSimpleCompletionDetector, + ) + def test_completion_queue_is_the_default_when_no_mode_is_requested(self): """Default machine construction should preserve the queue detector.""" machine = Machine("example", 1) @@ -113,6 +128,23 @@ class FluxDirect: with self.assertRaisesRegex(AtsError, "unsupported for FluxDirect"): create_completion_detector(FluxDirect(), "completion_queue") + with self.assertRaisesRegex(AtsError, "unsupported for FluxDirect"): + create_completion_detector(FluxDirect(), "completion_queue_simple") + + def test_short_aliases_select_expected_detectors(self): + """Short aliases should normalize to the expected detector types.""" + machine = Machine("example", 1, completion_detection_mode="poll") + self.assertEqual(machine.completion_detection_mode, "legacy_poll") + self.assertIsInstance(machine._completionDetector, LegacyPollCompletionDetector) + + machine = Machine("example", 1, completion_detection_mode="reap") + self.assertEqual(machine.completion_detection_mode, "completion_queue") + self.assertIsInstance(machine._completionDetector, CompletionQueueCompletionDetector) + + machine = Machine("example", 1, completion_detection_mode="queue") + self.assertEqual(machine.completion_detection_mode, "completion_queue_simple") + self.assertIsInstance(machine._completionDetector, CompletionQueueSimpleCompletionDetector) + def test_queue_mode_does_not_call_child_poll(self): """Queue mode should trust reaper-populated return codes.""" machine = Machine( @@ -141,6 +173,20 @@ def test_legacy_mode_still_calls_child_poll(self): self.assertEqual(machine._pollChild(test), 0) child.poll.assert_called_once_with() + def test_simple_queue_mode_still_calls_child_poll(self): + """Simple queue mode should still use Popen.poll() for completion checks.""" + machine = Machine( + "example", + 1, + completion_detection_mode="completion_queue_simple", + ) + child = Mock() + child.returncode = 0 + test = SimpleNamespace(child=child) + + self.assertEqual(machine._pollChild(test), 0) + child.poll.assert_called_once_with() + def test_completion_queue_reaper_sets_child_returncode(self): """Queue-mode reaper should publish subprocess-style return codes.""" machine = _DetectorMachineStub() From bf7de27aa48a505e3d98c85df781690bce96fed4 Mon Sep 17 00:00:00 2001 From: "Peter B. Robinson" Date: Fri, 22 May 2026 15:15:59 -0700 Subject: [PATCH 12/21] some cleanup during inital review --- ats/completion_legacy_poll.py | 2 +- ats/completion_queue.py | 2 +- ats/machines.py | 2 ++ docs/source/scheduler_extensions.rst | 4 ++-- 4 files changed, 6 insertions(+), 4 deletions(-) diff --git a/ats/completion_legacy_poll.py b/ats/completion_legacy_poll.py index 52e4896..1795895 100644 --- a/ats/completion_legacy_poll.py +++ b/ats/completion_legacy_poll.py @@ -7,7 +7,7 @@ class LegacyPollCompletionDetector(CompletionDetector): - """Preserve the plain ATS polling behavior from the ``ale3d`` branch.""" + """Legacy polling behavior.""" mode_name = "legacy_poll" diff --git a/ats/completion_queue.py b/ats/completion_queue.py index 532ef44..1e1d030 100644 --- a/ats/completion_queue.py +++ b/ats/completion_queue.py @@ -329,7 +329,7 @@ def poll_queued_completion_tests(self, completion_limit=None): machine._incrementCompletionStat("_pollQueuedCompletionTests_completed") else: result_kind = "selected_none_completed" - machine._incrementCompletionStat("_pollQueuedCompletionTests_selected_none_completed") + machine._incrementCompletionStat("_pollQueuedCompletionTests_selected_none_completed") return completed finally: machine._recordCompletionInternalSpan( diff --git a/ats/machines.py b/ats/machines.py index 642b336..ad626b9 100644 --- a/ats/machines.py +++ b/ats/machines.py @@ -469,11 +469,13 @@ def _finishCompletedTest(self, test): test.ats_returncode_observed_us = time.time_ns() // 1000 test.setEndDateTime() test.statusCode = test.child.returncode + # If the user set ignoreReturnCode to True then set statusCode to 0. ignoreReturnCode = test.options.get('ignoreReturnCode', False) if ignoreReturnCode: test.statusCode = 0 if test.statusCode == 0: status = PASSED + # This checks for flux timeouts since ATS' method for determining timeouts doesnt work with flux elif "flux" in configuration.MACHINE_TYPE and test.statusCode == 142: # Flux reports scheduler-enforced timeouts as return code 142. status = TIMEDOUT diff --git a/docs/source/scheduler_extensions.rst b/docs/source/scheduler_extensions.rst index 901a089..6dfd8a9 100644 --- a/docs/source/scheduler_extensions.rst +++ b/docs/source/scheduler_extensions.rst @@ -62,9 +62,9 @@ ATS ships three detector types: * ``ats.completion_queue.CompletionQueueCompletionDetector`` owns child reaping with a dedicated ``waitpid`` reaper and records completed tests into a queue; * ``ats.completion_queue_simple.CompletionQueueSimpleCompletionDetector`` - preserves the original watcher-thread-per-child queue strategy; + spawns a thread per active running test to notice when the test is completed * ``ats.completion_legacy_poll.LegacyPollCompletionDetector`` preserves the - plain sleep-then-poll behavior from the ``ale3d`` ATS branch. + legacy sleep-then-poll behavior The normalizer also accepts short aliases: From e52ad92f81dbb3cd13c541e5e29aecd0f465f065 Mon Sep 17 00:00:00 2001 From: "Peter B. Robinson" Date: Fri, 22 May 2026 16:18:53 -0700 Subject: [PATCH 13/21] notice and print warnings when reaper thread reaps processes its not supposed to. --- .gitignore | 2 + ats/completion_detector.py | 11 +++ ats/completion_legacy_poll.py | 4 + ats/completion_queue.py | 102 ++++++++++++++++++---- ats/management.py | 2 + test/test_completion_detector_examples.py | 46 ++++++++++ 6 files changed, 152 insertions(+), 15 deletions(-) diff --git a/.gitignore b/.gitignore index 0ee97f0..ce799a1 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,5 @@ __pycache__/ *.log* *out +*.swp +*.swo diff --git a/ats/completion_detector.py b/ats/completion_detector.py index 5da4c97..dc82b9d 100644 --- a/ats/completion_detector.py +++ b/ats/completion_detector.py @@ -98,6 +98,17 @@ def owns_child_reaping(self): """ return False + @abstractmethod + def logCompletionWarnings(self, logger): + """Print any detector-specific end-of-run completion warnings. + + Args: + logger (callable): Logging function compatible with ``ats.log.log``. + + Returns: + None: Implementations may emit zero or more warnings. + """ + @abstractmethod def check_running(self): """Update machine running state according to one detector strategy. diff --git a/ats/completion_legacy_poll.py b/ats/completion_legacy_poll.py index 1795895..d0d4c3b 100644 --- a/ats/completion_legacy_poll.py +++ b/ats/completion_legacy_poll.py @@ -145,3 +145,7 @@ def check_running(self): self.poll_running_tests( allow_running_checks=True, ) + + def logCompletionWarnings(self, logger): + """Legacy polling has no detector-specific completion warnings.""" + del logger diff --git a/ats/completion_queue.py b/ats/completion_queue.py index 1e1d030..86e72ba 100644 --- a/ats/completion_queue.py +++ b/ats/completion_queue.py @@ -25,6 +25,10 @@ def __init__(self, machine): self._reaper_lock = threading.Lock() self._reaper_condition = threading.Condition(self._reaper_lock) self._registered_tests_by_pid = {} + self._unexpected_reaps = [] + self._unexpected_reaps_count = 0 + self._unexpected_reaps_dropped = 0 + self._unexpected_reaps_lock = threading.Lock() def owns_child_reaping(self): """Queue mode owns child reaping through the detector reaper.""" @@ -49,6 +53,88 @@ def _wait_status_to_returncode(self, wait_status): return -os.WTERMSIG(wait_status) return wait_status + def _handle_reaped_pid(self, pid, wait_status): + """Route one reaped child to the matching registered test if present.""" + machine = self.machine + with self._reaper_condition: + test = self._registered_tests_by_pid.pop(pid, None) + + if test is None: + machine._incrementCompletionStat("completion_queue_reaper_unknown_pid") + self._recordUnexpectedCompletionReap(pid, wait_status) + return + + child = getattr(test, "child", None) + if child is None: + machine._incrementCompletionStat("completion_queue_reaper_missing_child") + return + + child.returncode = self._wait_status_to_returncode(wait_status) + machine._incrementCompletionStat("completion_queue_reaper_reaped") + self.record_completion_signal(test) + + def _recordUnexpectedCompletionReap(self, pid, wait_status): + """Remember one unexpectedly reaped child for end-of-run warnings.""" + observed_us = time.time_ns() // 1000 + if os.WIFEXITED(wait_status): + outcome = "exit %d" % os.WEXITSTATUS(wait_status) + elif os.WIFSIGNALED(wait_status): + outcome = "signal %d" % os.WTERMSIG(wait_status) + else: + outcome = "wait_status %d" % wait_status + sample = { + "pid": pid, + "wait_status": wait_status, + "outcome": outcome, + "observed_us": observed_us, + } + with self._unexpected_reaps_lock: + self._unexpected_reaps_count += 1 + if len(self._unexpected_reaps) < 8: + self._unexpected_reaps.append(sample) + else: + self._unexpected_reaps_dropped += 1 + + def _unexpectedReapsSnapshot(self): + """Return a snapshot of unexpected completion-reaper activity.""" + with self._unexpected_reaps_lock: + return { + "count": self._unexpected_reaps_count, + "samples": list(self._unexpected_reaps), + "dropped": self._unexpected_reaps_dropped, + } + + def logCompletionWarnings(self, logger): + """Print end-of-run warnings for suspicious queue-reaper events.""" + snapshot = self._unexpectedReapsSnapshot() + if snapshot["count"] <= 0: + logger( + "WARNING: reap has a hypothetical race condition, but you dodged it!" + " Use queue as your completion detector if you want to stop living on the edge." + ) + return + logger( + "WARNING: completion_queue reaped %d child process(es) that were not " + "registered ATS tests. This indicates the queue-mode reaper hit the " + "known waitpid(-1) race and may have consumed another ATS subprocess " + "exit status. This is especially an issue if wait_status!=0 for the pid." % snapshot["count"] + ) + for sample in snapshot["samples"]: + logger( + "WARNING: unexpected reaped pid=%d outcome=%s wait_status=%d observed_us=%d" + % ( + sample["pid"], + sample["outcome"], + sample["wait_status"], + sample["observed_us"], + ) + ) + if snapshot["dropped"]: + logger( + "WARNING: %d additional unexpected reaped child event(s) were not " + "listed individually." % snapshot["dropped"] + ) + def _reaper_loop(self): """Reap registered queue-mode children and enqueue their completions.""" machine = self.machine @@ -73,21 +159,7 @@ def _reaper_loop(self): time.sleep(0.01) continue - with self._reaper_condition: - test = self._registered_tests_by_pid.pop(pid, None) - - if test is None: - machine._incrementCompletionStat("completion_queue_reaper_unknown_pid") - continue - - child = getattr(test, "child", None) - if child is None: - machine._incrementCompletionStat("completion_queue_reaper_missing_child") - continue - - child.returncode = self._wait_status_to_returncode(wait_status) - machine._incrementCompletionStat("completion_queue_reaper_reaped") - self.record_completion_signal(test) + self._handle_reaped_pid(pid, wait_status) def completion_drain_limit(self): """Return the configured maximum completions drained per wakeup. diff --git a/ats/management.py b/ats/management.py index e27bbe5..5304f6d 100644 --- a/ats/management.py +++ b/ats/management.py @@ -315,6 +315,8 @@ def finalReport(self): log("""ATS SUMMARY %s""" % datestamp(long_format=True), echo=True) successful_run = self.summary(log) self._summary2(log) + if hasattr(self, "machine"): + self.machine._completionDetector.logCompletionWarnings(log) return successful_run def finalBanner(self): diff --git a/test/test_completion_detector_examples.py b/test/test_completion_detector_examples.py index c45517a..5169c3c 100644 --- a/test/test_completion_detector_examples.py +++ b/test/test_completion_detector_examples.py @@ -233,6 +233,52 @@ def test_completion_queue_drain_avoids_fallback_completion_rescan(self): self.assertEqual(machine.health_scan_calls, 1) self.assertEqual(machine.running, [still_running]) + def test_completion_queue_records_unexpected_reaped_children(self): + """Queue mode should record when waitpid reaps a non-test child.""" + machine = _DetectorMachineStub() + detector = CompletionQueueCompletionDetector(machine) + + detector._handle_reaped_pid(4242, 0) + + self.assertEqual(machine.stats["completion_queue_reaper_unknown_pid"], 1) + snapshot = detector._unexpectedReapsSnapshot() + self.assertEqual(snapshot["count"], 1) + self.assertEqual(snapshot["samples"][0]["pid"], 4242) + self.assertEqual(snapshot["samples"][0]["outcome"], "exit 0") + + def test_completion_queue_logs_unexpected_completion_reap_warning(self): + """Queue detector should summarize unexpected reaper events at end of run.""" + machine = _DetectorMachineStub() + detector = CompletionQueueCompletionDetector(machine) + messages = [] + + def collect(message, **kwargs): + del kwargs + messages.append(message) + + detector._recordUnexpectedCompletionReap(111, 0) + detector._recordUnexpectedCompletionReap(222, 9) + detector.logCompletionWarnings(collect) + + self.assertEqual(len(messages), 3) + self.assertIn("completion_queue reaped 2 child process(es)", messages[0]) + self.assertIn("pid=111", messages[1]) + self.assertIn("outcome=exit 0", messages[1]) + self.assertIn("pid=222", messages[2]) + self.assertIn("outcome=signal 9", messages[2]) + + def test_legacy_detector_has_no_completion_warning_output(self): + """Legacy detector should satisfy the warning interface with a no-op.""" + machine = Machine("example", 1, completion_detection_mode="legacy_poll") + messages = [] + + def collect(message, **kwargs): + del kwargs + messages.append(message) + + machine._completionDetector.logCompletionWarnings(collect) + self.assertEqual(messages, []) + if __name__ == "__main__": unittest.main() From 68969d85fe7a941bf31b182edad689d5f485168c Mon Sep 17 00:00:00 2001 From: "Peter B. Robinson" Date: Tue, 26 May 2026 09:43:53 -0700 Subject: [PATCH 14/21] Clean up unneeded return, add TODOs --- ats/completion_queue.py | 5 +---- ats/machines.py | 2 ++ 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/ats/completion_queue.py b/ats/completion_queue.py index 86e72ba..a36bce9 100644 --- a/ats/completion_queue.py +++ b/ats/completion_queue.py @@ -177,9 +177,7 @@ def completion_drain_limit(self): def wait_for_completion_signal(self): """Wait one polling interval for queued completion signals. - Returns: - list: Always returns an empty list because queue-mode wakeups only - signal that queued completions may be available. + Returns nothing """ start_us = time.time_ns() // 1000 machine = self.machine @@ -188,7 +186,6 @@ def wait_for_completion_signal(self): try: machine._incrementCompletionStat("_waitForCompletionSignal_queue_event_wait") machine._completionEvent.wait(machine.naptime) - return [] finally: machine._recordCompletionInternalSpan( "_waitForCompletionSignal", diff --git a/ats/machines.py b/ats/machines.py index ad626b9..022e94a 100644 --- a/ats/machines.py +++ b/ats/machines.py @@ -475,6 +475,7 @@ def _finishCompletedTest(self, test): test.statusCode = 0 if test.statusCode == 0: status = PASSED + # TODO: move this machine specific check to flux-specific machine files # This checks for flux timeouts since ATS' method for determining timeouts doesnt work with flux elif "flux" in configuration.MACHINE_TYPE and test.statusCode == 142: # Flux reports scheduler-enforced timeouts as return code 142. @@ -544,6 +545,7 @@ def _completeTest(self, test, status): self.testEnded(test, status) return True + # TODO: move Slurm specific checks to slurm machine file def _detectRunningSlurmError(self, test): """Check a still-running test for known SLURM launch/runtime failures. From 22a80ac07620ef73dca78578dd1dff2c1b274b0e Mon Sep 17 00:00:00 2001 From: "Peter B. Robinson" Date: Tue, 26 May 2026 11:04:12 -0700 Subject: [PATCH 15/21] updated docstring for wait_for_completion_signal --- ats/completion_queue.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/ats/completion_queue.py b/ats/completion_queue.py index a36bce9..2eddbe0 100644 --- a/ats/completion_queue.py +++ b/ats/completion_queue.py @@ -177,7 +177,9 @@ def completion_drain_limit(self): def wait_for_completion_signal(self): """Wait one polling interval for queued completion signals. - Returns nothing + Returns: + None: Will have waited for {machine.naptime} seconds, and updated appropriate + statistics and timing data. """ start_us = time.time_ns() // 1000 machine = self.machine From 88ba01dfc2fbfd18b023b0fae3f5d81ddb1aac3d Mon Sep 17 00:00:00 2001 From: "Peter B. Robinson" Date: Wed, 27 May 2026 12:58:23 -0700 Subject: [PATCH 16/21] add a TODO --- ats/completion_queue.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ats/completion_queue.py b/ats/completion_queue.py index 2eddbe0..a458027 100644 --- a/ats/completion_queue.py +++ b/ats/completion_queue.py @@ -7,7 +7,7 @@ from ats.atsut import AtsError, PASSED from ats.completion_detector import CompletionDetector - +# TODO: When we move to RHEL 5 or RHEL 6, migrate this to use pidfd instead of a reaper thread strategy. class CompletionQueueCompletionDetector(CompletionDetector): """Drain explicitly signaled completions from a machine-owned queue.""" From 52a98302ad349ccc890ac0b13d19da990b05b7af Mon Sep 17 00:00:00 2001 From: Peter B Robinson Date: Thu, 28 May 2026 05:30:40 -0700 Subject: [PATCH 17/21] Rename completion detectors and modes --- ats/completion_detector.py | 43 +++++-------- ats/completion_legacy_poll.py | 47 ++++---------- ats/completion_queue.py | 16 +++-- ats/completion_queue_simple.py | 15 +++-- ats/machines.py | 10 +-- ats/management.py | 7 ++- docs/source/scheduler_extensions.rst | 33 +++++----- test/test_completion_detector_examples.py | 77 +++++++++++------------ 8 files changed, 112 insertions(+), 136 deletions(-) diff --git a/ats/completion_detector.py b/ats/completion_detector.py index dc82b9d..305bc72 100644 --- a/ats/completion_detector.py +++ b/ats/completion_detector.py @@ -5,13 +5,6 @@ from ats.atsut import AtsError -_COMPLETION_DETECTION_MODE_ALIASES = { - "poll": "legacy_poll", - "reap": "completion_queue", - "queue": "completion_queue_simple", -} - - def normalize_completion_detection_mode(mode): """Normalize a completion-detection mode string. @@ -20,10 +13,9 @@ def normalize_completion_detection_mode(mode): Returns: str: Normalized lowercase mode name, defaulting to - ``"completion_queue"``. + ``"waitpid_reaper"``. """ - normalized_mode = str(mode or "completion_queue").strip().lower() or "completion_queue" - return _COMPLETION_DETECTION_MODE_ALIASES.get(normalized_mode, normalized_mode) + return str(mode or "waitpid_reaper").strip().lower() or "waitpid_reaper" def _validate_completion_detection_mode_for_machine(machine, normalized_mode): @@ -42,13 +34,13 @@ def _validate_completion_detection_mode_for_machine(machine, normalized_mode): machine_class = machine.__class__.__name__ machine_module = machine.__class__.__module__ if ( - normalized_mode in ("completion_queue", "completion_queue_simple") + normalized_mode in ("waitpid_reaper", "per_test_watcher") and machine_class == "FluxDirect" and machine_module.endswith("flux_direct") ): raise AtsError( - "queue-based completion detection is unsupported for FluxDirect. " - "Use 'legacy_poll' or its alias 'poll' for this experimental machine." + "threaded completion detection is unsupported for FluxDirect. " + "Use 'poll' for this experimental machine." ) @@ -69,7 +61,7 @@ def __init__(self, machine): """ self.machine = machine - def prepare_for_launch(self, test): + def register_launched_test(self, test): """Prepare one launched test for detector-specific completion work. Args: @@ -80,7 +72,7 @@ def prepare_for_launch(self, test): setup. """ - def close_for_test(self, test): + def unregister_finished_test(self, test): """Release detector-owned state associated with one finished test. Args: @@ -133,20 +125,19 @@ def create_completion_detector(machine, mode): """ normalized_mode = normalize_completion_detection_mode(mode) _validate_completion_detection_mode_for_machine(machine, normalized_mode) - if normalized_mode == "completion_queue": - from ats.completion_queue import CompletionQueueCompletionDetector + if normalized_mode == "waitpid_reaper": + from ats.completion_queue import WaitpidReaperCompletionDetector - return CompletionQueueCompletionDetector(machine) - if normalized_mode == "completion_queue_simple": - from ats.completion_queue_simple import CompletionQueueSimpleCompletionDetector + return WaitpidReaperCompletionDetector(machine) + if normalized_mode == "per_test_watcher": + from ats.completion_queue_simple import PerTestWatcherCompletionDetector - return CompletionQueueSimpleCompletionDetector(machine) - if normalized_mode == "legacy_poll": - from ats.completion_legacy_poll import LegacyPollCompletionDetector + return PerTestWatcherCompletionDetector(machine) + if normalized_mode == "poll": + from ats.completion_legacy_poll import PollingCompletionDetector - return LegacyPollCompletionDetector(machine) + return PollingCompletionDetector(machine) raise AtsError( "Unknown completion detection mode %r. Expected one of: " - "'completion_queue', 'completion_queue_simple', 'legacy_poll' " - "(aliases: 'reap', 'queue', 'poll')." % mode + "'waitpid_reaper', 'per_test_watcher', 'poll'." % mode ) diff --git a/ats/completion_legacy_poll.py b/ats/completion_legacy_poll.py index d0d4c3b..9468727 100644 --- a/ats/completion_legacy_poll.py +++ b/ats/completion_legacy_poll.py @@ -1,4 +1,4 @@ -"""Legacy polling detector for ATS machines.""" +"""Polling completion detector for ATS machines.""" import time @@ -6,10 +6,10 @@ from ats.atsut import AtsError, PASSED -class LegacyPollCompletionDetector(CompletionDetector): - """Legacy polling behavior.""" +class PollingCompletionDetector(CompletionDetector): + """Sleep-then-poll completion behavior.""" - mode_name = "legacy_poll" + mode_name = "poll" def preserve_new_running_tests(self, remaining, seen_ids): """Keep tests appended to ``machine.running`` during completion callbacks. @@ -33,19 +33,13 @@ def preserve_new_running_tests(self, remaining, seen_ids): def poll_running_tests( self, allow_running_checks, - prioritized=None, - stop_after_completion=False, completion_limit=None, ): - """Poll running tests, optionally prioritizing likely completions. + """Poll running tests in scheduler order. Args: allow_running_checks (bool): When ``False``, skip timeout and runtime error checks for children that have not yet exited. - prioritized (iterable|None): Optional running-test candidates to - check before the rest of ``machine.running``. - stop_after_completion (bool): When ``True``, stop after the first - completed test is handled. completion_limit (int|None): Maximum number of completions to process before returning control to the scheduler. @@ -62,27 +56,12 @@ def poll_running_tests( else: machine._incrementCompletionStat("_pollRunningTests_allow_running_checks_false") - prioritized = list(prioritized or []) - prioritized_count = len(prioritized) ordered_count = 0 completed = 0 result_kind = "completed_none" try: - ordered = [] - seen_ids = set() - for test in prioritized: - test_id = id(test) - if test_id in seen_ids: - continue - ordered.append(test) - seen_ids.add(test_id) - for test in machine.running: - test_id = id(test) - if test_id in seen_ids: - continue - ordered.append(test) - seen_ids.add(test_id) - + ordered = list(machine.running) + seen_ids = {id(test) for test in ordered} ordered_count = len(ordered) machine._incrementCompletionStat("_pollRunningTests_total_ordered", ordered_count) @@ -95,13 +74,11 @@ def poll_running_tests( completed += 1 if test.status is not PASSED and configuration.options.oneFailure: raise AtsError("Test failed in oneFailure mode.") - if stop_after_completion or ( - completion_limit is not None and completed >= completion_limit - ): + if completion_limit is not None and completed >= completion_limit: remaining.extend(ordered[index + 1:]) self.preserve_new_running_tests(remaining, seen_ids) machine.running = remaining - result_kind = "stopped_after_completion" + result_kind = "stopped_after_completion_limit" machine._incrementCompletionStat("_pollRunningTests_stopped_after_completion") machine._incrementCompletionStat("_pollRunningTests_total_completed", completed) return completed @@ -123,9 +100,7 @@ def poll_running_tests( metadata={ "mode": getattr(machine, "completion_detection_mode", ""), "allow_running_checks": bool(allow_running_checks), - "prioritized_count": prioritized_count, "ordered_count": ordered_count, - "stop_after_completion": bool(stop_after_completion), "completion_limit": completion_limit, "completed_count": completed, "result": result_kind, @@ -147,5 +122,5 @@ def check_running(self): ) def logCompletionWarnings(self, logger): - """Legacy polling has no detector-specific completion warnings.""" - del logger + """Polling mode has no detector-specific completion warnings.""" + return None diff --git a/ats/completion_queue.py b/ats/completion_queue.py index a458027..d840b5b 100644 --- a/ats/completion_queue.py +++ b/ats/completion_queue.py @@ -8,10 +8,10 @@ from ats.completion_detector import CompletionDetector # TODO: When we move to RHEL 5 or RHEL 6, migrate this to use pidfd instead of a reaper thread strategy. -class CompletionQueueCompletionDetector(CompletionDetector): +class WaitpidReaperCompletionDetector(CompletionDetector): """Drain explicitly signaled completions from a machine-owned queue.""" - mode_name = "completion_queue" + mode_name = "waitpid_reaper" def __init__(self, machine): """Initialize queue-mode reaper state for one ATS machine. @@ -109,12 +109,12 @@ def logCompletionWarnings(self, logger): snapshot = self._unexpectedReapsSnapshot() if snapshot["count"] <= 0: logger( - "WARNING: reap has a hypothetical race condition, but you dodged it!" - " Use queue as your completion detector if you want to stop living on the edge." + "WARNING: waitpid_reaper did not hit its known waitpid(-1) race in this run, " + "but the risk remains. Use per_test_watcher if you need the safer path." ) return logger( - "WARNING: completion_queue reaped %d child process(es) that were not " + "WARNING: waitpid_reaper reaped %d child process(es) that were not " "registered ATS tests. This indicates the queue-mode reaper hit the " "known waitpid(-1) race and may have consumed another ATS subprocess " "exit status. This is especially an issue if wait_status!=0 for the pid." % snapshot["count"] @@ -204,7 +204,7 @@ def wait_for_completion_signal(self): }, ) - def prepare_for_launch(self, test): + def register_launched_test(self, test): """Register one launched child with the queued completion reaper. Args: @@ -224,7 +224,7 @@ def prepare_for_launch(self, test): self._reaper_condition.notify() self.machine._incrementCompletionStat("completion_queue_reaper_registered") - def close_for_test(self, test): + def unregister_finished_test(self, test): """Clear reaper bookkeeping for one finished test. Args: @@ -270,8 +270,6 @@ def record_completion_signal(self, test): Args: test: ATS test object associated with the completion signal. - observed_us (int|None): Signal timestamp in microseconds. Uses the - current time when omitted. Returns: None: Internal timestamps, queue state, and statistics are updated. diff --git a/ats/completion_queue_simple.py b/ats/completion_queue_simple.py index 6887106..be91586 100644 --- a/ats/completion_queue_simple.py +++ b/ats/completion_queue_simple.py @@ -2,25 +2,26 @@ import threading -from ats.completion_queue import CompletionQueueCompletionDetector +from ats.completion_queue import WaitpidReaperCompletionDetector -class CompletionQueueSimpleCompletionDetector(CompletionQueueCompletionDetector): +class PerTestWatcherCompletionDetector(WaitpidReaperCompletionDetector): """Preserve the original queued completion watcher-thread strategy.""" - mode_name = "completion_queue_simple" + mode_name = "per_test_watcher" def owns_child_reaping(self): """Simple queue mode leaves child reaping to ``subprocess.Popen``.""" return False - def prepare_for_launch(self, test): + def register_launched_test(self, test): """Start one watcher thread that waits for this child to exit.""" child = getattr(test, "child", None) if child is None: return watcher = getattr(test, "_completionWatcher", None) if watcher is not None: + self.machine._incrementCompletionStat("per_test_watcher_already_registered") return def watch_for_completion(): @@ -28,7 +29,9 @@ def watch_for_completion(): try: child.wait() except Exception: + self.machine._incrementCompletionStat("per_test_watcher_wait_error") return + self.machine._incrementCompletionStat("per_test_watcher_wait_completed") self.record_completion_signal(test) watcher = threading.Thread( @@ -38,8 +41,10 @@ def watch_for_completion(): ) test._completionWatcher = watcher watcher.start() + self.machine._incrementCompletionStat("per_test_watcher_registered") - def close_for_test(self, test): + def unregister_finished_test(self, test): """Clear watcher bookkeeping for one finished test.""" if hasattr(test, "_completionWatcher"): test._completionWatcher = None + self.machine._incrementCompletionStat("per_test_watcher_cleared") diff --git a/ats/machines.py b/ats/machines.py index 022e94a..b5b2292 100644 --- a/ats/machines.py +++ b/ats/machines.py @@ -28,7 +28,7 @@ def __init__(self, completion_detection_mode=None): Args: completion_detection_mode (str|None): Requested completion-detector - mode. When omitted, ATS falls back to ``"completion_queue"``. + mode. When omitted, ATS falls back to ``"waitpid_reaper"``. Returns: None: Completion detector state and hooks are initialized. @@ -110,7 +110,7 @@ def configureCompletionDetector(self, completion_detection_mode=None): Args: completion_detection_mode (str|None): Requested detector mode. When - omitted, ATS falls back to ``"completion_queue"``. + omitted, ATS falls back to ``"waitpid_reaper"``. Returns: object: Newly created completion detector strategy instance. @@ -438,6 +438,8 @@ def scan_running_tests_for_health(self): Returns: int: Number of running tests completed by health checks. """ + # Work on a snapshot so completion callbacks can append to + # ``self.running`` without disturbing this scan. ordered = list(self.running) seen_ids = {id(test) for test in ordered} remaining = [] @@ -541,7 +543,7 @@ def _completeTest(self, test, status): print(line) print(line, file=outhandle) - self._completionDetector.close_for_test(test) + self._completionDetector.unregister_finished_test(test) self.testEnded(test, status) return True @@ -966,7 +968,7 @@ def _launch(self, test): else: test.child = subprocess.Popen(test.commandList, cwd=test.directory, stdout = subprocess.PIPE, stderr=subprocess.STDOUT, env=E, stdin=testStdin) - self._completionDetector.prepare_for_launch(test) + self._completionDetector.register_launched_test(test) test.set(RUNNING, test.commandLine) self.running.append(test) self.numberTestsRunning += 1 diff --git a/ats/management.py b/ats/management.py index 5304f6d..a23b00c 100644 --- a/ats/management.py +++ b/ats/management.py @@ -304,8 +304,11 @@ def finalReport(self): successful_run = True if self.testlist: - log("=================================================\n" - "ATS RESULTS %s""" % datestamp(long_format=True), echo=True) + log( + "=================================================\n" + "ATS RESULTS %s" % datestamp(long_format=True), + echo=True, + ) log('-------------------------------------------------', echo = True) self.report() diff --git a/docs/source/scheduler_extensions.rst b/docs/source/scheduler_extensions.rst index 6dfd8a9..e24911b 100644 --- a/docs/source/scheduler_extensions.rst +++ b/docs/source/scheduler_extensions.rst @@ -59,18 +59,21 @@ aggregated completion statistics. ATS ships three detector types: -* ``ats.completion_queue.CompletionQueueCompletionDetector`` owns child reaping - with a dedicated ``waitpid`` reaper and records completed tests into a queue; -* ``ats.completion_queue_simple.CompletionQueueSimpleCompletionDetector`` - spawns a thread per active running test to notice when the test is completed -* ``ats.completion_legacy_poll.LegacyPollCompletionDetector`` preserves the - legacy sleep-then-poll behavior - -The normalizer also accepts short aliases: - -* ``reap`` -> ``completion_queue`` -* ``queue`` -> ``completion_queue_simple`` -* ``poll`` -> ``legacy_poll`` +* ``ats.completion_queue.WaitpidReaperCompletionDetector`` owns child reaping + with a dedicated ``waitpid`` reaper and records completed tests into a queue. + It has the lowest steady-state overhead, but it still carries the known + ``waitpid(-1)`` race and is unsupported for ``FluxDirect``. +* ``ats.completion_queue_simple.PerTestWatcherCompletionDetector`` spawns one + watcher thread per running test. That avoids the ``waitpid(-1)`` race, but + it scales with the number of active children because each child keeps its own + waiting thread. +* ``ats.completion_legacy_poll.PollingCompletionDetector`` preserves the plain + sleep-then-poll behavior. It is the simplest comparison baseline, but + completion latency and polling work both scale with the scheduler interval. + +Completion counters and timing spans are opt-in. Machines only update the +aggregated counters when ``completion_detection_stats`` is enabled, and they +only emit internal span hooks when ``completion_detection_spans`` is enabled. The ATS initialization path accepts ``completion_detection_mode`` and passes it through machine construction: @@ -81,7 +84,7 @@ through machine construction: ats.manager.init( clas="...", - completion_detection_mode="completion_queue", + completion_detection_mode="waitpid_reaper", ) Machine constructors also accept the same argument directly and instantiate the @@ -94,7 +97,7 @@ matching detector: machine = Machine( "generic", -1, - completion_detection_mode="completion_queue", + completion_detection_mode="waitpid_reaper", ) Custom machine subclasses should pass the mode through to ``Machine`` so the @@ -105,7 +108,7 @@ selection stays explicit at construction time: from ats import machines class MyMachine(machines.Machine): - def __init__(self, name, npMaxH, completion_detection_mode="completion_queue"): + def __init__(self, name, npMaxH, completion_detection_mode="waitpid_reaper"): super(MyMachine, self).__init__( name, npMaxH, diff --git a/test/test_completion_detector_examples.py b/test/test_completion_detector_examples.py index 5169c3c..a367f47 100644 --- a/test/test_completion_detector_examples.py +++ b/test/test_completion_detector_examples.py @@ -9,10 +9,10 @@ from ats import configuration from ats.atsut import AtsError, PASSED -from ats.completion_queue import CompletionQueueCompletionDetector +from ats.completion_queue import WaitpidReaperCompletionDetector from ats.completion_detector import create_completion_detector -from ats.completion_legacy_poll import LegacyPollCompletionDetector -from ats.completion_queue_simple import CompletionQueueSimpleCompletionDetector +from ats.completion_legacy_poll import PollingCompletionDetector +from ats.completion_queue_simple import PerTestWatcherCompletionDetector from ats.machines import Machine if not hasattr(configuration, "options"): @@ -33,7 +33,7 @@ class _DetectorMachineStub: def __init__(self, naptime=0.01): self.naptime = naptime self.running = [] - self.completion_detection_mode = "completion_queue" + self.completion_detection_mode = "waitpid_reaper" self.completion_fast_path_drain_limit = 128 self._completionEvent = threading.Event() self._completionQueue = deque() @@ -72,13 +72,13 @@ def test_constructor_argument_selects_requested_completion_detector(self): machine = Machine( "example", 1, - completion_detection_mode="completion_queue", + completion_detection_mode="waitpid_reaper", ) - self.assertEqual(machine.completion_detection_mode, "completion_queue") + self.assertEqual(machine.completion_detection_mode, "waitpid_reaper") self.assertIsInstance( machine._completionDetector, - CompletionQueueCompletionDetector, + WaitpidReaperCompletionDetector, ) def test_constructor_argument_selects_legacy_completion_detector(self): @@ -86,13 +86,13 @@ def test_constructor_argument_selects_legacy_completion_detector(self): machine = Machine( "example", 1, - completion_detection_mode="legacy_poll", + completion_detection_mode="poll", ) - self.assertEqual(machine.completion_detection_mode, "legacy_poll") + self.assertEqual(machine.completion_detection_mode, "poll") self.assertIsInstance( machine._completionDetector, - LegacyPollCompletionDetector, + PollingCompletionDetector, ) def test_constructor_argument_selects_simple_queue_completion_detector(self): @@ -100,57 +100,56 @@ def test_constructor_argument_selects_simple_queue_completion_detector(self): machine = Machine( "example", 1, - completion_detection_mode="completion_queue_simple", + completion_detection_mode="per_test_watcher", ) - self.assertEqual(machine.completion_detection_mode, "completion_queue_simple") + self.assertEqual(machine.completion_detection_mode, "per_test_watcher") self.assertIsInstance( machine._completionDetector, - CompletionQueueSimpleCompletionDetector, + PerTestWatcherCompletionDetector, ) def test_completion_queue_is_the_default_when_no_mode_is_requested(self): """Default machine construction should preserve the queue detector.""" machine = Machine("example", 1) - self.assertEqual(machine.completion_detection_mode, "completion_queue") + self.assertEqual(machine.completion_detection_mode, "waitpid_reaper") self.assertIsInstance( machine._completionDetector, - CompletionQueueCompletionDetector, + WaitpidReaperCompletionDetector, ) - def test_flux_direct_rejects_completion_queue_mode(self): + def test_flux_direct_rejects_threaded_completion_modes(self): """FluxDirect should reject the queued completion detector mode.""" class FluxDirect: __module__ = "ats.atsMachines.FutureMachines.flux_direct" with self.assertRaisesRegex(AtsError, "unsupported for FluxDirect"): - create_completion_detector(FluxDirect(), "completion_queue") + create_completion_detector(FluxDirect(), "waitpid_reaper") with self.assertRaisesRegex(AtsError, "unsupported for FluxDirect"): - create_completion_detector(FluxDirect(), "completion_queue_simple") + create_completion_detector(FluxDirect(), "per_test_watcher") - def test_short_aliases_select_expected_detectors(self): - """Short aliases should normalize to the expected detector types.""" + def test_old_detector_names_are_rejected(self): + """Old detector names should no longer be accepted.""" machine = Machine("example", 1, completion_detection_mode="poll") - self.assertEqual(machine.completion_detection_mode, "legacy_poll") - self.assertIsInstance(machine._completionDetector, LegacyPollCompletionDetector) + self.assertEqual(machine.completion_detection_mode, "poll") + self.assertIsInstance(machine._completionDetector, PollingCompletionDetector) - machine = Machine("example", 1, completion_detection_mode="reap") - self.assertEqual(machine.completion_detection_mode, "completion_queue") - self.assertIsInstance(machine._completionDetector, CompletionQueueCompletionDetector) - - machine = Machine("example", 1, completion_detection_mode="queue") - self.assertEqual(machine.completion_detection_mode, "completion_queue_simple") - self.assertIsInstance(machine._completionDetector, CompletionQueueSimpleCompletionDetector) + with self.assertRaisesRegex(AtsError, "Unknown completion detection mode"): + Machine("example", 1, completion_detection_mode="legacy_poll") + with self.assertRaisesRegex(AtsError, "Unknown completion detection mode"): + Machine("example", 1, completion_detection_mode="completion_queue") + with self.assertRaisesRegex(AtsError, "Unknown completion detection mode"): + Machine("example", 1, completion_detection_mode="completion_queue_simple") def test_queue_mode_does_not_call_child_poll(self): """Queue mode should trust reaper-populated return codes.""" machine = Machine( "example", 1, - completion_detection_mode="completion_queue", + completion_detection_mode="waitpid_reaper", ) child = Mock() child.returncode = 17 @@ -164,7 +163,7 @@ def test_legacy_mode_still_calls_child_poll(self): machine = Machine( "example", 1, - completion_detection_mode="legacy_poll", + completion_detection_mode="poll", ) child = Mock() child.returncode = 0 @@ -178,7 +177,7 @@ def test_simple_queue_mode_still_calls_child_poll(self): machine = Machine( "example", 1, - completion_detection_mode="completion_queue_simple", + completion_detection_mode="per_test_watcher", ) child = Mock() child.returncode = 0 @@ -190,7 +189,7 @@ def test_simple_queue_mode_still_calls_child_poll(self): def test_completion_queue_reaper_sets_child_returncode(self): """Queue-mode reaper should publish subprocess-style return codes.""" machine = _DetectorMachineStub() - detector = CompletionQueueCompletionDetector(machine) + detector = WaitpidReaperCompletionDetector(machine) child = subprocess.Popen( [sys.executable, "-c", "import sys; sys.exit(7)"], ) @@ -200,7 +199,7 @@ def test_completion_queue_reaper_sets_child_returncode(self): status=PASSED, ) - detector.prepare_for_launch(test) + detector.register_launched_test(test) self.assertTrue(machine._completionEvent.wait(5.0)) deadline = time.time() + 5.0 @@ -213,7 +212,7 @@ def test_completion_queue_reaper_sets_child_returncode(self): def test_completion_queue_drain_avoids_fallback_completion_rescan(self): """Queue mode should only finalize queued completions and still scan health.""" machine = _DetectorMachineStub() - detector = CompletionQueueCompletionDetector(machine) + detector = WaitpidReaperCompletionDetector(machine) queued_test = SimpleNamespace( child=SimpleNamespace(returncode=0), ats_completion_signal_us=None, @@ -236,7 +235,7 @@ def test_completion_queue_drain_avoids_fallback_completion_rescan(self): def test_completion_queue_records_unexpected_reaped_children(self): """Queue mode should record when waitpid reaps a non-test child.""" machine = _DetectorMachineStub() - detector = CompletionQueueCompletionDetector(machine) + detector = WaitpidReaperCompletionDetector(machine) detector._handle_reaped_pid(4242, 0) @@ -249,7 +248,7 @@ def test_completion_queue_records_unexpected_reaped_children(self): def test_completion_queue_logs_unexpected_completion_reap_warning(self): """Queue detector should summarize unexpected reaper events at end of run.""" machine = _DetectorMachineStub() - detector = CompletionQueueCompletionDetector(machine) + detector = WaitpidReaperCompletionDetector(machine) messages = [] def collect(message, **kwargs): @@ -261,7 +260,7 @@ def collect(message, **kwargs): detector.logCompletionWarnings(collect) self.assertEqual(len(messages), 3) - self.assertIn("completion_queue reaped 2 child process(es)", messages[0]) + self.assertIn("waitpid_reaper reaped 2 child process(es)", messages[0]) self.assertIn("pid=111", messages[1]) self.assertIn("outcome=exit 0", messages[1]) self.assertIn("pid=222", messages[2]) @@ -269,7 +268,7 @@ def collect(message, **kwargs): def test_legacy_detector_has_no_completion_warning_output(self): """Legacy detector should satisfy the warning interface with a no-op.""" - machine = Machine("example", 1, completion_detection_mode="legacy_poll") + machine = Machine("example", 1, completion_detection_mode="poll") messages = [] def collect(message, **kwargs): From 56f5333021d94b1d7991e117ea4c04464c74bedf Mon Sep 17 00:00:00 2001 From: Peter B Robinson Date: Wed, 1 Jul 2026 10:53:55 -0700 Subject: [PATCH 18/21] Rename completion detector modules --- ats/completion_detector.py | 6 +- ...mple.py => completion_per_test_watcher.py} | 4 +- ...tion_legacy_poll.py => completion_poll.py} | 0 ..._queue.py => completion_waitpid_reaper.py} | 2 +- docs/source/scheduler_extensions.rst | 6 +- test/test_completion_detector_examples.py | 75 ++++++++----------- 6 files changed, 39 insertions(+), 54 deletions(-) rename ats/{completion_queue_simple.py => completion_per_test_watcher.py} (92%) rename ats/{completion_legacy_poll.py => completion_poll.py} (100%) rename ats/{completion_queue.py => completion_waitpid_reaper.py} (99%) diff --git a/ats/completion_detector.py b/ats/completion_detector.py index 305bc72..97fc28f 100644 --- a/ats/completion_detector.py +++ b/ats/completion_detector.py @@ -126,15 +126,15 @@ def create_completion_detector(machine, mode): normalized_mode = normalize_completion_detection_mode(mode) _validate_completion_detection_mode_for_machine(machine, normalized_mode) if normalized_mode == "waitpid_reaper": - from ats.completion_queue import WaitpidReaperCompletionDetector + from ats.completion_waitpid_reaper import WaitpidReaperCompletionDetector return WaitpidReaperCompletionDetector(machine) if normalized_mode == "per_test_watcher": - from ats.completion_queue_simple import PerTestWatcherCompletionDetector + from ats.completion_per_test_watcher import PerTestWatcherCompletionDetector return PerTestWatcherCompletionDetector(machine) if normalized_mode == "poll": - from ats.completion_legacy_poll import PollingCompletionDetector + from ats.completion_poll import PollingCompletionDetector return PollingCompletionDetector(machine) raise AtsError( diff --git a/ats/completion_queue_simple.py b/ats/completion_per_test_watcher.py similarity index 92% rename from ats/completion_queue_simple.py rename to ats/completion_per_test_watcher.py index be91586..2746b98 100644 --- a/ats/completion_queue_simple.py +++ b/ats/completion_per_test_watcher.py @@ -1,8 +1,8 @@ -"""Queued completion detector using one watcher thread per child.""" +"""Per-test watcher completion detector for ATS machines.""" import threading -from ats.completion_queue import WaitpidReaperCompletionDetector +from ats.completion_waitpid_reaper import WaitpidReaperCompletionDetector class PerTestWatcherCompletionDetector(WaitpidReaperCompletionDetector): diff --git a/ats/completion_legacy_poll.py b/ats/completion_poll.py similarity index 100% rename from ats/completion_legacy_poll.py rename to ats/completion_poll.py diff --git a/ats/completion_queue.py b/ats/completion_waitpid_reaper.py similarity index 99% rename from ats/completion_queue.py rename to ats/completion_waitpid_reaper.py index d840b5b..cfdef9f 100644 --- a/ats/completion_queue.py +++ b/ats/completion_waitpid_reaper.py @@ -1,4 +1,4 @@ -"""Queued completion detector for ATS machines.""" +"""Waitpid-reaper completion detector for ATS machines.""" import os import threading diff --git a/docs/source/scheduler_extensions.rst b/docs/source/scheduler_extensions.rst index e24911b..be1e585 100644 --- a/docs/source/scheduler_extensions.rst +++ b/docs/source/scheduler_extensions.rst @@ -59,15 +59,15 @@ aggregated completion statistics. ATS ships three detector types: -* ``ats.completion_queue.WaitpidReaperCompletionDetector`` owns child reaping +* ``ats.completion_waitpid_reaper.WaitpidReaperCompletionDetector`` owns child reaping with a dedicated ``waitpid`` reaper and records completed tests into a queue. It has the lowest steady-state overhead, but it still carries the known ``waitpid(-1)`` race and is unsupported for ``FluxDirect``. -* ``ats.completion_queue_simple.PerTestWatcherCompletionDetector`` spawns one +* ``ats.completion_per_test_watcher.PerTestWatcherCompletionDetector`` spawns one watcher thread per running test. That avoids the ``waitpid(-1)`` race, but it scales with the number of active children because each child keeps its own waiting thread. -* ``ats.completion_legacy_poll.PollingCompletionDetector`` preserves the plain +* ``ats.completion_poll.PollingCompletionDetector`` preserves the plain sleep-then-poll behavior. It is the simplest comparison baseline, but completion latency and polling work both scale with the scheduler interval. diff --git a/test/test_completion_detector_examples.py b/test/test_completion_detector_examples.py index a367f47..02d4099 100644 --- a/test/test_completion_detector_examples.py +++ b/test/test_completion_detector_examples.py @@ -9,10 +9,10 @@ from ats import configuration from ats.atsut import AtsError, PASSED -from ats.completion_queue import WaitpidReaperCompletionDetector +from ats.completion_waitpid_reaper import WaitpidReaperCompletionDetector from ats.completion_detector import create_completion_detector -from ats.completion_legacy_poll import PollingCompletionDetector -from ats.completion_queue_simple import PerTestWatcherCompletionDetector +from ats.completion_per_test_watcher import PerTestWatcherCompletionDetector +from ats.completion_poll import PollingCompletionDetector from ats.machines import Machine if not hasattr(configuration, "options"): @@ -67,8 +67,8 @@ def scan_running_tests_for_health(self): class CompletionDetectorExamplesTest(unittest.TestCase): """Keep the scheduler-extension completion-detector examples executable.""" - def test_constructor_argument_selects_requested_completion_detector(self): - """Constructor selection example should instantiate the requested detector.""" + def test_constructor_argument_selects_waitpid_reaper_completion_detector(self): + """Constructor selection should instantiate the waitpid reaper detector.""" machine = Machine( "example", 1, @@ -81,8 +81,8 @@ def test_constructor_argument_selects_requested_completion_detector(self): WaitpidReaperCompletionDetector, ) - def test_constructor_argument_selects_legacy_completion_detector(self): - """Constructor selection should also support the legacy detector.""" + def test_constructor_argument_selects_polling_completion_detector(self): + """Constructor selection should also support the polling detector.""" machine = Machine( "example", 1, @@ -95,8 +95,8 @@ def test_constructor_argument_selects_legacy_completion_detector(self): PollingCompletionDetector, ) - def test_constructor_argument_selects_simple_queue_completion_detector(self): - """Constructor selection should support the simple queue detector.""" + def test_constructor_argument_selects_per_test_watcher_completion_detector(self): + """Constructor selection should support the per-test watcher detector.""" machine = Machine( "example", 1, @@ -109,8 +109,8 @@ def test_constructor_argument_selects_simple_queue_completion_detector(self): PerTestWatcherCompletionDetector, ) - def test_completion_queue_is_the_default_when_no_mode_is_requested(self): - """Default machine construction should preserve the queue detector.""" + def test_waitpid_reaper_is_the_default_when_no_mode_is_requested(self): + """Default machine construction should preserve the waitpid reaper detector.""" machine = Machine("example", 1) self.assertEqual(machine.completion_detection_mode, "waitpid_reaper") @@ -120,7 +120,7 @@ def test_completion_queue_is_the_default_when_no_mode_is_requested(self): ) def test_flux_direct_rejects_threaded_completion_modes(self): - """FluxDirect should reject the queued completion detector mode.""" + """FluxDirect should reject the threaded detector modes.""" class FluxDirect: __module__ = "ats.atsMachines.FutureMachines.flux_direct" @@ -131,21 +131,8 @@ class FluxDirect: with self.assertRaisesRegex(AtsError, "unsupported for FluxDirect"): create_completion_detector(FluxDirect(), "per_test_watcher") - def test_old_detector_names_are_rejected(self): - """Old detector names should no longer be accepted.""" - machine = Machine("example", 1, completion_detection_mode="poll") - self.assertEqual(machine.completion_detection_mode, "poll") - self.assertIsInstance(machine._completionDetector, PollingCompletionDetector) - - with self.assertRaisesRegex(AtsError, "Unknown completion detection mode"): - Machine("example", 1, completion_detection_mode="legacy_poll") - with self.assertRaisesRegex(AtsError, "Unknown completion detection mode"): - Machine("example", 1, completion_detection_mode="completion_queue") - with self.assertRaisesRegex(AtsError, "Unknown completion detection mode"): - Machine("example", 1, completion_detection_mode="completion_queue_simple") - - def test_queue_mode_does_not_call_child_poll(self): - """Queue mode should trust reaper-populated return codes.""" + def test_waitpid_reaper_mode_does_not_call_child_poll(self): + """Waitpid reaper mode should trust reaper-populated return codes.""" machine = Machine( "example", 1, @@ -158,8 +145,8 @@ def test_queue_mode_does_not_call_child_poll(self): self.assertEqual(machine._pollChild(test), 17) child.poll.assert_not_called() - def test_legacy_mode_still_calls_child_poll(self): - """Legacy polling mode should continue to use Popen.poll().""" + def test_polling_mode_still_calls_child_poll(self): + """Polling mode should continue to use ``Popen.poll()``.""" machine = Machine( "example", 1, @@ -172,8 +159,8 @@ def test_legacy_mode_still_calls_child_poll(self): self.assertEqual(machine._pollChild(test), 0) child.poll.assert_called_once_with() - def test_simple_queue_mode_still_calls_child_poll(self): - """Simple queue mode should still use Popen.poll() for completion checks.""" + def test_per_test_watcher_mode_still_calls_child_poll(self): + """Per-test watcher mode should still use ``Popen.poll()`` for completion checks.""" machine = Machine( "example", 1, @@ -186,8 +173,8 @@ def test_simple_queue_mode_still_calls_child_poll(self): self.assertEqual(machine._pollChild(test), 0) child.poll.assert_called_once_with() - def test_completion_queue_reaper_sets_child_returncode(self): - """Queue-mode reaper should publish subprocess-style return codes.""" + def test_waitpid_reaper_sets_child_returncode(self): + """Waitpid reaper should publish subprocess-style return codes.""" machine = _DetectorMachineStub() detector = WaitpidReaperCompletionDetector(machine) child = subprocess.Popen( @@ -209,8 +196,8 @@ def test_completion_queue_reaper_sets_child_returncode(self): self.assertEqual(child.returncode, 7) self.assertEqual(detector.drain_completion_queue(), [test]) - def test_completion_queue_drain_avoids_fallback_completion_rescan(self): - """Queue mode should only finalize queued completions and still scan health.""" + def test_waitpid_reaper_drain_avoids_fallback_completion_rescan(self): + """Waitpid reaper should only finalize queued completions and still scan health.""" machine = _DetectorMachineStub() detector = WaitpidReaperCompletionDetector(machine) queued_test = SimpleNamespace( @@ -232,8 +219,8 @@ def test_completion_queue_drain_avoids_fallback_completion_rescan(self): self.assertEqual(machine.health_scan_calls, 1) self.assertEqual(machine.running, [still_running]) - def test_completion_queue_records_unexpected_reaped_children(self): - """Queue mode should record when waitpid reaps a non-test child.""" + def test_waitpid_reaper_records_unexpected_reaped_children(self): + """Waitpid reaper should record when waitpid reaps a non-test child.""" machine = _DetectorMachineStub() detector = WaitpidReaperCompletionDetector(machine) @@ -245,14 +232,13 @@ def test_completion_queue_records_unexpected_reaped_children(self): self.assertEqual(snapshot["samples"][0]["pid"], 4242) self.assertEqual(snapshot["samples"][0]["outcome"], "exit 0") - def test_completion_queue_logs_unexpected_completion_reap_warning(self): - """Queue detector should summarize unexpected reaper events at end of run.""" + def test_waitpid_reaper_logs_unexpected_completion_reap_warning(self): + """Waitpid reaper should summarize unexpected reaper events at end of run.""" machine = _DetectorMachineStub() detector = WaitpidReaperCompletionDetector(machine) messages = [] - def collect(message, **kwargs): - del kwargs + def collect(message, **_kwargs): messages.append(message) detector._recordUnexpectedCompletionReap(111, 0) @@ -266,13 +252,12 @@ def collect(message, **kwargs): self.assertIn("pid=222", messages[2]) self.assertIn("outcome=signal 9", messages[2]) - def test_legacy_detector_has_no_completion_warning_output(self): - """Legacy detector should satisfy the warning interface with a no-op.""" + def test_polling_detector_has_no_completion_warning_output(self): + """Polling detector should satisfy the warning interface with a no-op.""" machine = Machine("example", 1, completion_detection_mode="poll") messages = [] - def collect(message, **kwargs): - del kwargs + def collect(message, **_kwargs): messages.append(message) machine._completionDetector.logCompletionWarnings(collect) From 29657d635b9822bd7b33aa7929d0d3cadc06565e Mon Sep 17 00:00:00 2001 From: Peter B Robinson Date: Tue, 7 Jul 2026 13:29:11 -0700 Subject: [PATCH 19/21] Address completion detector review comments --- test/test_completion_detector_examples.py | 81 ++++------------------- 1 file changed, 13 insertions(+), 68 deletions(-) diff --git a/test/test_completion_detector_examples.py b/test/test_completion_detector_examples.py index 02d4099..c245c5c 100644 --- a/test/test_completion_detector_examples.py +++ b/test/test_completion_detector_examples.py @@ -5,7 +5,6 @@ import time from types import SimpleNamespace import unittest -from unittest.mock import Mock from ats import configuration from ats.atsut import AtsError, PASSED @@ -28,7 +27,7 @@ class _DetectorMachineStub: - """Minimal machine stub for queue-detector focused unit tests.""" + """Minimal machine stub for completion-detector focused unit tests.""" def __init__(self, naptime=0.01): self.naptime = naptime @@ -67,8 +66,8 @@ def scan_running_tests_for_health(self): class CompletionDetectorExamplesTest(unittest.TestCase): """Keep the scheduler-extension completion-detector examples executable.""" - def test_constructor_argument_selects_waitpid_reaper_completion_detector(self): - """Constructor selection should instantiate the waitpid reaper detector.""" + def test_constructor_argument_selects_waitpid_reaper_completion_detector_class(self): + """Constructor selection should instantiate ``WaitpidReaperCompletionDetector``.""" machine = Machine( "example", 1, @@ -81,8 +80,8 @@ def test_constructor_argument_selects_waitpid_reaper_completion_detector(self): WaitpidReaperCompletionDetector, ) - def test_constructor_argument_selects_polling_completion_detector(self): - """Constructor selection should also support the polling detector.""" + def test_constructor_argument_selects_polling_completion_detector_class(self): + """Constructor selection should instantiate ``PollingCompletionDetector``.""" machine = Machine( "example", 1, @@ -95,8 +94,8 @@ def test_constructor_argument_selects_polling_completion_detector(self): PollingCompletionDetector, ) - def test_constructor_argument_selects_per_test_watcher_completion_detector(self): - """Constructor selection should support the per-test watcher detector.""" + def test_constructor_argument_selects_per_test_watcher_completion_detector_class(self): + """Constructor selection should instantiate ``PerTestWatcherCompletionDetector``.""" machine = Machine( "example", 1, @@ -109,8 +108,8 @@ def test_constructor_argument_selects_per_test_watcher_completion_detector(self) PerTestWatcherCompletionDetector, ) - def test_waitpid_reaper_is_the_default_when_no_mode_is_requested(self): - """Default machine construction should preserve the waitpid reaper detector.""" + def test_waitpid_reaper_completion_detector_is_the_default_when_no_mode_is_requested(self): + """Default machine construction should use ``WaitpidReaperCompletionDetector``.""" machine = Machine("example", 1) self.assertEqual(machine.completion_detection_mode, "waitpid_reaper") @@ -131,50 +130,8 @@ class FluxDirect: with self.assertRaisesRegex(AtsError, "unsupported for FluxDirect"): create_completion_detector(FluxDirect(), "per_test_watcher") - def test_waitpid_reaper_mode_does_not_call_child_poll(self): - """Waitpid reaper mode should trust reaper-populated return codes.""" - machine = Machine( - "example", - 1, - completion_detection_mode="waitpid_reaper", - ) - child = Mock() - child.returncode = 17 - test = SimpleNamespace(child=child) - - self.assertEqual(machine._pollChild(test), 17) - child.poll.assert_not_called() - - def test_polling_mode_still_calls_child_poll(self): - """Polling mode should continue to use ``Popen.poll()``.""" - machine = Machine( - "example", - 1, - completion_detection_mode="poll", - ) - child = Mock() - child.returncode = 0 - test = SimpleNamespace(child=child) - - self.assertEqual(machine._pollChild(test), 0) - child.poll.assert_called_once_with() - - def test_per_test_watcher_mode_still_calls_child_poll(self): - """Per-test watcher mode should still use ``Popen.poll()`` for completion checks.""" - machine = Machine( - "example", - 1, - completion_detection_mode="per_test_watcher", - ) - child = Mock() - child.returncode = 0 - test = SimpleNamespace(child=child) - - self.assertEqual(machine._pollChild(test), 0) - child.poll.assert_called_once_with() - def test_waitpid_reaper_sets_child_returncode(self): - """Waitpid reaper should publish subprocess-style return codes.""" + """``WaitpidReaperCompletionDetector`` should publish subprocess-style return codes.""" machine = _DetectorMachineStub() detector = WaitpidReaperCompletionDetector(machine) child = subprocess.Popen( @@ -197,7 +154,7 @@ def test_waitpid_reaper_sets_child_returncode(self): self.assertEqual(detector.drain_completion_queue(), [test]) def test_waitpid_reaper_drain_avoids_fallback_completion_rescan(self): - """Waitpid reaper should only finalize queued completions and still scan health.""" + """``WaitpidReaperCompletionDetector`` should only finalize queued completions and scan health.""" machine = _DetectorMachineStub() detector = WaitpidReaperCompletionDetector(machine) queued_test = SimpleNamespace( @@ -220,7 +177,7 @@ def test_waitpid_reaper_drain_avoids_fallback_completion_rescan(self): self.assertEqual(machine.running, [still_running]) def test_waitpid_reaper_records_unexpected_reaped_children(self): - """Waitpid reaper should record when waitpid reaps a non-test child.""" + """``WaitpidReaperCompletionDetector`` should record when waitpid reaps a non-test child.""" machine = _DetectorMachineStub() detector = WaitpidReaperCompletionDetector(machine) @@ -233,7 +190,7 @@ def test_waitpid_reaper_records_unexpected_reaped_children(self): self.assertEqual(snapshot["samples"][0]["outcome"], "exit 0") def test_waitpid_reaper_logs_unexpected_completion_reap_warning(self): - """Waitpid reaper should summarize unexpected reaper events at end of run.""" + """``WaitpidReaperCompletionDetector`` should summarize unexpected reaper events at end of run.""" machine = _DetectorMachineStub() detector = WaitpidReaperCompletionDetector(machine) messages = [] @@ -252,17 +209,5 @@ def collect(message, **_kwargs): self.assertIn("pid=222", messages[2]) self.assertIn("outcome=signal 9", messages[2]) - def test_polling_detector_has_no_completion_warning_output(self): - """Polling detector should satisfy the warning interface with a no-op.""" - machine = Machine("example", 1, completion_detection_mode="poll") - messages = [] - - def collect(message, **_kwargs): - messages.append(message) - - machine._completionDetector.logCompletionWarnings(collect) - self.assertEqual(messages, []) - - if __name__ == "__main__": unittest.main() From 1b679fb03fcbb38e6893bf90e38e43b7a7301584 Mon Sep 17 00:00:00 2001 From: Peter B Robinson Date: Thu, 9 Jul 2026 10:22:09 -0700 Subject: [PATCH 20/21] Default to per-test completion watcher --- ats/completion_detector.py | 4 ++-- ats/completion_per_test_watcher.py | 9 ++++++++- ats/machines.py | 4 ++-- test/test_completion_detector_examples.py | 8 ++++---- 4 files changed, 16 insertions(+), 9 deletions(-) diff --git a/ats/completion_detector.py b/ats/completion_detector.py index 97fc28f..280cac9 100644 --- a/ats/completion_detector.py +++ b/ats/completion_detector.py @@ -13,9 +13,9 @@ def normalize_completion_detection_mode(mode): Returns: str: Normalized lowercase mode name, defaulting to - ``"waitpid_reaper"``. + ``"per_test_watcher"``. """ - return str(mode or "waitpid_reaper").strip().lower() or "waitpid_reaper" + return str(mode or "per_test_watcher").strip().lower() or "per_test_watcher" def _validate_completion_detection_mode_for_machine(machine, normalized_mode): diff --git a/ats/completion_per_test_watcher.py b/ats/completion_per_test_watcher.py index 2746b98..cefbdd7 100644 --- a/ats/completion_per_test_watcher.py +++ b/ats/completion_per_test_watcher.py @@ -6,7 +6,14 @@ class PerTestWatcherCompletionDetector(WaitpidReaperCompletionDetector): - """Preserve the original queued completion watcher-thread strategy.""" + """Use one ``Popen.wait`` watcher thread per launched test. + + This mode preserves ATS' historical completion behavior: each watcher + waits on the exact child process owned by its test, then enqueues that + test for the shared fast-path drain. It avoids the process-wide + ``waitpid(-1)`` reaping used by ``waitpid_reaper`` while still avoiding + repeated full scans of the running-test list. + """ mode_name = "per_test_watcher" diff --git a/ats/machines.py b/ats/machines.py index b5b2292..ce12218 100644 --- a/ats/machines.py +++ b/ats/machines.py @@ -28,7 +28,7 @@ def __init__(self, completion_detection_mode=None): Args: completion_detection_mode (str|None): Requested completion-detector - mode. When omitted, ATS falls back to ``"waitpid_reaper"``. + mode. When omitted, ATS falls back to ``"per_test_watcher"``. Returns: None: Completion detector state and hooks are initialized. @@ -110,7 +110,7 @@ def configureCompletionDetector(self, completion_detection_mode=None): Args: completion_detection_mode (str|None): Requested detector mode. When - omitted, ATS falls back to ``"waitpid_reaper"``. + omitted, ATS falls back to ``"per_test_watcher"``. Returns: object: Newly created completion detector strategy instance. diff --git a/test/test_completion_detector_examples.py b/test/test_completion_detector_examples.py index c245c5c..d1269d1 100644 --- a/test/test_completion_detector_examples.py +++ b/test/test_completion_detector_examples.py @@ -108,14 +108,14 @@ def test_constructor_argument_selects_per_test_watcher_completion_detector_class PerTestWatcherCompletionDetector, ) - def test_waitpid_reaper_completion_detector_is_the_default_when_no_mode_is_requested(self): - """Default machine construction should use ``WaitpidReaperCompletionDetector``.""" + def test_per_test_watcher_completion_detector_is_the_default_when_no_mode_is_requested(self): + """Default construction should use the detector without the waitpid reaper race.""" machine = Machine("example", 1) - self.assertEqual(machine.completion_detection_mode, "waitpid_reaper") + self.assertEqual(machine.completion_detection_mode, "per_test_watcher") self.assertIsInstance( machine._completionDetector, - WaitpidReaperCompletionDetector, + PerTestWatcherCompletionDetector, ) def test_flux_direct_rejects_threaded_completion_modes(self): From 7027717bc765a003acc38e3daac3cde7506950ed Mon Sep 17 00:00:00 2001 From: "Peter B. Robinson" Date: Thu, 9 Jul 2026 10:33:39 -0700 Subject: [PATCH 21/21] better documentation --- ats/completion_per_test_watcher.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/ats/completion_per_test_watcher.py b/ats/completion_per_test_watcher.py index cefbdd7..f692fb9 100644 --- a/ats/completion_per_test_watcher.py +++ b/ats/completion_per_test_watcher.py @@ -8,11 +8,11 @@ class PerTestWatcherCompletionDetector(WaitpidReaperCompletionDetector): """Use one ``Popen.wait`` watcher thread per launched test. - This mode preserves ATS' historical completion behavior: each watcher + Each watcher waits on the exact child process owned by its test, then enqueues that - test for the shared fast-path drain. It avoids the process-wide - ``waitpid(-1)`` reaping used by ``waitpid_reaper`` while still avoiding - repeated full scans of the running-test list. + test for the shared fast-path drain. + This detector is thread safe, but may not scale beyond a few nodes of + concurrently running tests. """ mode_name = "per_test_watcher"