diff --git a/controllers/array_action/array_mediator_svc.py b/controllers/array_action/array_mediator_svc.py index 2e2a7fbb9..6573a48a1 100644 --- a/controllers/array_action/array_mediator_svc.py +++ b/controllers/array_action/array_mediator_svc.py @@ -1827,7 +1827,9 @@ def _get_replication_mode(self, volume_group_id): replication_local_location = volume_group_replication.local_location location_attr_name = f"location{replication_local_location}_replication_mode" - return getattr(volume_group_replication, location_attr_name, None) + if hasattr(volume_group_replication, location_attr_name): + return getattr(volume_group_replication, location_attr_name) + return None @staticmethod def _getattr_as_str(obj, attr, default=array_settings.UNKNOWN): @@ -1941,7 +1943,9 @@ def _get_replication_policy(self, volume_group_id): volume_group_replication = self._lsvolumegroupreplication(volume_group_id) if not volume_group_replication: return None - return volume_group_replication.replication_policy_name + if hasattr(volume_group_replication, 'replication_policy_name'): + return volume_group_replication.replication_policy_name + return None def _assign_partition_replication_policy(self, volume_group_id, requested_policy_name): try: @@ -2238,14 +2242,14 @@ def _demote_ear_replication_volume(self, volume_group_name): # Check if this is first demote attempt or retry if volume_group_name not in self._demote_state_map: # First attempt - create checkpoint and track state - self._demote_state_map[volume_group_name] = time.time() + self._demote_state_map[volume_group_name] = time.monotonic() logger.info("First demote attempt for volume group {}, creating checkpoint".format(volume_group_name)) self._chvolumegroupreplication(volume_group_name, checkpoint=array_settings.CHECKPOINT) else: # Retry - checkpoint already created, just check status first_attempt_time = self._demote_state_map[volume_group_name] - elapsed_time = time.time() - first_attempt_time + elapsed_time = time.monotonic() - first_attempt_time logger.info("Retry demote for volume group {}, checking checkpoint status " "(elapsed time: {:.1f}s)".format(volume_group_name, elapsed_time)) diff --git a/controllers/servers/csi/sync_lock.py b/controllers/servers/csi/sync_lock.py index 9c369d73c..f1c302357 100644 --- a/controllers/servers/csi/sync_lock.py +++ b/controllers/servers/csi/sync_lock.py @@ -1,4 +1,5 @@ import threading +import time from collections import defaultdict from controllers.common.csi_logger import get_stdout_logger @@ -8,19 +9,42 @@ ids_in_use = defaultdict(set) ids_in_use_lock = threading.Lock() +ids_last_access = {} # Track last access time for each lock entry +STALE_LOCK_TIMEOUT = 600 # 10 minutes in seconds def _add_to_ids_in_use(lock_key, object_id): ids_in_use[lock_key].add(object_id) + ids_last_access[(lock_key, object_id)] = time.monotonic() def _remove_from_ids_in_use(lock_key, object_id): if object_id in ids_in_use[lock_key]: ids_in_use[lock_key].remove(object_id) + ids_last_access.pop((lock_key, object_id), None) else: logger.error("could not find lock to release for {}: {}".format(lock_key, object_id)) +def _cleanup_stale_locks(): + """Remove lock entries that haven't been accessed recently.""" + current_time = time.monotonic() + stale_entries = [] + + for (lock_key, object_id), last_access in ids_last_access.items(): + if current_time - last_access > STALE_LOCK_TIMEOUT: + stale_entries.append((lock_key, object_id)) + + for lock_key, object_id in stale_entries: + logger.warning("Cleaning up stale lock for {}: {} (last accessed {} seconds ago)".format( + lock_key, object_id, current_time - ids_last_access.get((lock_key, object_id), current_time))) + if object_id in ids_in_use[lock_key]: + ids_in_use[lock_key].remove(object_id) + ids_last_access.pop((lock_key, object_id), None) + + return len(stale_entries) + + class SyncLock: def __init__(self, lock_key, object_id, action_name): self.lock_key = lock_key diff --git a/controllers/tests/array_action/svc/array_mediator_svc_test.py b/controllers/tests/array_action/svc/array_mediator_svc_test.py index f6482aa34..120fe8288 100644 --- a/controllers/tests/array_action/svc/array_mediator_svc_test.py +++ b/controllers/tests/array_action/svc/array_mediator_svc_test.py @@ -2534,6 +2534,22 @@ def test_get_volume_group_success(self): object_id=common_settings.INTERNAL_VOLUME_GROUP_ID) self.assertEqual(0, len(volume_group.volumes)) + def test_get_volume_group_id_uses_uid_when_present(self): + cli_vg = self._mock_cli_volume_group(uid=common_settings.VOLUME_GROUP_UID) + self.svc.client.svcinfo.lsvolumegroup.return_value = Mock(as_single_element=cli_vg) + + volume_group = self.svc.get_volume_group(common_settings.INTERNAL_VOLUME_GROUP_ID) + + self.assertEqual(common_settings.VOLUME_GROUP_UID, volume_group.id) + + def test_get_volume_group_id_falls_back_to_id_when_uid_absent(self): + cli_vg = self._mock_cli_volume_group() # no uid field — older array firmware + self.svc.client.svcinfo.lsvolumegroup.return_value = Mock(as_single_element=cli_vg) + + volume_group = self.svc.get_volume_group(common_settings.INTERNAL_VOLUME_GROUP_ID) + + self.assertEqual(common_settings.INTERNAL_VOLUME_GROUP_ID, volume_group.id) + def test_get_volume_group_not_found_failed(self): self._prepare_lsvolumegroup(no_return=True) with self.assertRaises(array_errors.ObjectNotFoundError):