From d46441bc122665c2fc9676f72e23d964fb4f5d25 Mon Sep 17 00:00:00 2001 From: David Dias Date: Wed, 27 May 2026 17:35:31 +0100 Subject: [PATCH 01/14] BP-899: Use Redis pipeline when writing --- CHANGELOG.md | 4 ++++ dal/movaidb/database.py | 18 ++++++++++++++++-- pyproject.toml | 2 +- 3 files changed, 21 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1fd550ee..0d03c772 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,9 @@ # Changelog +## vTBD +- [BP-899](https://movai.atlassian.net/browse/BP-899): Not possible to obtain active_flow + - Use Redis pipeline when writing + ## v3.23.6 - [BP-1703](https://movai.atlassian.net/browse/BP-1703): Apply DEVICE_NAME to robot name on init, rather than a temporary name diff --git a/dal/movaidb/database.py b/dal/movaidb/database.py index 1bd607b0..0a0783fd 100755 --- a/dal/movaidb/database.py +++ b/dal/movaidb/database.py @@ -480,12 +480,23 @@ def set( xx=False, validate=True, # TODO: remove unused parameter ) -> None: - """Set key values in database.""" + """Set key values in database, always using a Redis pipeline. + + Args: + _input (dict): The input dict to be saved in the database. + pickl (bool, optional): Whether to pickle values before saving. Defaults to True. + pipe (Pipeline, optional): Redis pipeline to use for the operation. Defaults to None. + ex (int, optional): Expiration time in seconds. Defaults to None. + px (int, optional): Expiration time in milliseconds. Defaults to None. + nx (bool, optional): Only set the key if it does not already exist. Defaults to False. + xx (bool, optional): Only set the key if it already exists. Defaults to False. + + """ # here we validate our dict and get the keys kvs = self.dict_to_keys(_input) - db_set = pipe if isinstance(pipe, Pipeline) else self.db_write + db_set = pipe if isinstance(pipe, Pipeline) else self.db_write.pipeline() # Save each key value in redis according to template value type for key, value, source in kvs: if pickl and source not in ["hash", "list"]: @@ -519,6 +530,9 @@ def set( except Exception as e: LOGGER.error("Something went wrong while saving this in Redis: %s", e) + if not isinstance(pipe, Pipeline): + db_set.execute() + def delete(self, _input: dict, pipe=None) -> Optional[int]: """ deletes _input diff --git a/pyproject.toml b/pyproject.toml index a4660420..f1f2e79c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -64,7 +64,7 @@ dal = [ line-length = 100 [tool.bumpversion] -current_version = "3.23.6.2" +current_version = "3.23.7.0" parse = "(?P\\d+)\\.(?P\\d+)\\.(?P\\d+)?(\\.(?P\\d+))?" serialize = ["{major}.{minor}.{patch}.{build}"] From 9752ed007d94393f674b342b28e5377dd5158a6e Mon Sep 17 00:00:00 2001 From: daniela-costa7 Date: Mon, 1 Jun 2026 13:06:48 +0100 Subject: [PATCH 02/14] Restructure imports to minimize memory usage by gdnode --- dal/data/schema.py | 2 +- dal/models/__init__.py | 10 ------- dal/models/baseuser.py | 2 +- dal/models/internaluser.py | 2 +- dal/movaidb/db_schema.py | 2 +- dal/scopes/__init__.py | 4 +-- dal/scopes/scope.py | 26 +++++++++++++---- dal/scopes/translation.py | 4 +-- dal/scopes/translation_constants.py | 9 ++++++ dal/tools/backup.py | 15 ++++++---- dal/validation/__init__.py | 28 +++++++++++++++++-- tests/unit/tools/test_extract_i18n.py | 2 +- tests/unit/with_db/scopes/test_translation.py | 2 +- 13 files changed, 74 insertions(+), 34 deletions(-) create mode 100644 dal/scopes/translation_constants.py diff --git a/dal/data/schema.py b/dal/data/schema.py index 78ba6742..f76b9f73 100644 --- a/dal/data/schema.py +++ b/dal/data/schema.py @@ -12,7 +12,7 @@ from .tree import TreeNode, DictNode, ObjectNode, PropertyNode, CallableNode from .serialization import ObjectDeserializer from .version import VersionNode -from dal.validation import REDIS_SCHEMA_FOLDER_PATH +from dal.validation.constants import REDIS_SCHEMA_FOLDER_PATH class SchemaNode(DictNode): diff --git a/dal/models/__init__.py b/dal/models/__init__.py index 3a320dd8..21d4135f 100644 --- a/dal/models/__init__.py +++ b/dal/models/__init__.py @@ -10,13 +10,11 @@ from .configuration import Configuration from .aclobject import AclObject from .application import Application -from .baseuser import BaseUser from .callback import Callback from .container import Container from .flow import Flow from .flowlinks import FlowLinks from .form import Form -from .internaluser import InternalUser from .ldapconfig import LdapConfig from .lock import Lock from .message import Message @@ -25,8 +23,6 @@ from .nodeinst import NodeInst from .package import Package from .ports import Ports -from .remoteuser import RemoteUser -from .role import Role from .scopestree import ( scopes, ScopeInstanceVersionNode, @@ -36,7 +32,6 @@ ScopesTree, ) from .system import System -from .user import User from .var import Var from .widget import Widget @@ -71,14 +66,12 @@ "Configuration", "AclObject", "Application", - "BaseUser", "Callback", "Configuration", "Container", "Flow", "FlowLinks", "Form", - "InternalUser", "LdapConfig", "Lock", "Message", @@ -87,8 +80,6 @@ "NodeInst", "Package", "Ports", - "RemoteUser", - "Role", "scopes", "ScopeInstanceVersionNode", "ScopePropertyNode", @@ -96,7 +87,6 @@ "ScopeObjectNode", "ScopesTree", "System", - "User", "Var", "Widget", ] diff --git a/dal/models/baseuser.py b/dal/models/baseuser.py index 571deb57..de9472b5 100644 --- a/dal/models/baseuser.py +++ b/dal/models/baseuser.py @@ -27,7 +27,7 @@ from dal.models.model import Model from dal.models.acl import NewACLManager from dal.models.acl import ResourceType -from dal.scopes.translation import DEFAULT_LANGUAGE +from dal.scopes.translation_constants import DEFAULT_LANGUAGE class BaseUser(Model): diff --git a/dal/models/internaluser.py b/dal/models/internaluser.py index 813232c0..c925de79 100644 --- a/dal/models/internaluser.py +++ b/dal/models/internaluser.py @@ -14,7 +14,7 @@ from dal.models.model import Model from dal.models.baseuser import BaseUser from dal.models.user import User -from dal.scopes.translation import DEFAULT_LANGUAGE +from dal.scopes.translation_constants import DEFAULT_LANGUAGE class InternalUser(BaseUser): diff --git a/dal/movaidb/db_schema.py b/dal/movaidb/db_schema.py index 2b9d05ed..1b0ab7d1 100644 --- a/dal/movaidb/db_schema.py +++ b/dal/movaidb/db_schema.py @@ -2,7 +2,7 @@ from typing import Dict from dal.plugins.classes import Resource -from dal.validation import REDIS_SCHEMA_FOLDER_PATH +from dal.validation.constants import REDIS_SCHEMA_FOLDER_PATH class DBSchema(dict): diff --git a/dal/scopes/__init__.py b/dal/scopes/__init__.py index ca67c64b..72701952 100644 --- a/dal/scopes/__init__.py +++ b/dal/scopes/__init__.py @@ -23,9 +23,9 @@ from .statemachine import StateMachine, SMVars from .structures import Struct from .system import System -from .translation import Translation from .user import User from .widget import Widget +from .translation_constants import DEFAULT_LANGUAGE from dal.utils import ( UsageSearchResult, DirectNodeUsageItem, @@ -62,11 +62,11 @@ "SMVars", "Struct", "System", - "Translation", "UsageSearchResult", "User", "Widget", "Alert", + "DEFAULT_LANGUAGE", "get_usage_search_scope_map", ] diff --git a/dal/scopes/scope.py b/dal/scopes/scope.py index f90a94f6..04c79664 100644 --- a/dal/scopes/scope.py +++ b/dal/scopes/scope.py @@ -13,13 +13,10 @@ """ from typing import List from functools import cached_property - -from dal.validation.validator import Validator from movai_core_shared.exceptions import DoesNotExist, AlreadyExist from .structures import Struct from dal.movaidb import MovaiDB from dal.movaidb.db_schema import DBSchema -from dal.validation import JsonValidator SCOPES_TO_VALIDATE: List[str] = ["Translation", "Alert"] @@ -29,13 +26,30 @@ class Scope(Struct): """Scope main class. Attributes: - validator (JsonValidator): Validator for the scope. + validator (JsonValidator): Validator for the scope """ permissions = ["create", "read", "update", "delete"] - validator: Validator = JsonValidator() + validator = None + + @classmethod + def get_validator(cls): + """Lazy-load validator only when actually needed. + + If a subclass defines its own validator, use that instead. + """ + # Check if this specific class has its own validator + if "validator" in cls.__dict__ and cls.__dict__["validator"] is not None: + return cls.__dict__["validator"] + + # Otherwise, lazy-load the default validator + if cls.validator is None: + from dal.validation import JsonValidator + + cls.validator = JsonValidator() + return cls.validator def __init__(self, scope, name, version, new=False, db="global"): self.__dict__["name"] = name @@ -156,4 +170,4 @@ def validate_format(cls, scope, data: dict): """ if scope in SCOPES_TO_VALIDATE: - cls.validator.validate(scope, data) + cls.get_validator().validate(scope, data) diff --git a/dal/scopes/translation.py b/dal/scopes/translation.py index f502af33..b35b4437 100644 --- a/dal/scopes/translation.py +++ b/dal/scopes/translation.py @@ -1,8 +1,8 @@ from dal.validation.validator import TranslationValidator from .scope import Scope - -DEFAULT_LANGUAGE = "en" +# pylint: disable=unused-import +from .translation_constants import DEFAULT_LANGUAGE class Translation(Scope): diff --git a/dal/scopes/translation_constants.py b/dal/scopes/translation_constants.py new file mode 100644 index 00000000..f2942ac6 --- /dev/null +++ b/dal/scopes/translation_constants.py @@ -0,0 +1,9 @@ +""" + Copyright (C) Mov.ai - All Rights Reserved + Unauthorized copying of this file, via any medium is strictly prohibited + Proprietary and confidential + + Translation constants - separated to avoid loading heavy validator modules. +""" + +DEFAULT_LANGUAGE = "en" diff --git a/dal/tools/backup.py b/dal/tools/backup.py index 0d939550..a0591dc0 100644 --- a/dal/tools/backup.py +++ b/dal/tools/backup.py @@ -79,12 +79,17 @@ class Factory: def get_class(scope): """Get scope class.""" if scope not in Factory.CLASSES_CACHE: - mod = import_module("dal.scopes") - - try: + # Translation is not in package-level imports to avoid heavy validator load + # Import directly when needed + if scope == "Translation": + mod = import_module("dal.scopes.translation") Factory.CLASSES_CACHE[scope] = getattr(mod, scope) - except AttributeError as exc: - raise BackupException(f"Scope does not exists {scope}") from exc + else: + mod = import_module("dal.scopes") + try: + Factory.CLASSES_CACHE[scope] = getattr(mod, scope) + except AttributeError as exc: + raise BackupException(f"Scope does not exists {scope}") from exc return Factory.CLASSES_CACHE[scope] diff --git a/dal/validation/__init__.py b/dal/validation/__init__.py index 1bbb3f1a..11d32ee7 100644 --- a/dal/validation/__init__.py +++ b/dal/validation/__init__.py @@ -6,10 +6,32 @@ Developers: - Moawiya Mograbi (moawiya@mov.ai) - 2022 """ +from typing import TYPE_CHECKING + from .constants import REDIS_SCHEMA_FOLDER_PATH, JSON_SCHEMA_FOLDER_PATH -from .schema import Schema -from .validator import JsonValidator -from .template import Template + +# Import for type checking only - actual imports are lazy-loaded via __getattr__ +if TYPE_CHECKING: + from .schema import Schema + from .template import Template + from .validator import JsonValidator + + +def __getattr__(name): + """Lazy-load validation classes to avoid loading jsonschema unless actually needed.""" + if name == "Schema": + from .schema import Schema + + return Schema + elif name == "JsonValidator": + from .validator import JsonValidator + + return JsonValidator + elif name == "Template": + from .template import Template + + return Template + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") __all__ = [ diff --git a/tests/unit/tools/test_extract_i18n.py b/tests/unit/tools/test_extract_i18n.py index ac9379c1..67bf6d08 100644 --- a/tests/unit/tools/test_extract_i18n.py +++ b/tests/unit/tools/test_extract_i18n.py @@ -88,7 +88,7 @@ def test_gen_and_import(self, tmp_path, global_db): main() from dal.tools.backup import Importer - from dal.scopes import Translation + from dal.scopes.translation import Translation importer = Importer( tmp_path, diff --git a/tests/unit/with_db/scopes/test_translation.py b/tests/unit/with_db/scopes/test_translation.py index 8e796e67..49eaf3bd 100644 --- a/tests/unit/with_db/scopes/test_translation.py +++ b/tests/unit/with_db/scopes/test_translation.py @@ -4,7 +4,7 @@ class TestTranslation: def test_translation(self, global_db, metadata_folder): from dal.tools.backup import Importer - from dal.scopes import Translation + from dal.scopes.translation import Translation tool = Importer( metadata_folder, From e26ad12ad3880e6a05e57c0d48f0ea6d917de7da Mon Sep 17 00:00:00 2001 From: daniela-costa7 Date: Mon, 1 Jun 2026 16:17:14 +0100 Subject: [PATCH 03/14] dynamic lazy imports in init files --- dal/models/__init__.py | 86 +++++++++++++++++++++------ dal/scopes/__init__.py | 115 +++++++++++++++++++++++++++---------- dal/validation/__init__.py | 27 ++++----- 3 files changed, 168 insertions(+), 60 deletions(-) diff --git a/dal/models/__init__.py b/dal/models/__init__.py index 21d4135f..9ce0e0f9 100644 --- a/dal/models/__init__.py +++ b/dal/models/__init__.py @@ -6,23 +6,16 @@ Developers: - Moawiya Mograbi (moawiya@mov.ai) - 2022 """ -from .acl import ACLManager, NewACLManager -from .configuration import Configuration -from .aclobject import AclObject -from .application import Application -from .callback import Callback -from .container import Container -from .flow import Flow -from .flowlinks import FlowLinks -from .form import Form -from .ldapconfig import LdapConfig -from .lock import Lock -from .message import Message +import importlib +from typing import TYPE_CHECKING + +# Eagerly import critical base classes and singletons to avoid initialization issues from .model import Model from .node import Node from .nodeinst import NodeInst -from .package import Package -from .ports import Ports +from .container import Container +from .flow import Flow +from .flowlinks import FlowLinks from .scopestree import ( scopes, ScopeInstanceVersionNode, @@ -31,9 +24,63 @@ ScopeObjectNode, ScopesTree, ) -from .system import System -from .var import Var -from .widget import Widget + +# Import for type checking only - actual imports are lazy-loaded via __getattr__ +if TYPE_CHECKING: + from .acl import ACLManager, NewACLManager + from .aclobject import AclObject + from .application import Application + from .baseuser import BaseUser + from .callback import Callback + from .configuration import Configuration + from .form import Form + from .internaluser import InternalUser + from .ldapconfig import LdapConfig + from .lock import Lock + from .message import Message + from .package import Package + from .ports import Ports + from .remoteuser import RemoteUser + from .role import Role + from .system import System + from .user import User + from .var import Var + from .widget import Widget + +# Mapping of attribute names to their module paths (excluding eagerly loaded ones) +_LAZY_IMPORTS = { + "ACLManager": ".acl", + "NewACLManager": ".acl", + "Configuration": ".configuration", + "AclObject": ".aclobject", + "Application": ".application", + "BaseUser": ".baseuser", + "Callback": ".callback", + "Form": ".form", + "InternalUser": ".internaluser", + "LdapConfig": ".ldapconfig", + "Lock": ".lock", + "Message": ".message", + "Package": ".package", + "Ports": ".ports", + "RemoteUser": ".remoteuser", + "Role": ".role", + "System": ".system", + "User": ".user", + "Var": ".var", + "Widget": ".widget", +} + + +def __getattr__(name): + """Dynamically import classes on first access to reduce memory usage.""" + if name in _LAZY_IMPORTS: + module = importlib.import_module(_LAZY_IMPORTS[name], package=__name__) + attr = getattr(module, name) + # Cache it in globals for faster subsequent access + globals()[name] = attr + return attr + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") try: @@ -66,12 +113,14 @@ "Configuration", "AclObject", "Application", + "BaseUser", "Callback", "Configuration", "Container", "Flow", "FlowLinks", "Form", + "InternalUser", "LdapConfig", "Lock", "Message", @@ -80,6 +129,8 @@ "NodeInst", "Package", "Ports", + "RemoteUser", + "Role", "scopes", "ScopeInstanceVersionNode", "ScopePropertyNode", @@ -87,6 +138,7 @@ "ScopeObjectNode", "ScopesTree", "System", + "User", "Var", "Widget", ] diff --git a/dal/scopes/__init__.py b/dal/scopes/__init__.py index 72701952..b19207bb 100644 --- a/dal/scopes/__init__.py +++ b/dal/scopes/__init__.py @@ -6,36 +6,91 @@ Developers: - Moawiya Mograbi (moawiya@mov.ai) - 2022 """ -from .application import Application -from .alert import Alert -from .callback import Callback -from .configuration import Config, Configuration -from .fleetrobot import FleetRobot -from .flow import Flow -from .form import Form -from .message import Message -from .node import Node -from .package import Package -from .ports import Ports -from .robot import Robot -from .role import Role -from .scope import Scope -from .statemachine import StateMachine, SMVars -from .structures import Struct -from .system import System -from .user import User -from .widget import Widget -from .translation_constants import DEFAULT_LANGUAGE -from dal.utils import ( - UsageSearchResult, - DirectNodeUsageItem, - IndirectNodeUsageItem, - DirectFlowUsageItem, - IndirectFlowUsageItem, - NodeFlowUsage, - FlowFlowUsage, - get_usage_search_scope_map, -) +import importlib +from typing import TYPE_CHECKING + +# Import for type checking only - actual imports are lazy-loaded via __getattr__ +if TYPE_CHECKING: + from .alert import Alert + from .application import Application + from .callback import Callback + from .configuration import Config, Configuration + from .fleetrobot import FleetRobot + from .flow import Flow + from .form import Form + from .message import Message + from .node import Node + from .package import Package + from .ports import Ports + from .robot import Robot + from .role import Role + from .scope import Scope + from .statemachine import SMVars, StateMachine + from .structures import Struct + from .system import System + from .translation_constants import DEFAULT_LANGUAGE + from .user import User + from .widget import Widget + from dal.utils import ( + DirectFlowUsageItem, + DirectNodeUsageItem, + FlowFlowUsage, + IndirectFlowUsageItem, + IndirectNodeUsageItem, + NodeFlowUsage, + UsageSearchResult, + get_usage_search_scope_map, + ) + +# Mapping of attribute names to their module paths +_LAZY_IMPORTS = { + "Application": ".application", + "Alert": ".alert", + "Callback": ".callback", + "Config": ".configuration", + "Configuration": ".configuration", + "FleetRobot": ".fleetrobot", + "Flow": ".flow", + "Form": ".form", + "Message": ".message", + "Node": ".node", + "Package": ".package", + "Ports": ".ports", + "Robot": ".robot", + "Role": ".role", + "Scope": ".scope", + "StateMachine": ".statemachine", + "SMVars": ".statemachine", + "Struct": ".structures", + "System": ".system", + "User": ".user", + "Widget": ".widget", + "DEFAULT_LANGUAGE": ".translation_constants", + "UsageSearchResult": "dal.utils", + "DirectNodeUsageItem": "dal.utils", + "IndirectNodeUsageItem": "dal.utils", + "DirectFlowUsageItem": "dal.utils", + "IndirectFlowUsageItem": "dal.utils", + "NodeFlowUsage": "dal.utils", + "FlowFlowUsage": "dal.utils", + "get_usage_search_scope_map": "dal.utils", +} + + +def __getattr__(name): + """Dynamically import classes on first access to reduce memory usage.""" + if name in _LAZY_IMPORTS: + module_path = _LAZY_IMPORTS[name] + if module_path.startswith("."): + module = importlib.import_module(module_path, package=__name__) + else: + module = importlib.import_module(module_path) + attr = getattr(module, name) + # Cache it in globals for faster subsequent access + globals()[name] = attr + return attr + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") + __all__ = [ "Application", diff --git a/dal/validation/__init__.py b/dal/validation/__init__.py index 11d32ee7..483a15f8 100644 --- a/dal/validation/__init__.py +++ b/dal/validation/__init__.py @@ -6,6 +6,7 @@ Developers: - Moawiya Mograbi (moawiya@mov.ai) - 2022 """ +import importlib from typing import TYPE_CHECKING from .constants import REDIS_SCHEMA_FOLDER_PATH, JSON_SCHEMA_FOLDER_PATH @@ -16,21 +17,21 @@ from .template import Template from .validator import JsonValidator +_LAZY_IMPORTS = { + "Schema": ".schema", + "Template": ".template", + "JsonValidator": ".validator", +} -def __getattr__(name): - """Lazy-load validation classes to avoid loading jsonschema unless actually needed.""" - if name == "Schema": - from .schema import Schema - - return Schema - elif name == "JsonValidator": - from .validator import JsonValidator - return JsonValidator - elif name == "Template": - from .template import Template - - return Template +def __getattr__(name): + """Dynamically import classes on first access to reduce memory usage.""" + if name in _LAZY_IMPORTS: + module = importlib.import_module(_LAZY_IMPORTS[name], package=__name__) + attr = getattr(module, name) + # Cache it in globals for faster subsequent access + globals()[name] = attr + return attr raise AttributeError(f"module {__name__!r} has no attribute {name!r}") From 9b092a0977173a8f6ac25ad43375332aeb022310 Mon Sep 17 00:00:00 2001 From: David Dias Date: Mon, 1 Jun 2026 16:20:01 +0100 Subject: [PATCH 04/14] Cleanup helpers and Callback imports --- dal/helpers/__init__.py | 20 --- dal/models/callback.py | 269 +-------------------------------------- dal/scopes/callback.py | 256 +------------------------------------ dal/scopes/flow.py | 2 +- dal/scopes/node.py | 2 +- dal/scopes/package.py | 2 +- dal/scopes/structures.py | 2 +- 7 files changed, 12 insertions(+), 541 deletions(-) diff --git a/dal/helpers/__init__.py b/dal/helpers/__init__.py index 5a137dec..e69de29b 100644 --- a/dal/helpers/__init__.py +++ b/dal/helpers/__init__.py @@ -1,20 +0,0 @@ -""" - Copyright (C) Mov.ai - All Rights Reserved - Unauthorized copying of this file, via any medium is strictly prohibited - Proprietary and confidential - - Developers: - - Moawiya Mograbi (moawiya@mov.ai) - 2022 -""" - -# from .parsers import ParamParser, get_string_from_template -from .flow.gflow import GFlow -from .helpers import Helpers, flatten - -__all__ = [ - # "ParamParser", - "GFlow", - # "get_string_from_template", - "Helpers", - "flatten", -] diff --git a/dal/models/callback.py b/dal/models/callback.py index e512d805..7fd705b6 100644 --- a/dal/models/callback.py +++ b/dal/models/callback.py @@ -12,14 +12,10 @@ import importlib import inspect import pkgutil -import pydoc -import sys from typing import Any, Dict, List -import rospkg from movai_core_shared.logger import Log from .scopestree import scopes -from dal.scopes.system import System from .model import Model @@ -58,6 +54,8 @@ def has_permission( @staticmethod def get_modules() -> List: """Get list of modules""" + import pydoc + modules_list = [x[1] for x in pkgutil.iter_modules()] modules = {} @@ -79,6 +77,8 @@ def onerror(modname): @staticmethod def get_methods(module: str) -> Dict: """get methods of module""" + import pydoc + try: object = importlib.import_module(module) @@ -153,267 +153,6 @@ def get_full_modules() -> Dict: return module_description - # ported blindly - @staticmethod - def _get_modules(jump_over_modules) -> dict: - """new, recursive way to get modules - - This kinda looks like: - - pkg1/ - | - + __init__.py - \\ mod_dred.py - mod1.py - - - { - 'pkg1' : { - 'isPkg' : True, - 'functions' : ['foo1','bar1'], - 'classes' : ['class1'], - 'constants' : ['ID1','factor1'], - 'modules' : { - 'mod_dred' : { - 'isPkg': False, - 'functions' : ['bar2','foo2'], - 'classes' : [], - 'constants' : [] - # no modules here - } - } - }, - 'mod1' : { - 'isPkg' : False, - 'functions' : [], - 'constants' : [], - 'classes' : ['MegaClass'] - # again, no modules here - } - } - Args: - jump_over_modules (list): the list of modules not to expand - - """ - - modules = {} - - def expand_package(ret_dict: dict, pkg: pkgutil.ModuleInfo, parent: str = ""): - """check package for nested modules""" - path = pkg[0].path + "/" + pkg[1] - - # expand it - this = parent + "." + pkg[1] - if this[0] == ".": - this = this[1:] - - # iterate contents of this package - for x in pkgutil.iter_modules([path]): - # ignore '_*' modules - if x[1].startswith("_") or x[1] == "init_local_db" or x[1] == "tf_monitor": - continue - - # shouldn't be python2 for sure - ret_dict[x[1]] = {"isPkg": x[2]} - - # always expand module - v = expand_module(ret_dict[x[1]], this + "." + x[1]) - # and maybe expand package - if x[2] and v: # on error expanding module, completely ignore it - ret_dict[x[1]]["modules"] = {} - expand_package(ret_dict[x[1]]["modules"], x, this) - - if not v: - # remove it from the list if erroneous - del ret_dict[x[1]] - - def expand_module(ret_dict: dict, mod: str) -> bool: - """get methods, classes and constants from the module - - return True on success - return False on module not found or error parsing the module - """ - - i_mod = None - try: - i_mod = importlib.import_module(mod) - except ModuleNotFoundError: - # this is an actual error - return False - except: - # this may actually not be an error ... - # but we can't still work with it - # so, acknowledge it exists - # (this happens with API2.Scopes) - return True - - try: - # i_mod = importlib.import_module(mod) - - # now read its internals - # dammed ros - a_all = None - try: - a_all = getattr(i_mod, "__all__", None) - except: - # ros sends another exception instead of using default value... - # #python2 - pass - - ret_dict["classes"] = [] - ret_dict["functions"] = [] - ret_dict["consts"] = [] - - # FIXME probably needs optimizing, still figuring how to... - # when intersecting the class/function/data list with the __all__ list - - # get classes - for key, value in inspect.getmembers(i_mod, inspect.isclass): - # ignore __var__ like __these__ - # probably simpler than re.match() - if key[:2] == "__" and key[-2:] == "__": - continue - - if a_all is not None: - # if there is an __all__ - # and it's advertised there - if key in a_all: - # add it - ret_dict["classes"].append(key) - # ignore the value tho, only need key - else: - # just add it - ret_dict["classes"].append(key) - - # get methods - for key, value in inspect.getmembers(i_mod, inspect.isroutine): - # ignore __var__ like __these__ - if key[:2] == "__" and key[-2:] == "__": - continue - - if a_all is not None: - # if there is an __all__ - # and it's advertised there - if key in a_all: - # add it - ret_dict["functions"].append(key) - # ignore the value tho, only need key - else: - # just add it - ret_dict["functions"].append(key) - - # get data/constants - for key, value in inspect.getmembers(i_mod, pydoc.isdata): - # ignore __var__ like __these__ - if key[:2] == "__" and key[-2:] == "__": - continue - - if a_all is not None: - # if there is an __all__ - # and it's advertised there - if key in a_all: - # add it - ret_dict["consts"].append(key) - # ignore the value tho, only need key - else: - # just add it - # or don't ? - ret_dict["consts"].append(key) - - # so it's done ? - del i_mod - except rospkg.common.ResourceNotFound: - # ros throws this exception somewhere in inspect.getmembers - # called on a ros (python2) module - # which means we can't get the module members, - # but still add to the list of modules - return True # early return - except: # something? - return False - return True - - # iterate every module found - - builtin_modules = [("", name, False) for name in sys.builtin_module_names] - for x in set(list(pkgutil.iter_modules()) + builtin_modules): - # now ... - # x[0] -> FileFinder(path_where_modules_was_found) - # x[1] -> module/package name - # x[2] -> if it is a package (True=package,False=module) - - # this finds all modules, including python2, so we need to filter that - - # filter x[0].path for python2 (ignore those) - # not anymore - # if '/python2.' in x[0].path: - # # ignore this one - # continue - - # ignore _pkgs _starting _with '_' - if x[1].startswith("_") or x[1] == "init_local_db": - # nope - - continue - # create the template dict - modules[x[1]] = {"isPkg": x[2]} - - # i guess always expand module - v = expand_module(modules[x[1]], x[1]) - if x[1] in jump_over_modules: - continue - - # and maybe expand the package, if it's one - if x[2] and v: - # package - modules[x[1]]["modules"] = {} - expand_package(modules[x[1]]["modules"], x) - - if not v: # on error - del modules[x[1]] - - # and return it - return modules - - @staticmethod - def export_modules(jump_over_modules=None) -> None: - """Get modules and save them to db (System) - - this takes about 4 seconds to run - - and prints a LOT of stuff to the terminal - - currently (14-10 on spawner) uses about 19kb of memory (?) - Args: - jump_over_modules (list): the list of modules not to expand - """ - if jump_over_modules is None: - jump_over_modules = ["scipy", "twisted"] - - data = Callback._get_modules(jump_over_modules) - - try: - # currently using the old api - mods = System("PyModules", db="local") # scopes('local').System['PyModules', 'cache'] - except Exception: # pylint: disable=broad-except - mods = System( - "PyModules", new=True, db="local" - ) # scopes('local').create('System', 'PyModules') - - mods.Value = data - - del mods, data # not that it's gonna make a difference ... - - @staticmethod - def fetch_modules_api() -> Dict: - """to be called from rest api""" - try: - return System( - "PyModules", db="local" - ).Value # scopes('local').System['PyModules', 'cache'].Value - except Exception: # pylint: disable=broad-except - # non existent yet - return {} - def template_depends(self, force: bool = False) -> Dict: """get all the objects that depend on this callback""" # FIXME either implement or wait for api to be able to handle reverse dependency lookup diff --git a/dal/scopes/callback.py b/dal/scopes/callback.py index cf9534bb..8329f8e1 100644 --- a/dal/scopes/callback.py +++ b/dal/scopes/callback.py @@ -10,10 +10,8 @@ Module that implements a Callback scope class https://github.com/python/cpython/blob/master/Lib/pydoc.py """ -import sys import inspect import pkgutil -import pydoc import importlib from dal.movaidb import MovaiDB @@ -73,6 +71,8 @@ def remove(self, force=False): @staticmethod def get_modules() -> list: + import pydoc + modules_list = [x[1] for x in pkgutil.iter_modules()] modules = {} @@ -92,6 +92,8 @@ def onerror(modname): @staticmethod def get_methods(module: str) -> dict: + import pydoc + try: object = importlib.import_module(module) @@ -181,256 +183,6 @@ def get_full_modules() -> dict: return module_description - @staticmethod - def _get_modules() -> dict: - """new, recursive way to get modules""" - - modules = {} - - # use to catch an exception below - import rospkg.common - - """ - This kinda looks like: - - pkg1/ - | - + __init__.py - \ mod_dred.py - mod1.py - - - { - 'pkg1' : { - 'isPkg' : True, - 'functions' : ['foo1','bar1'], - 'classes' : ['class1'], - 'constants' : ['ID1','factor1'], - 'modules' : { - 'mod_dred' : { - 'isPkg': False, - 'functions' : ['bar2','foo2'], - 'classes' : [], - 'constants' : [] - # no modules here - } - } - }, - 'mod1' : { - 'isPkg' : False, - 'functions' : [], - 'constants' : [], - 'classes' : ['MegaClass'] - # again, no modules here - } - } - - """ - - def expand_package(ret_dict: dict, pkg: pkgutil.ModuleInfo, parent: str = ""): - """check package for nested modules""" - path = pkg[0].path + "/" + pkg[1] - - # expand it - this = parent + "." + pkg[1] - if this[0] == ".": - this = this[1:] - - # iterate contents of this package - for x in pkgutil.iter_modules([path]): - # ignore '_*' modules - if x[1].startswith("_") or x[1] == "init_local_db": - continue - - # shouldn't be python2 for sure - ret_dict[x[1]] = {"isPkg": x[2]} - - # always expand module - v = expand_module(ret_dict[x[1]], this + "." + x[1]) - # and maybe expand package - if x[2] and v: # on error expanding module, completely ignore it - ret_dict[x[1]]["modules"] = {} - expand_package(ret_dict[x[1]]["modules"], x, this) - - if not v: - # remove it from the list if erroneous - del ret_dict[x[1]] - - def expand_module(ret_dict: dict, mod: str) -> bool: - """get methods, classes and constants from the module - - return True on success - return False on module not found or error parsing the module - """ - - i_mod = None - try: - i_mod = importlib.import_module(mod) - except ModuleNotFoundError: - # this is an actual error - return False - except: - # this may actually not be an error ... - # but we can't still work with it - # so, acknowledge it exists - # (this happens with API2.Scopes) - return True - - try: - # i_mod = importlib.import_module(mod) - - # now read its internals - # dammed ros - a_all = None - try: - a_all = getattr(i_mod, "__all__", None) - except: - # ros sends another exception instead of using default value... - # #python2 - pass - - ret_dict["classes"] = [] - ret_dict["functions"] = [] - ret_dict["consts"] = [] - - # FIXME probably needs optimizing, still figuring how to... - # when intersecting the class/function/data list with the __all__ list - - # get classes - for key, value in inspect.getmembers(i_mod, inspect.isclass): - # ignore __var__ like __these__ - # probably simpler than re.match() - if key[:2] == "__" and key[-2:] == "__": - continue - - if a_all is not None: - # if there is an __all__ - # and it's advertised there - if key in a_all: - # add it - ret_dict["classes"].append(key) - # ignore the value tho, only need key - else: - # just add it - ret_dict["classes"].append(key) - - # get methods - for key, value in inspect.getmembers(i_mod, inspect.isroutine): - # ignore __var__ like __these__ - if key[:2] == "__" and key[-2:] == "__": - continue - - if a_all is not None: - # if there is an __all__ - # and it's advertised there - if key in a_all: - # add it - ret_dict["functions"].append(key) - # ignore the value tho, only need key - else: - # just add it - ret_dict["functions"].append(key) - - # get data/constants - for key, value in inspect.getmembers(i_mod, pydoc.isdata): - # ignore __var__ like __these__ - if key[:2] == "__" and key[-2:] == "__": - continue - - if a_all is not None: - # if there is an __all__ - # and it's advertised there - if key in a_all: - # add it - ret_dict["consts"].append(key) - # ignore the value tho, only need key - else: - # just add it - # or don't ? - ret_dict["consts"].append(key) - - # so it's done ? - del i_mod - except rospkg.common.ResourceNotFound: - # ros throws this exception somewhere in inspect.getmembers - # called on a ros (python2) module - # which means we can't get the module members, - # but still add to the list of modules - return True # early return - except: # something? - return False - return True - - # iterate every module found - - builtin_modules = [("", name, False) for name in sys.builtin_module_names] - for x in list(pkgutil.iter_modules()) + builtin_modules: - # now ... - # x[0] -> FileFinder(path_where_modules_was_found) - # x[1] -> module/package name - # x[2] -> if it is a package (True=package,False=module) - - # this finds all modules, including python2, so we need to filter that - - # filter x[0].path for python2 (ignore those) - # not anymore - # if '/python2.' in x[0].path: - # # ignore this one - # continue - - # ignore _pkgs _starting _with '_' - if x[1].startswith("_"): - # nope - continue - - # create the template dict - modules[x[1]] = {"isPkg": x[2]} - - # i guess always expand module - v = expand_module(modules[x[1]], x[1]) - - # and maybe expand the package, if it's one - if x[2] and v: - # package - modules[x[1]]["modules"] = {} - expand_package(modules[x[1]]["modules"], x) - - if not v: # on error - del modules[x[1]] - - # not using re anymore - del rospkg.common - - # and return it - return modules - - @staticmethod - def export_modules(): - """Get modules and save them to Layer1 - - this takes about 4 seconds to run - - and prints a LOT of stuff to the terminal - - currently (14-10 on spawner) uses about 19kb of memory (?) - """ - # useless - - root = Callback._get_modules() - # +DEBUG - """ - import json - print(json.dumps(root,indent = 2)) - del json - """ - # -DEBUG - - # and now save to database - MovaiDB("local").set({"System": {"PyModules": {"Value": root}}}) - # python frees the class - - # nothing to return - @staticmethod def fetch_modules_api(): """Retrieve saved modules from Layer1, called from REST API""" diff --git a/dal/scopes/flow.py b/dal/scopes/flow.py index 1c8743a7..9a64b26d 100644 --- a/dal/scopes/flow.py +++ b/dal/scopes/flow.py @@ -13,7 +13,7 @@ import uuid from itertools import product -from dal.helpers import flatten +from dal.helpers.helpers import flatten from movai_core_shared.consts import ( CONFIG_REGEX, LINK_REGEX, diff --git a/dal/scopes/node.py b/dal/scopes/node.py index 7b33c857..ba836f9e 100644 --- a/dal/scopes/node.py +++ b/dal/scopes/node.py @@ -29,7 +29,7 @@ DirectNodeUsageItem, IndirectNodeUsageItem, ) -from dal.helpers import Helpers +from dal.helpers.helpers import Helpers from movai_core_shared.logger import Log from typing import Dict diff --git a/dal/scopes/package.py b/dal/scopes/package.py index 49bf4d15..2f9f3c26 100644 --- a/dal/scopes/package.py +++ b/dal/scopes/package.py @@ -15,7 +15,7 @@ from dal.scopes.scope import Scope from dal.scopes.structures import Struct -from dal.helpers import Helpers +from dal.helpers.helpers import Helpers from movai_core_shared.logger import Log LOGGER = Log.get_logger("spawner.mov.ai") diff --git a/dal/scopes/structures.py b/dal/scopes/structures.py index 9ea6e6c7..133c7789 100644 --- a/dal/scopes/structures.py +++ b/dal/scopes/structures.py @@ -13,7 +13,7 @@ from movai_core_shared.exceptions import AlreadyExist from dal.movaidb import MovaiDB -from dal.helpers import Helpers +from dal.helpers.helpers import Helpers class List(list): From a33c201646bed6ffb7edb9c0ed4231413460ac2d Mon Sep 17 00:00:00 2001 From: David Dias Date: Mon, 1 Jun 2026 16:26:46 +0100 Subject: [PATCH 05/14] Only import pyyaml and deepdiff when needed --- dal/models/configuration.py | 4 +++- dal/movaidb/database.py | 3 ++- dal/plugins/resources/file/file.py | 3 ++- dal/scopes/alert.py | 3 ++- dal/scopes/configuration.py | 13 ++----------- 5 files changed, 11 insertions(+), 15 deletions(-) diff --git a/dal/models/configuration.py b/dal/models/configuration.py index 87273945..935a4df5 100644 --- a/dal/models/configuration.py +++ b/dal/models/configuration.py @@ -8,7 +8,6 @@ - Moawiya Mograni (moawiya@mov.ai) - 2023 """ import pickle -import yaml from dal.movaidb import MovaiDB from .model import Model from dal.helpers.cache import ThreadSafeCache @@ -20,6 +19,8 @@ def __init__(self): self._map = {} def yaml_load(self, config_name, yaml_str): + import yaml + with self.__class__._lock: # Lock for thread safety map_key = config_name if map_key in self._map: @@ -53,6 +54,7 @@ def _get_db_yaml(self) -> str: def get_value(self, cached=True) -> dict: """Returns a dictionary with the configuration values""" + import yaml if self.Type == "xml": # Yaml is the name of the field diff --git a/dal/movaidb/database.py b/dal/movaidb/database.py index 0a0783fd..0dd6aec8 100755 --- a/dal/movaidb/database.py +++ b/dal/movaidb/database.py @@ -21,7 +21,6 @@ import dal import redis import random -from deepdiff import DeepDiff from redis.client import Pipeline from redis.connection import Connection @@ -1164,6 +1163,8 @@ def validate_path(path: str, structure: dict) -> Optional[str]: @staticmethod def calc_scope_update(old_dict: dict, new_dict: dict, structure: dict) -> List[Dict[str, Any]]: """Calculate scope updates dicts""" + from deepdiff import DeepDiff + # Translate dict to list of paths old_dict_paths = [p for p in MovaiDB.dict_to_paths(old_dict)] # Translate dict to list of paths diff --git a/dal/plugins/resources/file/file.py b/dal/plugins/resources/file/file.py index d9069df2..9e50aa1d 100644 --- a/dal/plugins/resources/file/file.py +++ b/dal/plugins/resources/file/file.py @@ -9,7 +9,6 @@ from io import BytesIO, StringIO from json import JSONDecodeError, load from os import listdir, path, getenv, getcwd -import yaml from dal.plugins.classes import Plugin, Resource, ResourcePlugin, ResourceException __DRIVER_NAME__ = "Filesystem Plugin" @@ -64,6 +63,8 @@ def read_yaml(self, url: str): """ read a yaml file, returns a dict """ + import yaml + local_path = self._get_local_path(url) with open(local_path, "r") as fd: diff --git a/dal/scopes/alert.py b/dal/scopes/alert.py index fdc26a7e..1675ee63 100644 --- a/dal/scopes/alert.py +++ b/dal/scopes/alert.py @@ -8,7 +8,6 @@ from dal.scopes.robot import Robot from movai_core_shared.logger import Log from movai_core_shared.consts import DeactivationType -from movai_core_shared.messages.alert_data import AlertActivationData try: from movai_core_enterprise.message_client_handlers._alert_metrics import AlertMetricsFactory @@ -71,6 +70,8 @@ def activate(self, **kwargs): **kwargs: Parameters to fill placeholders in Title, Info, and Action fields. """ + from movai_core_shared.messages.alert_data import AlertActivationData + # Verify that all necessary activation fields were provided self.validate_parameters("Title", self.Title, **kwargs) self.validate_parameters("Info", self.Info, **kwargs) diff --git a/dal/scopes/configuration.py b/dal/scopes/configuration.py index fc2f9553..545661a6 100644 --- a/dal/scopes/configuration.py +++ b/dal/scopes/configuration.py @@ -9,8 +9,6 @@ Module that implements Configuration scope class """ import pickle -import yaml -from box import Box from dal.helpers.cache import ThreadSafeCache from .scope import Scope @@ -41,6 +39,8 @@ def _get_db_yaml(self) -> str: def get_value(self) -> dict: """Returns a dictionary with the configuration values""" + import yaml + if self.Type == "xml": # Yaml is the name of the field return self._get_db_yaml() @@ -70,12 +70,3 @@ def get_param(self, param: str): '"%s" is not a valid parameter in configuration "%s"' % (param, self.name) ) return value - - -class Config(Box): - """Config with dot accessible elements""" - - def __init__(self, name): - # raises DoesNotExist in case Configuration name does not exist - config = Configuration(name).get_value() - super().__init__(Box(config)) From 28c6292ee5c9a752a60104fa684e45d26037e7b7 Mon Sep 17 00:00:00 2001 From: David Dias Date: Mon, 1 Jun 2026 16:44:56 +0100 Subject: [PATCH 06/14] Only import rosmsg if needed --- dal/scopes/message.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/dal/scopes/message.py b/dal/scopes/message.py index ae3ee636..10a045ff 100644 --- a/dal/scopes/message.py +++ b/dal/scopes/message.py @@ -14,7 +14,6 @@ import os import rospkg import genmsg -import rosmsg from dal.scopes.scope import Scope from dal.movaidb import MovaiDB @@ -34,6 +33,7 @@ def is_valid(self): @classmethod def get_packages(cls, msg_type="all", db="global") -> list: """Gives a list of all packages containing messages of a type""" + import rosmsg if msg_type not in ["all", "msg", "srv", "action"]: raise Exception( @@ -82,6 +82,7 @@ def get_packages(cls, msg_type="all", db="global") -> list: @classmethod def get_msgs(cls, package: str, msg_type="all", db="global") -> list: """Gives a list of all messages of some type in a package""" + import rosmsg if msg_type not in ["all", "msg", "srv", "action"]: raise Exception( @@ -138,6 +139,8 @@ def get_msgs(cls, package: str, msg_type="all", db="global") -> list: @classmethod def get_all(cls, db="global") -> dict: """Gives the all available messages of all types organized by package""" + import rosmsg + rospack = rospkg.RosPack() full_dict = {} @@ -165,6 +168,7 @@ def get_all(cls, db="global") -> dict: @classmethod def export_portdata(cls, db="global") -> dict: """Gives all messages/services available organized by type. Possibly intended to replace get_all(), but kept for legacy reasons""" + import rosmsg rospack = rospkg.RosPack() @@ -280,6 +284,8 @@ def fetch_portdata_messages() -> dict: @staticmethod def get_structure(message: str) -> dict: """Gives the full structure of a message given the 'package/message' input""" + import rosmsg + rospack = rospkg.RosPack() package, msg_name = message.split("/") try: # MESSAGE From 5cd44ba809878c681088489f0ba77230b145cda8 Mon Sep 17 00:00:00 2001 From: David Dias Date: Mon, 1 Jun 2026 16:45:53 +0100 Subject: [PATCH 07/14] Only import AlertData if needed --- dal/scopes/robot.py | 35 ++++++++++++++++++++++++++--------- 1 file changed, 26 insertions(+), 9 deletions(-) diff --git a/dal/scopes/robot.py b/dal/scopes/robot.py index 6422bdba..9ed9f5bf 100644 --- a/dal/scopes/robot.py +++ b/dal/scopes/robot.py @@ -9,7 +9,7 @@ Module that implements Robot namespace """ -from typing import List, Optional +from typing import TYPE_CHECKING, List, Optional import asyncio import uuid import pickle @@ -23,11 +23,6 @@ ) from movai_core_shared.envvars import SPAWNER_BIND_ADDR, DEVICE_NAME from movai_core_shared.logger import Log -from movai_core_shared.messages.alert_data import ( - AlertActivationData, - AlertDeactivationData, - AlertData, -) from dal.scopes.scope import Scope from dal.movaidb import MovaiDB @@ -35,6 +30,14 @@ from datetime import datetime from .configuration import Configuration +if TYPE_CHECKING: + from movai_core_shared.messages.alert_data import ( + AlertActivationData, + AlertDeactivationData, + AlertData, + ) + + LOGGER = Log.get_logger("dal.mov.ai") @@ -146,7 +149,7 @@ def set_role(self, role: Role): def add_active_alert( self, alert_id: str, - data: AlertActivationData, + data: "AlertActivationData", ): """Add an active alert to the Robot""" if "ActiveAlerts" not in self.fleet.__dict__: @@ -159,8 +162,14 @@ def add_active_alert( def pop_alert( self, alert_id: str, deactivation_type: str = DeactivationType.REQUESTED - ) -> Optional[AlertData]: + ) -> Optional["AlertData"]: """Remove an active alert from the Robot""" + from movai_core_shared.messages.alert_data import ( + AlertActivationData, + AlertDeactivationData, + AlertData, + ) + if "ActiveAlerts" in self.fleet.__dict__: if alert_id in self.fleet.ActiveAlerts: alert = self.fleet.ActiveAlerts.pop(alert_id) @@ -176,8 +185,16 @@ def pop_alert( **deactivation.model_dump(), ) - def clear_alerts(self, deactivation_type: str = DeactivationType.REQUESTED) -> List[AlertData]: + def clear_alerts( + self, deactivation_type: str = DeactivationType.REQUESTED + ) -> List["AlertData"]: """Clear all active alerts from the Robot""" + from movai_core_shared.messages.alert_data import ( + AlertActivationData, + AlertDeactivationData, + AlertData, + ) + if "ActiveAlerts" in self.__dict__: LOGGER.warning(f"Clearing all alerts from robot {self.RobotName}") alert_metrics = [] From cc06e3abb16d4d144c57bc13728370da71cee35c Mon Sep 17 00:00:00 2001 From: David Dias Date: Mon, 1 Jun 2026 16:48:48 +0100 Subject: [PATCH 08/14] Clear dal/utils/__init__.py --- dal/scopes/__init__.py | 26 -------------------------- dal/utils/__init__.py | 39 --------------------------------------- 2 files changed, 65 deletions(-) diff --git a/dal/scopes/__init__.py b/dal/scopes/__init__.py index b19207bb..29520fd3 100644 --- a/dal/scopes/__init__.py +++ b/dal/scopes/__init__.py @@ -31,16 +31,6 @@ from .translation_constants import DEFAULT_LANGUAGE from .user import User from .widget import Widget - from dal.utils import ( - DirectFlowUsageItem, - DirectNodeUsageItem, - FlowFlowUsage, - IndirectFlowUsageItem, - IndirectNodeUsageItem, - NodeFlowUsage, - UsageSearchResult, - get_usage_search_scope_map, - ) # Mapping of attribute names to their module paths _LAZY_IMPORTS = { @@ -66,14 +56,6 @@ "User": ".user", "Widget": ".widget", "DEFAULT_LANGUAGE": ".translation_constants", - "UsageSearchResult": "dal.utils", - "DirectNodeUsageItem": "dal.utils", - "IndirectNodeUsageItem": "dal.utils", - "DirectFlowUsageItem": "dal.utils", - "IndirectFlowUsageItem": "dal.utils", - "NodeFlowUsage": "dal.utils", - "FlowFlowUsage": "dal.utils", - "get_usage_search_scope_map": "dal.utils", } @@ -97,17 +79,11 @@ def __getattr__(name): "Callback", "Config", "Configuration", - "DirectFlowUsageItem", - "DirectNodeUsageItem", "FleetRobot", "Flow", - "FlowFlowUsage", "Form", - "IndirectFlowUsageItem", - "IndirectNodeUsageItem", "Message", "Node", - "NodeFlowUsage", "Package", "Ports", "Robot", @@ -117,12 +93,10 @@ def __getattr__(name): "SMVars", "Struct", "System", - "UsageSearchResult", "User", "Widget", "Alert", "DEFAULT_LANGUAGE", - "get_usage_search_scope_map", ] try: diff --git a/dal/utils/__init__.py b/dal/utils/__init__.py index f31a6f70..e69de29b 100644 --- a/dal/utils/__init__.py +++ b/dal/utils/__init__.py @@ -1,39 +0,0 @@ -""" - Copyright (C) Mov.ai - All Rights Reserved - Unauthorized copying of this file, via any medium is strictly prohibited - Proprietary and confidential - - Utility modules for DAL. -""" -from .usage_search.usage_types import ( - UsageSearchResult, - DirectNodeUsageItem, - IndirectNodeUsageItem, - DirectFlowUsageItem, - IndirectFlowUsageItem, - NodeFlowUsage, - FlowFlowUsage, -) - - -# Lazy import to avoid circular dependency -def get_usage_search_scope_map(): - """Get the mapping of scope types that support usage search. - - This is a re-export with lazy loading to avoid circular imports. - """ - from .usage_search.scope_map import get_usage_search_scope_map as _get_map - - return _get_map() - - -__all__ = [ - "UsageSearchResult", - "DirectNodeUsageItem", - "IndirectNodeUsageItem", - "DirectFlowUsageItem", - "IndirectFlowUsageItem", - "NodeFlowUsage", - "FlowFlowUsage", - "get_usage_search_scope_map", -] From 2847b8c1e459025c426e46be0408735b65b96253 Mon Sep 17 00:00:00 2001 From: David Dias Date: Mon, 1 Jun 2026 17:02:25 +0100 Subject: [PATCH 09/14] LazyImport callback builtins --- dal/utils/callback.py | 93 +++++++++++++++++++++++++------------------ 1 file changed, 54 insertions(+), 39 deletions(-) diff --git a/dal/utils/callback.py b/dal/utils/callback.py index 7a94802b..550355db 100644 --- a/dal/utils/callback.py +++ b/dal/utils/callback.py @@ -19,33 +19,16 @@ from movai_core_shared.logger import Log from movai_core_shared.exceptions import TransitionException -# Imports from DAL - -from dal.models.callback import Callback as CallbackModel - from dal.models.lock import Lock -from dal.models.container import Container from dal.models.nodeinst import NodeInst -from dal.scopes.package import Package -from dal.models.ports import Ports from dal.models.var import Var from dal.models.scopestree import ScopesTree, scopes -from dal.scopes.configuration import Configuration -from dal.scopes.fleetrobot import FleetRobot -from dal.scopes.message import Message from dal.scopes.robot import Robot -from dal.scopes.statemachine import StateMachine -from dal.scopes.alert import Alert +# Check if enterprise modules are available try: - from movai_core_enterprise.models.annotation import Annotation - from movai_core_enterprise.models.graphicscene import GraphicScene - from movai_core_enterprise.models.layout import Layout - from movai_core_enterprise.scopes.task import Task - from movai_core_enterprise.models.taskentry import TaskEntry - from movai_core_enterprise.models.tasktemplate import TaskTemplate - from movai_core_enterprise.message_client_handlers.metrics import Metrics + import movai_core_enterprise # pylint: disable=unused-import enterprise = True except ImportError: @@ -74,6 +57,27 @@ def __getattr__(self, name): return getattr(instance, name) +class LazyModule: + """Delay module/class import until first attribute access.""" + + def __init__(self, module_path, class_name=None): + self._module_path = module_path + self._class_name = class_name + self._cached = None + + def _load(self): + if self._cached is None: + module = importlib.import_module(self._module_path) + self._cached = getattr(module, self._class_name) if self._class_name else module + return self._cached + + def __getattr__(self, name): + return getattr(self._load(), name) + + def __call__(self, *args, **kwargs): + return self._load()(*args, **kwargs) + + class UserFunctions: """Class that provides functions to the callback execution""" @@ -98,7 +102,7 @@ def __init__( self.load_classes(_node_name, _port_name, _user) def load_classes(self, _node_name, _port_name, _user): - _robot_id: str = Callback.robot().name + _robot_id: str = UserCallback.robot().name class UserVar(Var): """Class for user to set and get vars""" @@ -123,36 +127,47 @@ def __init__(self, name, **kwargs): self.globals.update( { "scopes": scopes, - "Package": Package, - "Message": Message, - "Ports": Ports, - "StateMachine": StateMachine, # TODO implement model + "Package": LazyModule("dal.scopes.package", "Package"), + "Message": LazyModule("dal.scopes.message", "Message"), + "Ports": LazyModule("dal.scopes.ports", "Ports"), "Var": UserVar, - "Robot": Callback.robot(), - "FleetRobot": FleetRobot, + "Robot": UserCallback.robot(), + "FleetRobot": LazyModule("dal.scopes.fleetrobot", "FleetRobot"), "logger": logger, "PortName": _port_name, - "Callback": CallbackModel, + "Callback": LazyModule("dal.scopes.callback", "Callback"), "Lock": UserLock, "print": self.user_print, - "Scene": LazyInstantiation(Callback.scene), + "Scene": LazyInstantiation(UserCallback.scene), "NodeInst": NodeInst, - "Container": Container, - "Configuration": Configuration, + "Container": LazyModule("dal.scopes.container", "Container"), + "Configuration": LazyModule("dal.scopes.configuration", "Configuration"), } ) if enterprise: self.globals.update( { - "Alert": Alert, - "Annotation": Annotation, - "GraphicScene": GraphicScene, - "Layout": Layout, - "metrics": LazyInstantiation(Metrics), - "Task": Task, - "TaskEntry": TaskEntry, - "TaskTemplate": TaskTemplate, + "Alert": LazyModule("dal.scopes.alert", "Alert"), + "Annotation": LazyModule( + "movai_core_enterprise.models.annotation", "Annotation" + ), + "GraphicScene": LazyModule( + "movai_core_enterprise.models.graphicscene", "GraphicScene" + ), + "Layout": LazyModule("movai_core_enterprise.models.layout", "Layout"), + "metrics": LazyInstantiation( + LazyModule( + "movai_core_enterprise.message_client_handlers.metrics", "Metrics" + ) + ), + "Task": LazyModule("movai_core_enterprise.scopes.task", "Task"), + "TaskEntry": LazyModule( + "movai_core_enterprise.models.taskentry", "TaskEntry" + ), + "TaskTemplate": LazyModule( + "movai_core_enterprise.models.tasktemplate", "TaskTemplate" + ), } ) @@ -192,7 +207,7 @@ def run(self, cb_name, msg): exec(compiled_code, globais) -class Callback: +class UserCallback: """Callback class used by GD_Node to execute code Args: From c455a3d7674c59ca08dd0f78d465f223c7185a8e Mon Sep 17 00:00:00 2001 From: daniela-costa7 Date: Mon, 1 Jun 2026 17:16:35 +0100 Subject: [PATCH 10/14] Fix get_usage_search_scope_map imports --- dal/tools/usage_search.py | 2 +- tests/unit/with_db/test_usage_search.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/dal/tools/usage_search.py b/dal/tools/usage_search.py index 43e3e435..391e2794 100644 --- a/dal/tools/usage_search.py +++ b/dal/tools/usage_search.py @@ -2,7 +2,7 @@ from movai_core_shared.exceptions import DoesNotExist from dal.utils.usage_search.usage_types import UsageSearchResult, UsageData -from dal.utils import get_usage_search_scope_map +from dal.utils.usage_search.scope_map import get_usage_search_scope_map class Searcher: diff --git a/tests/unit/with_db/test_usage_search.py b/tests/unit/with_db/test_usage_search.py index f827a07a..600e0c09 100644 --- a/tests/unit/with_db/test_usage_search.py +++ b/tests/unit/with_db/test_usage_search.py @@ -11,7 +11,7 @@ IndirectNodeUsageItem, IndirectFlowUsageItem, ) -from dal.utils import get_usage_search_scope_map +from dal.utils.usage_search.scope_map import get_usage_search_scope_map def get_scope_instance(search_type, name): From 0d69c3b6f7305b88511a4d3fcb23913959699936 Mon Sep 17 00:00:00 2001 From: David Dias Date: Tue, 2 Jun 2026 14:45:03 +0100 Subject: [PATCH 11/14] Revert changes to models __init__ --- dal/models/__init__.py | 86 +++++++++++------------------------------- 1 file changed, 22 insertions(+), 64 deletions(-) diff --git a/dal/models/__init__.py b/dal/models/__init__.py index 9ce0e0f9..3a320dd8 100644 --- a/dal/models/__init__.py +++ b/dal/models/__init__.py @@ -6,16 +6,27 @@ Developers: - Moawiya Mograbi (moawiya@mov.ai) - 2022 """ -import importlib -from typing import TYPE_CHECKING - -# Eagerly import critical base classes and singletons to avoid initialization issues -from .model import Model -from .node import Node -from .nodeinst import NodeInst +from .acl import ACLManager, NewACLManager +from .configuration import Configuration +from .aclobject import AclObject +from .application import Application +from .baseuser import BaseUser +from .callback import Callback from .container import Container from .flow import Flow from .flowlinks import FlowLinks +from .form import Form +from .internaluser import InternalUser +from .ldapconfig import LdapConfig +from .lock import Lock +from .message import Message +from .model import Model +from .node import Node +from .nodeinst import NodeInst +from .package import Package +from .ports import Ports +from .remoteuser import RemoteUser +from .role import Role from .scopestree import ( scopes, ScopeInstanceVersionNode, @@ -24,63 +35,10 @@ ScopeObjectNode, ScopesTree, ) - -# Import for type checking only - actual imports are lazy-loaded via __getattr__ -if TYPE_CHECKING: - from .acl import ACLManager, NewACLManager - from .aclobject import AclObject - from .application import Application - from .baseuser import BaseUser - from .callback import Callback - from .configuration import Configuration - from .form import Form - from .internaluser import InternalUser - from .ldapconfig import LdapConfig - from .lock import Lock - from .message import Message - from .package import Package - from .ports import Ports - from .remoteuser import RemoteUser - from .role import Role - from .system import System - from .user import User - from .var import Var - from .widget import Widget - -# Mapping of attribute names to their module paths (excluding eagerly loaded ones) -_LAZY_IMPORTS = { - "ACLManager": ".acl", - "NewACLManager": ".acl", - "Configuration": ".configuration", - "AclObject": ".aclobject", - "Application": ".application", - "BaseUser": ".baseuser", - "Callback": ".callback", - "Form": ".form", - "InternalUser": ".internaluser", - "LdapConfig": ".ldapconfig", - "Lock": ".lock", - "Message": ".message", - "Package": ".package", - "Ports": ".ports", - "RemoteUser": ".remoteuser", - "Role": ".role", - "System": ".system", - "User": ".user", - "Var": ".var", - "Widget": ".widget", -} - - -def __getattr__(name): - """Dynamically import classes on first access to reduce memory usage.""" - if name in _LAZY_IMPORTS: - module = importlib.import_module(_LAZY_IMPORTS[name], package=__name__) - attr = getattr(module, name) - # Cache it in globals for faster subsequent access - globals()[name] = attr - return attr - raise AttributeError(f"module {__name__!r} has no attribute {name!r}") +from .system import System +from .user import User +from .var import Var +from .widget import Widget try: From 66dd12300d2c187a4bf72ad78ffa94bfba749b59 Mon Sep 17 00:00:00 2001 From: daniela-costa7 Date: Wed, 3 Jun 2026 12:23:02 +0100 Subject: [PATCH 12/14] Add constants dir to dal --- .../translation.py} | 0 dal/models/baseuser.py | 2 +- dal/models/internaluser.py | 2 +- dal/scopes/__init__.py | 6 +++--- dal/scopes/robot.py | 1 - dal/scopes/translation.py | 2 +- dal/tools/backup.py | 15 +++++---------- dal/utils/callback.py | 5 ++++- 8 files changed, 15 insertions(+), 18 deletions(-) rename dal/{scopes/translation_constants.py => constants/translation.py} (100%) diff --git a/dal/scopes/translation_constants.py b/dal/constants/translation.py similarity index 100% rename from dal/scopes/translation_constants.py rename to dal/constants/translation.py diff --git a/dal/models/baseuser.py b/dal/models/baseuser.py index de9472b5..2e925b15 100644 --- a/dal/models/baseuser.py +++ b/dal/models/baseuser.py @@ -27,7 +27,7 @@ from dal.models.model import Model from dal.models.acl import NewACLManager from dal.models.acl import ResourceType -from dal.scopes.translation_constants import DEFAULT_LANGUAGE +from dal.constants.translation import DEFAULT_LANGUAGE class BaseUser(Model): diff --git a/dal/models/internaluser.py b/dal/models/internaluser.py index c925de79..e36ac874 100644 --- a/dal/models/internaluser.py +++ b/dal/models/internaluser.py @@ -14,7 +14,7 @@ from dal.models.model import Model from dal.models.baseuser import BaseUser from dal.models.user import User -from dal.scopes.translation_constants import DEFAULT_LANGUAGE +from dal.constants.translation import DEFAULT_LANGUAGE class InternalUser(BaseUser): diff --git a/dal/scopes/__init__.py b/dal/scopes/__init__.py index 29520fd3..065c842e 100644 --- a/dal/scopes/__init__.py +++ b/dal/scopes/__init__.py @@ -28,7 +28,7 @@ from .statemachine import SMVars, StateMachine from .structures import Struct from .system import System - from .translation_constants import DEFAULT_LANGUAGE + from .translation import Translation from .user import User from .widget import Widget @@ -55,7 +55,7 @@ "System": ".system", "User": ".user", "Widget": ".widget", - "DEFAULT_LANGUAGE": ".translation_constants", + "Translation": ".translation", } @@ -96,7 +96,7 @@ def __getattr__(name): "User", "Widget", "Alert", - "DEFAULT_LANGUAGE", + "Translation", ] try: diff --git a/dal/scopes/robot.py b/dal/scopes/robot.py index 9ed9f5bf..1341d43f 100644 --- a/dal/scopes/robot.py +++ b/dal/scopes/robot.py @@ -33,7 +33,6 @@ if TYPE_CHECKING: from movai_core_shared.messages.alert_data import ( AlertActivationData, - AlertDeactivationData, AlertData, ) diff --git a/dal/scopes/translation.py b/dal/scopes/translation.py index b35b4437..6b2bb094 100644 --- a/dal/scopes/translation.py +++ b/dal/scopes/translation.py @@ -2,7 +2,7 @@ from .scope import Scope # pylint: disable=unused-import -from .translation_constants import DEFAULT_LANGUAGE +from dal.constants.translation import DEFAULT_LANGUAGE class Translation(Scope): diff --git a/dal/tools/backup.py b/dal/tools/backup.py index a0591dc0..0d939550 100644 --- a/dal/tools/backup.py +++ b/dal/tools/backup.py @@ -79,17 +79,12 @@ class Factory: def get_class(scope): """Get scope class.""" if scope not in Factory.CLASSES_CACHE: - # Translation is not in package-level imports to avoid heavy validator load - # Import directly when needed - if scope == "Translation": - mod = import_module("dal.scopes.translation") + mod = import_module("dal.scopes") + + try: Factory.CLASSES_CACHE[scope] = getattr(mod, scope) - else: - mod = import_module("dal.scopes") - try: - Factory.CLASSES_CACHE[scope] = getattr(mod, scope) - except AttributeError as exc: - raise BackupException(f"Scope does not exists {scope}") from exc + except AttributeError as exc: + raise BackupException(f"Scope does not exists {scope}") from exc return Factory.CLASSES_CACHE[scope] diff --git a/dal/utils/callback.py b/dal/utils/callback.py index 550355db..23cb198b 100644 --- a/dal/utils/callback.py +++ b/dal/utils/callback.py @@ -13,7 +13,7 @@ import time import importlib from os import getenv -from typing import Any, Dict +from typing import TYPE_CHECKING, Any, Dict from asyncio import CancelledError from movai_core_shared.logger import Log @@ -26,6 +26,9 @@ from dal.scopes.robot import Robot +if TYPE_CHECKING: + from movai_core_enterprise.models.graphicscene import GraphicScene + # Check if enterprise modules are available try: import movai_core_enterprise # pylint: disable=unused-import From a90edecb0c168cc9e016c78ce98e03de5020825c Mon Sep 17 00:00:00 2001 From: daniela-costa7 Date: Wed, 3 Jun 2026 12:27:04 +0100 Subject: [PATCH 13/14] Version bumpage --- CHANGELOG.md | 3 +++ pyproject.toml | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0d03c772..e86b45a8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,9 @@ # Changelog ## vTBD +- [BP-1704](https://movai.atlassian.net/browse/BP-1704): Optimize and remove unused imports + - Use lazy imports for dal/scopes + - Remove unused code from dal.scopes.callback - [BP-899](https://movai.atlassian.net/browse/BP-899): Not possible to obtain active_flow - Use Redis pipeline when writing diff --git a/pyproject.toml b/pyproject.toml index f1f2e79c..325e1927 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -35,7 +35,7 @@ dependencies = [ "py3rosmsgs==1.18.2", "cachetools==5.3.1", "astunparse==1.6.3; python_version < '3.9'", - "movai-core-shared>=3.9.0.1" + "movai-core-shared>=3.9.2.1" ] [project.urls] From 36600a34cea4c55c5a768f68dc7375422da32ee6 Mon Sep 17 00:00:00 2001 From: daniela-costa7 Date: Wed, 3 Jun 2026 15:35:50 +0100 Subject: [PATCH 14/14] clear validation init --- dal/helpers/flow/gflow.py | 2 +- dal/models/flowlinks.py | 2 +- dal/scopes/scope.py | 2 +- dal/validation/__init__.py | 44 ------------------------- tests/unit/validation/test_schema.py | 4 +-- tests/unit/validation/test_validator.py | 2 +- 6 files changed, 6 insertions(+), 50 deletions(-) diff --git a/dal/helpers/flow/gflow.py b/dal/helpers/flow/gflow.py index da9fcc07..1e0b404b 100644 --- a/dal/helpers/flow/gflow.py +++ b/dal/helpers/flow/gflow.py @@ -18,7 +18,7 @@ MOVAI_TRANSITIONTO, ) from movai_core_shared.exceptions import RemapValidationError -from dal.validation import Template +from dal.validation.template import Template if TYPE_CHECKING: from dal.models import Flow diff --git a/dal/models/flowlinks.py b/dal/models/flowlinks.py index 16e888ec..6222a4cc 100644 --- a/dal/models/flowlinks.py +++ b/dal/models/flowlinks.py @@ -9,7 +9,7 @@ import re import uuid from .scopestree import ScopePropertyNode, ScopeNode -from dal.validation import Template +from dal.validation.template import Template class FlowLinks(ScopePropertyNode): diff --git a/dal/scopes/scope.py b/dal/scopes/scope.py index 04c79664..8651416b 100644 --- a/dal/scopes/scope.py +++ b/dal/scopes/scope.py @@ -46,7 +46,7 @@ def get_validator(cls): # Otherwise, lazy-load the default validator if cls.validator is None: - from dal.validation import JsonValidator + from dal.validation.validator import JsonValidator cls.validator = JsonValidator() return cls.validator diff --git a/dal/validation/__init__.py b/dal/validation/__init__.py index 483a15f8..e69de29b 100644 --- a/dal/validation/__init__.py +++ b/dal/validation/__init__.py @@ -1,44 +0,0 @@ -""" - Copyright (C) Mov.ai - All Rights Reserved - Unauthorized copying of this file, via any medium is strictly prohibited - Proprietary and confidential - - Developers: - - Moawiya Mograbi (moawiya@mov.ai) - 2022 -""" -import importlib -from typing import TYPE_CHECKING - -from .constants import REDIS_SCHEMA_FOLDER_PATH, JSON_SCHEMA_FOLDER_PATH - -# Import for type checking only - actual imports are lazy-loaded via __getattr__ -if TYPE_CHECKING: - from .schema import Schema - from .template import Template - from .validator import JsonValidator - -_LAZY_IMPORTS = { - "Schema": ".schema", - "Template": ".template", - "JsonValidator": ".validator", -} - - -def __getattr__(name): - """Dynamically import classes on first access to reduce memory usage.""" - if name in _LAZY_IMPORTS: - module = importlib.import_module(_LAZY_IMPORTS[name], package=__name__) - attr = getattr(module, name) - # Cache it in globals for faster subsequent access - globals()[name] = attr - return attr - raise AttributeError(f"module {__name__!r} has no attribute {name!r}") - - -__all__ = [ - "Schema", - "JsonValidator", - "Template", - "REDIS_SCHEMA_FOLDER_PATH", - "JSON_SCHEMA_FOLDER_PATH", -] diff --git a/tests/unit/validation/test_schema.py b/tests/unit/validation/test_schema.py index ce2eb987..1c97a4be 100644 --- a/tests/unit/validation/test_schema.py +++ b/tests/unit/validation/test_schema.py @@ -3,8 +3,8 @@ from pathlib import Path import urllib.parse -from dal.validation import Schema -from dal.validation import JSON_SCHEMA_FOLDER_PATH +from dal.validation.schema import Schema +from dal.validation.constants import JSON_SCHEMA_FOLDER_PATH from dal.classes.filesystem import FileSystem diff --git a/tests/unit/validation/test_validator.py b/tests/unit/validation/test_validator.py index 39880028..2fd63fc3 100644 --- a/tests/unit/validation/test_validator.py +++ b/tests/unit/validation/test_validator.py @@ -2,7 +2,7 @@ import pytest import time -from dal.validation import JsonValidator +from dal.validation.validator import JsonValidator import dal.exceptions from dal.classes.filesystem import FileSystem