diff --git a/CHANGELOG.md b/CHANGELOG.md index 1fd550ee..e86b45a8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,12 @@ # Changelog +## vTBD +- [BP-1704](https://movai.atlassian.net/browse/BP-1704): Optimize and remove unused imports + - Use lazy imports for dal/scopes + - Remove unused code from dal.scopes.callback +- [BP-899](https://movai.atlassian.net/browse/BP-899): Not possible to obtain active_flow + - Use Redis pipeline when writing + ## v3.23.6 - [BP-1703](https://movai.atlassian.net/browse/BP-1703): Apply DEVICE_NAME to robot name on init, rather than a temporary name diff --git a/dal/constants/translation.py b/dal/constants/translation.py new file mode 100644 index 00000000..f2942ac6 --- /dev/null +++ b/dal/constants/translation.py @@ -0,0 +1,9 @@ +""" + Copyright (C) Mov.ai - All Rights Reserved + Unauthorized copying of this file, via any medium is strictly prohibited + Proprietary and confidential + + Translation constants - separated to avoid loading heavy validator modules. +""" + +DEFAULT_LANGUAGE = "en" diff --git a/dal/data/schema.py b/dal/data/schema.py index 78ba6742..f76b9f73 100644 --- a/dal/data/schema.py +++ b/dal/data/schema.py @@ -12,7 +12,7 @@ from .tree import TreeNode, DictNode, ObjectNode, PropertyNode, CallableNode from .serialization import ObjectDeserializer from .version import VersionNode -from dal.validation import REDIS_SCHEMA_FOLDER_PATH +from dal.validation.constants import REDIS_SCHEMA_FOLDER_PATH class SchemaNode(DictNode): diff --git a/dal/helpers/__init__.py b/dal/helpers/__init__.py index 5a137dec..e69de29b 100644 --- a/dal/helpers/__init__.py +++ b/dal/helpers/__init__.py @@ -1,20 +0,0 @@ -""" - Copyright (C) Mov.ai - All Rights Reserved - Unauthorized copying of this file, via any medium is strictly prohibited - Proprietary and confidential - - Developers: - - Moawiya Mograbi (moawiya@mov.ai) - 2022 -""" - -# from .parsers import ParamParser, get_string_from_template -from .flow.gflow import GFlow -from .helpers import Helpers, flatten - -__all__ = [ - # "ParamParser", - "GFlow", - # "get_string_from_template", - "Helpers", - "flatten", -] diff --git a/dal/helpers/flow/gflow.py b/dal/helpers/flow/gflow.py index da9fcc07..1e0b404b 100644 --- a/dal/helpers/flow/gflow.py +++ b/dal/helpers/flow/gflow.py @@ -18,7 +18,7 @@ MOVAI_TRANSITIONTO, ) from movai_core_shared.exceptions import RemapValidationError -from dal.validation import Template +from dal.validation.template import Template if TYPE_CHECKING: from dal.models import Flow diff --git a/dal/models/baseuser.py b/dal/models/baseuser.py index 571deb57..2e925b15 100644 --- a/dal/models/baseuser.py +++ b/dal/models/baseuser.py @@ -27,7 +27,7 @@ from dal.models.model import Model from dal.models.acl import NewACLManager from dal.models.acl import ResourceType -from dal.scopes.translation import DEFAULT_LANGUAGE +from dal.constants.translation import DEFAULT_LANGUAGE class BaseUser(Model): diff --git a/dal/models/callback.py b/dal/models/callback.py index e512d805..7fd705b6 100644 --- a/dal/models/callback.py +++ b/dal/models/callback.py @@ -12,14 +12,10 @@ import importlib import inspect import pkgutil -import pydoc -import sys from typing import Any, Dict, List -import rospkg from movai_core_shared.logger import Log from .scopestree import scopes -from dal.scopes.system import System from .model import Model @@ -58,6 +54,8 @@ def has_permission( @staticmethod def get_modules() -> List: """Get list of modules""" + import pydoc + modules_list = [x[1] for x in pkgutil.iter_modules()] modules = {} @@ -79,6 +77,8 @@ def onerror(modname): @staticmethod def get_methods(module: str) -> Dict: """get methods of module""" + import pydoc + try: object = importlib.import_module(module) @@ -153,267 +153,6 @@ def get_full_modules() -> Dict: return module_description - # ported blindly - @staticmethod - def _get_modules(jump_over_modules) -> dict: - """new, recursive way to get modules - - This kinda looks like: - - pkg1/ - | - + __init__.py - \\ mod_dred.py - mod1.py - - - { - 'pkg1' : { - 'isPkg' : True, - 'functions' : ['foo1','bar1'], - 'classes' : ['class1'], - 'constants' : ['ID1','factor1'], - 'modules' : { - 'mod_dred' : { - 'isPkg': False, - 'functions' : ['bar2','foo2'], - 'classes' : [], - 'constants' : [] - # no modules here - } - } - }, - 'mod1' : { - 'isPkg' : False, - 'functions' : [], - 'constants' : [], - 'classes' : ['MegaClass'] - # again, no modules here - } - } - Args: - jump_over_modules (list): the list of modules not to expand - - """ - - modules = {} - - def expand_package(ret_dict: dict, pkg: pkgutil.ModuleInfo, parent: str = ""): - """check package for nested modules""" - path = pkg[0].path + "/" + pkg[1] - - # expand it - this = parent + "." + pkg[1] - if this[0] == ".": - this = this[1:] - - # iterate contents of this package - for x in pkgutil.iter_modules([path]): - # ignore '_*' modules - if x[1].startswith("_") or x[1] == "init_local_db" or x[1] == "tf_monitor": - continue - - # shouldn't be python2 for sure - ret_dict[x[1]] = {"isPkg": x[2]} - - # always expand module - v = expand_module(ret_dict[x[1]], this + "." + x[1]) - # and maybe expand package - if x[2] and v: # on error expanding module, completely ignore it - ret_dict[x[1]]["modules"] = {} - expand_package(ret_dict[x[1]]["modules"], x, this) - - if not v: - # remove it from the list if erroneous - del ret_dict[x[1]] - - def expand_module(ret_dict: dict, mod: str) -> bool: - """get methods, classes and constants from the module - - return True on success - return False on module not found or error parsing the module - """ - - i_mod = None - try: - i_mod = importlib.import_module(mod) - except ModuleNotFoundError: - # this is an actual error - return False - except: - # this may actually not be an error ... - # but we can't still work with it - # so, acknowledge it exists - # (this happens with API2.Scopes) - return True - - try: - # i_mod = importlib.import_module(mod) - - # now read its internals - # dammed ros - a_all = None - try: - a_all = getattr(i_mod, "__all__", None) - except: - # ros sends another exception instead of using default value... - # #python2 - pass - - ret_dict["classes"] = [] - ret_dict["functions"] = [] - ret_dict["consts"] = [] - - # FIXME probably needs optimizing, still figuring how to... - # when intersecting the class/function/data list with the __all__ list - - # get classes - for key, value in inspect.getmembers(i_mod, inspect.isclass): - # ignore __var__ like __these__ - # probably simpler than re.match() - if key[:2] == "__" and key[-2:] == "__": - continue - - if a_all is not None: - # if there is an __all__ - # and it's advertised there - if key in a_all: - # add it - ret_dict["classes"].append(key) - # ignore the value tho, only need key - else: - # just add it - ret_dict["classes"].append(key) - - # get methods - for key, value in inspect.getmembers(i_mod, inspect.isroutine): - # ignore __var__ like __these__ - if key[:2] == "__" and key[-2:] == "__": - continue - - if a_all is not None: - # if there is an __all__ - # and it's advertised there - if key in a_all: - # add it - ret_dict["functions"].append(key) - # ignore the value tho, only need key - else: - # just add it - ret_dict["functions"].append(key) - - # get data/constants - for key, value in inspect.getmembers(i_mod, pydoc.isdata): - # ignore __var__ like __these__ - if key[:2] == "__" and key[-2:] == "__": - continue - - if a_all is not None: - # if there is an __all__ - # and it's advertised there - if key in a_all: - # add it - ret_dict["consts"].append(key) - # ignore the value tho, only need key - else: - # just add it - # or don't ? - ret_dict["consts"].append(key) - - # so it's done ? - del i_mod - except rospkg.common.ResourceNotFound: - # ros throws this exception somewhere in inspect.getmembers - # called on a ros (python2) module - # which means we can't get the module members, - # but still add to the list of modules - return True # early return - except: # something? - return False - return True - - # iterate every module found - - builtin_modules = [("", name, False) for name in sys.builtin_module_names] - for x in set(list(pkgutil.iter_modules()) + builtin_modules): - # now ... - # x[0] -> FileFinder(path_where_modules_was_found) - # x[1] -> module/package name - # x[2] -> if it is a package (True=package,False=module) - - # this finds all modules, including python2, so we need to filter that - - # filter x[0].path for python2 (ignore those) - # not anymore - # if '/python2.' in x[0].path: - # # ignore this one - # continue - - # ignore _pkgs _starting _with '_' - if x[1].startswith("_") or x[1] == "init_local_db": - # nope - - continue - # create the template dict - modules[x[1]] = {"isPkg": x[2]} - - # i guess always expand module - v = expand_module(modules[x[1]], x[1]) - if x[1] in jump_over_modules: - continue - - # and maybe expand the package, if it's one - if x[2] and v: - # package - modules[x[1]]["modules"] = {} - expand_package(modules[x[1]]["modules"], x) - - if not v: # on error - del modules[x[1]] - - # and return it - return modules - - @staticmethod - def export_modules(jump_over_modules=None) -> None: - """Get modules and save them to db (System) - - this takes about 4 seconds to run - - and prints a LOT of stuff to the terminal - - currently (14-10 on spawner) uses about 19kb of memory (?) - Args: - jump_over_modules (list): the list of modules not to expand - """ - if jump_over_modules is None: - jump_over_modules = ["scipy", "twisted"] - - data = Callback._get_modules(jump_over_modules) - - try: - # currently using the old api - mods = System("PyModules", db="local") # scopes('local').System['PyModules', 'cache'] - except Exception: # pylint: disable=broad-except - mods = System( - "PyModules", new=True, db="local" - ) # scopes('local').create('System', 'PyModules') - - mods.Value = data - - del mods, data # not that it's gonna make a difference ... - - @staticmethod - def fetch_modules_api() -> Dict: - """to be called from rest api""" - try: - return System( - "PyModules", db="local" - ).Value # scopes('local').System['PyModules', 'cache'].Value - except Exception: # pylint: disable=broad-except - # non existent yet - return {} - def template_depends(self, force: bool = False) -> Dict: """get all the objects that depend on this callback""" # FIXME either implement or wait for api to be able to handle reverse dependency lookup diff --git a/dal/models/configuration.py b/dal/models/configuration.py index 87273945..935a4df5 100644 --- a/dal/models/configuration.py +++ b/dal/models/configuration.py @@ -8,7 +8,6 @@ - Moawiya Mograni (moawiya@mov.ai) - 2023 """ import pickle -import yaml from dal.movaidb import MovaiDB from .model import Model from dal.helpers.cache import ThreadSafeCache @@ -20,6 +19,8 @@ def __init__(self): self._map = {} def yaml_load(self, config_name, yaml_str): + import yaml + with self.__class__._lock: # Lock for thread safety map_key = config_name if map_key in self._map: @@ -53,6 +54,7 @@ def _get_db_yaml(self) -> str: def get_value(self, cached=True) -> dict: """Returns a dictionary with the configuration values""" + import yaml if self.Type == "xml": # Yaml is the name of the field diff --git a/dal/models/flowlinks.py b/dal/models/flowlinks.py index 16e888ec..6222a4cc 100644 --- a/dal/models/flowlinks.py +++ b/dal/models/flowlinks.py @@ -9,7 +9,7 @@ import re import uuid from .scopestree import ScopePropertyNode, ScopeNode -from dal.validation import Template +from dal.validation.template import Template class FlowLinks(ScopePropertyNode): diff --git a/dal/models/internaluser.py b/dal/models/internaluser.py index 813232c0..e36ac874 100644 --- a/dal/models/internaluser.py +++ b/dal/models/internaluser.py @@ -14,7 +14,7 @@ from dal.models.model import Model from dal.models.baseuser import BaseUser from dal.models.user import User -from dal.scopes.translation import DEFAULT_LANGUAGE +from dal.constants.translation import DEFAULT_LANGUAGE class InternalUser(BaseUser): diff --git a/dal/movaidb/database.py b/dal/movaidb/database.py index 1bd607b0..0dd6aec8 100755 --- a/dal/movaidb/database.py +++ b/dal/movaidb/database.py @@ -21,7 +21,6 @@ import dal import redis import random -from deepdiff import DeepDiff from redis.client import Pipeline from redis.connection import Connection @@ -480,12 +479,23 @@ def set( xx=False, validate=True, # TODO: remove unused parameter ) -> None: - """Set key values in database.""" + """Set key values in database, always using a Redis pipeline. + + Args: + _input (dict): The input dict to be saved in the database. + pickl (bool, optional): Whether to pickle values before saving. Defaults to True. + pipe (Pipeline, optional): Redis pipeline to use for the operation. Defaults to None. + ex (int, optional): Expiration time in seconds. Defaults to None. + px (int, optional): Expiration time in milliseconds. Defaults to None. + nx (bool, optional): Only set the key if it does not already exist. Defaults to False. + xx (bool, optional): Only set the key if it already exists. Defaults to False. + + """ # here we validate our dict and get the keys kvs = self.dict_to_keys(_input) - db_set = pipe if isinstance(pipe, Pipeline) else self.db_write + db_set = pipe if isinstance(pipe, Pipeline) else self.db_write.pipeline() # Save each key value in redis according to template value type for key, value, source in kvs: if pickl and source not in ["hash", "list"]: @@ -519,6 +529,9 @@ def set( except Exception as e: LOGGER.error("Something went wrong while saving this in Redis: %s", e) + if not isinstance(pipe, Pipeline): + db_set.execute() + def delete(self, _input: dict, pipe=None) -> Optional[int]: """ deletes _input @@ -1150,6 +1163,8 @@ def validate_path(path: str, structure: dict) -> Optional[str]: @staticmethod def calc_scope_update(old_dict: dict, new_dict: dict, structure: dict) -> List[Dict[str, Any]]: """Calculate scope updates dicts""" + from deepdiff import DeepDiff + # Translate dict to list of paths old_dict_paths = [p for p in MovaiDB.dict_to_paths(old_dict)] # Translate dict to list of paths diff --git a/dal/movaidb/db_schema.py b/dal/movaidb/db_schema.py index 2b9d05ed..1b0ab7d1 100644 --- a/dal/movaidb/db_schema.py +++ b/dal/movaidb/db_schema.py @@ -2,7 +2,7 @@ from typing import Dict from dal.plugins.classes import Resource -from dal.validation import REDIS_SCHEMA_FOLDER_PATH +from dal.validation.constants import REDIS_SCHEMA_FOLDER_PATH class DBSchema(dict): diff --git a/dal/plugins/resources/file/file.py b/dal/plugins/resources/file/file.py index d9069df2..9e50aa1d 100644 --- a/dal/plugins/resources/file/file.py +++ b/dal/plugins/resources/file/file.py @@ -9,7 +9,6 @@ from io import BytesIO, StringIO from json import JSONDecodeError, load from os import listdir, path, getenv, getcwd -import yaml from dal.plugins.classes import Plugin, Resource, ResourcePlugin, ResourceException __DRIVER_NAME__ = "Filesystem Plugin" @@ -64,6 +63,8 @@ def read_yaml(self, url: str): """ read a yaml file, returns a dict """ + import yaml + local_path = self._get_local_path(url) with open(local_path, "r") as fd: diff --git a/dal/scopes/__init__.py b/dal/scopes/__init__.py index ca67c64b..065c842e 100644 --- a/dal/scopes/__init__.py +++ b/dal/scopes/__init__.py @@ -6,53 +6,84 @@ Developers: - Moawiya Mograbi (moawiya@mov.ai) - 2022 """ -from .application import Application -from .alert import Alert -from .callback import Callback -from .configuration import Config, Configuration -from .fleetrobot import FleetRobot -from .flow import Flow -from .form import Form -from .message import Message -from .node import Node -from .package import Package -from .ports import Ports -from .robot import Robot -from .role import Role -from .scope import Scope -from .statemachine import StateMachine, SMVars -from .structures import Struct -from .system import System -from .translation import Translation -from .user import User -from .widget import Widget -from dal.utils import ( - UsageSearchResult, - DirectNodeUsageItem, - IndirectNodeUsageItem, - DirectFlowUsageItem, - IndirectFlowUsageItem, - NodeFlowUsage, - FlowFlowUsage, - get_usage_search_scope_map, -) +import importlib +from typing import TYPE_CHECKING + +# Import for type checking only - actual imports are lazy-loaded via __getattr__ +if TYPE_CHECKING: + from .alert import Alert + from .application import Application + from .callback import Callback + from .configuration import Config, Configuration + from .fleetrobot import FleetRobot + from .flow import Flow + from .form import Form + from .message import Message + from .node import Node + from .package import Package + from .ports import Ports + from .robot import Robot + from .role import Role + from .scope import Scope + from .statemachine import SMVars, StateMachine + from .structures import Struct + from .system import System + from .translation import Translation + from .user import User + from .widget import Widget + +# Mapping of attribute names to their module paths +_LAZY_IMPORTS = { + "Application": ".application", + "Alert": ".alert", + "Callback": ".callback", + "Config": ".configuration", + "Configuration": ".configuration", + "FleetRobot": ".fleetrobot", + "Flow": ".flow", + "Form": ".form", + "Message": ".message", + "Node": ".node", + "Package": ".package", + "Ports": ".ports", + "Robot": ".robot", + "Role": ".role", + "Scope": ".scope", + "StateMachine": ".statemachine", + "SMVars": ".statemachine", + "Struct": ".structures", + "System": ".system", + "User": ".user", + "Widget": ".widget", + "Translation": ".translation", +} + + +def __getattr__(name): + """Dynamically import classes on first access to reduce memory usage.""" + if name in _LAZY_IMPORTS: + module_path = _LAZY_IMPORTS[name] + if module_path.startswith("."): + module = importlib.import_module(module_path, package=__name__) + else: + module = importlib.import_module(module_path) + attr = getattr(module, name) + # Cache it in globals for faster subsequent access + globals()[name] = attr + return attr + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") + __all__ = [ "Application", "Callback", "Config", "Configuration", - "DirectFlowUsageItem", - "DirectNodeUsageItem", "FleetRobot", "Flow", - "FlowFlowUsage", "Form", - "IndirectFlowUsageItem", - "IndirectNodeUsageItem", "Message", "Node", - "NodeFlowUsage", "Package", "Ports", "Robot", @@ -62,12 +93,10 @@ "SMVars", "Struct", "System", - "Translation", - "UsageSearchResult", "User", "Widget", "Alert", - "get_usage_search_scope_map", + "Translation", ] try: diff --git a/dal/scopes/alert.py b/dal/scopes/alert.py index fdc26a7e..1675ee63 100644 --- a/dal/scopes/alert.py +++ b/dal/scopes/alert.py @@ -8,7 +8,6 @@ from dal.scopes.robot import Robot from movai_core_shared.logger import Log from movai_core_shared.consts import DeactivationType -from movai_core_shared.messages.alert_data import AlertActivationData try: from movai_core_enterprise.message_client_handlers._alert_metrics import AlertMetricsFactory @@ -71,6 +70,8 @@ def activate(self, **kwargs): **kwargs: Parameters to fill placeholders in Title, Info, and Action fields. """ + from movai_core_shared.messages.alert_data import AlertActivationData + # Verify that all necessary activation fields were provided self.validate_parameters("Title", self.Title, **kwargs) self.validate_parameters("Info", self.Info, **kwargs) diff --git a/dal/scopes/callback.py b/dal/scopes/callback.py index cf9534bb..8329f8e1 100644 --- a/dal/scopes/callback.py +++ b/dal/scopes/callback.py @@ -10,10 +10,8 @@ Module that implements a Callback scope class https://github.com/python/cpython/blob/master/Lib/pydoc.py """ -import sys import inspect import pkgutil -import pydoc import importlib from dal.movaidb import MovaiDB @@ -73,6 +71,8 @@ def remove(self, force=False): @staticmethod def get_modules() -> list: + import pydoc + modules_list = [x[1] for x in pkgutil.iter_modules()] modules = {} @@ -92,6 +92,8 @@ def onerror(modname): @staticmethod def get_methods(module: str) -> dict: + import pydoc + try: object = importlib.import_module(module) @@ -181,256 +183,6 @@ def get_full_modules() -> dict: return module_description - @staticmethod - def _get_modules() -> dict: - """new, recursive way to get modules""" - - modules = {} - - # use to catch an exception below - import rospkg.common - - """ - This kinda looks like: - - pkg1/ - | - + __init__.py - \ mod_dred.py - mod1.py - - - { - 'pkg1' : { - 'isPkg' : True, - 'functions' : ['foo1','bar1'], - 'classes' : ['class1'], - 'constants' : ['ID1','factor1'], - 'modules' : { - 'mod_dred' : { - 'isPkg': False, - 'functions' : ['bar2','foo2'], - 'classes' : [], - 'constants' : [] - # no modules here - } - } - }, - 'mod1' : { - 'isPkg' : False, - 'functions' : [], - 'constants' : [], - 'classes' : ['MegaClass'] - # again, no modules here - } - } - - """ - - def expand_package(ret_dict: dict, pkg: pkgutil.ModuleInfo, parent: str = ""): - """check package for nested modules""" - path = pkg[0].path + "/" + pkg[1] - - # expand it - this = parent + "." + pkg[1] - if this[0] == ".": - this = this[1:] - - # iterate contents of this package - for x in pkgutil.iter_modules([path]): - # ignore '_*' modules - if x[1].startswith("_") or x[1] == "init_local_db": - continue - - # shouldn't be python2 for sure - ret_dict[x[1]] = {"isPkg": x[2]} - - # always expand module - v = expand_module(ret_dict[x[1]], this + "." + x[1]) - # and maybe expand package - if x[2] and v: # on error expanding module, completely ignore it - ret_dict[x[1]]["modules"] = {} - expand_package(ret_dict[x[1]]["modules"], x, this) - - if not v: - # remove it from the list if erroneous - del ret_dict[x[1]] - - def expand_module(ret_dict: dict, mod: str) -> bool: - """get methods, classes and constants from the module - - return True on success - return False on module not found or error parsing the module - """ - - i_mod = None - try: - i_mod = importlib.import_module(mod) - except ModuleNotFoundError: - # this is an actual error - return False - except: - # this may actually not be an error ... - # but we can't still work with it - # so, acknowledge it exists - # (this happens with API2.Scopes) - return True - - try: - # i_mod = importlib.import_module(mod) - - # now read its internals - # dammed ros - a_all = None - try: - a_all = getattr(i_mod, "__all__", None) - except: - # ros sends another exception instead of using default value... - # #python2 - pass - - ret_dict["classes"] = [] - ret_dict["functions"] = [] - ret_dict["consts"] = [] - - # FIXME probably needs optimizing, still figuring how to... - # when intersecting the class/function/data list with the __all__ list - - # get classes - for key, value in inspect.getmembers(i_mod, inspect.isclass): - # ignore __var__ like __these__ - # probably simpler than re.match() - if key[:2] == "__" and key[-2:] == "__": - continue - - if a_all is not None: - # if there is an __all__ - # and it's advertised there - if key in a_all: - # add it - ret_dict["classes"].append(key) - # ignore the value tho, only need key - else: - # just add it - ret_dict["classes"].append(key) - - # get methods - for key, value in inspect.getmembers(i_mod, inspect.isroutine): - # ignore __var__ like __these__ - if key[:2] == "__" and key[-2:] == "__": - continue - - if a_all is not None: - # if there is an __all__ - # and it's advertised there - if key in a_all: - # add it - ret_dict["functions"].append(key) - # ignore the value tho, only need key - else: - # just add it - ret_dict["functions"].append(key) - - # get data/constants - for key, value in inspect.getmembers(i_mod, pydoc.isdata): - # ignore __var__ like __these__ - if key[:2] == "__" and key[-2:] == "__": - continue - - if a_all is not None: - # if there is an __all__ - # and it's advertised there - if key in a_all: - # add it - ret_dict["consts"].append(key) - # ignore the value tho, only need key - else: - # just add it - # or don't ? - ret_dict["consts"].append(key) - - # so it's done ? - del i_mod - except rospkg.common.ResourceNotFound: - # ros throws this exception somewhere in inspect.getmembers - # called on a ros (python2) module - # which means we can't get the module members, - # but still add to the list of modules - return True # early return - except: # something? - return False - return True - - # iterate every module found - - builtin_modules = [("", name, False) for name in sys.builtin_module_names] - for x in list(pkgutil.iter_modules()) + builtin_modules: - # now ... - # x[0] -> FileFinder(path_where_modules_was_found) - # x[1] -> module/package name - # x[2] -> if it is a package (True=package,False=module) - - # this finds all modules, including python2, so we need to filter that - - # filter x[0].path for python2 (ignore those) - # not anymore - # if '/python2.' in x[0].path: - # # ignore this one - # continue - - # ignore _pkgs _starting _with '_' - if x[1].startswith("_"): - # nope - continue - - # create the template dict - modules[x[1]] = {"isPkg": x[2]} - - # i guess always expand module - v = expand_module(modules[x[1]], x[1]) - - # and maybe expand the package, if it's one - if x[2] and v: - # package - modules[x[1]]["modules"] = {} - expand_package(modules[x[1]]["modules"], x) - - if not v: # on error - del modules[x[1]] - - # not using re anymore - del rospkg.common - - # and return it - return modules - - @staticmethod - def export_modules(): - """Get modules and save them to Layer1 - - this takes about 4 seconds to run - - and prints a LOT of stuff to the terminal - - currently (14-10 on spawner) uses about 19kb of memory (?) - """ - # useless - - root = Callback._get_modules() - # +DEBUG - """ - import json - print(json.dumps(root,indent = 2)) - del json - """ - # -DEBUG - - # and now save to database - MovaiDB("local").set({"System": {"PyModules": {"Value": root}}}) - # python frees the class - - # nothing to return - @staticmethod def fetch_modules_api(): """Retrieve saved modules from Layer1, called from REST API""" diff --git a/dal/scopes/configuration.py b/dal/scopes/configuration.py index fc2f9553..545661a6 100644 --- a/dal/scopes/configuration.py +++ b/dal/scopes/configuration.py @@ -9,8 +9,6 @@ Module that implements Configuration scope class """ import pickle -import yaml -from box import Box from dal.helpers.cache import ThreadSafeCache from .scope import Scope @@ -41,6 +39,8 @@ def _get_db_yaml(self) -> str: def get_value(self) -> dict: """Returns a dictionary with the configuration values""" + import yaml + if self.Type == "xml": # Yaml is the name of the field return self._get_db_yaml() @@ -70,12 +70,3 @@ def get_param(self, param: str): '"%s" is not a valid parameter in configuration "%s"' % (param, self.name) ) return value - - -class Config(Box): - """Config with dot accessible elements""" - - def __init__(self, name): - # raises DoesNotExist in case Configuration name does not exist - config = Configuration(name).get_value() - super().__init__(Box(config)) diff --git a/dal/scopes/flow.py b/dal/scopes/flow.py index 1c8743a7..9a64b26d 100644 --- a/dal/scopes/flow.py +++ b/dal/scopes/flow.py @@ -13,7 +13,7 @@ import uuid from itertools import product -from dal.helpers import flatten +from dal.helpers.helpers import flatten from movai_core_shared.consts import ( CONFIG_REGEX, LINK_REGEX, diff --git a/dal/scopes/message.py b/dal/scopes/message.py index ae3ee636..10a045ff 100644 --- a/dal/scopes/message.py +++ b/dal/scopes/message.py @@ -14,7 +14,6 @@ import os import rospkg import genmsg -import rosmsg from dal.scopes.scope import Scope from dal.movaidb import MovaiDB @@ -34,6 +33,7 @@ def is_valid(self): @classmethod def get_packages(cls, msg_type="all", db="global") -> list: """Gives a list of all packages containing messages of a type""" + import rosmsg if msg_type not in ["all", "msg", "srv", "action"]: raise Exception( @@ -82,6 +82,7 @@ def get_packages(cls, msg_type="all", db="global") -> list: @classmethod def get_msgs(cls, package: str, msg_type="all", db="global") -> list: """Gives a list of all messages of some type in a package""" + import rosmsg if msg_type not in ["all", "msg", "srv", "action"]: raise Exception( @@ -138,6 +139,8 @@ def get_msgs(cls, package: str, msg_type="all", db="global") -> list: @classmethod def get_all(cls, db="global") -> dict: """Gives the all available messages of all types organized by package""" + import rosmsg + rospack = rospkg.RosPack() full_dict = {} @@ -165,6 +168,7 @@ def get_all(cls, db="global") -> dict: @classmethod def export_portdata(cls, db="global") -> dict: """Gives all messages/services available organized by type. Possibly intended to replace get_all(), but kept for legacy reasons""" + import rosmsg rospack = rospkg.RosPack() @@ -280,6 +284,8 @@ def fetch_portdata_messages() -> dict: @staticmethod def get_structure(message: str) -> dict: """Gives the full structure of a message given the 'package/message' input""" + import rosmsg + rospack = rospkg.RosPack() package, msg_name = message.split("/") try: # MESSAGE diff --git a/dal/scopes/node.py b/dal/scopes/node.py index 7b33c857..ba836f9e 100644 --- a/dal/scopes/node.py +++ b/dal/scopes/node.py @@ -29,7 +29,7 @@ DirectNodeUsageItem, IndirectNodeUsageItem, ) -from dal.helpers import Helpers +from dal.helpers.helpers import Helpers from movai_core_shared.logger import Log from typing import Dict diff --git a/dal/scopes/package.py b/dal/scopes/package.py index 49bf4d15..2f9f3c26 100644 --- a/dal/scopes/package.py +++ b/dal/scopes/package.py @@ -15,7 +15,7 @@ from dal.scopes.scope import Scope from dal.scopes.structures import Struct -from dal.helpers import Helpers +from dal.helpers.helpers import Helpers from movai_core_shared.logger import Log LOGGER = Log.get_logger("spawner.mov.ai") diff --git a/dal/scopes/robot.py b/dal/scopes/robot.py index 6422bdba..1341d43f 100644 --- a/dal/scopes/robot.py +++ b/dal/scopes/robot.py @@ -9,7 +9,7 @@ Module that implements Robot namespace """ -from typing import List, Optional +from typing import TYPE_CHECKING, List, Optional import asyncio import uuid import pickle @@ -23,11 +23,6 @@ ) from movai_core_shared.envvars import SPAWNER_BIND_ADDR, DEVICE_NAME from movai_core_shared.logger import Log -from movai_core_shared.messages.alert_data import ( - AlertActivationData, - AlertDeactivationData, - AlertData, -) from dal.scopes.scope import Scope from dal.movaidb import MovaiDB @@ -35,6 +30,13 @@ from datetime import datetime from .configuration import Configuration +if TYPE_CHECKING: + from movai_core_shared.messages.alert_data import ( + AlertActivationData, + AlertData, + ) + + LOGGER = Log.get_logger("dal.mov.ai") @@ -146,7 +148,7 @@ def set_role(self, role: Role): def add_active_alert( self, alert_id: str, - data: AlertActivationData, + data: "AlertActivationData", ): """Add an active alert to the Robot""" if "ActiveAlerts" not in self.fleet.__dict__: @@ -159,8 +161,14 @@ def add_active_alert( def pop_alert( self, alert_id: str, deactivation_type: str = DeactivationType.REQUESTED - ) -> Optional[AlertData]: + ) -> Optional["AlertData"]: """Remove an active alert from the Robot""" + from movai_core_shared.messages.alert_data import ( + AlertActivationData, + AlertDeactivationData, + AlertData, + ) + if "ActiveAlerts" in self.fleet.__dict__: if alert_id in self.fleet.ActiveAlerts: alert = self.fleet.ActiveAlerts.pop(alert_id) @@ -176,8 +184,16 @@ def pop_alert( **deactivation.model_dump(), ) - def clear_alerts(self, deactivation_type: str = DeactivationType.REQUESTED) -> List[AlertData]: + def clear_alerts( + self, deactivation_type: str = DeactivationType.REQUESTED + ) -> List["AlertData"]: """Clear all active alerts from the Robot""" + from movai_core_shared.messages.alert_data import ( + AlertActivationData, + AlertDeactivationData, + AlertData, + ) + if "ActiveAlerts" in self.__dict__: LOGGER.warning(f"Clearing all alerts from robot {self.RobotName}") alert_metrics = [] diff --git a/dal/scopes/scope.py b/dal/scopes/scope.py index f90a94f6..8651416b 100644 --- a/dal/scopes/scope.py +++ b/dal/scopes/scope.py @@ -13,13 +13,10 @@ """ from typing import List from functools import cached_property - -from dal.validation.validator import Validator from movai_core_shared.exceptions import DoesNotExist, AlreadyExist from .structures import Struct from dal.movaidb import MovaiDB from dal.movaidb.db_schema import DBSchema -from dal.validation import JsonValidator SCOPES_TO_VALIDATE: List[str] = ["Translation", "Alert"] @@ -29,13 +26,30 @@ class Scope(Struct): """Scope main class. Attributes: - validator (JsonValidator): Validator for the scope. + validator (JsonValidator): Validator for the scope """ permissions = ["create", "read", "update", "delete"] - validator: Validator = JsonValidator() + validator = None + + @classmethod + def get_validator(cls): + """Lazy-load validator only when actually needed. + + If a subclass defines its own validator, use that instead. + """ + # Check if this specific class has its own validator + if "validator" in cls.__dict__ and cls.__dict__["validator"] is not None: + return cls.__dict__["validator"] + + # Otherwise, lazy-load the default validator + if cls.validator is None: + from dal.validation.validator import JsonValidator + + cls.validator = JsonValidator() + return cls.validator def __init__(self, scope, name, version, new=False, db="global"): self.__dict__["name"] = name @@ -156,4 +170,4 @@ def validate_format(cls, scope, data: dict): """ if scope in SCOPES_TO_VALIDATE: - cls.validator.validate(scope, data) + cls.get_validator().validate(scope, data) diff --git a/dal/scopes/structures.py b/dal/scopes/structures.py index 9ea6e6c7..133c7789 100644 --- a/dal/scopes/structures.py +++ b/dal/scopes/structures.py @@ -13,7 +13,7 @@ from movai_core_shared.exceptions import AlreadyExist from dal.movaidb import MovaiDB -from dal.helpers import Helpers +from dal.helpers.helpers import Helpers class List(list): diff --git a/dal/scopes/translation.py b/dal/scopes/translation.py index f502af33..6b2bb094 100644 --- a/dal/scopes/translation.py +++ b/dal/scopes/translation.py @@ -1,8 +1,8 @@ from dal.validation.validator import TranslationValidator from .scope import Scope - -DEFAULT_LANGUAGE = "en" +# pylint: disable=unused-import +from dal.constants.translation import DEFAULT_LANGUAGE class Translation(Scope): diff --git a/dal/tools/usage_search.py b/dal/tools/usage_search.py index 43e3e435..391e2794 100644 --- a/dal/tools/usage_search.py +++ b/dal/tools/usage_search.py @@ -2,7 +2,7 @@ from movai_core_shared.exceptions import DoesNotExist from dal.utils.usage_search.usage_types import UsageSearchResult, UsageData -from dal.utils import get_usage_search_scope_map +from dal.utils.usage_search.scope_map import get_usage_search_scope_map class Searcher: diff --git a/dal/utils/__init__.py b/dal/utils/__init__.py index f31a6f70..e69de29b 100644 --- a/dal/utils/__init__.py +++ b/dal/utils/__init__.py @@ -1,39 +0,0 @@ -""" - Copyright (C) Mov.ai - All Rights Reserved - Unauthorized copying of this file, via any medium is strictly prohibited - Proprietary and confidential - - Utility modules for DAL. -""" -from .usage_search.usage_types import ( - UsageSearchResult, - DirectNodeUsageItem, - IndirectNodeUsageItem, - DirectFlowUsageItem, - IndirectFlowUsageItem, - NodeFlowUsage, - FlowFlowUsage, -) - - -# Lazy import to avoid circular dependency -def get_usage_search_scope_map(): - """Get the mapping of scope types that support usage search. - - This is a re-export with lazy loading to avoid circular imports. - """ - from .usage_search.scope_map import get_usage_search_scope_map as _get_map - - return _get_map() - - -__all__ = [ - "UsageSearchResult", - "DirectNodeUsageItem", - "IndirectNodeUsageItem", - "DirectFlowUsageItem", - "IndirectFlowUsageItem", - "NodeFlowUsage", - "FlowFlowUsage", - "get_usage_search_scope_map", -] diff --git a/dal/utils/callback.py b/dal/utils/callback.py index 7a94802b..23cb198b 100644 --- a/dal/utils/callback.py +++ b/dal/utils/callback.py @@ -13,39 +13,25 @@ import time import importlib from os import getenv -from typing import Any, Dict +from typing import TYPE_CHECKING, Any, Dict from asyncio import CancelledError from movai_core_shared.logger import Log from movai_core_shared.exceptions import TransitionException -# Imports from DAL - -from dal.models.callback import Callback as CallbackModel - from dal.models.lock import Lock -from dal.models.container import Container from dal.models.nodeinst import NodeInst -from dal.scopes.package import Package -from dal.models.ports import Ports from dal.models.var import Var from dal.models.scopestree import ScopesTree, scopes -from dal.scopes.configuration import Configuration -from dal.scopes.fleetrobot import FleetRobot -from dal.scopes.message import Message from dal.scopes.robot import Robot -from dal.scopes.statemachine import StateMachine -from dal.scopes.alert import Alert -try: - from movai_core_enterprise.models.annotation import Annotation +if TYPE_CHECKING: from movai_core_enterprise.models.graphicscene import GraphicScene - from movai_core_enterprise.models.layout import Layout - from movai_core_enterprise.scopes.task import Task - from movai_core_enterprise.models.taskentry import TaskEntry - from movai_core_enterprise.models.tasktemplate import TaskTemplate - from movai_core_enterprise.message_client_handlers.metrics import Metrics + +# Check if enterprise modules are available +try: + import movai_core_enterprise # pylint: disable=unused-import enterprise = True except ImportError: @@ -74,6 +60,27 @@ def __getattr__(self, name): return getattr(instance, name) +class LazyModule: + """Delay module/class import until first attribute access.""" + + def __init__(self, module_path, class_name=None): + self._module_path = module_path + self._class_name = class_name + self._cached = None + + def _load(self): + if self._cached is None: + module = importlib.import_module(self._module_path) + self._cached = getattr(module, self._class_name) if self._class_name else module + return self._cached + + def __getattr__(self, name): + return getattr(self._load(), name) + + def __call__(self, *args, **kwargs): + return self._load()(*args, **kwargs) + + class UserFunctions: """Class that provides functions to the callback execution""" @@ -98,7 +105,7 @@ def __init__( self.load_classes(_node_name, _port_name, _user) def load_classes(self, _node_name, _port_name, _user): - _robot_id: str = Callback.robot().name + _robot_id: str = UserCallback.robot().name class UserVar(Var): """Class for user to set and get vars""" @@ -123,36 +130,47 @@ def __init__(self, name, **kwargs): self.globals.update( { "scopes": scopes, - "Package": Package, - "Message": Message, - "Ports": Ports, - "StateMachine": StateMachine, # TODO implement model + "Package": LazyModule("dal.scopes.package", "Package"), + "Message": LazyModule("dal.scopes.message", "Message"), + "Ports": LazyModule("dal.scopes.ports", "Ports"), "Var": UserVar, - "Robot": Callback.robot(), - "FleetRobot": FleetRobot, + "Robot": UserCallback.robot(), + "FleetRobot": LazyModule("dal.scopes.fleetrobot", "FleetRobot"), "logger": logger, "PortName": _port_name, - "Callback": CallbackModel, + "Callback": LazyModule("dal.scopes.callback", "Callback"), "Lock": UserLock, "print": self.user_print, - "Scene": LazyInstantiation(Callback.scene), + "Scene": LazyInstantiation(UserCallback.scene), "NodeInst": NodeInst, - "Container": Container, - "Configuration": Configuration, + "Container": LazyModule("dal.scopes.container", "Container"), + "Configuration": LazyModule("dal.scopes.configuration", "Configuration"), } ) if enterprise: self.globals.update( { - "Alert": Alert, - "Annotation": Annotation, - "GraphicScene": GraphicScene, - "Layout": Layout, - "metrics": LazyInstantiation(Metrics), - "Task": Task, - "TaskEntry": TaskEntry, - "TaskTemplate": TaskTemplate, + "Alert": LazyModule("dal.scopes.alert", "Alert"), + "Annotation": LazyModule( + "movai_core_enterprise.models.annotation", "Annotation" + ), + "GraphicScene": LazyModule( + "movai_core_enterprise.models.graphicscene", "GraphicScene" + ), + "Layout": LazyModule("movai_core_enterprise.models.layout", "Layout"), + "metrics": LazyInstantiation( + LazyModule( + "movai_core_enterprise.message_client_handlers.metrics", "Metrics" + ) + ), + "Task": LazyModule("movai_core_enterprise.scopes.task", "Task"), + "TaskEntry": LazyModule( + "movai_core_enterprise.models.taskentry", "TaskEntry" + ), + "TaskTemplate": LazyModule( + "movai_core_enterprise.models.tasktemplate", "TaskTemplate" + ), } ) @@ -192,7 +210,7 @@ def run(self, cb_name, msg): exec(compiled_code, globais) -class Callback: +class UserCallback: """Callback class used by GD_Node to execute code Args: diff --git a/dal/validation/__init__.py b/dal/validation/__init__.py index 1bbb3f1a..e69de29b 100644 --- a/dal/validation/__init__.py +++ b/dal/validation/__init__.py @@ -1,21 +0,0 @@ -""" - Copyright (C) Mov.ai - All Rights Reserved - Unauthorized copying of this file, via any medium is strictly prohibited - Proprietary and confidential - - Developers: - - Moawiya Mograbi (moawiya@mov.ai) - 2022 -""" -from .constants import REDIS_SCHEMA_FOLDER_PATH, JSON_SCHEMA_FOLDER_PATH -from .schema import Schema -from .validator import JsonValidator -from .template import Template - - -__all__ = [ - "Schema", - "JsonValidator", - "Template", - "REDIS_SCHEMA_FOLDER_PATH", - "JSON_SCHEMA_FOLDER_PATH", -] diff --git a/pyproject.toml b/pyproject.toml index a4660420..325e1927 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -35,7 +35,7 @@ dependencies = [ "py3rosmsgs==1.18.2", "cachetools==5.3.1", "astunparse==1.6.3; python_version < '3.9'", - "movai-core-shared>=3.9.0.1" + "movai-core-shared>=3.9.2.1" ] [project.urls] @@ -64,7 +64,7 @@ dal = [ line-length = 100 [tool.bumpversion] -current_version = "3.23.6.2" +current_version = "3.23.7.0" parse = "(?P\\d+)\\.(?P\\d+)\\.(?P\\d+)?(\\.(?P\\d+))?" serialize = ["{major}.{minor}.{patch}.{build}"] diff --git a/tests/unit/tools/test_extract_i18n.py b/tests/unit/tools/test_extract_i18n.py index ac9379c1..67bf6d08 100644 --- a/tests/unit/tools/test_extract_i18n.py +++ b/tests/unit/tools/test_extract_i18n.py @@ -88,7 +88,7 @@ def test_gen_and_import(self, tmp_path, global_db): main() from dal.tools.backup import Importer - from dal.scopes import Translation + from dal.scopes.translation import Translation importer = Importer( tmp_path, diff --git a/tests/unit/validation/test_schema.py b/tests/unit/validation/test_schema.py index ce2eb987..1c97a4be 100644 --- a/tests/unit/validation/test_schema.py +++ b/tests/unit/validation/test_schema.py @@ -3,8 +3,8 @@ from pathlib import Path import urllib.parse -from dal.validation import Schema -from dal.validation import JSON_SCHEMA_FOLDER_PATH +from dal.validation.schema import Schema +from dal.validation.constants import JSON_SCHEMA_FOLDER_PATH from dal.classes.filesystem import FileSystem diff --git a/tests/unit/validation/test_validator.py b/tests/unit/validation/test_validator.py index 39880028..2fd63fc3 100644 --- a/tests/unit/validation/test_validator.py +++ b/tests/unit/validation/test_validator.py @@ -2,7 +2,7 @@ import pytest import time -from dal.validation import JsonValidator +from dal.validation.validator import JsonValidator import dal.exceptions from dal.classes.filesystem import FileSystem diff --git a/tests/unit/with_db/scopes/test_translation.py b/tests/unit/with_db/scopes/test_translation.py index 8e796e67..49eaf3bd 100644 --- a/tests/unit/with_db/scopes/test_translation.py +++ b/tests/unit/with_db/scopes/test_translation.py @@ -4,7 +4,7 @@ class TestTranslation: def test_translation(self, global_db, metadata_folder): from dal.tools.backup import Importer - from dal.scopes import Translation + from dal.scopes.translation import Translation tool = Importer( metadata_folder, diff --git a/tests/unit/with_db/test_usage_search.py b/tests/unit/with_db/test_usage_search.py index f827a07a..600e0c09 100644 --- a/tests/unit/with_db/test_usage_search.py +++ b/tests/unit/with_db/test_usage_search.py @@ -11,7 +11,7 @@ IndirectNodeUsageItem, IndirectFlowUsageItem, ) -from dal.utils import get_usage_search_scope_map +from dal.utils.usage_search.scope_map import get_usage_search_scope_map def get_scope_instance(search_type, name):