EE 2.5.1 BP-1704: Optimize and remove unused imports

CHANGELOG.md

@@ -1,5 +1,12 @@
 # Changelog
 
+## vTBD
+- [BP-1704](https://movai.atlassian.net/browse/BP-1704): Optimize and remove unused imports
+  - Use lazy imports for dal/scopes
+  - Remove unused code from dal.scopes.callback
+- [BP-899](https://movai.atlassian.net/browse/BP-899): Not possible to obtain active_flow
+  - Use Redis pipeline when writing
+
 ## v3.23.6
 - [BP-1703](https://movai.atlassian.net/browse/BP-1703): Apply DEVICE_NAME to robot name on init, rather than a temporary name
 

dal/constants/translation.py

@@ -0,0 +1,9 @@
+"""
+   Copyright (C) Mov.ai  - All Rights Reserved
+   Unauthorized copying of this file, via any medium is strictly prohibited
+   Proprietary and confidential
+
+   Translation constants - separated to avoid loading heavy validator modules.
+"""
+
+DEFAULT_LANGUAGE = "en"

dal/data/schema.py

@@ -12,7 +12,7 @@
 from .tree import TreeNode, DictNode, ObjectNode, PropertyNode, CallableNode
 from .serialization import ObjectDeserializer
 from .version import VersionNode
-from dal.validation import REDIS_SCHEMA_FOLDER_PATH
+from dal.validation.constants import REDIS_SCHEMA_FOLDER_PATH
 
 
 class SchemaNode(DictNode):

dal/helpers/__init__.py

@@ -1,20 +0,0 @@
-"""
-   Copyright (C) Mov.ai  - All Rights Reserved
-   Unauthorized copying of this file, via any medium is strictly prohibited
-   Proprietary and confidential
-
-   Developers:
-   - Moawiya Mograbi  (moawiya@mov.ai) - 2022
-"""
-
-# from .parsers import ParamParser, get_string_from_template
-from .flow.gflow import GFlow
-from .helpers import Helpers, flatten
-
-__all__ = [
-    # "ParamParser",
-    "GFlow",
-    # "get_string_from_template",
-    "Helpers",
-    "flatten",
-]

dal/helpers/flow/gflow.py

@@ -18,7 +18,7 @@
     MOVAI_TRANSITIONTO,
 )
 from movai_core_shared.exceptions import RemapValidationError
-from dal.validation import Template
+from dal.validation.template import Template
 
 if TYPE_CHECKING:
     from dal.models import Flow

dal/models/baseuser.py

@@ -27,7 +27,7 @@
 from dal.models.model import Model
 from dal.models.acl import NewACLManager
 from dal.models.acl import ResourceType
-from dal.scopes.translation import DEFAULT_LANGUAGE
+from dal.constants.translation import DEFAULT_LANGUAGE
 
 
 class BaseUser(Model):

dal/models/callback.py

@@ -12,14 +12,10 @@
 import importlib
 import inspect
 import pkgutil
-import pydoc
-import sys
 from typing import Any, Dict, List
-import rospkg
 from movai_core_shared.logger import Log
 
 from .scopestree import scopes
-from dal.scopes.system import System
 
 from .model import Model
 
@@ -58,6 +54,8 @@ def has_permission(
     @staticmethod
     def get_modules() -> List:
         """Get list of modules"""
+        import pydoc
+
         modules_list = [x[1] for x in pkgutil.iter_modules()]
 
         modules = {}
@@ -79,6 +77,8 @@ def onerror(modname):
     @staticmethod
     def get_methods(module: str) -> Dict:
         """get methods of module"""
+        import pydoc
+
         try:
             object = importlib.import_module(module)
 
@@ -153,267 +153,6 @@ def get_full_modules() -> Dict:
 
         return module_description
 
-    # ported blindly
-    @staticmethod
-    def _get_modules(jump_over_modules) -> dict:
-        """new, recursive way to get modules
-
-        This kinda looks like:
-
-        pkg1/
-         |
-         +  __init__.py
-         \\  mod_dred.py
-        mod1.py
-
-
-        {
-            'pkg1' : {
-                'isPkg' : True,
-                'functions' : ['foo1','bar1'],
-                'classes' : ['class1'],
-                'constants' : ['ID1','factor1'],
-                'modules' : {
-                    'mod_dred' : {
-                        'isPkg': False,
-                        'functions' : ['bar2','foo2'],
-                        'classes' : [],
-                        'constants' : []
-                        # no modules here
-                    }
-                }
-            },
-            'mod1' : {
-                'isPkg' : False,
-                'functions' : [],
-                'constants' : [],
-                'classes' : ['MegaClass']
-                # again, no modules here
-            }
-        }
-        Args:
-            jump_over_modules (list): the list of modules not to expand
-
-        """
-
-        modules = {}
-
-        def expand_package(ret_dict: dict, pkg: pkgutil.ModuleInfo, parent: str = ""):
-            """check package for nested modules"""
-            path = pkg[0].path + "/" + pkg[1]
-
-            # expand it
-            this = parent + "." + pkg[1]
-            if this[0] == ".":
-                this = this[1:]
-
-            # iterate contents of this package
-            for x in pkgutil.iter_modules([path]):
-                # ignore '_*' modules
-                if x[1].startswith("_") or x[1] == "init_local_db" or x[1] == "tf_monitor":
-                    continue
-
-                # shouldn't be python2 for sure
-                ret_dict[x[1]] = {"isPkg": x[2]}
-
-                # always expand module
-                v = expand_module(ret_dict[x[1]], this + "." + x[1])
-                # and maybe expand package
-                if x[2] and v:  # on error expanding module, completely ignore it
-                    ret_dict[x[1]]["modules"] = {}
-                    expand_package(ret_dict[x[1]]["modules"], x, this)
-
-                if not v:
-                    # remove it from the list if erroneous
-                    del ret_dict[x[1]]
-
-        def expand_module(ret_dict: dict, mod: str) -> bool:
-            """get methods, classes and constants from the module
-
-            return True on success
-            return False on module not found or error parsing the module
-            """
-
-            i_mod = None
-            try:
-                i_mod = importlib.import_module(mod)
-            except ModuleNotFoundError:
-                # this is an actual error
-                return False
-            except:
-                # this may actually not be an error ...
-                # but we can't still work with it
-                # so, acknowledge it exists
-                # (this happens with API2.Scopes)
-                return True
-
-            try:
-                # i_mod = importlib.import_module(mod)
-
-                # now read its internals
-                # dammed ros
-                a_all = None
-                try:
-                    a_all = getattr(i_mod, "__all__", None)
-                except:
-                    # ros sends another exception instead of using default value...
-                    # #python2
-                    pass
-
-                ret_dict["classes"] = []
-                ret_dict["functions"] = []
-                ret_dict["consts"] = []
-
-                # FIXME probably needs optimizing, still figuring how to...
-                # when intersecting the class/function/data list with the __all__ list
-
-                # get classes
-                for key, value in inspect.getmembers(i_mod, inspect.isclass):
-                    # ignore __var__ like __these__
-                    # probably simpler than re.match()
-                    if key[:2] == "__" and key[-2:] == "__":
-                        continue
-
-                    if a_all is not None:
-                        # if there is an __all__
-                        # and it's advertised there
-                        if key in a_all:
-                            # add it
-                            ret_dict["classes"].append(key)
-                            # ignore the value tho, only need key
-                    else:
-                        # just add it
-                        ret_dict["classes"].append(key)
-
-                # get methods
-                for key, value in inspect.getmembers(i_mod, inspect.isroutine):
-                    # ignore __var__ like __these__
-                    if key[:2] == "__" and key[-2:] == "__":
-                        continue
-
-                    if a_all is not None:
-                        # if there is an __all__
-                        # and it's advertised there
-                        if key in a_all:
-                            # add it
-                            ret_dict["functions"].append(key)
-                            # ignore the value tho, only need key
-                    else:
-                        # just add it
-                        ret_dict["functions"].append(key)
-
-                # get data/constants
-                for key, value in inspect.getmembers(i_mod, pydoc.isdata):
-                    # ignore __var__ like __these__
-                    if key[:2] == "__" and key[-2:] == "__":
-                        continue
-
-                    if a_all is not None:
-                        # if there is an __all__
-                        # and it's advertised there
-                        if key in a_all:
-                            # add it
-                            ret_dict["consts"].append(key)
-                            # ignore the value tho, only need key
-                    else:
-                        # just add it
-                        # or don't ?
-                        ret_dict["consts"].append(key)
-
-                # so it's done ?
-                del i_mod
-            except rospkg.common.ResourceNotFound:
-                # ros throws this exception somewhere in inspect.getmembers
-                # called on a ros (python2) module
-                # which means we can't get the module members,
-                # but still add to the list of modules
-                return True  # early return
-            except:  # something?
-                return False
-            return True
-
-        # iterate every module found
-
-        builtin_modules = [("", name, False) for name in sys.builtin_module_names]
-        for x in set(list(pkgutil.iter_modules()) + builtin_modules):
-            # now ...
-            # x[0] -> FileFinder(path_where_modules_was_found)
-            # x[1] -> module/package name
-            # x[2] -> if it is a package (True=package,False=module)
-
-            # this finds all modules, including python2, so we need to filter that
-
-            # filter x[0].path for python2 (ignore those)
-            # not anymore
-            # if '/python2.' in x[0].path:
-            #    # ignore this one
-            #    continue
-
-            # ignore _pkgs _starting _with '_'
-            if x[1].startswith("_") or x[1] == "init_local_db":
-                # nope
-
-                continue
-            # create the template dict
-            modules[x[1]] = {"isPkg": x[2]}
-
-            # i guess always expand module
-            v = expand_module(modules[x[1]], x[1])
-            if x[1] in jump_over_modules:
-                continue
-
-            # and maybe expand the package, if it's one
-            if x[2] and v:
-                # package
-                modules[x[1]]["modules"] = {}
-                expand_package(modules[x[1]]["modules"], x)
-
-            if not v:  # on error
-                del modules[x[1]]
-
-        # and return it
-        return modules
-
-    @staticmethod
-    def export_modules(jump_over_modules=None) -> None:
-        """Get modules and save them to db (System)
-
-        this takes about 4 seconds to run
-
-        and prints a LOT of stuff to the terminal
-
-        currently (14-10 on spawner) uses about 19kb of memory (?)
-        Args:
-            jump_over_modules (list): the list of modules not to expand
-        """
-        if jump_over_modules is None:
-            jump_over_modules = ["scipy", "twisted"]
-
-        data = Callback._get_modules(jump_over_modules)
-
-        try:
-            # currently using the old api
-            mods = System("PyModules", db="local")  # scopes('local').System['PyModules', 'cache']
-        except Exception:  # pylint: disable=broad-except
-            mods = System(
-                "PyModules", new=True, db="local"
-            )  # scopes('local').create('System', 'PyModules')
-
-        mods.Value = data
-
-        del mods, data  # not that it's gonna make a difference ...
-
-    @staticmethod
-    def fetch_modules_api() -> Dict:
-        """to be called from rest api"""
-        try:
-            return System(
-                "PyModules", db="local"
-            ).Value  # scopes('local').System['PyModules', 'cache'].Value
-        except Exception:  # pylint: disable=broad-except
-            # non existent yet
-            return {}
-
     def template_depends(self, force: bool = False) -> Dict:
         """get all the objects that depend on this callback"""
         # FIXME either implement or wait for api to be able to handle reverse dependency lookup

dal/models/configuration.py

@@ -8,7 +8,6 @@
    - Moawiya Mograni (moawiya@mov.ai) - 2023
 """
 import pickle
-import yaml
 from dal.movaidb import MovaiDB
 from .model import Model
 from dal.helpers.cache import ThreadSafeCache
@@ -20,6 +19,8 @@ def __init__(self):
         self._map = {}
 
     def yaml_load(self, config_name, yaml_str):
+        import yaml
+
         with self.__class__._lock:  # Lock for thread safety
             map_key = config_name
             if map_key in self._map:
@@ -53,6 +54,7 @@ def _get_db_yaml(self) -> str:
 
     def get_value(self, cached=True) -> dict:
         """Returns a dictionary with the configuration values"""
+        import yaml
 
         if self.Type == "xml":
             # Yaml is the name of the field