From b0d9f2836543eb549541043faa78d969d38aba2a Mon Sep 17 00:00:00 2001 From: Asa Rentschler Date: Fri, 30 Jan 2026 09:02:58 -0500 Subject: [PATCH 1/5] Added solver timing. --- pyproject.toml | 2 +- src/velocity/__init__.py | 6 +++--- src/velocity/__main__.py | 9 ++++----- src/velocity/_build.py | 2 +- src/velocity/_graph.py | 9 +++++++++ 5 files changed, 18 insertions(+), 10 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 6e7159c..d758e09 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "olcf-velocity" -version = "0.4.dev" +version = "0.4.0dev" description = "A container build manager" readme = "README.md" requires-python = ">=3.10" diff --git a/src/velocity/__init__.py b/src/velocity/__init__.py index 44bf721..d1d31ed 100644 --- a/src/velocity/__init__.py +++ b/src/velocity/__init__.py @@ -7,7 +7,7 @@ from velocity._config import config # noqa: E402 from velocity._graph import ImageRepo # noqa: E402 from velocity._build import ImageBuilder # noqa: E402 -from velocity._print import TextBlock, header_print, indent_print # noqa: E402 +from velocity._print import TextBlock, bare_print, header_print, indent_print # noqa: E402 # config functions @@ -85,7 +85,7 @@ def set_distro(distro: str) -> None: def build( targets: str, - name: str = None, + name: str | None = None, dry_run: bool = False, leave_tags: bool = False, verbose: bool = False, @@ -120,7 +120,7 @@ def build( header_print([TextBlock("Build Order:")]) for r in recipe: indent_print([TextBlock(f"{r.name}@{r.version}-{r.id}", fore=Fore.MAGENTA, style=Style.BRIGHT)]) - print() # newline + bare_print([]) # newline # prep builder builder = ImageBuilder( diff --git a/src/velocity/__main__.py b/src/velocity/__main__.py index e54736b..6ce28ef 100644 --- a/src/velocity/__main__.py +++ b/src/velocity/__main__.py @@ -159,7 +159,7 @@ recipe = imageRepo.create_build_recipe(args.targets)[0] # print build specs - header_print([TextBlock("Build Order:")]) + header_print([TextBlock("Build recipe:")]) for r in recipe: indent_print([TextBlock(f"{r.name}@{r.version}-{r.id}", fore=Fore.MAGENTA, style=Style.BRIGHT)]) print() # newline @@ -197,7 +197,7 @@ deps.sort() for t in deps: indent_print([TextBlock(t.version, fore=Fore.YELLOW, style=Style.BRIGHT)]) - print() # add newline + bare_print([]) # add newline elif args.subcommand == "spec": # get recipe @@ -249,14 +249,13 @@ def spec_print(seed: str, indent: int, fdt: dict, rs: tuple[Image]): # print specs for tl in top_level_entries: spec_print(tl.name, 0, flat_dep_tree, recipe) - print() # add newline + bare_print([]) # add newline else: parser.print_help() - print() # add newline + bare_print([]) # add newline except KeyboardInterrupt: bare_print([ TextBlock("\b\b==> ", fore=Fore.YELLOW, style=Style.BRIGHT), TextBlock("Keyboard Interrupt", fore=Fore.MAGENTA, style=Style.BRIGHT) ]) - diff --git a/src/velocity/_build.py b/src/velocity/_build.py index 02dccd8..c7bad65 100644 --- a/src/velocity/_build.py +++ b/src/velocity/_build.py @@ -94,7 +94,7 @@ class ImageBuilder(metaclass=OurMeta): def __init__( self, bt: tuple[Image], - build_name: str = None, + build_name: str | None = None, dry_run: bool = False, remove_tags: bool = True, clean_build_dir: bool = False, diff --git a/src/velocity/_graph.py b/src/velocity/_graph.py index f55dca5..1b73175 100644 --- a/src/velocity/_graph.py +++ b/src/velocity/_graph.py @@ -1,9 +1,11 @@ """Graph library and tools for dependency graph""" +from colorama import Fore, Style from loguru import logger from re import split as re_split, fullmatch as re_fullmatch, Match as ReMatch from pathlib import Path from hashlib import sha256 +from timeit import default_timer as timer from typing_extensions import Self from yaml import safe_load as yaml_safe_load from copy import deepcopy @@ -17,6 +19,7 @@ ) from ._config import config from ._exceptions import InvalidImageVersionError, CannotFindDependency, EdgeViolatesDAG, NoAvailableBuild +from ._print import TextBlock, header_print, indent_print from ._tools import OurMeta, trace_function @@ -716,6 +719,9 @@ def import_from_dir(self, path: str) -> None: def create_build_recipe(self, targets: list[str]) -> tuple[tuple, ImageGraph]: """Create an ordered build recipe of images.""" + header_print([TextBlock("Creating build recipe:")]) + start = timer() + images: set[Image] = deepcopy(self.images) build_targets: list[Target] = list() @@ -853,4 +859,7 @@ def create_build_recipe(self, targets: list[str]) -> tuple[tuple, ImageGraph]: if di.satisfies(dep): bt_ig.add_edge(image, di) + end = timer() + indent_print([TextBlock(f"created recipe in {round(end - start, 2)}s\n", fore=Fore.MAGENTA, style=Style.BRIGHT)]) + return bt, bt_ig From 1d7c3814984c3d0a5ff895b3173438a6cbfef84c Mon Sep 17 00:00:00 2001 From: Asa Rentschler Date: Mon, 2 Feb 2026 11:26:02 -0500 Subject: [PATCH 2/5] Huge performance increse of the build recipe solver. --- src/velocity/__main__.py | 5 +- src/velocity/_exceptions.py | 10 +- src/velocity/_graph.py | 496 ++++++++++++++++++------------------ 3 files changed, 253 insertions(+), 258 deletions(-) diff --git a/src/velocity/__main__.py b/src/velocity/__main__.py index 6ce28ef..42ab985 100644 --- a/src/velocity/__main__.py +++ b/src/velocity/__main__.py @@ -4,6 +4,7 @@ from colorama import Fore, Style from importlib.metadata import version from loguru import logger +from networkx import neighbors as nx_neighbors from re import fullmatch as re_fullmatch import sys @@ -207,14 +208,14 @@ flat_dep_tree = dict() for r in recipe: flat_dep_tree[r.name] = set() - deps = set(graph.get_dependencies(r)) + deps = set(nx_neighbors(graph, r)) for o in deps.intersection(set(recipe)): flat_dep_tree[r.name].add(o.name) # get top level entries top_level_entries = set() deps = set() for r in recipe: - deps.update(graph.get_dependencies(r)) + deps.update(set(nx_neighbors(graph, r))) for r in recipe: if r not in deps: top_level_entries.add(r) diff --git a/src/velocity/_exceptions.py b/src/velocity/_exceptions.py index 449d168..b7b036f 100644 --- a/src/velocity/_exceptions.py +++ b/src/velocity/_exceptions.py @@ -20,8 +20,8 @@ def __init__(self, *args) -> None: class EdgeViolatesDAG(Exception): """Edge breaks the DAG requirement of a graph.""" - def __init__(self, u_of_edge, v_of_edge, cycle) -> None: - super().__init__(f"Addition of edge {u_of_edge} -> {v_of_edge} violates graph DAG requirement!") + def __init__(self, cycle) -> None: + super().__init__("Addition of edge(s) violates graph DAG requirement!") for c in cycle: print(f"{c[0]} -> {c[1]}", file=stderr) @@ -83,3 +83,9 @@ class InvalidCLIArgumentFormat(Exception): def __init__(self, *args) -> None: super().__init__(*args) + +class SpecSyntaxError(Exception): + """Invalid spec syntax.""" + + def __init__(self, *args) -> None: + super().__init__(*args) diff --git a/src/velocity/_graph.py b/src/velocity/_graph.py index 1b73175..4336a84 100644 --- a/src/velocity/_graph.py +++ b/src/velocity/_graph.py @@ -1,27 +1,57 @@ """Graph library and tools for dependency graph""" -from colorama import Fore, Style -from loguru import logger -from re import split as re_split, fullmatch as re_fullmatch, Match as ReMatch -from pathlib import Path -from hashlib import sha256 -from timeit import default_timer as timer -from typing_extensions import Self -from yaml import safe_load as yaml_safe_load from copy import deepcopy from enum import Enum +from hashlib import sha256 +from pathlib import Path +from re import ( + Match as ReMatch, + Pattern as RePattern, + compile as re_compile, + fullmatch as re_fullmatch, + split as re_split, +) +from timeit import default_timer as timer + +from colorama import Fore, Style +from loguru import logger from networkx import ( DiGraph as nx_DiGraph, - is_directed_acyclic_graph as nx_is_directed_acyclic_graph, find_cycle as nx_find_cycle, + is_directed_acyclic_graph as nx_is_directed_acyclic_graph, neighbors as nx_neighbors, - has_path as nx_has_path, ) +from typing_extensions import Self +from yaml import safe_load as yaml_safe_load + from ._config import config -from ._exceptions import InvalidImageVersionError, CannotFindDependency, EdgeViolatesDAG, NoAvailableBuild +from ._exceptions import ( + CannotFindDependency, + EdgeViolatesDAG, + InvalidImageVersionError, + NoAvailableBuild, + SpecSyntaxError, +) from ._print import TextBlock, header_print, indent_print from ._tools import OurMeta, trace_function +# variables +EMPTY_STR: str = "" + +TARGET_REGEX: RePattern = re_compile( + r"^(?P[a-zA-Z0-9-]+)(?:(?:@(?P[^:\s]+)(?!@))?(?:@?(?P:)(?P\S+)?)?)?$" +) +SYSTEM_REGEX: RePattern = re_compile(r"^system=(?P[a-zA-Z0-9]+)$") +BACKEND_REGEX: RePattern = re_compile(r"^backend=(?P[a-zA-Z0-9]+)$") +DISTRO_REGEX: RePattern = re_compile(r"^distro=(?P[a-zA-Z0-9]+)$") +DEPENDENCY_REGEX: RePattern = re_compile(r"^\^(?P[a-zA-Z0-9-]+)$") + +VARIABLE_REGEX: RePattern = re_compile(r"^(?P[^=]+)=(?P.*)$") + +VERSION_REGEX: RePattern = re_compile( + r"^(?P[0-9]+)(?:\.(?P[0-9]+)(:?\.(?P[0-9]+))?)?(?:-(?P[a-zA-Z0-9]+))?$" +) + @trace_function def get_permutations(idx: int, sets: list[list]): @@ -38,8 +68,8 @@ def get_permutations(idx: int, sets: list[list]): for i in sets[idx]: permutations.append({i}) else: + sub_permutations = get_permutations(idx + 1, sets) for i in sets[idx]: - sub_permutations = get_permutations(idx + 1, sets) for sub in sub_permutations: permutations.append(sub.union({i})) return permutations @@ -51,14 +81,11 @@ class Version(metaclass=OurMeta): def __init__(self, version_specifier: str) -> None: self.vs = version_specifier try: - version_dict: dict = re_fullmatch( - r"^(?P[0-9]+)(?:\.(?P[0-9]+)(:?\.(?P[0-9]+))?)?(?:-(?P[a-zA-Z0-9]+))?$", - version_specifier, - ).groupdict() - self.major: int | None = int(version_dict["major"]) if version_dict["major"] is not None else None - self.minor: int | None = int(version_dict["minor"]) if version_dict["minor"] is not None else None - self.patch: int | None = int(version_dict["patch"]) if version_dict["patch"] is not None else None - self.suffix: str | None = str(version_dict["suffix"]) if version_dict["suffix"] is not None else None + version_dict: dict = VERSION_REGEX.fullmatch(version_specifier).groupdict() + self.major: int | None = int(version_dict["major"]) if version_dict["major"] else None + self.minor: int | None = int(version_dict["minor"]) if version_dict["minor"] else None + self.patch: int | None = int(version_dict["patch"]) if version_dict["patch"] else None + self.suffix: str | None = str(version_dict["suffix"]) if version_dict["suffix"] else None except AttributeError: self.major: int | None = None self.minor: int | None = None @@ -72,10 +99,10 @@ def __init__(self, version_specifier: str) -> None: def vcs(self) -> str: """Version Comparison String""" return "{:#>9}.{:#>9}.{:#>9}.{:~<9}".format( - self.major if self.major is not None else "#", - self.minor if self.minor is not None else "#", - self.patch if self.patch is not None else "#", - self.suffix if self.suffix is not None else "~", + self.major if self.major else "#", + self.minor if self.minor else "#", + self.patch if self.patch else "#", + self.suffix if self.suffix else "~", ) def preferred(self, other) -> bool: @@ -86,13 +113,13 @@ def preferred(self, other) -> bool: def _vcs_t(self) -> str: """Version Comparison String Truncated""" vl: int = 0 - if self.major is None: + if not self.major: pass - elif self.minor is None: + elif not self.minor: vl = 9 - elif self.patch is None: + elif not self.patch: vl = 19 - elif self.suffix is None: + elif not self.suffix: vl = 29 else: vl = 39 @@ -107,7 +134,7 @@ def _cut_vcses_to_size(cls, one: Self, two: Self) -> tuple[str, str]: def __eq__(self, other) -> bool: if not isinstance(other, Version): raise TypeError( - f"'>' not supported between instances of " f"'{type(self).__name__}' and '{type(other).__name__}'" + f"'>' not supported between instances of '{type(self).__name__}' and '{type(other).__name__}'" ) s_vcs, o_vcs = self._cut_vcses_to_size(self, other) return s_vcs == o_vcs @@ -115,7 +142,7 @@ def __eq__(self, other) -> bool: def __ne__(self, other) -> bool: if not isinstance(other, Version): raise TypeError( - f"'>' not supported between instances of " f"'{type(self).__name__}' and '{type(other).__name__}'" + f"'>' not supported between instances of '{type(self).__name__}' and '{type(other).__name__}'" ) s_vcs, o_vcs = self._cut_vcses_to_size(self, other) return s_vcs != o_vcs @@ -123,28 +150,28 @@ def __ne__(self, other) -> bool: def __gt__(self, other) -> bool: if not isinstance(other, Version): raise TypeError( - f"'>' not supported between instances of " f"'{type(self).__name__}' and '{type(other).__name__}'" + f"'>' not supported between instances of '{type(self).__name__}' and '{type(other).__name__}'" ) return self.vcs > other.vcs and not self.__eq__(other) def __ge__(self, other) -> bool: if not isinstance(other, Version): raise TypeError( - f"'>' not supported between instances of " f"'{type(self).__name__}' and '{type(other).__name__}'" + f"'>' not supported between instances of '{type(self).__name__}' and '{type(other).__name__}'" ) return self.vcs > other.vcs or self.__eq__(other) def __lt__(self, other) -> bool: if not isinstance(other, Version): raise TypeError( - f"'>' not supported between instances of " f"'{type(self).__name__}' and '{type(other).__name__}'" + f"'>' not supported between instances of '{type(self).__name__}' and '{type(other).__name__}'" ) return self.vcs < other.vcs and not self.__eq__(other) def __le__(self, other) -> bool: if not isinstance(other, Version): raise TypeError( - f"'>' not supported between instances of " f"'{type(self).__name__}' and '{type(other).__name__}'" + f"'>' not supported between instances of '{type(self).__name__}' and '{type(other).__name__}'" ) return self.vcs < other.vcs or self.__eq__(other) @@ -175,44 +202,41 @@ def __init__(self, name: str, version: str, system: str, backend: str, distro: s # metadata self.path: Path = Path(path) + # extra + self.name_version_regex: RePattern = re_compile( + r"^(?P{})(?:(?:@(?P[\d\.]+)(?!@))?(?:@?(?P:)(?P[\d\.]+)?)?)?$".format(self.name) + ) + def satisfies(self, spec: str) -> bool: """Test if this node satisfies the given spec.""" # return true if spec has no condition - if re_fullmatch(r"^\s*$", spec): + if spec.isspace(): return True # else evaluate conditional - name_version_regex: str = ( - r"^(?P{})(?:(?:@(?P[\d\.]+)(?!@))?(?:@?(?P:)(?P[\d\.]+)?)?)?$".format(self.name) - ) - system_regex: str = r"^system=(?P[a-zA-Z0-9]+)$" - backend_regex: str = r"^backend=(?P[a-zA-Z0-9]+)$" - distro_regex: str = r"^distro=(?P[a-zA-Z0-9]+)$" - dependency_regex: str = r"^\^(?P[a-zA-Z0-9-]+)$" - ss: list[str] = re_split(r"\s+", spec.strip()) for part in ss: # name and version - res: ReMatch | None = re_fullmatch(name_version_regex, part) - if res is not None: + res: ReMatch | None = self.name_version_regex.fullmatch(part) + if res: gd: dict = res.groupdict() - if gd["left"] is not None and gd["right"] is None: # n@v: or n@v - if gd["colen"] is not None: + if gd["left"] and not gd["right"]: # n@v: or n@v + if gd["colen"]: if Version(gd["left"]) > self.version: return False else: if Version(gd["left"]) != self.version: return False - elif gd["left"] is None and gd["right"] is not None: # n@:v - if gd["colen"] is not None: + elif not gd["left"] and gd["right"]: # n@:v + if gd["colen"]: if Version(gd["right"]) < self.version: return False else: return False - elif gd["left"] is None and gd["right"] is None: # n - if gd["colen"] is not None: + elif not gd["left"] and not gd["right"]: # n + if gd["colen"]: return False else: # n@v:v if Version(gd["left"]) > self.version or self.version > Version(gd["right"]): @@ -220,29 +244,29 @@ def satisfies(self, spec: str) -> bool: continue # part has been handled so continue # system - res = re_fullmatch(system_regex, part) - if res is not None: + res = SYSTEM_REGEX.fullmatch(part) + if res: if res.group("system") != self.system: return False continue # part has been handled so continue # backend - res = re_fullmatch(backend_regex, part) - if res is not None: + res = BACKEND_REGEX.fullmatch(part) + if res: if res.group("backend") != self.backend: return False continue # part has been handled so continue # distro - res = re_fullmatch(distro_regex, part) - if res is not None: + res = DISTRO_REGEX.fullmatch(part) + if res: if res.group("distro") != self.distro: return False continue # part has been handled so continue # dependencies - res = re_fullmatch(dependency_regex, part) - if res is not None: + res = DEPENDENCY_REGEX.fullmatch(part) + if res: matched = False for dep in self.dependencies: if res.group("name") == dep: @@ -264,7 +288,7 @@ def apply_constraint(self, conditional: str, _type: str, spec: str) -> bool: self.dependencies.add(spec) return True elif _type == "variable": - parts = re_fullmatch(r"^(?P[^=]+)=(?P.*)$", spec).groupdict() + parts = VARIABLE_REGEX.fullmatch(spec).groupdict() self.variables[parts["key"]] = parts["value"] elif _type == "argument": self.arguments.add(spec) @@ -299,8 +323,7 @@ def hash(self) -> str: hash_list.append(self.underlay) hash_str: str = "|".join(str(x) for x in hash_list) - #if self.name == "ucx": - # logger.debug(hash_list) + logger.debug(f"hash string for {self.name}: {hash_str}") return sha256(hash_str.encode()).hexdigest() @property @@ -318,6 +341,7 @@ def __eq__(self, other) -> bool: def __lt__(self, other) -> bool: if isinstance(other, Image): + # need to consider the name of the images as well return not self.version.preferred(other.version) return False @@ -359,17 +383,22 @@ class ImageGraph(nx_DiGraph, metaclass=OurMeta): def __init__(self, **kwargs) -> None: super().__init__(**kwargs) - def add_edge(self, u_of_edge: Image, v_of_edge: Image, **kwargs) -> None: + def add_edges_from(self, edges: list[tuple[Image, Image]], **kwargs) -> None: + logger.info("Adding edges to graph") # check that edge endpoints are in graph - if self.has_node(u_of_edge) and self.has_node(v_of_edge): - super().add_edge(u_of_edge, v_of_edge, **kwargs) - else: - raise CannotFindDependency("Cannot find dependency {} for {}".format(v_of_edge, u_of_edge)) + nodes = set() + for edge in edges: + nodes.update(edge) + for node in nodes: + if not self.has_node(node): + raise CannotFindDependency("Cannot find dependency '{}'.".format(node)) + + super().add_edges_from(edges, **kwargs) # check that graph is still a DAG if not nx_is_directed_acyclic_graph(self): cycle = nx_find_cycle(self) - raise EdgeViolatesDAG(u_of_edge, v_of_edge, cycle) + raise EdgeViolatesDAG(cycle) def get_similar_nodes(self, node: Image) -> set: """Get all nodes with the same name.""" @@ -379,43 +408,11 @@ def get_similar_nodes(self, node: Image) -> set: similar.add(n) return similar - def get_dependencies(self, node: Image) -> set[Image]: - """Get all dependencies for an image.""" - # had to add this wrapper because nx.neighbors was dropping the attributes of some nodes (cuda, python) - deps = set() - for node in nx_neighbors(self, node): - for n in self.nodes: - if n == node: - deps.add(n) - # TODO will contact the maintainers of networkx at some point - return deps - - def is_above(self, u_node: Image, v_node: Image) -> bool: - """Test if one node is above another in the dependency tree.""" - return nx_has_path(self, u_node, v_node) - def _is_valid_build_tuple(self, bt: tuple[Image]) -> bool: - """Verify that all the dependencies of a build tuple can be met.""" - valid = True - - # check for similar images - for i0 in range(len(bt)): - for i2 in bt[i0 + 1 :]: - if bt[i0].satisfies(i2.name): - valid = False - - # check that all images exist - for node in bt: - if not self.has_node(node): - valid = False - - # break prematurely if the first two checks fail - if not valid: - return valid - + """Verify that all the dependencies in a build tuple are met.""" # check that deps in build tuple for node in bt: - deps = self.get_dependencies(node) + deps = set(nx_neighbors(self, node)) # group deps grouped = dict() @@ -427,12 +424,13 @@ def _is_valid_build_tuple(self, bt: tuple[Image]) -> bool: # check that the needed dependency exists for g in grouped: if grouped[g].isdisjoint(bt): - valid = False + return False - return valid + return True - def create_build_recipe(self, targets: list[Target]) -> tuple: + def solve_build(self, targets: list[Target]) -> tuple: """Create a build recipe.""" + logger.info("solving for build recipe") # check if all the targets exist for node in targets: if len(self.get_similar_nodes(node.node)) < 1: @@ -440,31 +438,31 @@ def create_build_recipe(self, targets: list[Target]) -> tuple: # init build set and priority list build_set = set() - priority_list = list() + build_names = set() # add similar to build set + logger.info("loading similar images") for target in targets: build_set.update(self.get_similar_nodes(target.node)) - # make sure to update priority - if target.node.name not in priority_list: - priority_list.append(target.node.name) + build_names.add(target.node.name) # add deps to build set + logger.info("adding dependencies to build") while True: build_set_length = len(build_set) for node in build_set.copy(): - dependencies = self.get_dependencies(node) - for d in dependencies: - build_set.add(d) - if d.name not in priority_list: - priority_list.append(d.name) + dependencies = set(nx_neighbors(self, node)) + # logger.info(dependencies) + build_set.update(dependencies) + build_names.update([_.name for _ in dependencies]) # loop until all dependencies are added if build_set_length == len(build_set): break # apply constraints + logger.info("apply target constraints") for target in targets: for node in build_set.copy(): if node.satisfies(target.node.name): @@ -475,52 +473,43 @@ def create_build_recipe(self, targets: list[Target]) -> tuple: elif target.op == DepOp.LE and node.version > target.node.version: build_set.remove(node) + logger.info("group and prioritize build") # group deps - grouped = dict() + grouped: dict = dict() for node in build_set: if node.name not in grouped: grouped[node.name] = set() grouped[node.name].add(node) # sort deps so that the highest versions of images further up the dep tree will be chosen - prioritized_list_group = list() - for group in priority_list: + prioritized_list_group: list[list] = list() + for group in build_names: tmp = list(grouped[group]) tmp.sort(reverse=True) prioritized_list_group.append(tmp) + # get permutations + logger.info("get build permutations") permutations = get_permutations(0, prioritized_list_group) # return valid build tuple + logger.info("examining build permutations") for p in permutations: - # clean up permutation - clean_p = set() - for n in p: - for t in targets: - con_targ = None - for nt in [x for x in p]: - if nt.satisfies(t.node.name): - con_targ = nt - break - if self.is_above(con_targ, n): - clean_p.add(n) - break - - if self._is_valid_build_tuple(tuple(clean_p)): + if self._is_valid_build_tuple(tuple(p)): # order build build_list = list() processed = set() - unprocessed = clean_p.copy() + unprocessed = p.copy() while len(unprocessed) > 0: level_holder = list() for node in unprocessed.copy(): - deps = set(self.get_dependencies(node)).intersection(clean_p) + deps = set(nx_neighbors(self, node)).intersection(p) if deps.issubset(processed): level_holder.append(node) level_holder.sort() + processed.update(level_holder) + build_list.extend(level_holder) for node in level_holder: - processed.add(node) unprocessed.remove(node) - build_list.append(node) return tuple(build_list) @@ -532,13 +521,17 @@ class ImageRepo(metaclass=OurMeta): """Image repository.""" def __init__(self) -> None: + logger.info("creating image repo") self.images: set[Image] = set() - # constraint(image, condition, type, spec, scope) + + # constraint(image, condition, type, spec, scope(only for versions and dependencies)) + logger.info("loading global image constraints") self.constraints: list[tuple[str, str, str, str, str]] = list() - if config.get("constraints", warn_on_miss=False) is not None: + if cstrs := config.get("constraints", warn_on_miss=False): + logger.debug(f"global constraints: {cstrs}") # arguments - if "arguments" in config.get("constraints"): - for argument in config.get("constraints")["arguments"]: + if "arguments" in cstrs: + for argument in cstrs["arguments"]: if isinstance(argument["value"], list): specs = argument["value"] else: @@ -547,24 +540,17 @@ def __init__(self) -> None: ] for spec in specs: self.constraints.append( - ( - "", - argument["when"] if "when" in argument else "", - "argument", - spec, - "global", - ) + (EMPTY_STR, argument["when"] if "when" in argument else EMPTY_STR, "argument", spec) ) # variables - if "variables" in config.get("constraints"): - for variable in config.get("constraints")["variables"]: + if "variables" in cstrs: + for variable in cstrs["variables"]: self.constraints.append( ( - "", - variable["when"] if "when" in variable else "", + EMPTY_STR, + variable["when"] if "when" in variable else EMPTY_STR, "variable", "{}={}".format(variable["name"], variable["value"]), - "global", ) ) @@ -573,22 +559,26 @@ def import_from_dir(self, path: str) -> None: p = Path(path) if not p.is_dir(): raise NotADirectoryError(f"Image path {path} is not a directory!") + logger.info(f"importing images from {p}") - for name in [x for x in p.iterdir() if x.is_dir() and x.name[0] != "."]: + imported_names = set() + for image_path in [_ for _ in p.iterdir() if _.is_dir() and _.name[0] != "."]: + name = image_path.name # check for duplicate image - duplicate_image = False - for _i in self.images: - if _i.name == name.name: - duplicate_image = name - break - if duplicate_image: - logger.info("The image definition in '{}' is being skipped because it has the same name as an already imported image.".format(duplicate_image)) + if name in imported_names: + logger.warn( + "The image definition in '{}' is being skipped because it has the same name as an already imported image.".format( + name + ) + ) continue + imported_names.add(name) # process metadata - with open(name.joinpath("specs.yaml"), "r") as fi: + logger.info(f"importing image {name}") + with open(image_path.joinpath("specs.yaml"), "r") as f: try: - specs_file = yaml_safe_load(fi) + specs_file = yaml_safe_load(f) # add versions for version in specs_file["versions"]: if isinstance(version["spec"], list): @@ -599,18 +589,17 @@ def import_from_dir(self, path: str) -> None: ] for spec in specs: image = Image( - name.name, + name, spec, config.get("velocity:system"), config.get("velocity:backend"), config.get("velocity:distro"), - str(name), + str(image_path), ) if "when" in version: - if image.satisfies(version["when"]): - self.images.add(image) - else: - self.images.add(image) + if not image.satisfies(version["when"]): + continue + self.images.add(image) # add constraints # dependencies if "dependencies" in specs_file: @@ -622,10 +611,12 @@ def import_from_dir(self, path: str) -> None: dependency["spec"], ] for spec in specs: + if "^" in spec: + raise SpecSyntaxError("'^' not allowed in dependency spec.") self.constraints.append( ( - name.name, - dependency["when"] if "when" in dependency else "", + name, + dependency["when"] if "when" in dependency else EMPTY_STR, "dependency", spec, dependency["scope"] if "scope" in dependency else "image", @@ -634,22 +625,14 @@ def import_from_dir(self, path: str) -> None: # templates if "templates" in specs_file: for template in specs_file["templates"]: - if isinstance(template["name"], list): - specs = template["name"] - else: - specs = [ + self.constraints.append( + ( + name, + template["when"] if "when" in template else EMPTY_STR, + "template", template["name"], - ] - for spec in specs: - self.constraints.append( - ( - name.name, - template["when"] if "when" in template else "", - "template", - spec, - template["scope"] if "scope" in template else "image", - ) ) + ) # arguments if "arguments" in specs_file: for argument in specs_file["arguments"]: @@ -662,11 +645,10 @@ def import_from_dir(self, path: str) -> None: for spec in specs: self.constraints.append( ( - name.name, - argument["when"] if "when" in argument else "", + name, + argument["when"] if "when" in argument else EMPTY_STR, "argument", spec, - argument["scope"] if "scope" in argument else "image", ) ) # variables @@ -674,11 +656,10 @@ def import_from_dir(self, path: str) -> None: for variable in specs_file["variables"]: self.constraints.append( ( - name.name, - variable["when"] if "when" in variable else "", + name, + variable["when"] if "when" in variable else EMPTY_STR, "variable", "{}={}".format(variable["name"], variable["value"]), - variable["scope"] if "scope" in variable else "image", ) ) # files @@ -693,11 +674,10 @@ def import_from_dir(self, path: str) -> None: for spec in specs: self.constraints.append( ( - name.name, + name, file["when"] if "when" in file else "", "file", spec, - file["scope"] if "scope" in file else "image", ) ) # prologs @@ -705,11 +685,10 @@ def import_from_dir(self, path: str) -> None: for prolog in specs_file["prologs"]: self.constraints.append( ( - name.name, + name, prolog["when"] if "when" in prolog else "", "prolog", prolog["script"], - prolog["scope"] if "scope" in prolog else "image", ) ) except TypeError as e: @@ -722,58 +701,58 @@ def create_build_recipe(self, targets: list[str]) -> tuple[tuple, ImageGraph]: header_print([TextBlock("Creating build recipe:")]) start = timer() + logger.info("start image recipe creation") images: set[Image] = deepcopy(self.images) + logger.info("parsing targets") build_targets: list[Target] = list() for target in targets: - res = re_fullmatch( - r"^(?P[a-zA-Z0-9-]+)(?:(?:@(?P[^:\s]+)(?!@))?(?:@?(?P:)(?P\S+)?)?)?$", - target, - ) - if res is not None: + if res := TARGET_REGEX.fullmatch(target): gd: dict = res.groupdict() + system = config.get("velocity:system") + backend = config.get("velocity:backend") + distro = config.get("velocity:distro") + # n@v: or n@v - if gd["left"] is not None and gd["right"] is None: + if gd["left"] and not gd["right"]: t = Image( gd["name"], gd["left"], - config.get("velocity:system"), - config.get("velocity:backend"), - config.get("velocity:distro"), - "", + system, + backend, + distro, + EMPTY_STR, ) - if gd["colen"] is not None: + if gd["colon"]: build_targets.append(Target(t, DepOp.GE)) else: build_targets.append(Target(t, DepOp.EQ)) # n@:v - elif gd["left"] is None and gd["right"] is not None: - if gd["colen"] is not None: - t = Image( - gd["name"], - gd["right"], - config.get("velocity:system"), - config.get("velocity:backend"), - config.get("velocity:distro"), - "", - ) - build_targets.append(Target(t, DepOp.LE)) - else: + elif not gd["left"] and gd["right"]: + if not gd["colon"]: raise InvalidImageVersionError("Invalid version '{}'.".format(target)) + t = Image( + gd["name"], + gd["right"], + system, + backend, + distro, + EMPTY_STR, + ) + build_targets.append(Target(t, DepOp.LE)) # n - elif gd["left"] is None and gd["right"] is None: - if gd["colen"] is not None: + elif not gd["left"] and not gd["right"]: + if gd["colon"]: raise InvalidImageVersionError("Invalid version '{}'.".format(target)) - else: - t = Image( - gd["name"], - "", - config.get("velocity:system"), - config.get("velocity:backend"), - config.get("velocity:distro"), - "", - ) - build_targets.append(Target(t, DepOp.UN)) + t = Image( + gd["name"], + "", + config.get("velocity:system"), + config.get("velocity:backend"), + config.get("velocity:distro"), + EMPTY_STR, + ) + build_targets.append(Target(t, DepOp.UN)) # n@v:v else: t = Image( @@ -782,7 +761,7 @@ def create_build_recipe(self, targets: list[str]) -> tuple[tuple, ImageGraph]: config.get("velocity:system"), config.get("velocity:backend"), config.get("velocity:distro"), - "", + EMPTY_STR, ) build_targets.append(Target(t, DepOp.GE)) t = Image( @@ -791,75 +770,84 @@ def create_build_recipe(self, targets: list[str]) -> tuple[tuple, ImageGraph]: config.get("velocity:system"), config.get("velocity:backend"), config.get("velocity:distro"), - "", + EMPTY_STR, ) build_targets.append(Target(t, DepOp.LE)) else: raise NoAvailableBuild("No available build!") # pre-burner graph + logger.info("setting up pre-burner graph") for constraint in self.constraints: for image in images: - if image.apply_constraint("{} {}".format(constraint[0], constraint[1]), constraint[2], constraint[3]): - images_changed = True + image.apply_constraint("{} {}".format(constraint[0], constraint[1]), constraint[2], constraint[3]) ig = ImageGraph() - for image in images: - ig.add_node(image) + ig.add_nodes_from(images) + edges: list[tuple] = list() for image in images: for dep in image.dependencies: for di in images: if di.satisfies(dep): - ig.add_edge(image, di) + edges.append((image, di)) + ig.add_edges_from(edges) - bt: tuple[Image] = ig.create_build_recipe(build_targets) + logger.info("creating pre-burner recipe") + bt: tuple[Image] = ig.solve_build(build_targets) # apply constraints for the build scope + logger.info("applying constraints for the build") images_changed: bool = True while images_changed: images_changed = False for constraint in self.constraints: - if constraint[4] == "build": - for targ in bt: - if targ.satisfies(constraint[1]): + if constraint[2] == "dependency" and constraint[4] == "build": + for b_target in bt: + if b_target.satisfies(constraint[1]): for image in images: if image.apply_constraint(constraint[0], constraint[2], constraint[3]): images_changed = True - # "image" or "universal" - else: - for image in images: - if image.apply_constraint( - "{} {}".format(constraint[0], constraint[1]), constraint[2], constraint[3] - ): - images_changed = True + for constraint in self.constraints: + for image in images: + image.apply_constraint("{} {}".format(constraint[0], constraint[1]), constraint[2], constraint[3]) # create graph + logger.info("creating final image graph") ig = ImageGraph() - for image in images: - ig.add_node(image) + ig.add_nodes_from(images) + edges = list() for image in images: for dep in image.dependencies: for di in images: if di.satisfies(dep): - ig.add_edge(image, di) + edges.append((image, di)) + ig.add_edges_from(edges) - bt: tuple[Image] = ig.create_build_recipe(build_targets) + logger.info("creating final recipe") + bt: tuple[Image] = ig.solve_build(build_targets) # update images so that their hash includes the layers below them + logger.info("updating image hashes with dependencies") cumulative_deps: int = 0 for b in bt: b.underlay = cumulative_deps cumulative_deps = cumulative_deps + int(b.id, 16) + logger.info("creating recipe image graph") bt_ig = ImageGraph() - for image in bt: - bt_ig.add_node(image) + bt_ig.add_nodes_from(bt) + edges = list() for image in bt: for dep in image.dependencies: for di in bt: if di.satisfies(dep): - bt_ig.add_edge(image, di) + edges.append((image, di)) + bt_ig.add_edges_from(edges) + + logger.info("build recipe created") end = timer() - indent_print([TextBlock(f"created recipe in {round(end - start, 2)}s\n", fore=Fore.MAGENTA, style=Style.BRIGHT)]) + indent_print( + [TextBlock(f"created recipe in {round(end - start, 2)}s\n", fore=Fore.MAGENTA, style=Style.BRIGHT)] + ) return bt, bt_ig From 1e316a8b654484d57de68edcd594dac89acd44f5 Mon Sep 17 00:00:00 2001 From: Asa Rentschler Date: Mon, 2 Feb 2026 11:54:51 -0500 Subject: [PATCH 3/5] Fixed Version bug due to conditionals. --- src/velocity/_graph.py | 25 ++++++++++++------------- 1 file changed, 12 insertions(+), 13 deletions(-) diff --git a/src/velocity/_graph.py b/src/velocity/_graph.py index 4336a84..524df2a 100644 --- a/src/velocity/_graph.py +++ b/src/velocity/_graph.py @@ -8,7 +8,6 @@ Match as ReMatch, Pattern as RePattern, compile as re_compile, - fullmatch as re_fullmatch, split as re_split, ) from timeit import default_timer as timer @@ -82,10 +81,10 @@ def __init__(self, version_specifier: str) -> None: self.vs = version_specifier try: version_dict: dict = VERSION_REGEX.fullmatch(version_specifier).groupdict() - self.major: int | None = int(version_dict["major"]) if version_dict["major"] else None - self.minor: int | None = int(version_dict["minor"]) if version_dict["minor"] else None - self.patch: int | None = int(version_dict["patch"]) if version_dict["patch"] else None - self.suffix: str | None = str(version_dict["suffix"]) if version_dict["suffix"] else None + self.major: int | None = int(version_dict["major"]) if version_dict["major"] is not None else None + self.minor: int | None = int(version_dict["minor"]) if version_dict["minor"] is not None else None + self.patch: int | None = int(version_dict["patch"]) if version_dict["patch"] is not None else None + self.suffix: str | None = str(version_dict["suffix"]) if version_dict["suffix"] is not None else None except AttributeError: self.major: int | None = None self.minor: int | None = None @@ -99,10 +98,10 @@ def __init__(self, version_specifier: str) -> None: def vcs(self) -> str: """Version Comparison String""" return "{:#>9}.{:#>9}.{:#>9}.{:~<9}".format( - self.major if self.major else "#", - self.minor if self.minor else "#", - self.patch if self.patch else "#", - self.suffix if self.suffix else "~", + self.major if self.major is not None else "#", + self.minor if self.minor is not None else "#", + self.patch if self.patch is not None else "#", + self.suffix if self.suffix is not None else "~", ) def preferred(self, other) -> bool: @@ -113,13 +112,13 @@ def preferred(self, other) -> bool: def _vcs_t(self) -> str: """Version Comparison String Truncated""" vl: int = 0 - if not self.major: + if self.major is None: pass - elif not self.minor: + elif self.minor is None: vl = 9 - elif not self.patch: + elif self.patch is None: vl = 19 - elif not self.suffix: + elif self.suffix is None: vl = 29 else: vl = 39 From 93016a29dbb762d36f9c5ebbaa7d81133173a61c Mon Sep 17 00:00:00 2001 From: Asa Rentschler Date: Mon, 9 Feb 2026 08:28:46 -0500 Subject: [PATCH 4/5] Added a sleep to the build loop to lower CPU usage. --- src/velocity/_build.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/velocity/_build.py b/src/velocity/_build.py index c7bad65..0913217 100644 --- a/src/velocity/_build.py +++ b/src/velocity/_build.py @@ -9,6 +9,7 @@ from timeit import default_timer as timer from queue import SimpleQueue from subprocess import PIPE, Popen +from time import sleep from colorama import Fore, Style from loguru import logger @@ -32,7 +33,7 @@ def read_pipe(pipe: PIPE, topic: SimpleQueue, prefix: str, log: SimpleQueue) -> topic.put(ln.strip("\n")) log.put("{} {}".format(prefix, ln.strip("\n"))) except UnicodeDecodeError as e: - # some ubuntu container builds were printing out wierd characters + # some ubuntu container builds were printing out weird characters logger.error(e) @@ -62,6 +63,7 @@ def run(cmd: str, log_file: Path = None, verbose: bool = False, critical: bool = if file and not log.empty(): file.write(log.get() + "\n") file.flush() # TODO is this needed? + sleep(0.01) # Sleep 10ms to reduce CPU usage out.join() err.join() From 90f2815e58ad0de5c50eebdd2609ac7cdbe2d84d Mon Sep 17 00:00:00 2001 From: Asa Rentschler Date: Mon, 23 Feb 2026 12:42:59 -0500 Subject: [PATCH 5/5] Added support for not operator and updated image hash generation. --- src/velocity/_graph.py | 89 +++++++++++++++++++----------------------- 1 file changed, 41 insertions(+), 48 deletions(-) diff --git a/src/velocity/_graph.py b/src/velocity/_graph.py index 524df2a..b8c79ef 100644 --- a/src/velocity/_graph.py +++ b/src/velocity/_graph.py @@ -5,7 +5,6 @@ from hashlib import sha256 from pathlib import Path from re import ( - Match as ReMatch, Pattern as RePattern, compile as re_compile, split as re_split, @@ -215,69 +214,56 @@ def satisfies(self, spec: str) -> bool: # else evaluate conditional ss: list[str] = re_split(r"\s+", spec.strip()) - + result = True for part in ss: + # not + not_flag = False + if part[0] == "!": + not_flag = True + part = part[1:] + # name and version - res: ReMatch | None = self.name_version_regex.fullmatch(part) - if res: + if res := self.name_version_regex.fullmatch(part): gd: dict = res.groupdict() - if gd["left"] and not gd["right"]: # n@v: or n@v + if gd["left"] and gd["right"]: # n@v:v + result = (Version(gd["left"]) <= self.version <= Version(gd["right"])) ^ not_flag + elif gd["left"] and not gd["right"]: # n@v: or n@v if gd["colen"]: - if Version(gd["left"]) > self.version: - return False + result = (Version(gd["left"]) <= self.version) ^ not_flag else: - if Version(gd["left"]) != self.version: - return False + result = (Version(gd["left"]) == self.version) ^ not_flag elif not gd["left"] and gd["right"]: # n@:v + result = (self.version <= Version(gd["right"])) ^ not_flag + else: # n if gd["colen"]: - if Version(gd["right"]) < self.version: - return False - else: - return False - elif not gd["left"] and not gd["right"]: # n - if gd["colen"]: - return False - else: # n@v:v - if Version(gd["left"]) > self.version or self.version > Version(gd["right"]): - return False - continue # part has been handled so continue + raise SpecSyntaxError("Invalid syntax '{}'.".format(spec)) + result = True ^ not_flag # system - res = SYSTEM_REGEX.fullmatch(part) - if res: - if res.group("system") != self.system: - return False - continue # part has been handled so continue + elif res := SYSTEM_REGEX.fullmatch(part): + result = (res.group("system") == self.system) ^ not_flag # backend - res = BACKEND_REGEX.fullmatch(part) - if res: - if res.group("backend") != self.backend: - return False - continue # part has been handled so continue + elif res := BACKEND_REGEX.fullmatch(part): + result = (res.group("backend") == self.backend) ^ not_flag # distro - res = DISTRO_REGEX.fullmatch(part) - if res: - if res.group("distro") != self.distro: - return False - continue # part has been handled so continue + elif res := DISTRO_REGEX.fullmatch(part): + result = (res.group("distro") == self.distro) ^ not_flag # dependencies - res = DEPENDENCY_REGEX.fullmatch(part) - if res: - matched = False - for dep in self.dependencies: - if res.group("name") == dep: - matched = True - if matched: - continue # match is found so continue + elif res := DEPENDENCY_REGEX.fullmatch(part): + result = (res.group("name") in self.dependencies) ^ not_flag # if we get here this part has not been handled - return False + else: + result = False - # all parts were handled - return True + # if result is False break + if not result: + break + + return result def apply_constraint(self, conditional: str, _type: str, spec: str) -> bool: """Evaluate and apply constraints. Return True if a constraint changes the dependencies.""" @@ -310,14 +296,21 @@ def hash(self) -> str: # hash_list.append(self.backend) # disable backend for now because it should not make a difference in the image hash_list.append(self.distro) hash_list.append(",".join(str(x) for x in sorted(self.dependencies))) - hash_list.append(",".join(str(x) for x in sorted(self.variables))) + hash_list.append(",".join(x + "=" + str(self.variables[x]) for x in sorted(self.variables))) hash_list.append(",".join(str(x) for x in sorted(self.arguments))) + tf = Path(self.path).joinpath("templates", "{}.vtmp".format(self.template)) if tf.is_file(): hash_list.append(sha256(tf.read_bytes()).hexdigest()) else: hash_list.append(None) - hash_list.append(",".join(str(x) for x in sorted(self.files))) + + for file in sorted(self.files): + hash_list.append(file) + tf = Path(self.path).joinpath("files", file) + if tf.is_file(): + hash_list.append(sha256(tf.read_bytes()).hexdigest()) + hash_list.append(self.prolog) hash_list.append(self.underlay)