Skip to content

feat: migrate frontend to C++ and integrate LLM inference pipeline #2

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
RicardoRodriguezArg wants to merge 1 commit into
base: main
Choose a base branch
from
Draft

feat: migrate frontend to C++ and integrate LLM inference pipeline #2

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
The table of contents is too big for display.
Diff view
Diff view
  •  
  •  
  •  
55 changes: 55 additions & 0 deletions src/CMakeLists.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,55 @@
cmake_minimum_required(VERSION 3.23)
project(agent CXX)

set(CMAKE_CXX_STANDARD 23)
set(CMAKE_CXX_STANDARD_REQUIRED ON)
set(CMAKE_EXPORT_COMPILE_COMMANDS ON)

set(OPENVINO_GENAI_SRC
"${CMAKE_SOURCE_DIR}/resources/openvino.genai-2026.1.0.0"
)
set(OPENVINO_GENAI_INCLUDE "${OPENVINO_GENAI_SRC}/src/cpp/include")

set(OPENVINO_GENAI_LIB
"$ENV{HOME}/virtual_enviroments/kamerdyner/lib/python3.12/site-packages/openvino_genai/libopenvino_genai.so.2610"
)
set(OPENVINO_TOKENIZERS_LIB
"$ENV{HOME}/virtual_enviroments/kamerdyner/lib/python3.12/site-packages/openvino_tokenizers/lib/libopenvino_tokenizers.so"
)

find_package(OpenVINO REQUIRED PATHS /usr/lib/cmake/openvino2026.1.0)
find_package(tinyxml2 REQUIRED)

add_library(openvino_genai SHARED IMPORTED)
set_target_properties(openvino_genai PROPERTIES
IMPORTED_LOCATION "${OPENVINO_GENAI_LIB}"
INTERFACE_INCLUDE_DIRECTORIES "${OPENVINO_GENAI_INCLUDE}"
)
target_link_libraries(openvino_genai INTERFACE openvino::runtime)

# El root del proyecto es el include base para todos los subdirectorios
# Permite includes del estilo: #include "inference/foo.h", #include "frontend/bar.h"
include_directories(${CMAKE_SOURCE_DIR})

add_subdirectory(inference)
add_subdirectory(frontend)

add_library(openvino_tokenizers SHARED IMPORTED)
set_target_properties(openvino_tokenizers PROPERTIES
IMPORTED_LOCATION "${OPENVINO_TOKENIZERS_LIB}"
)

install(FILES config/inference/model_inference_config.xml
DESTINATION agent/config/inference
RENAME model_inferences_setting.xml)

add_executable(robot_router main.cpp)
target_compile_definitions(robot_router PRIVATE
MODEL_INFERENCE_CONFIG_PATH="${CMAKE_INSTALL_PREFIX}/agent/config/inference/model_inferences_setting.xml"
)
target_link_libraries(robot_router PRIVATE frontend_lib openvino_tokenizers)
set_target_properties(robot_router PROPERTIES
BUILD_RPATH "$ENV{HOME}/virtual_enviroments/kamerdyner/lib/python3.12/site-packages/openvino_genai;\
$ENV{HOME}/virtual_enviroments/kamerdyner/lib/python3.12/site-packages/openvino_tokenizers/lib;\
$ENV{HOME}/virtual_enviroments/kamerdyner/lib/python3.12/site-packages/openvino/libs"
)
3 changes: 3 additions & 0 deletions src/commands/base.h
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
namespace butler::commands {
struct ExecutiveCommandType{};
}
0 src/frontend/frontend/__init__.py → src/commands/builder.h
View file
Open in desktop
File renamed without changes.
25 changes: 25 additions & 0 deletions src/commands/commons.h
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,25 @@
#ifndef BUTLER_COMMANDS_HIGH_LEVEL_COMMANDS_H
#define BUTLER_COMMANDS_HIGH_LEVEL_COMMANDS_H

#include <cstdint>


namespace butler::commands {
enum class Priority: std::uint8_t
{
kLow=0,
kLevelOne=1
};

enum class Type: std::uint8_t
{
kStop,
kMoveTo
};



} // namespace butler::commands


#endif // BUTLER_COMMANDS_HIGH_LEVEL_COMMANDS_H
0 src/frontend/frontend/commands/__init__.py → src/commands/factory.h
View file
Open in desktop
File renamed without changes.
0 src/frontend/frontend/frontend/__init__.py → src/commands/move_to.h
View file
Open in desktop
File renamed without changes.
199 changes: 199 additions & 0 deletions src/config/executive/behave_tree/command_executor_tree.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,199 @@
<root BTCPP_format="4">

<!--
IMPORTANT FOR ORCHESTRATOR:
This tree processes ONE command per execution cycle.
The orchestrator MUST reset the tree (BT::Tree::haltTree()) after each
command completes or fails, before ticking it again for the next command.
Failing to do so will cause SequenceWithMemory to skip GetNextCommand
and ClearBoard on subsequent ticks, leaving stale state on the blackboard.
-->

<BehaviorTree ID="AgentCommandExecutorTree">

<SequenceWithMemory name="ExecuteCommandFromList">

<GetNextCommand
name="GetNextHighLevelCommand"
commands="{commands}"
command="{current_command}"
command_type="{current_command_type}" />

<ClearBoardFromPreviousCommandExecution
name="ClearBoardFromPreviousCommandExecution"
command="{current_command}"
feedback="{current_command_feedback}"
status="{current_command_status}"
error="{current_command_error}" />

<Switch3 name="HighLevelCommandDispatcher"
variable="{current_command_type}"
case_1="go_to_and_return"
case_2="go_to_wait"
case_3="get_state">

<!-- case_1: go_to(landmark) + wait(time) + go_to(home) -->
<Sequence name="GoToAndReturnHomeSequence">

<ValidateLandmark
name="ValidateGoToLandmark"
command="{current_command}"
landmarks="{landmarks}" />

<ExecuteCommandWithFeedback
name="ExecuteGoToLandmarkCommand"
command="{current_command}"
phase="go_to_landmark"
feedback="{current_command_feedback}"
status="{current_command_status}"
error="{current_command_error}" />

<ValidateArrivalToLandmark
name="ValidateArrivalToLandmark"
command="{current_command}"
landmarks="{landmarks}" />

<ExecuteCommandWithFeedback
name="ExecuteWaitAtLandmarkCommand"
command="{current_command}"
phase="wait"
feedback="{current_command_feedback}"
status="{current_command_status}"
error="{current_command_error}" />

<ExecuteCommandWithFeedback
name="ExecuteReturnHomeCommand"
command="{current_command}"
phase="return_home"
feedback="{current_command_feedback}"
status="{current_command_status}"
error="{current_command_error}" />

<ValidateArrivalToLandmark
name="ValidateArrivalAtHome"
command="{current_command}"
landmarks="{landmarks}" />

<CommunicateReturnHome
name="CommunicateReturnHome"
command="{current_command}"
landmarks="{landmarks}" />

</Sequence>

<!-- case_2: go_to(landmark) + wait indefinitely -->
<Sequence name="GoToSpecificLandmarkAndWaitSequence">

<ValidateLandmark
name="ValidateGoToLandmark"
command="{current_command}"
landmarks="{landmarks}" />

<ExecuteCommandWithFeedback
name="ExecuteGoToLandmarkCommand"
command="{current_command}"
phase="go_to_landmark"
feedback="{current_command_feedback}"
status="{current_command_status}"
error="{current_command_error}" />

<ValidateArrivalToLandmark
name="ValidateArrivalToLandmark"
command="{current_command}"
landmarks="{landmarks}" />

<CommunicateWaitingStatus
name="CommunicateWaitingStatus"
command="{current_command}"
landmarks="{landmarks}" />

</Sequence>

<!-- case_3: read state of a component -->
<Sequence name="GetStateFlow">

<ValidateComponent
name="ValidateGetStateComponent"
command="{current_command}"
components="{components}" />

<ExecuteCommandWithFeedback
name="ExecuteGetStateCommand"
command="{current_command}"
phase="get_state"
feedback="{current_command_feedback}"
status="{current_command_status}"
error="{current_command_error}" />

</Sequence>

<!-- default: command type not recognized -->
<Sequence name="CommandNotListed">

<CommunicateCommandWasNotListed
name="CommunicateCommandWasNotListed"
command="{current_command}" />

</Sequence>

</Switch3>

</SequenceWithMemory>

</BehaviorTree>

<TreeNodesModel>

<Action ID="GetNextCommand">
<input_port name="commands" type="std::vector&lt;Command&gt;" />
<output_port name="command" type="Command" />
<output_port name="command_type" type="std::string" />
</Action>

<Action ID="ClearBoardFromPreviousCommandExecution">
<input_port name="command" type="Command" />
<inout_port name="feedback" type="std::string" />
<inout_port name="status" type="std::string" />
<inout_port name="error" type="std::string" />
</Action>

<Action ID="ExecuteCommandWithFeedback">
<input_port name="command" type="Command" />
<input_port name="phase" type="std::string" />
<output_port name="feedback" type="std::string" />
<output_port name="status" type="std::string" />
<output_port name="error" type="std::string" />
</Action>

<Condition ID="ValidateLandmark">
<input_port name="command" type="Command" />
<input_port name="landmarks" type="std::vector&lt;Landmark&gt;" />
</Condition>

<Condition ID="ValidateArrivalToLandmark">
<input_port name="command" type="Command" />
<input_port name="landmarks" type="std::vector&lt;Landmark&gt;" />
</Condition>

<Condition ID="ValidateComponent">
<input_port name="command" type="Command" />
<input_port name="components" type="std::vector&lt;Component&gt;" />
</Condition>

<Action ID="CommunicateReturnHome">
<input_port name="command" type="Command" />
<input_port name="landmarks" type="std::vector&lt;Landmark&gt;" />
</Action>

<Action ID="CommunicateWaitingStatus">
<input_port name="command" type="Command" />
<input_port name="landmarks" type="std::vector&lt;Landmark&gt;" />
</Action>

<Action ID="CommunicateCommandWasNotListed">
<input_port name="command" type="Command" />
</Action>

</TreeNodesModel>

</root>
11 changes: 11 additions & 0 deletions src/config/inference/model_inference_config.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@
<?xml version="1.0" encoding="UTF-8"?>
<model_inference>
<model_path>/home/operador/Documents/tmp/agent/models/llm/distilled-1b-robot-router/exported/openvino_int4</model_path>
<device>CPU</device>
<modelfile_path>/home/operador/Documents/brain-ws/docker/models/gpu/Modelfile</modelfile_path>
<generation>
<max_new_tokens>150</max_new_tokens>
<do_sample>false</do_sample>
<apply_chat_template>true</apply_chat_template>
</generation>
</model_inference>
88 changes: 88 additions & 0 deletions src/definitions.h
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,88 @@
#ifndef BUTLER_DEFINITIONS_H
#define BUTLER_DEFINITIONS_H

#include <cstdint>
#include <functional>
#include <string>
#include <string_view>
#include <type_traits>
#include <unordered_map>


namespace butler::features{
enum class FeatureType : std::uint8_t {kLandmarks};
}

template<>
struct std::hash<butler::features::FeatureType> {
std::size_t operator()(butler::features::FeatureType e) const noexcept {
return std::hash<std::underlying_type_t<butler::features::FeatureType>>{}(
static_cast<std::underlying_type_t<butler::features::FeatureType>>(e));
}
};

namespace butler::robot_state {

static constepxr std::uint8_t kMaxCommandAllowedPerExecution{2u};

enum class GeneralState : std::uint8_t { Unknown, kFullyOperational };

enum class ComponentState : std::uint8_t { Unknown, Operational, Failure, _Count };

enum class ComponentId : std::uint8_t {
amcl_localization_pose,
amcl_particle_cloud,
amcl_transform,
map_server,
map_occupancy_grid,
tf_broadcast,
tf_static_broadcast,
nav2_planner_server,
nav2_controller_server,
nav2_behavior_server,
nav2_bt_navigator,
nav2_costmap_global,
nav2_costmap_local,
_Count
};

struct EnumHash {
template<typename E>
std::size_t operator()(E e) const noexcept {
return std::hash<std::underlying_type_t<E>>{}(
static_cast<std::underlying_type_t<E>>(e));
}
};

}

namespace butler::robot_state::components {

template<butler::robot_state::ComponentId, typename> struct LifecycleNodeStrategy;
template<butler::robot_state::ComponentId, typename> struct TopicFreshnessStrategy;
template<butler::robot_state::ComponentId, typename> struct TfStrategy;

// ── AMCL ─────────────────────────────────────────────────────────────────────
template<typename StrategyType> using AmclLocalizationPose = TopicFreshnessStrategy<ComponentId::amcl_localization_pose, StrategyType>;
template<typename StrategyType> using AmclParticleCloud = TopicFreshnessStrategy<ComponentId::amcl_particle_cloud, StrategyType>;
template<typename StrategyType> using AmclTransform = TfStrategy<ComponentId::amcl_transform, StrategyType>;

// ── Map ──────────────────────────────────────────────────────────────────────
template<typename StrategyType> using MapServer = LifecycleNodeStrategy<ComponentId::map_server, StrategyType>;
template<typename StrategyType> using MapOccupancyGrid = TopicFreshnessStrategy<ComponentId::map_occupancy_grid, StrategyType>;

// ── TF ───────────────────────────────────────────────────────────────────────
template<typename StrategyType> using TfBroadcast = TfStrategy<ComponentId::tf_broadcast, StrategyType>;
template<typename StrategyType> using TfStaticBroadcast = TfStrategy<ComponentId::tf_static_broadcast, StrategyType>;

// ── Nav2 ─────────────────────────────────────────────────────────────────────
template<typename StrategyType> using Nav2PlannerServer = LifecycleNodeStrategy<ComponentId::nav2_planner_server, StrategyType>;
template<typename StrategyType> using Nav2ControllerServer = LifecycleNodeStrategy<ComponentId::nav2_controller_server, StrategyType>;
template<typename StrategyType> using Nav2BehaviorServer = LifecycleNodeStrategy<ComponentId::nav2_behavior_server, StrategyType>;
template<typename StrategyType> using Nav2BtNavigator = LifecycleNodeStrategy<ComponentId::nav2_bt_navigator, StrategyType>;
template<typename StrategyType> using Nav2CostmapGlobal = LifecycleNodeStrategy<ComponentId::nav2_costmap_global, StrategyType>;
template<typename StrategyType> using Nav2CostmapLocal = LifecycleNodeStrategy<ComponentId::nav2_costmap_local, StrategyType>;

} // namespace butler::robot_state::components

#endif // BUTLER_DEFINITIONS_H
Empty file added src/executive/CMakeLists.txt
View file
Open in desktop
Empty file.
Empty file added src/executive/builder.h
View file
Open in desktop
Empty file.
Loading