From 28f9312fa7ed95f89b5d0ef9f416b35bd9151057 Mon Sep 17 00:00:00 2001 From: Reason-Wang Date: Thu, 4 Sep 2025 19:46:35 +0000 Subject: [PATCH 1/2] Refine docs --- .gitignore | 1 + README.md | 4 +- agentfly/agents/__init__.py | 3 +- agentfly/agents/agent_base.py | 5 +- agentfly/agents/llm_backends/llm_backends.py | 3 +- agentfly/agents/specialized/openai_agent.py | 4 +- agentfly/agents/templates/__init__.py | 0 agentfly/templates/__init__.py | 10 + agentfly/{agents => }/templates/constants.py | 0 agentfly/templates/global_policy.py | 5 + agentfly/{agents => }/templates/preprocess.py | 0 .../{agents => }/templates/system_policy.py | 0 agentfly/{agents => }/templates/templates.py | 108 +++- .../{agents => }/templates/tool_policy.py | 0 agentfly/{agents => }/templates/utils.py | 7 +- .../templates/vision_processor.py | 0 .../test_text_templates_full_align.py | 2 +- agentfly/tools/tool_base.py | 7 +- agentfly/tools/utils/schema.py | 5 +- docs/api_references/agents/agent.rst | 6 - docs/api_references/agents/index.rst | 93 +-- docs/api_references/agents/llm_backends.rst | 53 +- docs/api_references/agents/rollout.rst | 11 + .../chat_template/README.md | 0 .../chat_template/advanced_features.md | 0 .../chat_template/basic_usage.md | 68 +- docs/api_references/chat_template/chat.rst | 8 + .../chat_template/core_components.md | 175 +++++ .../chat_template/custom_templates.md | 12 +- docs/api_references/chat_template/examples.md | 242 +++++++ .../chat_template/index.rst | 83 +-- .../api_references/chat_template/template.rst | 8 + .../chat_template/vision_templates.md | 0 docs/api_references/tools/index.rst | 107 +--- docs/api_references/tools/tool.rst | 16 +- docs/features/chat_template/architecture.md | 168 ----- docs/features/chat_template/examples.md | 601 ------------------ docs/index.rst | 1 + docs/start/first_agent.md | 10 + docs/start/first_tool_reward.md | 6 +- docs/start/first_training.md | 5 +- pyproject.toml | 2 + 42 files changed, 641 insertions(+), 1198 deletions(-) delete mode 100644 agentfly/agents/templates/__init__.py create mode 100644 agentfly/templates/__init__.py rename agentfly/{agents => }/templates/constants.py (100%) create mode 100644 agentfly/templates/global_policy.py rename agentfly/{agents => }/templates/preprocess.py (100%) rename agentfly/{agents => }/templates/system_policy.py (100%) rename agentfly/{agents => }/templates/templates.py (93%) rename agentfly/{agents => }/templates/tool_policy.py (100%) rename agentfly/{agents => }/templates/utils.py (98%) rename agentfly/{agents => }/templates/vision_processor.py (100%) create mode 100644 docs/api_references/agents/rollout.rst rename docs/{features => api_references}/chat_template/README.md (100%) rename docs/{features => api_references}/chat_template/advanced_features.md (100%) rename docs/{features => api_references}/chat_template/basic_usage.md (75%) create mode 100644 docs/api_references/chat_template/chat.rst create mode 100644 docs/api_references/chat_template/core_components.md rename docs/{features => api_references}/chat_template/custom_templates.md (96%) create mode 100644 docs/api_references/chat_template/examples.md rename docs/{features => api_references}/chat_template/index.rst (52%) create mode 100644 docs/api_references/chat_template/template.rst rename docs/{features => api_references}/chat_template/vision_templates.md (100%) delete mode 100644 docs/features/chat_template/architecture.md delete mode 100644 docs/features/chat_template/examples.md diff --git a/.gitignore b/.gitignore index f354bfc..c0918cf 100644 --- a/.gitignore +++ b/.gitignore @@ -142,3 +142,4 @@ test_outputs/ agentfly/data/ *.ipynb +./*.jpg \ No newline at end of file diff --git a/README.md b/README.md index 4d50bd0..7e73d95 100644 --- a/README.md +++ b/README.md @@ -54,11 +54,11 @@ Please refer to [installation.md](docs/start/installation.md) for custmoized ins ```python # Really small example to build an agent and run from agentfly.agents import HFAgent -from agentfly.tools import calculate +from agentfly.tools import calculator messages = [{"role": "user", "content": "What is the result of 1 + 1?"}] agent = HFAgent( model_name_or_path="Qwen/Qwen2.5-3B-Instruct", - tools=[calculate], + tools=[calculator], template="qwen2.5", backend="async_vllm", ) diff --git a/agentfly/agents/__init__.py b/agentfly/agents/__init__.py index b3d2564..ec8166e 100644 --- a/agentfly/agents/__init__.py +++ b/agentfly/agents/__init__.py @@ -2,5 +2,4 @@ from .specialized.code_agent import CodeAgent from .specialized.think_agent import ThinkAgent from .specialized.gui_agent import GUIAgent -from .specialized.hf_agent import HFAgent -from .templates.utils import process_vision_info, tokenize_conversation, tokenize_conversations \ No newline at end of file +from .specialized.hf_agent import HFAgent \ No newline at end of file diff --git a/agentfly/agents/agent_base.py b/agentfly/agents/agent_base.py index 4284e4d..c2685bc 100644 --- a/agentfly/agents/agent_base.py +++ b/agentfly/agents/agent_base.py @@ -2,7 +2,7 @@ from collections import defaultdict import json from .utils.messages import MessagesList -from .templates.templates import get_template +from ..templates.templates import get_template from ..__init__ import AGENT_DATA_DIR from .llm_backends import ( AsyncVLLMBackend, @@ -15,8 +15,7 @@ from typing import Any, Callable, Dict, List, Optional, Tuple, Union import numpy as np import torch -from .templates.utils import tokenize_conversations -from .templates.vision_processor import is_vision_template +from ..templates import tokenize_conversations from .chain.chain_base import ChainRollout import os import transformers diff --git a/agentfly/agents/llm_backends/llm_backends.py b/agentfly/agents/llm_backends/llm_backends.py index 340e985..72331d2 100644 --- a/agentfly/agents/llm_backends/llm_backends.py +++ b/agentfly/agents/llm_backends/llm_backends.py @@ -17,8 +17,7 @@ from ...utils.verl import pad_tensor_to_rank_size from vllm import LLM, AsyncLLMEngine, SamplingParams, AsyncEngineArgs import openai -from ..templates.templates import Chat -from ..templates.vision_processor import get_processor +from ...templates import Chat import logging import PIL diff --git a/agentfly/agents/specialized/openai_agent.py b/agentfly/agents/specialized/openai_agent.py index ae0f37a..38f725e 100644 --- a/agentfly/agents/specialized/openai_agent.py +++ b/agentfly/agents/specialized/openai_agent.py @@ -6,8 +6,8 @@ from ...tools import answer_qa from ...tools.tool_base import tool from ..agent_base import BaseAgent -from ..llm_backend import ClientBackend -from ..backend_config import ClientConfig +from ..llm_backends import ClientBackend +from ..llm_backends.backend_configs import ClientConfig from tenacity import retry, wait_random_exponential, stop_after_attempt from termcolor import colored import json diff --git a/agentfly/agents/templates/__init__.py b/agentfly/agents/templates/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/agentfly/templates/__init__.py b/agentfly/templates/__init__.py new file mode 100644 index 0000000..d2b2176 --- /dev/null +++ b/agentfly/templates/__init__.py @@ -0,0 +1,10 @@ +from .templates import Template, Chat, get_template, register_template +from .utils import ( + process_vision_info, + tokenize_conversation, + tokenize_conversations, + compare_hf_template, +) +from .tool_policy import ToolPolicy, JsonFormatter +from .system_policy import SystemPolicy +from .global_policy import GlobalPolicy \ No newline at end of file diff --git a/agentfly/agents/templates/constants.py b/agentfly/templates/constants.py similarity index 100% rename from agentfly/agents/templates/constants.py rename to agentfly/templates/constants.py diff --git a/agentfly/templates/global_policy.py b/agentfly/templates/global_policy.py new file mode 100644 index 0000000..1f3e013 --- /dev/null +++ b/agentfly/templates/global_policy.py @@ -0,0 +1,5 @@ +import dataclasses + +@dataclasses.dataclass +class GlobalPolicy: + prefix: str = None diff --git a/agentfly/agents/templates/preprocess.py b/agentfly/templates/preprocess.py similarity index 100% rename from agentfly/agents/templates/preprocess.py rename to agentfly/templates/preprocess.py diff --git a/agentfly/agents/templates/system_policy.py b/agentfly/templates/system_policy.py similarity index 100% rename from agentfly/agents/templates/system_policy.py rename to agentfly/templates/system_policy.py diff --git a/agentfly/agents/templates/templates.py b/agentfly/templates/templates.py similarity index 93% rename from agentfly/agents/templates/templates.py rename to agentfly/templates/templates.py index 6b8d3ea..ab5f2b0 100644 --- a/agentfly/agents/templates/templates.py +++ b/agentfly/templates/templates.py @@ -25,6 +25,7 @@ from .system_policy import Llama32DateProcessor, SystemPolicy from .tool_policy import ToolPolicy from .constants import ToolPlacement, Role +from .global_policy import GlobalPolicy Logger = logging.getLogger(__name__) @@ -36,15 +37,25 @@ console_handler.setFormatter(formatter) Logger.addHandler(console_handler) -@dataclasses.dataclass -class GlobalPolicy: - prefix: str = None - @dataclasses.dataclass class Template: - """A class that manages prompt templates and keeps all conversation history.""" - # Properties + """Class that holds all the components of a chat template. Convert messages to string prompts, tokenize messages to token ids, and generate jinja-based chat templates. + + Args: + name: The name of this template + system_template: The system template component + system_template_with_tools: The system template with tool usage component + system_message: The default system message + stop_words: The stop words where the model stops generating (usually EOS token) + tool_template: The tool response template component + user_template: The user template component + user_template_with_tools: The user template with tool usage component + assistant_template: The assistant template component + global_policy: The global policy, controls the behavior of the template + system_policy: The system message policy, controls the behavior of forming the system message + tool_policy: The tool policy for the template, controls the behavior of forming tools. + """ # The name of this template name: str # The template of the system prompt @@ -142,10 +153,7 @@ def _supports_tool_call(self) -> bool: return False def render(self, messages: List[Dict], tools=None, add_generation_prompt: bool = False) -> str: - """Render the template and return - 1. the final prompt string, - 2. the list of string *elements* that compose the prompt, and - 3. the corresponding list of *roles* (used by downstream post-processing). + """Render the template. The heavy lifting is delegated to small, single-purpose helpers so the high-level flow is immediately apparent: @@ -153,6 +161,16 @@ def render(self, messages: List[Dict], tools=None, add_generation_prompt: bool = 1. _insert_tools – decide where the tool catalogue lives 2. _encode_turns – encode every conversation turn 3. _maybe_add_generation_prompt – append the generation prefix if requested + + Args: + messages: The list of messages + tools: The list of tools + add_generation_prompt: Whether to add the generation prefix + + Returns: + prompt: The final prompt string + elements: The list of string *elements* that compose the prompt + roles: The corresponding list of *roles* (used by downstream post-processing) """ # Step 1 – decide tool placement & clone messages @@ -173,15 +191,14 @@ def _insert_tools(self, messages: List[Dict], tools): """Clone *messages* and compute where (and how) the tool catalogue should be injected. - Returns - ------- - work_messages : List[Dict] - A deepcopy of the original *messages* so we never mutate caller data. - tools_str : Optional[str] - The formatted tool catalogue or *None* if `tools` is falsy. - insert_tools_idx : int - Index of the *user* message that receives the catalogue, or -1 when - no injection is required. + Returns: + work_messages : List[Dict] + A deepcopy of the original *messages* so we never mutate caller data. + tools_str : Optional[str] + The formatted tool catalogue or *None* if `tools` is falsy. + insert_tools_idx : int + Index of the *user* message that receives the catalogue, or -1 when + no injection is required. """ work_messages = deepcopy(messages) @@ -418,6 +435,19 @@ def _split_assistant_message(self, assistant_message: str) -> List[str]: def encode(self, messages: List[Dict], tokenizer: PreTrainedTokenizer, return_tensors: str = None, tools=None, add_generation_prompt=False, processor=None) -> str: + """Encode the messages to token ids. + + Args: + messages: The list of messages + tokenizer: The tokenizer + return_tensors: The return tensors + tools: The list of tools + add_generation_prompt: Whether to add the generation prefix + processor: The processor for vision templates + + Returns: + inputs: The dictionary of input ids, attention mask, labels, and action mask + """ if processor is None and self.supports_vision(): raise ValueError(f"Processor is required for vision templates: {self.name}") @@ -587,6 +617,11 @@ def get_vision_inputs(self, messages: List[Dict]): return vision_inputs def jinja_template(self) -> str: + """Interface for getting the Jinja template. + + Returns: + The Jinja template string + """ if self.chat_template: return self.chat_template else: @@ -926,6 +961,13 @@ def dict(self): class Chat: def __init__(self, template: str, messages: List[List[str]]=None, tools=None, tokenizer: PreTrainedTokenizer = None): + """ + Args: + template: The name of the template to use. + messages: The messages to use for the chat. + tools: The tools to use for the chat. + tokenizer: The tokenizer to use for the chat. + """ self.template = get_template(template) self.messages = self.convert_to_hf_format_messages(messages) self.tokenizer = tokenizer @@ -967,9 +1009,19 @@ def convert_to_hf_format_messages(self, messages: Union[List[Dict], Dict[str, Li return hf_messages def set_messages(self, messages: List[Dict]): + """Set the messages for the chat.""" self.messages = self.convert_to_hf_format_messages(messages) def prompt(self, add_generation_prompt=False, tools=None) -> str: + """Get the prompt for the chat. + + Args: + add_generation_prompt: Whether to add the generation prompt. + tools: The tools to use for the chat. + + Returns: + The prompt for the chat. + """ self.flags['add_generation_prompt'] = add_generation_prompt prompt, _, _ = self.template.render(messages=self.messages, tools=tools, add_generation_prompt=add_generation_prompt) return prompt @@ -982,13 +1034,29 @@ def vision_inputs(self) -> List[Any]: return self.template.get_vision_inputs(self.messages) def tokenize(self, tokenizer: PreTrainedTokenizer = None, add_generation_prompt=False, tools=None, processor=None) -> List[int]: + """Tokenize the messages. + + Args: + tokenizer: The tokenizer to use for the chat. + add_generation_prompt: Whether to add the generation prompt. + tools: The tools to use for the chat. + processor: The processor to use for the chat. + + Returns: + The tokenized messages, a dictionary with the following items: + - input_ids + - attention_mask + - labels + - action_mask + - multi_modal_inputs + """ if tokenizer is None: tokenizer = self.tokenizer if tools is None: tools = self.tools return self.template.encode(messages=self.messages, tokenizer=tokenizer, return_tensors="pt", tools=tools, add_generation_prompt=add_generation_prompt, processor=processor) - def append(self, message: Union[Dict, List[Dict]]): + def append(self, message: Union[Dict]): self._convert_single_message_to_hf_format(message) self.messages.append(message) diff --git a/agentfly/agents/templates/tool_policy.py b/agentfly/templates/tool_policy.py similarity index 100% rename from agentfly/agents/templates/tool_policy.py rename to agentfly/templates/tool_policy.py diff --git a/agentfly/agents/templates/utils.py b/agentfly/templates/utils.py similarity index 98% rename from agentfly/agents/templates/utils.py rename to agentfly/templates/utils.py index ab6c07c..441f705 100644 --- a/agentfly/agents/templates/utils.py +++ b/agentfly/templates/utils.py @@ -10,10 +10,10 @@ import re import logging from .templates import Chat, get_template -from ... import AGENT_DATA_DIR +from .. import AGENT_DATA_DIR from typing import Any from .vision_processor import get_processor -# Set up logging that won't be overridden by other modules + LOGGER = logging.getLogger(__name__) ANSI_RE = re.compile(r'\x1b\[[0-9;]*m') # matches any ANSI color/style code @@ -269,7 +269,8 @@ def tokenize_conversations( concatenated_mm_inputs = {} if concatenate_mm_inputs: for key in batch_mm_inputs[0].keys(): - concatenated_mm_inputs[key] = torch.cat([mm_inputs[key] for mm_inputs in batch_mm_inputs if mm_inputs[key] is not None], dim=0) + if mm_inputs[key]: + concatenated_mm_inputs[key] = torch.cat([mm_inputs[key] for mm_inputs in batch_mm_inputs if mm_inputs[key] is not None], dim=0) inputs = dict( input_ids=batch_input_ids, diff --git a/agentfly/agents/templates/vision_processor.py b/agentfly/templates/vision_processor.py similarity index 100% rename from agentfly/agents/templates/vision_processor.py rename to agentfly/templates/vision_processor.py diff --git a/agentfly/tests/unit/agents/templates/test_text_templates_full_align.py b/agentfly/tests/unit/agents/templates/test_text_templates_full_align.py index 74cf1f3..652ffa4 100644 --- a/agentfly/tests/unit/agents/templates/test_text_templates_full_align.py +++ b/agentfly/tests/unit/agents/templates/test_text_templates_full_align.py @@ -8,7 +8,7 @@ """ -from .....agents.templates.utils import compare_hf_template +from .....templates import compare_hf_template from transformers import AutoTokenizer import pytest diff --git a/agentfly/tools/tool_base.py b/agentfly/tools/tool_base.py index 9129ffc..97047c2 100644 --- a/agentfly/tools/tool_base.py +++ b/agentfly/tools/tool_base.py @@ -13,6 +13,10 @@ import concurrent.futures from ..envs.manager.env_manager import EnvironmentManager +import logging + +logger = logging.getLogger(__name__) + # current_env = contextvars.ContextVar("current_env") class Tool: @@ -271,7 +275,8 @@ def decorator(func): func_name = func.__name__ final_name = name or func_name if name and name != func_name: - warnings.warn(f"Tool name {func_name!r} overridden by {name!r}") + logger.warning(f"Tool name {func_name!r} overridden by {name!r}") + # warnings.warn(f"Tool name {func_name!r} overridden by {name!r}") signature = extract_signatures(func) docs = parse_docstring(inspect.getdoc(func)) diff --git a/agentfly/tools/utils/schema.py b/agentfly/tools/utils/schema.py index 37016ce..2074178 100644 --- a/agentfly/tools/utils/schema.py +++ b/agentfly/tools/utils/schema.py @@ -2,6 +2,9 @@ import inspect import warnings from copy import deepcopy +import logging + +logger = logging.getLogger(__name__) def extract_signatures(func): sig = inspect.signature(func) @@ -153,7 +156,7 @@ def validate_schema(name, description, signature, docs): else: # May be should raise an error properties[param]['type'] = "unknown" - warnings.warn(f"Parameter {param} has no type in signature or docstring.") + logger.warning(f"Parameter {param} has no type in signature or docstring.") if "default" in signature[param]: properties[param]['default'] = signature[param]['default'] diff --git a/docs/api_references/agents/agent.rst b/docs/api_references/agents/agent.rst index db85c3f..9ed2c3f 100644 --- a/docs/api_references/agents/agent.rst +++ b/docs/api_references/agents/agent.rst @@ -11,10 +11,4 @@ The foundation class for all agents in AgentFly: :members: :show-inheritance: -Chain Generation ----------------- - -Base class for chain-based generation: -.. autoclass:: agentfly.agents.chain.chain_base.ChainRollout - :members: diff --git a/docs/api_references/agents/index.rst b/docs/api_references/agents/index.rst index 23c5f2c..853b2f1 100644 --- a/docs/api_references/agents/index.rst +++ b/docs/api_references/agents/index.rst @@ -9,7 +9,7 @@ Overview AgentFly provides a comprehensive agent system with a base class and specialized implementations for different use cases. All agents inherit from :py:class:`BaseAgent` and support tool calling, chain rollout, and various backends. -Base Agent +Structure ========== .. toctree:: @@ -17,97 +17,8 @@ Base Agent agent llm_backends + rollout -Core Classes -=========== - -BaseAgent ---------- - -The foundation class for all agents in AgentFly: - -.. autoclass:: agentfly.agents.agent_base.BaseAgent - :members: - :show-inheritance: - :special-members: __init__ - -AutoAgent ---------- - -Factory class for automatic agent creation: - -.. autoclass:: agentfly.agents.auto.AutoAgent - :members: - :show-inheritance: - -Specialized Agents -================== - -ReactAgent ----------- - -ReAct-style agent for reasoning and tool use: - -.. autoclass:: agentfly.agents.react.react_agent.ReactAgent - :members: - :show-inheritance: - -CodeAgent ---------- - -Specialized agent for code generation and execution: - -.. autoclass:: agentfly.agents.specialized.code_agent.CodeAgent - :members: - :show-inheritance: - -ThinkAgent ----------- - -Agent that uses thinking steps before taking actions: - -.. autoclass:: agentfly.agents.specialized.think_agent.ThinkAgent - :members: - :show-inheritance: - -GUIAgent ---------- - -Agent for GUI automation tasks: - -.. autoclass:: agentfly.agents.specialized.gui_agent.GUIAgent - :members: - :show-inheritance: - -HFAgent --------- - -Hugging Face model-based agent: - -.. autoclass:: agentfly.agents.specialized.hf_agent.HFAgent - :members: - :show-inheritance: - -OpenAIAgent ------------ - -OpenAI API-based agent: - -.. autoclass:: agentfly.agents.specialized.openai_agent.OpenAIAgent - :members: - :show-inheritance: - -Chain Generation -=============== - -ChainRollout ------------- - -Base class for chain-based generation: - -.. autoclass:: agentfly.agents.chain.chain_base.ChainRollout - :members: - :show-inheritance: Usage Examples ============= diff --git a/docs/api_references/agents/llm_backends.rst b/docs/api_references/agents/llm_backends.rst index 8f57918..ef0d7bb 100644 --- a/docs/api_references/agents/llm_backends.rst +++ b/docs/api_references/agents/llm_backends.rst @@ -1,27 +1,19 @@ LLM Backends -============ +============================ Overview --------- +----------------------------- AgentFly supports multiple LLM backends for text generation, each with their own configuration options. -This module provides configuration classes for different backend types including Transformers, vLLM, Verl, and OpenAI-compatible clients. +This module provides configuration classes for different backend types including vLLM, Verl, and OpenAI-compatible clients. +Among them, Verl backend is designed for internal training usage. The Verl backend is the core design that **decouples agent system and rl training**. Configuration Classes --------------------- - -Transformers Backend -~~~~~~~~~~~~~~~~~~~ - -Configuration for the Transformers backend using Hugging Face models: - -.. autoclass:: agentfly.agents.llm_backends.backend_configs.TransformersConfig - :show-inheritance: - :special-members: !__init__ +----------------------------- Async VLLM Backend -~~~~~~~~~~~~~~~~~~ +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Configuration for asynchronous vLLM backend with engine arguments: @@ -31,7 +23,7 @@ Configuration for asynchronous vLLM backend with engine arguments: Async Verl Backend -~~~~~~~~~~~~~~~~~~ +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Configuration for asynchronous Verl backend: @@ -40,7 +32,7 @@ Configuration for asynchronous Verl backend: :special-members: !__init__ Client Backend -~~~~~~~~~~~~~ +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Configuration for OpenAI-compatible client backends: @@ -49,10 +41,10 @@ Configuration for OpenAI-compatible client backends: :special-members: !__init__ Backend Implementations ----------------------- +------------------------------ Base Backend -~~~~~~~~~~~ +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Abstract base class for all LLM backends: @@ -60,26 +52,9 @@ Abstract base class for all LLM backends: :members: :show-inheritance: -Transformers Backend -~~~~~~~~~~~~~~~~~~~ - -HuggingFace Transformers implementation for local model inference: - -.. autoclass:: agentfly.agents.llm_backends.llm_backends.TransformersBackend - :members: - :show-inheritance: - -VLLM Backend -~~~~~~~~~~~~ - -vLLM implementation for high-performance model inference: - -.. autoclass:: agentfly.agents.llm_backends.llm_backends.VLLMBackend - :members: - :show-inheritance: Async VLLM Backend -~~~~~~~~~~~~~~~~~~ +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Asynchronous vLLM implementation for high-performance model inference: @@ -97,7 +72,7 @@ Asynchronous Verl implementation for distributed model inference: :show-inheritance: Client Backend -~~~~~~~~~~~~~ +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ OpenAI-compatible client backend for remote API inference: @@ -107,13 +82,13 @@ OpenAI-compatible client backend for remote API inference: Usage Examples --------------- +------------------------------ Backends are designed to work together with agents. Here are examples showing how to configure different backends when creating agents: Async VLLM Backend -~~~~~~~~~~~~~~~~~~ +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ .. code-block:: python diff --git a/docs/api_references/agents/rollout.rst b/docs/api_references/agents/rollout.rst new file mode 100644 index 0000000..3bfb6dc --- /dev/null +++ b/docs/api_references/agents/rollout.rst @@ -0,0 +1,11 @@ +Rollout +======= + + +Chain Generation +---------------- + +Base class for chain-based generation: + +.. autoclass:: agentfly.agents.chain.chain_base.ChainRollout + :members: \ No newline at end of file diff --git a/docs/features/chat_template/README.md b/docs/api_references/chat_template/README.md similarity index 100% rename from docs/features/chat_template/README.md rename to docs/api_references/chat_template/README.md diff --git a/docs/features/chat_template/advanced_features.md b/docs/api_references/chat_template/advanced_features.md similarity index 100% rename from docs/features/chat_template/advanced_features.md rename to docs/api_references/chat_template/advanced_features.md diff --git a/docs/features/chat_template/basic_usage.md b/docs/api_references/chat_template/basic_usage.md similarity index 75% rename from docs/features/chat_template/basic_usage.md rename to docs/api_references/chat_template/basic_usage.md index b376374..f1bd53e 100644 --- a/docs/features/chat_template/basic_usage.md +++ b/docs/api_references/chat_template/basic_usage.md @@ -7,9 +7,7 @@ The Chat Template System provides a simple yet powerful interface for creating a ## Importing the System ```python -from agentfly.agents.templates import Chat, get_template, Template -from agentfly.agents.templates.tool_policy import ToolPolicy, JsonFormatter -from agentfly.agents.templates.system_policy import SystemPolicy +from agentfly.templates import Chat, get_template, Template, ToolPolicy, JsonFormatter, SystemPolicy ``` ## Using Pre-built Templates @@ -28,6 +26,8 @@ The system comes with several pre-built templates: ### Basic Template Usage +`Template` is the basic template class, consists of different components and responsible for forming the prompt. While `Chat` is the class we recommand for users to obtain prompts. + ```python # Get a pre-built template template = get_template("qwen2.5") @@ -116,6 +116,8 @@ prompt_with_tools = chat.prompt(tools=tools) ### Tokenization +Use `Chat.tokenize` method to tokenize the messages with the specified chat template. + ```python # Tokenize the conversation inputs = chat.tokenize( @@ -136,12 +138,6 @@ inputs = chat.tokenize( ```python # Add a single message chat.append({"role": "user", "content": "Another question"}) - -# Add multiple messages -chat.append([ - {"role": "user", "content": "Question 1"}, - {"role": "assistant", "content": "Answer 1"} -]) ``` ## Template Configuration @@ -269,57 +265,3 @@ tools = [ "content": "Search results: [results here]" } ``` - -## Error Handling - -### Common Issues - -```python -try: - # Get a template that doesn't exist - template = get_template("nonexistent") -except KeyError as e: - print(f"Template not found: {e}") - -try: - # Create chat with invalid template - chat = Chat(template="invalid", messages=messages) -except KeyError as e: - print(f"Invalid template: {e}") -``` - -### Validation - -```python -# Check if template supports vision -if template.supports_vision(): - print("Template supports vision processing") - -# Check if template supports tool calls -if template._supports_tool_call(): - print("Template supports tool calls") -``` - -## Best Practices - -### 1. **Template Naming** -- Use descriptive names that indicate the model and capabilities -- Include version information when appropriate -- Use consistent naming conventions - -### 2. **Message Structure** -- Always use the standard role/content format -- For multi-modal content, use the list format with type specifications -- Ensure content types match the template's capabilities - -### 3. **Tool Integration** -- Define tools with clear, descriptive names and parameters -- Use appropriate tool placement strategies for your use case -- Test tool integration thoroughly before deployment - -### 4. **Vision Processing** -- Use vision-enabled templates for image/video content -- Ensure proper image formats and sizes -- Handle vision token expansion appropriately - -This basic usage guide should get you started with the Chat Template System. For more advanced features, see the [Advanced Features](./advanced_features.md) and [Vision Templates](./vision_templates.md) sections. diff --git a/docs/api_references/chat_template/chat.rst b/docs/api_references/chat_template/chat.rst new file mode 100644 index 0000000..f790ca1 --- /dev/null +++ b/docs/api_references/chat_template/chat.rst @@ -0,0 +1,8 @@ +Chat +====================== + + +.. autoclass:: agentfly.templates.Chat + :members: + :show-inheritance: + diff --git a/docs/api_references/chat_template/core_components.md b/docs/api_references/chat_template/core_components.md new file mode 100644 index 0000000..8e1ba9f --- /dev/null +++ b/docs/api_references/chat_template/core_components.md @@ -0,0 +1,175 @@ + + + + + + +# Core Components + +### Core Chat Template Components + +The Chat Template System is inspired by the art of building block toys - where complex structures are created by combining simple, standardized components. We identify some basic components from LLM's chat templates, and use them to form prompts from conversation messages. Below are some basic core compoenents: + +`system_template`: Specify system prompt is formatted in chat template. + +`system_template_with_tools`: Specify how tools along with system prompt is formatted in chat template + +`user_template`: Specify how user message is formatted in chat template + +`assistant_template`: Specify how assistant is formatted in chat template + +`tool_template`: Specify how tool response is formatted in chat template + +Assume we have the following chat template, and messages +``` +system_template = f"System: {system_message}\n" +system_template_with_tools = f"System: {system_message}\n#Tools: {tools}\n" +user_template = "User: {content}\n" +assistant_template = "User: {content}\n" + +messages = [ + {"role": "system", "content": "You are a helpful assistant."}, + {"role": "user", "content": "Hi, Can you help me search the information."}, + {"role": "assistant", "content": "tool call: search tool arguments: related query"} + {"role": "tool", "content": "Searched inforamtion..."} +] + +tools = [ + { + "name": "search", + "description": "Search the web." + } +] +``` + +**Formatted Prompt** + + : formatted system prompt; : formatted user message; : formatted assistant message; : formatted tool message; + +1. When combined, these create the complete prompt: + +System: You are a helpful assistant. + +User: Hi, how are you today. + +Assistant: I am good. How can I help you? + +Tool: Searched inforamtion... + +2. When tools are included, the `system_template_with_tools` is used: + +System: You are a helpful assistant. +
+#Tools: [{"name": "search", "description": "Search the web"}] + +User: Hi, Can you help me search the information. + +Assistant: tool call: search\ntool arguments: related query + +Tool: Searched inforamtion... + + +### High-Level Workflow + +``` +Messages + Tools → Template Processing → Vision Processing → LLM-Ready Inputs +``` + +The system follows a three-step rendering process: + +1. **Tool Insertion**: Decide where and how to inject tool definitions +2. **Turn Encoding**: Convert each conversation turn to its textual representation +3. **Generation Prompt**: Optionally append generation prefixes + +If we tokenize the input messages, the vision processor will do the following steps: + +- **Template** → Human-readable prompt with vision tokens +- **Vision Processor** → Token expansion and multi-modal inputs +- **Result** → LLM-ready inputs with proper tensor alignment + +### Core Class Components + +#### Template +The central class that manages: +- Message formatting templates +- Policy configurations +- Jinja template generation + +#### Chat +Recommended class for user usage: +- Store and format messages +- Get formatted prompt +- Tokenize formatted prompt + +### Advanced Features + +**1. Register & Obtain Template** + +Templates are created and retrieved through a global registry: +```python +# Registration +register_template(Template(name="custom", ...)) + +# Retrieval +template = get_template("custom") +``` + +**2. Fine-grained Behavior Control** + +Three levels of policy control: + +1. **Global Policy**: Template-wide settings (e.g., prefix tokens) +2. **System Policy**: System message behavior and content processing +3. **Tool Policy**: Tool placement, formatting, and content processing + +```python +# Tool formatting strategies +JsonFormatter(indent=4) +JsonCompactFormatter() +YamlFormatter() + +# Tool placement strategies +ToolPlacement.SYSTEM +ToolPlacement.FIRST_USER +ToolPlacement.LAST_USER +``` + +**3. Vision Process** + +Vision processors are automatically registered when vision tokens are detected: +```python +def _register_vision_processor(self): + """Automatically register a vision processor for this template""" + if self.image_token or self.video_token: + # Auto-registration based on template configuration +``` + + +**4. Jinja Template Generation** + +Templates can generate HuggingFace-compatible Jinja templates: +- Enables use with external systems (vLLM, etc.) +- Maintains consistency between Python and Jinja rendering +- Supports complex logic through Jinja macros diff --git a/docs/features/chat_template/custom_templates.md b/docs/api_references/chat_template/custom_templates.md similarity index 96% rename from docs/features/chat_template/custom_templates.md rename to docs/api_references/chat_template/custom_templates.md index b390515..8f90755 100644 --- a/docs/features/chat_template/custom_templates.md +++ b/docs/api_references/chat_template/custom_templates.md @@ -8,16 +8,18 @@ The Chat Template System is designed to be highly extensible, allowing you to cr ### Core Template Fields + + ```python from agentfly.agents.templates import Template template = Template( name="my-custom-template", # Unique identifier - system_template="{system_message}", # System message format - system_message="Default system message", - user_template="{content}", # User message format - assistant_template="{content}", # Assistant message format - tool_template="{observation}", # Tool response format + system_template="System: {system_message}", # System message format + system_message="You are a helpful assistant.", # Default system message + user_template="User: {content}", # User message format + assistant_template="Assistant: {content}", # Assistant message format + tool_template="Tool: {observation}", # Tool response format stop_words=[""] # Stop generation tokens ) ``` diff --git a/docs/api_references/chat_template/examples.md b/docs/api_references/chat_template/examples.md new file mode 100644 index 0000000..0bb036b --- /dev/null +++ b/docs/api_references/chat_template/examples.md @@ -0,0 +1,242 @@ +# Examples and Use Cases + +## Overview + +This section provides comprehensive examples of how to use the Chat Template System in various scenarios. Each example demonstrates different features and capabilities of the system. + +## Basic Examples + +### Example 1: Simple Chat Template + +```python +from agentfly.agents.templates import Chat, get_template + +# Get a pre-built template +template = get_template("qwen2.5") + +# Create a simple conversation +messages = [ + {"role": "user", "content": "Hello, how are you?"}, + {"role": "assistant", "content": "I'm doing well, thank you for asking! How can I help you today?"}, + {"role": "user", "content": "Can you explain what machine learning is?"} +] + +# Create chat instance +chat = Chat(template="qwen2.5", messages=messages) + +# Generate prompt +prompt = chat.prompt() +print("Generated Prompt:") +print(prompt) + +# Generate prompt with generation prompt (for inference) +prompt_with_gen = chat.prompt(add_generation_prompt=True) +print("\nPrompt with Generation Prompt:") +print(prompt_with_gen) +``` + +**Output:** +``` +Generated Prompt: +<|im_start|>system +You are Qwen, created by Alibaba Cloud. You are a helpful assistant.<|im_end|> +<|im_start|>user +Hello, how are you?<|im_end|> +<|im_start|>assistant +I'm doing well, thank you for asking! How can I help you today?<|im_end|> +<|im_start|>user +Can you explain what machine learning is?<|im_end|> + +Prompt with Generation Prompt: +<|im_start|>system +You are Qwen, created by Alibaba Cloud. You are a helpful assistant.<|im_end|> +<|im_start|>user +Hello, how are you?<|im_end|> +<|im_start|>assistant +I'm doing well, thank you for asking! How can I help you today?<|im_end|> +<|im_start|>user +Can you explain what machine learning is?<|im_end|> +<|im_start|>assistant +``` + +### Example 2: Chat with Tools + +```python +# Define tools +tools = [ + { + "function": { + "name": "search_web", + "description": "Search the web for current information", + "parameters": { + "type": "object", + "properties": { + "query": {"type": "string", "description": "Search query"}, + "max_results": {"type": "integer", "description": "Maximum number of results"} + }, + "required": ["query"] + } + } + }, + { + "function": { + "name": "calculate", + "description": "Perform mathematical calculations", + "parameters": { + "type": "object", + "properties": { + "expression": {"type": "string", "description": "Mathematical expression to evaluate"} + }, + "required": ["expression"] + } + } + } +] + +# Create chat with tools +chat = Chat(template="qwen2.5", messages=messages, tools=tools) + +# Generate prompt with tools +prompt_with_tools = chat.prompt(tools=tools) +print("Prompt with Tools:") +print(prompt_with_tools) +``` + +**Output:** +``` +Prompt with Tools: +<|im_start|>system +You are Qwen, created by Alibaba Cloud. You are a helpful assistant. + +# Tools + +You may call one or more functions to assist with the user query. + +You are provided with function signatures within XML tags: + +{"function": {"name": "search_web", "description": "Search the web for current information", "parameters": {"type": "object", "properties": {"query": {"type": "string", "description": "Search query"}, "max_results": {"type": "integer", "description": "Maximum number of results"}, "required": ["query"]}}} +{"function": {"name": "calculate", "description": "Perform mathematical calculations", "parameters": {"type": "object", "properties": {"expression": {"type": "string", "description": "Mathematical expression to evaluate"}, "required": ["expression"]}}} + + +<|im_end|> +<|im_start|>user +Hello, how are you?<|im_end|> +<|im_start|>assistant +I'm doing well, thank you for asking! How can I help you today?<|im_end|> +<|im_start|>user +Can you explain what machine learning is?<|im_end|> +``` + +### Example 3: Tokenization + +```python +from transformers import AutoTokenizer + +# Load tokenizer +tokenizer = AutoTokenizer.from_pretrained("Qwen/Qwen2.5-7B-Instruct") + +# Tokenize the conversation +inputs = chat.tokenize( + tokenizer=tokenizer, + add_generation_prompt=True, + tools=tools +) + +print("Tokenization Results:") +print(f"Input IDs shape: {inputs['input_ids'].shape}") +print(f"Attention mask shape: {inputs['attention_mask'].shape}") +print(f"Labels shape: {inputs['labels'].shape}") +print(f"Action mask shape: {inputs['action_mask'].shape}") + +# Show token alignment +print(f"\nFirst 20 tokens: {inputs['input_ids'][0][:20]}") +print(f"First 20 labels: {inputs['labels'][0][:20]}") +print(f"First 20 action mask: {inputs['action_mask'][0][:20]}") +``` + +## Advanced Examples + +### Example 4: Custom Template Creation + +```python +from agentfly.agents.templates import Template, register_template +from agentfly.agents.templates.tool_policy import ToolPolicy, JsonIndentedFormatter +from agentfly.agents.templates.constants import ToolPlacement + +# Create a custom coding assistant template +coding_template = Template( + name="coding-assistant", + + # System message + system_template="""<|im_start|>system +You are an expert coding assistant. You help users write, debug, and understand code. +Always provide clear explanations and follow best practices. +{system_message}<|im_end|> +""", + system_message="You are an expert coding assistant.", + + # Tool support for code execution + system_template_with_tools="""<|im_start|>system +You are an expert coding assistant with access to code execution tools. +Always think through the problem before writing code. +{system_message} + +Available Tools: +{tools}<|im_end|> +""", + + # User and assistant templates + user_template="<|im_start|>user\n{content}<|im_end|>\n", + user_template_with_tools="<|im_start|>user\n{content}\n\nTools: {tools}<|im_end|>\n", + assistant_template="<|im_start|>assistant\n{content}<|im_end|>\n", + tool_template="<|im_start|>tool\n{observation}<|im_end|>\n", + + # Stop words + stop_words=["<|im_end|>"], + + # Tool policy - place tools with first user message + tool_policy=ToolPolicy( + placement=ToolPlacement.FIRST_USER, + formatter=JsonIndentedFormatter(indent=2) + ) +) + +# Register the template +register_template(coding_template) + +# Test the template +coding_chat = Chat(template="coding-assistant", messages=[ + {"role": "user", "content": "Write a Python function to calculate fibonacci numbers"} +]) + +prompt = coding_chat.prompt() +print("Coding Assistant Template:") +print(prompt) +``` + +### Example 5: Vision Template Usage + +```python +# Create a vision-enabled chat +vision_messages = [ + { + "role": "user", + "content": [ + {"type": "text", "text": "What's in this image? Please describe it in detail."}, + {"type": "image", "image": "/path/to/sample_image.jpg"} + ] + } +] + +# Use a vision template +vision_chat = Chat(template="qwen2.5-vl", messages=vision_messages) + +# Generate prompt +vision_prompt = vision_chat.prompt() +print("Vision Template Prompt:") +print(vision_prompt) + +# Get vision inputs +vision_inputs = vision_chat.vision_inputs() +print(f"\nVision inputs: {list(vision_inputs.keys())}") +``` diff --git a/docs/features/chat_template/index.rst b/docs/api_references/chat_template/index.rst similarity index 52% rename from docs/features/chat_template/index.rst rename to docs/api_references/chat_template/index.rst index da7aba2..4cc1a19 100644 --- a/docs/features/chat_template/index.rst +++ b/docs/api_references/chat_template/index.rst @@ -1,83 +1,23 @@ Chat Template System Documentation -================================= +======================================= + +Structure +--------------------------------------- .. toctree:: :maxdepth: 2 - :hidden: - architecture + core_components basic_usage custom_templates advanced_features vision_templates examples - -Welcome to the comprehensive documentation for the Chat Template System - a powerful and flexible framework for creating conversation templates inspired by building block toys. - -Documentation Structure ------------------------- - -:doc:`Architecture & Design ` -^^^^^^^^^^^^^^^^^^^^ - -- **System Philosophy**: Building block approach to template design -- **Architecture Overview**: High-level system design and flow -- **Design Patterns**: Factory, Strategy, and Observer patterns -- **Extensibility Points**: How to extend the system -- **Key Design Decisions**: Rationale behind architectural choices - -:doc:`Basic Usage ` -^^^^^^^^^^^^^^^^^^^ - -- **Getting Started**: Quick start guide and imports -- **Pre-built Templates**: Available templates and their usage -- **Chat Operations**: Creating chats, generating prompts, tokenization -- **Template Configuration**: Basic template structure and fields -- **Message Formats**: Standard and multi-modal message structures -- **Error Handling**: Common issues and validation - -:doc:`Custom Templates ` -^^^^^^^^^^^^^^^^^^^ - -- **Template Components**: Core and advanced template fields -- **Template Creation**: Step-by-step template building -- **Policy Configuration**: System, tool, and global policies -- **Template Registration**: How to register and manage templates -- **Advanced Features**: Jinja templates, inheritance, copying -- **Best Practices**: Template design and testing guidelines - -:doc:`Advanced Features ` -^^^^^^^^^^^^^^^^ - -- **Tool Policy System**: Placement strategies and formatters -- **System Policy System**: Message control and content processors -- **Global Policy Configuration**: Template-wide settings -- **Policy Composition**: Combining and inheriting policies -- **Advanced Tool Integration**: Custom placement and validation -- **Performance Optimization**: Caching and lazy evaluation - -:doc:`Vision Templates ` -^^^^^^^^^^^^^^^ - -- **Vision Architecture**: Pipeline overview and key components -- **Creating Vision Templates**: Basic and advanced vision templates -- **Vision Processor Configuration**: Automatic registration and model inference -- **Processor Types**: Patch-based, Qwen-VL, LLaVA processors -- **Input Formats**: Image, video, and message formats -- **Token Calculation**: Image and video token computation -- **Advanced Features**: Custom processors and configuration options - -:doc:`Examples & Use Cases ` -^^^^^^^^^^^^^^^^^^^^ - -- **Basic Examples**: Simple chat, tools, tokenization -- **Advanced Examples**: Custom templates, vision usage, dynamic generation -- **Real-World Use Cases**: Customer support, education, content analysis -- **Testing & Validation**: Template comparison and validation -- **Complete Examples**: End-to-end implementation examples + template + chat Quick Start ------------ +--------------------------------------- .. code-block:: python @@ -166,7 +106,7 @@ Key Features Additional Resources ----------------------- -- **Source Code**: ``agents/agents/agents/templates/`` +- **Source Code**: ``agentfly/templates/`` - **API Reference**: Check the source code for detailed method documentation - **Issues & Discussions**: Use the project's issue tracker for questions @@ -174,8 +114,3 @@ Contributing ------------ The template system is designed to be extensible. See :doc:`Custom Templates ` for guidance on adding new template types and processors. - ---- - -*This documentation covers the complete Chat Template System. Start with the architecture to understand the design, then follow the usage guides to implement your own templates.* - diff --git a/docs/api_references/chat_template/template.rst b/docs/api_references/chat_template/template.rst new file mode 100644 index 0000000..6c109fb --- /dev/null +++ b/docs/api_references/chat_template/template.rst @@ -0,0 +1,8 @@ +Template +====================== + + +.. autoclass:: agentfly.templates.Template + :members: + :show-inheritance: + :exclude-members: __init__ diff --git a/docs/features/chat_template/vision_templates.md b/docs/api_references/chat_template/vision_templates.md similarity index 100% rename from docs/features/chat_template/vision_templates.md rename to docs/api_references/chat_template/vision_templates.md diff --git a/docs/api_references/tools/index.rst b/docs/api_references/tools/index.rst index 5cccb25..0d5e314 100644 --- a/docs/api_references/tools/index.rst +++ b/docs/api_references/tools/index.rst @@ -1,113 +1,16 @@ -.. _tools_index: -################### -Tools API Reference -################### - - -Overview -======== +Tools +============================== AgentFly provides a comprehensive tool system that enables agents to interact with external systems and APIs. Tools can be stateful (with environment management) or stateless, and support both synchronous and asynchronous execution. -Base Tool -========= - .. toctree:: :maxdepth: 2 - tool - -Core Classes -=========== - -Tool ----- - -The main tool wrapper class: - -.. autoclass:: agentfly.tools.tool_base.Tool - :members: - :show-inheritance: - :special-members: __call__ - -Tool Decorator -------------- - -The main decorator for creating tools: - -.. autofunction:: agentfly.tools.tool_base.tool - -Utility Functions -================ - -Tool Registration ----------------- - -.. autofunction:: agentfly.tools.register_tool - -Tool Retrieval --------------- - -.. autofunction:: agentfly.tools.get_tools_from_names - -Pre-built Tools -============== - -Code Tools ----------- - -.. autofunction:: agentfly.tools.src.code.tools.code_interpreter - -Search Tools ------------- - -.. autofunction:: agentfly.tools.src.search.google_search.google_search_serper - -.. autofunction:: agentfly.tools.src.search.dense_retriever.dense_retrieve - -.. autofunction:: agentfly.tools.src.search.async_dense_retriever.asyncdense_retrieve - -ALFWorld Tools --------------- - -.. autofunction:: agentfly.tools.src.alfworld.tools.alfworld_step - -.. autofunction:: agentfly.tools.src.alfworld.tools.alfworld_reset - -.. autofunction:: agentfly.tools.src.alfworld.tools.alfworld_get_task_objective - -.. autofunction:: agentfly.tools.src.alfworld.tools.alfworld_get_admissible_commands - -WebShop Tools -------------- - -.. autofunction:: agentfly.tools.src.webshop.tools.webshop_browser - -ScienceWorld Tools ------------------- - -.. autofunction:: agentfly.tools.src.scienceworld.tools.scienceworld_explorer - -ReAct Tools ------------ - -.. autofunction:: agentfly.tools.src.react.tools.answer_qa - -.. autofunction:: agentfly.tools.src.react.tools.answer_math - -Utility Tools -------------- - -.. autofunction:: agentfly.tools.src.calculate.tools.calculator - -.. autofunction:: agentfly.tools.src.ui.tools.pyautogui_code_generator - -Usage Examples -============= + tool Basic Tool Definition --------------------- +------------------------------- .. code-block:: python @@ -139,7 +42,7 @@ Stateful Tool with Environment return result Tool with Schema ---------------- +------------------------------ .. code-block:: python diff --git a/docs/api_references/tools/tool.rst b/docs/api_references/tools/tool.rst index 7d1580e..89ceb5c 100644 --- a/docs/api_references/tools/tool.rst +++ b/docs/api_references/tools/tool.rst @@ -1,8 +1,8 @@ Tool -============== +============================== Base Tool Class ---------------- +------------------------------ The main tool wrapper class: @@ -11,34 +11,34 @@ The main tool wrapper class: :special-members: __call__ Tool Decorator -------------- +------------------------------ The main decorator for creating tools: .. autofunction:: agentfly.tools.tool_base.tool Predefined Tools ---------------- +------------------------------ The following are predefined tool instances that can be used directly with agents. Code Interpreter -^^^^^^^^^^^^^^^ +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ .. autofunction:: agentfly.tools.src.code.tools.code_interpreter Google Search -^^^^^^^^^^^^ +^^^^^^^^^^^^^^^^^^^^^^^^^^^^ .. autofunction:: agentfly.tools.src.search.google_search.google_search_serper Calculator -^^^^^^^^^^ +^^^^^^^^^^^^^^^^^^^^^^^^^^ .. autofunction:: agentfly.tools.src.calculate.tools.calculator Answer Tools -^^^^^^^^^^^ +^^^^^^^^^^^^^^^^^^^^^^^^^^^ .. autofunction:: agentfly.tools.src.react.tools.answer_qa diff --git a/docs/features/chat_template/architecture.md b/docs/features/chat_template/architecture.md deleted file mode 100644 index ea283ad..0000000 --- a/docs/features/chat_template/architecture.md +++ /dev/null @@ -1,168 +0,0 @@ -# Architecture & Design Philosophy - -## Inspiration: Building Block Toys - -The Chat Template System is inspired by the art of building block toys - where complex structures are created by combining simple, standardized components. This philosophy manifests in several key design principles: - -### 1. **Modularity** -Just as building blocks can be snapped together in different configurations, template components are designed to be: -- **Interchangeable**: Different formatting strategies can be swapped without changing the core logic -- **Composable**: Complex templates are built from simple, reusable parts -- **Extensible**: New block types can be added without modifying existing ones - -### 2. **Separation of Concerns** -Each component has a single, well-defined responsibility: -- **Templates** handle message formatting and structure -- **Policies** control behavior and configuration -- **Processors** handle specialized input contents (images, videos, etc.) -- **Formatters** manage tool representation - -### 3. **Strategy Pattern** -The system uses the strategy pattern extensively, allowing different behaviors to be selected at runtime: -- **Tool Formatters**: JSON, YAML, custom formats -- **Tool Placement**: System, first user, last user... -- **Vision Processors**: Patch-based, model-specific implementations - -## System Architecture - -### High-Level Flow - -``` -Messages + Tools → Template Processing → Vision Processing → LLM-Ready Inputs -``` - -The system follows a three-step rendering process: - -1. **Tool Insertion**: Decide where and how to inject tool definitions -2. **Turn Encoding**: Convert each conversation turn to its textual representation -3. **Generation Prompt**: Optionally append generation prefixes - -### Core Components - -#### Template Class -The central orchestrator that manages: -- Message formatting templates -- Policy configurations -- Vision processor registration -- Jinja template generation - -#### Policy System -Three levels of policy control: - -1. **Global Policy**: Template-wide settings (e.g., prefix tokens) -2. **System Policy**: System message behavior and content processing -3. **Tool Policy**: Tool placement, formatting, and content processing - -#### Vision Processing -Separate from template processing to maintain clean separation: -- **Template** → Human-readable prompt with vision tokens -- **Vision Processor** → Token expansion and multi-modal inputs -- **Result** → LLM-ready inputs with proper tensor alignment - -### Design Patterns - -#### Factory Pattern -Templates are created and retrieved through a global registry: -```python -# Registration -register_template(Template(name="custom", ...)) - -# Retrieval -template = get_template("custom") -``` - -#### Strategy Pattern -Different behaviors are encapsulated in strategy classes: -```python -# Tool formatting strategies -JsonFormatter(indent=4) -JsonCompactFormatter() -YamlFormatter() - -# Tool placement strategies -ToolPlacement.SYSTEM -ToolPlacement.FIRST_USER -ToolPlacement.LAST_USER -``` - -#### Observer Pattern -Vision processors are automatically registered when vision tokens are detected: -```python -def _register_vision_processor(self): - """Automatically register a vision processor for this template""" - if self.image_token or self.video_token: - # Auto-registration based on template configuration -``` - -## Key Design Decisions - -### 1. **Template vs. Processor Separation** -- **Templates** handle text formatting and structure -- **Processors** handle specialized input types (images, videos, tools) -- This separation allows templates to focus on their core responsibility - -### 2. **Policy-Based Configuration** -Instead of hardcoded behavior, templates use configurable policies: -- **System Policy**: Controls when and how system messages appear -- **Tool Policy**: Manages tool integration strategy -- **Global Policy**: Template-wide behavior settings - -### 3. **Automatic Vision Registration** -Vision processors are automatically registered based on template configuration: -```python -def __post_init__(self): - if self.image_token or self.video_token: - self._register_vision_processor() -``` - -### 4. **Jinja Template Generation** -Templates can generate HuggingFace-compatible Jinja templates: -- Enables use with external systems (vLLM, etc.) -- Maintains consistency between Python and Jinja rendering -- Supports complex logic through Jinja macros - -## Extensibility Points - -### Adding New Template Types -1. Create a new `Template` instance -2. Configure templates, policies, and vision settings -3. Register with `register_template()` - -### Adding New Tool Formatters -1. Inherit from `ToolFormatter` base class -2. Implement `format()` and `jinja()` methods -3. Use in `ToolPolicy` configuration - -### Adding New Vision Processors -1. Inherit from `VisionProcessor` base class -2. Implement required abstract methods -3. Register with `register_processor()` - -### Adding New System Processors -1. Inherit from `SystemContentProcessor` base class -2. Implement `__call__()` and `jinja()` methods -3. Use in `SystemPolicy` configuration - -## Benefits of This Architecture - -### 1. **Maintainability** -- Clear separation of concerns -- Single responsibility principle -- Easy to modify individual components - -### 2. **Flexibility** -- Multiple strategies for each behavior -- Runtime configuration through policies -- Easy to add new capabilities - -### 3. **Reusability** -- Components can be shared across templates -- Policies can be reused and combined -- Vision processors work with any template - -### 4. **Consistency** -- Unified interface for all template types -- Consistent policy configuration -- Standardized vision processing pipeline - -This architecture makes the template system both powerful and easy to use, while maintaining the flexibility to handle diverse use cases and requirements. diff --git a/docs/features/chat_template/examples.md b/docs/features/chat_template/examples.md deleted file mode 100644 index dcce310..0000000 --- a/docs/features/chat_template/examples.md +++ /dev/null @@ -1,601 +0,0 @@ -# Examples and Use Cases - -## Overview - -This section provides comprehensive examples of how to use the Chat Template System in various scenarios. Each example demonstrates different features and capabilities of the system. - -## Basic Examples - -### Example 1: Simple Chat Template - -```python -from agentfly.agents.templates import Chat, get_template - -# Get a pre-built template -template = get_template("qwen2.5") - -# Create a simple conversation -messages = [ - {"role": "user", "content": "Hello, how are you?"}, - {"role": "assistant", "content": "I'm doing well, thank you for asking! How can I help you today?"}, - {"role": "user", "content": "Can you explain what machine learning is?"} -] - -# Create chat instance -chat = Chat(template="qwen2.5", messages=messages) - -# Generate prompt -prompt = chat.prompt() -print("Generated Prompt:") -print(prompt) - -# Generate prompt with generation prompt (for inference) -prompt_with_gen = chat.prompt(add_generation_prompt=True) -print("\nPrompt with Generation Prompt:") -print(prompt_with_gen) -``` - -**Output:** -``` -Generated Prompt: -<|im_start|>system -You are Qwen, created by Alibaba Cloud. You are a helpful assistant.<|im_end|> -<|im_start|>user -Hello, how are you?<|im_end|> -<|im_start|>assistant -I'm doing well, thank you for asking! How can I help you today?<|im_end|> -<|im_start|>user -Can you explain what machine learning is?<|im_end|> - -Prompt with Generation Prompt: -<|im_start|>system -You are Qwen, created by Alibaba Cloud. You are a helpful assistant.<|im_end|> -<|im_start|>user -Hello, how are you?<|im_end|> -<|im_start|>assistant -I'm doing well, thank you for asking! How can I help you today?<|im_end|> -<|im_start|>user -Can you explain what machine learning is?<|im_end|> -<|im_start|>assistant -``` - -### Example 2: Chat with Tools - -```python -# Define tools -tools = [ - { - "function": { - "name": "search_web", - "description": "Search the web for current information", - "parameters": { - "type": "object", - "properties": { - "query": {"type": "string", "description": "Search query"}, - "max_results": {"type": "integer", "description": "Maximum number of results"} - }, - "required": ["query"] - } - } - }, - { - "function": { - "name": "calculate", - "description": "Perform mathematical calculations", - "parameters": { - "type": "object", - "properties": { - "expression": {"type": "string", "description": "Mathematical expression to evaluate"} - }, - "required": ["expression"] - } - } - } -] - -# Create chat with tools -chat = Chat(template="qwen2.5", messages=messages, tools=tools) - -# Generate prompt with tools -prompt_with_tools = chat.prompt(tools=tools) -print("Prompt with Tools:") -print(prompt_with_tools) -``` - -**Output:** -``` -Prompt with Tools: -<|im_start|>system -You are Qwen, created by Alibaba Cloud. You are a helpful assistant. - -# Tools - -You may call one or more functions to assist with the user query. - -You are provided with function signatures within XML tags: - -{"function": {"name": "search_web", "description": "Search the web for current information", "parameters": {"type": "object", "properties": {"query": {"type": "string", "description": "Search query"}, "max_results": {"type": "integer", "description": "Maximum number of results"}, "required": ["query"]}}} -{"function": {"name": "calculate", "description": "Perform mathematical calculations", "parameters": {"type": "object", "properties": {"expression": {"type": "string", "description": "Mathematical expression to evaluate"}, "required": ["expression"]}}} - - -<|im_end|> -<|im_start|>user -Hello, how are you?<|im_end|> -<|im_start|>assistant -I'm doing well, thank you for asking! How can I help you today?<|im_end|> -<|im_start|>user -Can you explain what machine learning is?<|im_end|> -``` - -### Example 3: Tokenization - -```python -from transformers import AutoTokenizer - -# Load tokenizer -tokenizer = AutoTokenizer.from_pretrained("Qwen/Qwen2.5-7B-Instruct") - -# Tokenize the conversation -inputs = chat.tokenize( - tokenizer=tokenizer, - add_generation_prompt=True, - tools=tools -) - -print("Tokenization Results:") -print(f"Input IDs shape: {inputs['input_ids'].shape}") -print(f"Attention mask shape: {inputs['attention_mask'].shape}") -print(f"Labels shape: {inputs['labels'].shape}") -print(f"Action mask shape: {inputs['action_mask'].shape}") - -# Show token alignment -print(f"\nFirst 20 tokens: {inputs['input_ids'][0][:20]}") -print(f"First 20 labels: {inputs['labels'][0][:20]}") -print(f"First 20 action mask: {inputs['action_mask'][0][:20]}") -``` - -## Advanced Examples - -### Example 4: Custom Template Creation - -```python -from agentfly.agents.templates import Template, register_template -from agentfly.agents.templates.tool_policy import ToolPolicy, JsonIndentedFormatter -from agentfly.agents.templates.constants import ToolPlacement - -# Create a custom coding assistant template -coding_template = Template( - name="coding-assistant", - - # System message - system_template="""<|im_start|>system -You are an expert coding assistant. You help users write, debug, and understand code. -Always provide clear explanations and follow best practices. -{system_message}<|im_end|> -""", - system_message="You are an expert coding assistant.", - - # Tool support for code execution - system_template_with_tools="""<|im_start|>system -You are an expert coding assistant with access to code execution tools. -Always think through the problem before writing code. -{system_message} - -Available Tools: -{tools}<|im_end|> -""", - - # User and assistant templates - user_template="<|im_start|>user\n{content}<|im_end|>\n", - user_template_with_tools="<|im_start|>user\n{content}\n\nTools: {tools}<|im_end|>\n", - assistant_template="<|im_start|>assistant\n{content}<|im_end|>\n", - tool_template="<|im_start|>tool\n{observation}<|im_end|>\n", - - # Stop words - stop_words=["<|im_end|>"], - - # Tool policy - place tools with first user message - tool_policy=ToolPolicy( - placement=ToolPlacement.FIRST_USER, - formatter=JsonIndentedFormatter(indent=2) - ) -) - -# Register the template -register_template(coding_template) - -# Test the template -coding_chat = Chat(template="coding-assistant", messages=[ - {"role": "user", "content": "Write a Python function to calculate fibonacci numbers"} -]) - -prompt = coding_chat.prompt() -print("Coding Assistant Template:") -print(prompt) -``` - -### Example 5: Vision Template Usage - -```python -# Create a vision-enabled chat -vision_messages = [ - { - "role": "user", - "content": [ - {"type": "text", "text": "What's in this image? Please describe it in detail."}, - {"type": "image", "image": "/path/to/sample_image.jpg"} - ] - } -] - -# Use a vision template -vision_chat = Chat(template="qwen2.5-vl", messages=vision_messages) - -# Generate prompt -vision_prompt = vision_chat.prompt() -print("Vision Template Prompt:") -print(vision_prompt) - -# Get vision inputs -vision_inputs = vision_chat.vision_inputs() -print(f"\nVision inputs: {list(vision_inputs.keys())}") -``` - -### Example 6: Dynamic Template Generation - -```python -def create_specialized_template(base_name, capabilities): - """Create template based on capabilities""" - - # Base templates - system_base = "You are a helpful AI assistant." - user_base = "User: {content}" - assistant_base = "Assistant: {content}" - - # Add tool support if needed - if "tools" in capabilities: - system_base += "\n\nYou have access to tools: {tools}" - user_base += "\n\nAvailable tools: {tools}" - - # Add vision support if needed - if "vision" in capabilities: - system_base += "\n\nYou can process images and videos." - - return Template( - name=f"{base_name}-{'-'.join(capabilities)}", - system_template=system_base, - user_template=user_base, - assistant_template=assistant_base, - vision_start="" if "vision" in capabilities else None, - vision_end="" if "vision" in capabilities else None, - image_token="" if "vision" in capabilities else None, - video_token="