diff --git a/.gitignore b/.gitignore index 2d0caa2..1db93be 100644 --- a/.gitignore +++ b/.gitignore @@ -205,4 +205,5 @@ __marimo__/ .cursor/ # Example outputs -examples/*/output/ +examples/*/*output*/ +*.jsonl diff --git a/examples/turkish_tool_calling_v1/README.md b/examples/turkish_tool_calling_v1/README.md new file mode 100644 index 0000000..c3e0116 --- /dev/null +++ b/examples/turkish_tool_calling_v1/README.md @@ -0,0 +1,111 @@ +# Turkish Tool Calling v1 + +A synthetic Turkish tool-calling dataset generated using [ToolsGen](https://github.com/atasoglu/toolsgen) with Qwen models via OpenRouter. + +## Dataset Details + +- **Generated with**: ToolsGen +- **Total Samples**: 1,000 +- **Language**: Turkish +- **Format**: Single-turn conversations with tool calls + +### Models Used + +- **Problem Generator**: qwen/qwen3-235b-a22b-2507 (temp=1.0) +- **Tool Caller**: qwen/qwen3-235b-a22b-2507 (temp=0.0) +- **Judge**: qwen/qwen3-235b-a22b-2507 (temp=0.0) + +## Dataset Structure + +Each record contains: + +```json +{ + "id": "record_000000", + "language": "turkish", + "tools": [...], + "messages": [ + {"role": "user", "content": "İstanbul'da hava durumu nasıl?"} + ], + "assistant_calls": [ + { + "id": "call_...", + "type": "function", + "function": { + "name": "get_weather", + "arguments": "{\"location\": \"Istanbul, Turkey\"}" + } + } + ], + "problem_metadata": {...}, + "judge": { + "tool_relevance": 0.4, + "argument_quality": 0.38, + "clarity": 0.2, + "score": 0.98, + "verdict": "accept", + "rationale": "...", + "rubric_version": "0.1.0", + "model": "qwen/qwen3-235b-a22b-2507", + "temperature": 0.0 + }, + "quality_tags": [], + "tools_metadata": {"num_tools": 2} +} +``` + +## Generation Details + +### Configuration + +- **Strategy**: Random tool sampling +- **Tools per sample**: 1-8 (k_min=1, k_max=8) +- **Max attempts**: 1 +- **Train split**: 80% +- **Seed**: Random (1-10M range) + +### Quality Control + +All samples passed through an LLM-as-a-judge evaluation with a multi-dimensional rubric: + +- **Tool Relevance** (40%): Are the selected tools appropriate? +- **Argument Quality** (38%): Are arguments valid and plausible? +- **Clarity** (20%): Is the response complete and clear? + +Samples with `score >= 0.7` and `verdict == "accept"` are included. + +## Usage + +```python +from datasets import load_dataset + +dataset = load_dataset("atasoglu/turkish-tool-calling-v1") + +# Access a sample +sample = dataset["train"][0] +print(sample["messages"]) +print(sample["assistant_calls"]) +``` + +## Limitations + +- Single-turn conversations only +- Turkish language only +- Synthetic data generated by LLMs (may contain artifacts) +- No actual tool execution or validation +- Judge scores are model-based assessments + +## Citation + +```bibtex +@software{toolsgen2025, + title = {ToolsGen: Synthetic Tool-Calling Dataset Generator}, + author = {Ataşoğlu, Ahmet}, + year = {2025}, + url = {https://github.com/atasoglu/toolsgen} +} +``` + +## License + +MIT License diff --git a/examples/turkish_tool_calling_v1/config.py b/examples/turkish_tool_calling_v1/config.py new file mode 100644 index 0000000..1dd94a2 --- /dev/null +++ b/examples/turkish_tool_calling_v1/config.py @@ -0,0 +1,46 @@ +from random import randint +from toolsgen import ( + GenerationConfig, + ModelConfig, + RoleBasedModelConfig, +) + +seed = randint(1, 10_000_000) +print(f"Using seed: {seed}") + +openai_params = dict( + base_url="https://openrouter.ai/api/v1", +) + +gen_config = GenerationConfig( + num_samples=1000, + strategy="random", + seed=seed, + train_split=0.8, + language="turkish", + max_attempts=1, + k_min=1, + k_max=8, + shuffle_tools=True, +) + +role_config = RoleBasedModelConfig( + problem_generator=ModelConfig( + model="qwen/qwen3-235b-a22b-2507", + temperature=1.0, + openai_params=openai_params, + max_tokens=500, + ), + tool_caller=ModelConfig( + model="qwen/qwen3-235b-a22b-2507", + temperature=0, + openai_params=openai_params, + max_tokens=500, + ), + judge=ModelConfig( + model="qwen/qwen3-235b-a22b-2507", + temperature=0, + openai_params=openai_params, + max_tokens=500, + ), +) diff --git a/examples/turkish_tool_calling_v1/main.py b/examples/turkish_tool_calling_v1/main.py new file mode 100644 index 0000000..c9d3946 --- /dev/null +++ b/examples/turkish_tool_calling_v1/main.py @@ -0,0 +1,37 @@ +from pathlib import Path +from dotenv import load_dotenv +from preprocessing import load_tools_from_file +from config import gen_config, role_config +from toolsgen import generate_dataset +from uuid import uuid4 + +# Load environment variables from .env file +load_dotenv() + + +def main() -> None: + # Load tools from file + tools = list(load_tools_from_file("tools.jsonl")) + print("Loaded tools from file.") + print(f"Number of tools loaded: {len(tools)}") + + # Define output directory with timestamp + output_dir = Path(__file__).parent / f"output_{uuid4().hex}" + + # Generate dataset + manifest = generate_dataset(output_dir, gen_config, role_config, tools=tools) + + # Print summary + print( + f"\n✓ Generated {manifest['num_generated']}/{manifest['num_requested']} records" + ) + if manifest["num_failed"] > 0: + print(f" Failed: {manifest['num_failed']} attempts") + print(f" Problem Generator: {role_config.problem_generator.model}") + print(f" Tool Caller: {role_config.tool_caller.model}") + print(f" Judge: {role_config.judge.model}") + print(f" Output: {output_dir}") + + +if __name__ == "__main__": + main() diff --git a/examples/turkish_tool_calling_v1/postprocessing/__init__.py b/examples/turkish_tool_calling_v1/postprocessing/__init__.py new file mode 100644 index 0000000..64fc52d --- /dev/null +++ b/examples/turkish_tool_calling_v1/postprocessing/__init__.py @@ -0,0 +1,3 @@ +from .streamer import get_dirs, get_jsonl_files, read_lines, save_line + +__all__ = ["get_dirs", "get_jsonl_files", "read_lines", "save_line"] diff --git a/examples/turkish_tool_calling_v1/postprocessing/__main__.py b/examples/turkish_tool_calling_v1/postprocessing/__main__.py new file mode 100644 index 0000000..36f5096 --- /dev/null +++ b/examples/turkish_tool_calling_v1/postprocessing/__main__.py @@ -0,0 +1,44 @@ +import json +from .streamer import get_dirs, get_jsonl_files, read_lines, save_line +from pathlib import Path +from typing import Optional + + +def postprocess(line: str, max_newlines: int = 10) -> Optional[dict]: + """Postprocess the given line string by limiting consecutive newlines.""" + try: + if line.count(r"\n") > max_newlines: + raise ValueError("Too many newlines in the line.") + return json.loads(line) + except Exception: + return None + + +def main(): + success = 0 + failed = 0 + base_dir = Path.cwd() + output = base_dir / "postprocessed.jsonl" + output.touch(exist_ok=True) + dirs = get_dirs(base_dir) + for dir in dirs: + jsonl_files = get_jsonl_files(dir) + for jsonl_file in jsonl_files: + for line in read_lines(jsonl_file): + json_dict = postprocess(line) + if json_dict is not None: + success += 1 + json_dict["id"] = f"record_{success:06d}" + save_line(output, json.dumps(json_dict, ensure_ascii=False)) + else: + failed += 1 + print( + f"\rProcessed lines: {success + failed} (Success: {success}, Failed: {failed})", + end="\r", + ) + print(f"\nTotal processed lines: {success}") + print(f"Total failed lines: {failed}") + + +if __name__ == "__main__": + main() diff --git a/examples/turkish_tool_calling_v1/postprocessing/streamer.py b/examples/turkish_tool_calling_v1/postprocessing/streamer.py new file mode 100644 index 0000000..17fc53f --- /dev/null +++ b/examples/turkish_tool_calling_v1/postprocessing/streamer.py @@ -0,0 +1,32 @@ +from pathlib import Path +from typing import Generator + + +def get_jsonl_files(directory: Path) -> Generator[Path, None, None]: + """Get all JSONL files in the specified directory.""" + yield from directory.glob("*.jsonl") + + +def get_dirs(directory: Path) -> Generator[Path, None, None]: + """Get all subdirectories in the specified directory.""" + for d in directory.iterdir(): + if d.is_dir(): + yield d + + +def read_lines(file_path: Path) -> Generator[str, None, None]: + """Read lines from a file.""" + with file_path.open("r", encoding="utf-8") as f: + for line in f: + yield line.strip() + + +def save_line(file_path: Path, line: str): + """Save a line to a file.""" + with file_path.open("a", encoding="utf-8") as f: + f.write(line + "\n") + + +def count_newlines(s: str) -> int: + """Count the number of newline characters in a string.""" + return s.count("\n") diff --git a/examples/turkish_tool_calling_v1/preprocessing/__init__.py b/examples/turkish_tool_calling_v1/preprocessing/__init__.py new file mode 100644 index 0000000..597b242 --- /dev/null +++ b/examples/turkish_tool_calling_v1/preprocessing/__init__.py @@ -0,0 +1,7 @@ +from .streamer import ( + stream_tools_from_datasets, + save_tools_to_file, + load_tools_from_file, +) + +__all__ = ["stream_tools_from_datasets", "save_tools_to_file", "load_tools_from_file"] diff --git a/examples/turkish_tool_calling_v1/preprocessing/__main__.py b/examples/turkish_tool_calling_v1/preprocessing/__main__.py new file mode 100644 index 0000000..71a58d3 --- /dev/null +++ b/examples/turkish_tool_calling_v1/preprocessing/__main__.py @@ -0,0 +1,34 @@ +import time +from .streamer import stream_tools_from_datasets, save_tools_to_file +from dotenv import load_dotenv +from pathlib import Path +from typing import Generator + +load_dotenv() +example_dir = Path(__file__).parent.parent +file_path = example_dir / "tools.jsonl" + + +def stream_wrapper(stream: Generator[dict, None, None]) -> Generator[dict, None, None]: + total = 0 + start = time.time() + for tool in stream: + total += 1 + yield tool + print(f"Processed {total} tools...", end="\r") + end = time.time() + print(f"Finished processing {total} tool definitions in {end - start:.2f} seconds.") + + +def main(): + dataset_ids = [ + "argilla/Synth-APIGen-v0.1", + "Salesforce/xlam-function-calling-60k", + "argilla-warehouse/python-seed-tools", + ] + tools = stream_tools_from_datasets(dataset_ids, debug=False) + save_tools_to_file(stream_wrapper(tools), str(file_path)) + + +if __name__ == "__main__": + main() diff --git a/examples/turkish_tool_calling_v1/preprocessing/schema.py b/examples/turkish_tool_calling_v1/preprocessing/schema.py new file mode 100644 index 0000000..02ec3b7 --- /dev/null +++ b/examples/turkish_tool_calling_v1/preprocessing/schema.py @@ -0,0 +1,211 @@ +from __future__ import annotations + +import json +import re +from typing import Any, Dict, List, Literal, Optional, Union, get_args +from pydantic import ( + BaseModel, + ConfigDict, + Field, + ValidationInfo, + field_validator, + model_validator, +) + +NAME_PATTERN = re.compile(r"^[a-z0-9_]{1,64}$") +PROPERTY_PATTERN = re.compile(r"^[a-z0-9_]+$") +SchemaType = Literal["string", "number", "integer", "boolean", "array", "object"] +VALID_SCHEMA_TYPES = set(get_args(SchemaType)) + + +class OpenAIBaseModel(BaseModel): + """Base configuration shared by OpenAI tool schema models.""" + + model_config = ConfigDict( + extra="forbid", populate_by_name=True, str_strip_whitespace=True + ) + + +class ToolSchema(OpenAIBaseModel): + """Recursive JSON Schema node used to describe tool parameters.""" + + type: SchemaType + description: Optional[str] = None + default: Optional[Any] = None + enum: Optional[List[Any]] = None + items: Optional["ToolSchema"] = None + properties: Dict[str, "ToolSchema"] = Field(default_factory=dict) + required: List[str] = Field(default_factory=list) + additional_properties: Optional[Union[bool, "ToolSchema"]] = Field( + default=None, alias="additionalProperties" + ) + + @field_validator("type", mode="before") + @classmethod + def _normalize_type(cls, value: Any) -> SchemaType: + alias_map = { + "str": "string", + "float": "number", + "bool": "boolean", + "list": "array", + "dict": "object", + "int": "integer", + "set": "array", + "tuple": "array", + } + if isinstance(value, str): + lowered = value.lower().strip() + tokens = [ + token + for chunk in re.split(r"[\s,|/]+", lowered) + for token in re.findall(r"[a-z]+", chunk) + ] + for token in tokens: + canonical = alias_map.get(token, token) + if canonical in VALID_SCHEMA_TYPES: + return canonical # type: ignore[return-value] + return value # Pydantic enforces SchemaType after normalization + + @field_validator("enum") + @classmethod + def _check_enum(cls, values: Optional[List[Any]]) -> Optional[List[Any]]: + if isinstance(values, list) and not values: + raise ValueError("Enum must include at least one value.") + return values + + @field_validator("properties") + @classmethod + def _check_properties( + cls, properties: Dict[str, "ToolSchema"], info: ValidationInfo + ) -> Dict[str, "ToolSchema"]: + if not properties: + return {} + + cls._ensure_object_schema(info, "properties") + invalid = [name for name in properties if not PROPERTY_PATTERN.match(name)] + if invalid: + raise ValueError( + "Property names must use snake_case alphanumeric characters: " + + ", ".join(invalid) + ) + return properties + + @field_validator("required") + @classmethod + def _check_required(cls, required: List[str], info: ValidationInfo) -> List[str]: + if not required: + return [] + + cls._ensure_object_schema(info, "required") + defined_props = info.data.get("properties", {}) + missing = [name for name in required if name not in defined_props] + if missing: + raise ValueError( + f"Required keys lack property definitions: {', '.join(missing)}" + ) + return required + + @field_validator("additional_properties") + @classmethod + def _check_additional_properties( + cls, value: Optional[Union[bool, "ToolSchema"]], info: ValidationInfo + ) -> Optional[Union[bool, "ToolSchema"]]: + if value is None: + return None + + cls._ensure_object_schema(info, "additionalProperties") + return value + + @model_validator(mode="after") + def _check_array_items(self) -> "ToolSchema": + if self.type == "array" and self.items is None: + self.items = ToolSchema(type="string") + if self.type != "array" and self.items is not None: + raise ValueError("'items' can only be defined for array schemas.") + return self + + @staticmethod + def _ensure_object_schema(info: ValidationInfo, field_name: str) -> None: + if info.data.get("type") != "object": + raise ValueError(f"'{field_name}' can only be defined for object schemas.") + + +class ToolParameters(ToolSchema): + """Top-level parameter container for function tools.""" + + type: Literal["object"] = "object" + + +class ToolFunction(OpenAIBaseModel): + """OpenAI function specification with enforced best practices.""" + + name: str + description: str + parameters: ToolParameters = Field(default_factory=ToolParameters) + + @model_validator(mode="before") + @classmethod + def _unwrap_openai_spec(cls, data: Any) -> Any: + if ( + isinstance(data, dict) + and data.get("type") == "function" + and isinstance(data.get("function"), dict) + ): + return dict(data["function"]) + return data + + @model_validator(mode="before") + @classmethod + def _normalize_parameters(cls, data: Any) -> Any: + if isinstance(data, dict): + params = data.get("parameters") + if isinstance(params, dict): + schema_type = params.get("type") + has_properties = "properties" in params + schema_valid = ( + has_properties + and isinstance(schema_type, str) + and schema_type in VALID_SCHEMA_TYPES + ) + if not schema_valid: + data = dict(data) + data["parameters"] = {"type": "object", "properties": params} + return data + + @field_validator("parameters", mode="before") + @classmethod + def _coerce_parameters(cls, value: Any) -> Any: + if isinstance(value, dict): + schema_type = value.get("type") + has_properties = "properties" in value + schema_valid = ( + has_properties + and isinstance(schema_type, str) + and schema_type in VALID_SCHEMA_TYPES + ) + if not schema_valid: + return {"type": "object", "properties": value} + return value + + @field_validator("name") + @classmethod + def _check_name(cls, name: str) -> str: + if not NAME_PATTERN.match(name): + raise ValueError( + "Function names must be 1-64 characters long and snake_case (a-z0-9_)." + ) + return name + + def model_dump(self, *args: Any, **kwargs: Any) -> Dict[str, Any]: + if "exclude_none" not in kwargs: + kwargs["exclude_none"] = True + if "by_alias" not in kwargs: + kwargs["by_alias"] = True + function_payload = super().model_dump(*args, **kwargs) + return {"type": "function", "function": function_payload} + + def model_dump_json(self, *args: Any, **kwargs: Any) -> str: + return json.dumps(self.model_dump(*args, **kwargs), ensure_ascii=False) + + +ToolSchema.model_rebuild() diff --git a/examples/turkish_tool_calling_v1/preprocessing/streamer.py b/examples/turkish_tool_calling_v1/preprocessing/streamer.py new file mode 100644 index 0000000..84915e3 --- /dev/null +++ b/examples/turkish_tool_calling_v1/preprocessing/streamer.py @@ -0,0 +1,145 @@ +import json + +from .schema import ToolFunction +from datasets import load_dataset +from typing import Generator, Optional +from multiprocessing import Pool, cpu_count +from toolsgen import ToolSpec + + +def _process_sample(sample_data: tuple) -> tuple[list[dict], int]: + """Process a single sample and extract valid tools.""" + sample, column_id, debug = sample_data + tools_list = [] + failed_count = 0 + try: + tools = json.loads(sample[column_id]) + if not isinstance(tools, list): + tools = [tools] + for tool in tools: + try: + tool_definition = json.dumps(tool) + model = ToolFunction.model_validate_json(tool_definition) + tools_list.append(model.model_dump()) + except Exception as e: + failed_count += 1 + if debug: + print(f"Skipping invalid tool definition: {e}") + print(f"Tool definition: {json.dumps(tool, indent=2)}") + continue + except Exception as e: + failed_count += 1 + if debug: + print(f"Skipping invalid sample: {e}") + return tools_list, failed_count + + +def stream_tools_from_datasets( + dataset_ids: list[str] | str, + split: str = "train", + column_id: str = "tools", + debug: bool = False, + num_workers: Optional[int] = None, + batch_size: int = 100, +) -> Generator[dict, None, None]: + """Stream unique tool definitions from multiple datasets with parallel processing. + + **Note:** The datasets are expected to contain tool definitions in JSON format + within the specified column. Otherwise, those entries will be skipped. + + Args: + dataset_ids (list[str] | str): The dataset identifier(s). Can be a single string or list of strings. + split (str, optional): The dataset split to load. Defaults to "train". + column_id (str, optional): The column containing tool definitions. Defaults to "tools". + debug (bool, optional): Whether to enable debug mode. Defaults to False. + num_workers (Optional[int], optional): Number of worker processes. Defaults to CPU count. + batch_size (int, optional): Batch size for parallel processing. Defaults to 100. + + Yields: + Generator[dict, None, None]: Tool definitions as dictionaries. + """ + total_failed = 0 + if num_workers is None: + num_workers = cpu_count() + + # Convert single dataset_id to list for uniform processing + if isinstance(dataset_ids, str): + dataset_ids = [dataset_ids] + + seen_tools = set() + + for dataset_id in dataset_ids: + dataset = load_dataset(dataset_id, split=split, streaming=True) + + if num_workers == 1: + # Sequential processing + for sample in dataset: + tools_list, failed_count = _process_sample((sample, column_id, debug)) + total_failed += failed_count + for tool in tools_list: + tool_name = tool.get("function", {}).get("name") + if tool_name and tool_name not in seen_tools: + seen_tools.add(tool_name) + yield tool + else: + # Parallel processing + batch = [] + with Pool(num_workers) as pool: + for sample in dataset: + batch.append((sample, column_id, debug)) + if len(batch) >= batch_size: + results = pool.map(_process_sample, batch) + for tools_list, failed_count in results: + total_failed += failed_count + for tool in tools_list: + tool_name = tool.get("function", {}).get("name") + if tool_name and tool_name not in seen_tools: + seen_tools.add(tool_name) + yield tool + batch = [] + + # Process remaining batch + if batch: + results = pool.map(_process_sample, batch) + for tools_list, failed_count in results: + total_failed += failed_count + for tool in tools_list: + tool_name = tool.get("function", {}).get("name") + if tool_name and tool_name not in seen_tools: + seen_tools.add(tool_name) + yield tool + + # After processing each dataset, print the number of failed samples/tools + print(f"Total failed samples/tools: {total_failed}") + + +def save_tools_to_file( + tools: Generator[dict, None, None], + output_file: str, +) -> None: + """Save tool definitions to a JSONL file. + + Args: + tools (Generator[dict, None, None]): Generator of tool definitions. + output_file (str): Path to the output JSONL file. + """ + with open(output_file, "a", encoding="utf-8") as f: + for tool in tools: + f.write(json.dumps(tool, ensure_ascii=False) + "\n") + + +def load_tools_from_file( + input_file: str, +) -> Generator[ToolSpec, None, None]: + """Load tool definitions from a JSONL file. + + Args: + input_file (str): Path to the input JSONL file. + Yields: + Generator[ToolSpec, None, None]: Generator of ToolSpec instances. + """ + with open(input_file, "r", encoding="utf-8") as f: + for line in f: + tool_dict = json.loads(line) + tool_spec = ToolSpec.model_validate(tool_dict) + yield tool_spec diff --git a/examples/turkish_tool_calling_v1/push_to_hf.py b/examples/turkish_tool_calling_v1/push_to_hf.py new file mode 100644 index 0000000..1c242ba --- /dev/null +++ b/examples/turkish_tool_calling_v1/push_to_hf.py @@ -0,0 +1,45 @@ +import os +import json +from pathlib import Path +from datasets import Dataset +from dotenv import load_dotenv + + +def load_jsonl(path: Path) -> list[dict]: + records = [] + with open(path, "r", encoding="utf-8") as f: + for line in f: + if line.strip(): + records.append(json.loads(line)) + return records + + +def push_to_hub(jsonl_path: Path, repo_id: str, token: str = None): + # Load data + data = load_jsonl(jsonl_path) + + # Process fields for HF compatibility + for record in data: + # Keep simple types as-is: str, int, float + # Convert complex types (dict, list) to JSON strings + for key, value in record.items(): + if isinstance(value, (dict, list)): + record[key] = json.dumps(value, ensure_ascii=False) + # Keep str, int, float, bool as-is + + # Create dataset + dataset = Dataset.from_list(data) + + # Push to hub + dataset.push_to_hub(repo_id, token=token) + + +if __name__ == "__main__": + load_dotenv() + + base_dir = Path(__file__).parent + jsonl_path = base_dir / "postprocessed.jsonl" + repo_id = "atasoglu/turkish-tool-calling-10k" + token = os.getenv("HF_TOKEN") + + push_to_hub(jsonl_path, repo_id, token)