Merge pull request #9 from Ontos-AI/fix/wangbinqi/python-parse-result-payload

fix: sync parse result payload with current API schema

Author: suguanYang

src/knowhere/__init__.py

@@ -46,6 +46,10 @@
     ImageFileInfo,
     Manifest,
     ParseResult,
+    ProcessingCost,
+    ProcessingMetadata,
+    ProcessingTiming,
+    SlimChunk,
     Statistics,
     TableChunk,
     TableFileInfo,
@@ -91,6 +95,10 @@
     "FileIndex",
     "ImageFileInfo",
     "TableFileInfo",
+    "ProcessingCost",
+    "ProcessingMetadata",
+    "ProcessingTiming",
+    "SlimChunk",
     "BaseChunk",
     "TextChunk",
     "ImageChunk",

src/knowhere/_exceptions.py

@@ -387,11 +387,29 @@ def makeStatusError(
         response=response,
     )
 
-    if exception_class in (RateLimitError, ServiceUnavailableError, GatewayTimeoutError):
-        return exception_class(
+    if exception_class is RateLimitError:
+        return RateLimitError(
             status_code,
             **common_kwargs,
-            retry_after=retry_after,  # type: ignore[call-arg]
+            retry_after=retry_after,
+            limit=limit,
+            period=period,
+        )
+
+    if exception_class is ServiceUnavailableError:
+        return ServiceUnavailableError(
+            status_code,
+            **common_kwargs,
+            retry_after=retry_after,
+            limit=limit,
+            period=period,
+        )
+
+    if exception_class is GatewayTimeoutError:
+        return GatewayTimeoutError(
+            status_code,
+            **common_kwargs,
+            retry_after=retry_after,
             limit=limit,
             period=period,
         )

src/knowhere/lib/result_parser.py

@@ -16,6 +16,7 @@
     ImageChunk,
     Manifest,
     ParseResult,
+    SlimChunk,
     TableChunk,
     TextChunk,
     TextChunkTokens,
@@ -134,6 +135,7 @@ def _buildChunks(
                 type="image",
                 content=raw.get("content", ""),
                 path=raw.get("path"),
+                page_nums=metadata.get("page_nums", raw.get("page_nums")),
                 length=metadata.get("length", raw.get("length", 0)),
                 file_path=file_path,
                 original_name=metadata.get("original_name", raw.get("original_name")),
@@ -151,6 +153,7 @@ def _buildChunks(
                 type="table",
                 content=raw.get("content", ""),
                 path=raw.get("path"),
+                page_nums=metadata.get("page_nums", raw.get("page_nums")),
                 length=metadata.get("length", raw.get("length", 0)),
                 file_path=file_path,
                 original_name=metadata.get("original_name", raw.get("original_name")),
@@ -167,10 +170,12 @@ def _buildChunks(
                 type="text",
                 content=raw.get("content", ""),
                 path=raw.get("path"),
+                page_nums=metadata.get("page_nums", raw.get("page_nums")),
                 length=metadata.get("length", raw.get("length", 0)),
                 tokens=_parseTextChunkTokens(raw_tokens, chunk_id=chunk_id),
                 keywords=metadata.get("keywords", raw.get("keywords")),
                 summary=metadata.get("summary", raw.get("summary")),
+                connect_to=metadata.get("connect_to", raw.get("connect_to")),
                 relationships=metadata.get("relationships", raw.get("relationships")),
             )
 
@@ -230,12 +235,39 @@ def parseResultZip(
         json.loads(hierarchy_text) if hierarchy_text else None
     )
 
+    # -- Optimized sidecar files --
+    chunks_slim_text: Optional[str] = _readZipText(zf, "chunks_slim.json")
+    parsed_chunks_slim: Any = json.loads(chunks_slim_text) if chunks_slim_text else None
+    if isinstance(parsed_chunks_slim, dict) and "chunks" in parsed_chunks_slim:
+        raw_chunks_slim: List[Dict[str, Any]] = parsed_chunks_slim["chunks"]
+    elif isinstance(parsed_chunks_slim, list):
+        raw_chunks_slim = parsed_chunks_slim
+    else:
+        raw_chunks_slim = []
+    chunks_slim: Optional[List[SlimChunk]] = (
+        [SlimChunk.model_validate(chunk) for chunk in raw_chunks_slim]
+        if chunks_slim_text is not None
+        else None
+    )
+
+    toc_hierarchies_text: Optional[str] = _readZipText(zf, "toc_hierarchies.json")
+    toc_hierarchies: Optional[Any] = (
+        json.loads(toc_hierarchies_text) if toc_hierarchies_text else None
+    )
+
+    kb_csv: Optional[str] = _readZipText(zf, "kb.csv")
+    hierarchy_view_html: Optional[str] = _readZipText(zf, "hierarchy_view.html")
+
     zf.close()
 
     return ParseResult(
         manifest=manifest,
         chunks=chunks,
+        chunks_slim=chunks_slim,
         full_markdown=full_markdown,
         hierarchy=hierarchy,
+        toc_hierarchies=toc_hierarchies,
+        kb_csv=kb_csv,
+        hierarchy_view_html=hierarchy_view_html,
         raw_zip=zip_bytes,
     )

src/knowhere/types/__init__.py

@@ -13,6 +13,10 @@
     ImageFileInfo,
     Manifest,
     ParseResult,
+    ProcessingCost,
+    ProcessingMetadata,
+    ProcessingTiming,
+    SlimChunk,
     Statistics,
     TableChunk,
     TableFileInfo,
@@ -36,6 +40,10 @@
     "ImageFileInfo",
     "Manifest",
     "ParseResult",
+    "ProcessingCost",
+    "ProcessingMetadata",
+    "ProcessingTiming",
+    "SlimChunk",
     "Statistics",
     "TableChunk",
     "TableFileInfo",

src/knowhere/types/result.py

@@ -3,6 +3,7 @@
 from __future__ import annotations
 
 import os
+import json
 import re
 from pathlib import Path
 from typing import Any, Dict, List, Optional, Union
@@ -92,12 +93,39 @@ class FileIndex(BaseModel):
 
     chunks: Optional[str] = None
     markdown: Optional[str] = None
+    chunks_slim: Optional[str] = None
     kb_csv: Optional[str] = None
     hierarchy: Optional[str] = None
+    toc_hierarchies: Optional[str] = None
+    hierarchy_view_html: Optional[str] = None
     images: List[ImageFileInfo] = Field(default_factory=list)
     tables: List[TableFileInfo] = Field(default_factory=list)
 
 
+class ProcessingCost(BaseModel):
+    """Billing details emitted by manifest v2."""
+
+    micro_dollars: Optional[int] = None
+    credits: Optional[float] = None
+
+
+class ProcessingTiming(BaseModel):
+    """Timing details emitted by manifest v2."""
+
+    started_at: Optional[str] = None
+    completed_at: Optional[str] = None
+    duration_ms: Optional[int] = None
+
+
+class ProcessingMetadata(BaseModel):
+    """Worker-side processing metadata emitted by manifest v2."""
+
+    page_count: Optional[int] = None
+    billing_status: Optional[str] = None
+    cost: Optional[ProcessingCost] = None
+    timing: Optional[ProcessingTiming] = None
+
+
 class Manifest(BaseModel):
     """Top-level manifest describing the result ZIP contents."""
 
@@ -106,6 +134,7 @@ class Manifest(BaseModel):
     data_id: Optional[str] = None
     source_file_name: Optional[str] = None
     processing_date: Optional[str] = None
+    processing: Optional[ProcessingMetadata] = None
     checksum: Optional[Checksum] = None
     statistics: Optional[Statistics] = None
     files: Optional[FileIndex] = None
@@ -123,6 +152,7 @@ class BaseChunk(BaseModel):
     type: str
     content: str = ""
     path: Optional[str] = None
+    page_nums: Optional[List[int]] = None
 
 
 TextChunkTokens: TypeAlias = List[str]
@@ -136,6 +166,7 @@ class TextChunk(BaseChunk):
     tokens: Optional[TextChunkTokens] = None
     keywords: Optional[List[str]] = None
     summary: Optional[str] = None
+    connect_to: Optional[List[Dict[str, Any]]] = None
     relationships: Optional[List[Union[Dict[str, Any], str]]] = None
 
 
@@ -210,6 +241,15 @@ def save(self, directory: Union[str, Path]) -> Path:
 Chunk = Union[TextChunk, ImageChunk, TableChunk]
 
 
+class SlimChunk(BaseModel):
+    """Minimal chunk entry emitted in chunks_slim.json."""
+
+    type: str
+    path: Optional[str] = None
+    content: str = ""
+    summary: Optional[str] = None
+
+
 # ---------------------------------------------------------------------------
 # ParseResult — the top-level object returned to the user
 # ---------------------------------------------------------------------------
@@ -225,23 +265,35 @@ class ParseResult:
 
     manifest: Manifest
     chunks: List[Chunk]
+    chunks_slim: Optional[List[SlimChunk]]
     full_markdown: str
     hierarchy: Optional[Any]
+    toc_hierarchies: Optional[Any]
+    kb_csv: Optional[str]
+    hierarchy_view_html: Optional[str]
     raw_zip: bytes
 
     def __init__(
         self,
         *,
         manifest: Manifest,
         chunks: List[Chunk],
+        chunks_slim: Optional[List[SlimChunk]],
         full_markdown: str,
         hierarchy: Optional[Any],
+        toc_hierarchies: Optional[Any],
+        kb_csv: Optional[str],
+        hierarchy_view_html: Optional[str],
         raw_zip: bytes,
     ) -> None:
         self.manifest = manifest
         self.chunks = chunks
+        self.chunks_slim = chunks_slim
         self.full_markdown = full_markdown
         self.hierarchy = hierarchy
+        self.toc_hierarchies = toc_hierarchies
+        self.kb_csv = kb_csv
+        self.hierarchy_view_html = hierarchy_view_html
         self.raw_zip = raw_zip
 
     # -- convenience properties --
@@ -296,10 +348,58 @@ def save(self, directory: Union[str, Path]) -> Path:
         dir_path: Path = Path(directory)
         dir_path.mkdir(parents=True, exist_ok=True)
 
+        # Manifest / chunks
+        manifest_path: Path = dir_path / "manifest.json"
+        manifest_path.write_text(
+            self.manifest.model_dump_json(indent=2),
+            encoding="utf-8",
+        )
+
+        chunks_path: Path = dir_path / "chunks.json"
+        chunks_path.write_text(
+            json.dumps([chunk.model_dump() for chunk in self.chunks], indent=2),
+            encoding="utf-8",
+        )
+
+        if self.chunks_slim is not None:
+            chunks_slim_path: Path = dir_path / "chunks_slim.json"
+            chunks_slim_path.write_text(
+                json.dumps(
+                    {"chunks": [chunk.model_dump() for chunk in self.chunks_slim]},
+                    indent=2,
+                ),
+                encoding="utf-8",
+            )
+
         # Full markdown
         md_path: Path = dir_path / "full.md"
         md_path.write_text(self.full_markdown, encoding="utf-8")
 
+        if self.hierarchy is not None:
+            hierarchy_path: Path = dir_path / "hierarchy.json"
+            hierarchy_path.write_text(
+                json.dumps(self.hierarchy, indent=2),
+                encoding="utf-8",
+            )
+
+        if self.toc_hierarchies is not None:
+            toc_hierarchies_path: Path = dir_path / "toc_hierarchies.json"
+            toc_hierarchies_path.write_text(
+                json.dumps(self.toc_hierarchies, indent=2),
+                encoding="utf-8",
+            )
+
+        if self.kb_csv is not None:
+            kb_csv_path: Path = dir_path / "kb.csv"
+            kb_csv_path.write_text(self.kb_csv, encoding="utf-8")
+
+        if self.hierarchy_view_html is not None:
+            hierarchy_view_path: Path = dir_path / "hierarchy_view.html"
+            hierarchy_view_path.write_text(
+                self.hierarchy_view_html,
+                encoding="utf-8",
+            )
+
         # Images
         if self.image_chunks:
             images_dir: Path = dir_path / "images"