From cc5df232f763887fd8ce07a15c18c22aa71dd0bc Mon Sep 17 00:00:00 2001
From: Bill Herald <641463+bherald@users.noreply.github.com>
Date: Wed, 22 Jul 2026 16:28:31 -0400
Subject: [PATCH 1/4] feat: support OpenAI stop sequences

---
 c/colibri.c                   |  10 ++++
 c/openai_server.py            | 107 +++++++++++++++++++++++++++++-----
 c/tests/test_openai_server.py | 100 +++++++++++++++++++++++++++----
 docs/api.md                   |   8 ++-
 docs/serve_protocol.md        |   5 ++
 5 files changed, 203 insertions(+), 27 deletions(-)

diff --git a/c/colibri.c b/c/colibri.c
index 82eeac21b..03d08b320 100644
--- a/c/colibri.c
+++ b/c/colibri.c
@@ -5471,6 +5471,16 @@ static int mux_submit(Model *m, Tok *T, ServeCtx *ctx, ServeReq *req, GrDraft *g
     char *line=NULL; size_t cap=0; ssize_t nr=getline(&line,&cap,stdin);
     if(nr<0){ free(line); return -1; }
     if(nr && line[nr-1]=='\n') line[--nr]=0;
+    if(!strncmp(line,"STOP ",5)){
+        unsigned long long id=0; char tail;
+        if(sscanf(line+5,"%llu %c",&id,&tail)!=1 || id==0){
+            printf("ERROR 0 BAD_REQUEST\n"); fflush(stdout); free(line); return 0;
+        }
+        for(int i=0;i<nctx;i++) if(req[i].active && req[i].id==id){
+            mux_done(m,&ctx[i],&req[i]); free(line); return 0;
+        }
+        printf("ERROR %llu NOT_FOUND\n",id); fflush(stdout); free(line); return 0;
+    }
     if(!strncmp(line,"CANCEL ",7)){
         unsigned long long id=0; char tail;
         if(sscanf(line+7,"%llu %c",&id,&tail)!=1 || id==0){
diff --git a/c/openai_server.py b/c/openai_server.py
index 074f51533..aa50986a3 100644
--- a/c/openai_server.py
+++ b/c/openai_server.py
@@ -832,6 +832,72 @@ def anthropic_tools(body):
     'jws ::= ( " " | "\\t" | "\\n" | "\\r" )*\n'
 )
 
+
+def parse_stop_sequences(body):
+    value = body.get("stop")
+    if value is None:
+        return ()
+    if isinstance(value, str):
+        sequences = [value]
+    elif isinstance(value, list):
+        sequences = value
+    else:
+        raise APIError(400, "`stop` must be a string or an array of strings.",
+                       "stop", "invalid_value")
+    if not 1 <= len(sequences) <= 4:
+        raise APIError(400, "`stop` must contain between 1 and 4 sequences.",
+                       "stop", "invalid_value")
+    for index, sequence in enumerate(sequences):
+        if not isinstance(sequence, str) or not sequence:
+            raise APIError(400, "Each `stop` sequence must be a non-empty string.",
+                           f"stop.{index}", "invalid_value")
+    return tuple(sequences)
+
+
+class StopFilter:
+    """Stream text without exposing a full or partial stop sequence."""
+    def __init__(self, sequences, emit):
+        self.sequences = tuple(sequences)
+        self.emit = emit
+        self.pending = ""
+        self.matched = None
+
+    def feed(self, chunk):
+        if self.matched is not None:
+            return
+        text = self.pending + chunk
+        match = None
+        for order, sequence in enumerate(self.sequences):
+            offset = text.find(sequence)
+            candidate = (offset, order, sequence)
+            if offset >= 0 and (match is None or candidate[:2] < match[:2]):
+                match = candidate
+        if match is not None:
+            offset, _order, self.matched = match
+            if offset:
+                self.emit(text[:offset])
+            self.pending = ""
+            return
+
+        hold = 0
+        maximum = min(len(text), max((len(s) - 1 for s in self.sequences), default=0))
+        for size in range(1, maximum + 1):
+            suffix = text[-size:]
+            if any(sequence.startswith(suffix) for sequence in self.sequences):
+                hold = size
+        flush = len(text) - hold
+        if flush:
+            self.emit(text[:flush])
+        self.pending = text[flush:]
+
+    def finish(self):
+        if self.matched is None and self.pending:
+            self.emit(self.pending)
+        self.pending = ""
+
+    def stopped(self):
+        return self.matched is not None
+
 def generation_options(body, limit):
     if body.get("n", 1) != 1:
         raise APIError(400, "Colibri currently supports `n=1` only.", "n", "unsupported_value")
@@ -878,8 +944,7 @@ def generation_options(body, limit):
                            "tool_choice", "invalid_value")
         if choice != "none" and not (body.get("tools") or body.get("functions")):
             raise APIError(400, "`tool_choice` requires `tools`.", "tool_choice", "invalid_value")
-    if body.get("stop") is not None:
-        raise APIError(400, "Custom stop sequences are not supported yet.", "stop", "unsupported_parameter")
+    stop_sequences = parse_stop_sequences(body)
     if body.get("logprobs"):
         raise APIError(400, "Log probabilities are not supported yet.", "logprobs", "unsupported_parameter")
     if body.get("frequency_penalty", 0) or body.get("presence_penalty", 0):
@@ -944,7 +1009,7 @@ def generation_options(body, limit):
     if (isinstance(top_p, bool) or not isinstance(top_p, (int, float)) or
             not math.isfinite(top_p) or not 0 < top_p <= 1):
         raise APIError(400, "`top_p` must be greater than 0 and at most 1.", "top_p")
-    return maximum, float(temperature), float(top_p), grammar
+    return maximum, float(temperature), float(top_p), grammar, stop_sequences
 
 
 def read_engine_turn(stream, sentinel, on_bytes):
@@ -1108,7 +1173,7 @@ def _dispatch_stdout(self):
                 self._fail_pending(error)
 
     def generate(self, prompt, max_tokens, temperature, top_p, on_text, cache_slot=0,
-                 cancelled=None, grammar=None):
+                 cancelled=None, grammar=None, stopped=None):
         if isinstance(cache_slot, bool) or not isinstance(cache_slot, int) or not 0 <= cache_slot < self.kv_slots:
             raise APIError(400, "Invalid cache slot.", "cache_slot")
         payload = prompt.encode("utf-8")
@@ -1150,12 +1215,18 @@ def decode(data):
             raise
 
         cancel_sent = False
+        stop_sent = False
         while True:
             kind, value = events.get()
             if kind == "data":
-                if not cancel_sent:
+                if not cancel_sent and not stop_sent:
                     decode(value)
-                    if cancelled and cancelled():
+                    if stopped and stopped():
+                        stop_sent = True
+                        with self.write_lock:
+                            self.process.stdin.write(f"STOP {request_id}\n".encode())
+                            self.process.stdin.flush()
+                    elif cancelled and cancelled():
                         cancel_sent = True
                         with self.write_lock:
                             self.process.stdin.write(f"CANCEL {request_id}\n".encode())
@@ -1443,7 +1514,8 @@ def generation(self, body, prompt, request_id, chat, tools=None, tool_choice=Non
         if dbg >= 2:
             sys.stderr.write(f"\n===== PROMPT [{request_id}] =====\n{prompt}\n===== OUTPUT [{request_id}] =====\n")
             sys.stderr.flush()
-        maximum, temperature, top_p, grammar = generation_options(body, self.server.max_tokens)
+        maximum, temperature, top_p, grammar, stop_sequences = generation_options(
+            body, self.server.max_tokens)
         if grammar is not None and ARCH == "inkling":
             # inkling.c's serve loop speaks the 6-field SUBMIT header only; sending the
             # grammar payload extension would desync its stdin framing.
@@ -1475,9 +1547,11 @@ def generation(self, body, prompt, request_id, chat, tools=None, tool_choice=Non
             queue_headers = {"x-colibri-queue-wait-ms": str(round(queue_wait * 1000))}
             if not stream:
                 output = []
+                stop_filter = StopFilter(stop_sequences, output.append)
                 stats = self.server.engine.generate(
-                    prompt, maximum, temperature, top_p, output.append, cache_slot,
-                    self.client_disconnected, grammar=grammar)
+                    prompt, maximum, temperature, top_p, stop_filter.feed, cache_slot,
+                    self.client_disconnected, grammar=grammar, stopped=stop_filter.stopped)
+                stop_filter.finish()
                 text = "".join(output)
                 reasoning = ""
                 if ARCH == "inkling":
@@ -1597,9 +1671,11 @@ def emit_tools(chunk):
                     if flush:
                         emit(sp["buf"][:flush])
                         sp["buf"] = sp["buf"][flush:]
+                stop_filter = StopFilter(stop_sequences, emit_tools)
                 stats = self.server.engine.generate(
-                    prompt, maximum, temperature, top_p, emit_tools, cache_slot,
-                    lambda: not connected, grammar=grammar)
+                    prompt, maximum, temperature, top_p, stop_filter.feed, cache_slot,
+                    lambda: not connected, grammar=grammar, stopped=stop_filter.stopped)
+                stop_filter.finish()
                 if not sp["tool"] and sp["buf"]:
                     emit(sp["buf"])                     # no tool call happened: flush held tail
                 _content, calls = parse_tool_calls("".join(raw), tools)
@@ -1614,9 +1690,11 @@ def emit_plain(chunk):
                     if dbg_echo:
                         sys.stderr.write(chunk); sys.stderr.flush()
                     (splitter.feed if splitter else emit)(chunk)
+                stop_filter = StopFilter(stop_sequences, emit_plain)
                 stats = self.server.engine.generate(
-                    prompt, maximum, temperature, top_p, emit_plain, cache_slot,
-                    lambda: not connected, grammar=grammar)
+                    prompt, maximum, temperature, top_p, stop_filter.feed, cache_slot,
+                    lambda: not connected, grammar=grammar, stopped=stop_filter.stopped)
+                stop_filter.finish()
                 if splitter:
                     splitter.close()
                 finish = "length" if stats["length_limited"] else "stop"
@@ -1710,7 +1788,8 @@ def anthropic_messages(self, body, request_id):
         self.anthropic_generation(translated, prompt, request_id, tools, enable_thinking)
 
     def anthropic_generation(self, body, prompt, request_id, tools, enable_thinking):
-        maximum, temperature, top_p, grammar = generation_options(body, self.server.max_tokens)
+        maximum, temperature, top_p, grammar, _stop_sequences = generation_options(
+            body, self.server.max_tokens)
         cache_slot = body.get("cache_slot")
         if (cache_slot is not None and
                 (isinstance(cache_slot, bool) or not isinstance(cache_slot, int) or
diff --git a/c/tests/test_openai_server.py b/c/tests/test_openai_server.py
index 98da4c18d..24e6f47ee 100644
--- a/c/tests/test_openai_server.py
+++ b/c/tests/test_openai_server.py
@@ -11,19 +11,23 @@
 from pathlib import Path
 
 from openai_server import (APIError, APIHandler, APIServer, ClientCancelled, END, GenerationScheduler,
-                           READY, Engine, _engine_error, generation_options, parse_tool_calls,
-                           read_engine_turn, render_chat, serve)
+                           READY, Engine, StopFilter, _engine_error, generation_options,
+                           parse_tool_calls, read_engine_turn, render_chat, serve)
 
 
 class FakeEngine:
     def __init__(self):
         self.calls = []
+        self.stop_requests = 0
 
     def generate(self, prompt, maximum, temperature, top_p, on_text, cache_slot=0,
-                 cancelled=None, grammar=None):
+                 cancelled=None, grammar=None, stopped=None):
         self.calls.append((prompt, maximum, temperature, top_p, cache_slot, grammar))
-        on_text("Hé")
-        on_text("llo")
+        for chunk in ("Hé", "llo"):
+            on_text(chunk)
+            if stopped and stopped():
+                self.stop_requests += 1
+                break
         return {"prompt_tokens": 7, "completion_tokens": 2, "length_limited": False}
 
 
@@ -34,11 +38,11 @@ def __init__(self):
         self.release = threading.Event()
 
     def generate(self, prompt, maximum, temperature, top_p, on_text, cache_slot=0,
-                 cancelled=None, grammar=None):
+                 cancelled=None, grammar=None, stopped=None):
         self.entered.set()
         self.release.wait(2)
         return super().generate(prompt, maximum, temperature, top_p, on_text, cache_slot,
-                                cancelled)
+                                cancelled, grammar, stopped)
 
 
 class TemplateTest(unittest.TestCase):
@@ -71,11 +75,11 @@ def test_renders_thinking_prefix(self):
 
     def test_validates_generation_limits(self):
         self.assertEqual(generation_options({"max_tokens": 4, "temperature": 0, "top_p": 1}, 8),
-                         (4, 0.0, 1.0, None))
+                         (4, 0.0, 1.0, None, ()))
         # max_tokens above the server cap is clamped, not rejected (#260): OpenAI
         # clients default to large values; erroring breaks them.
         self.assertEqual(generation_options({"max_tokens": 9, "temperature": 0, "top_p": 1}, 8),
-                         (8, 0.0, 1.0, None))
+                         (8, 0.0, 1.0, None, ()))
         # non-positive / non-int max_tokens is still a hard error
         with self.assertRaises(APIError):
             generation_options({"max_tokens": 0}, 8)
@@ -84,7 +88,7 @@ def test_validates_generation_limits(self):
         with self.assertRaises(APIError):
             generation_options({"top_p": math.inf}, 8)
         self.assertEqual(generation_options({"temperature": None, "top_p": None}, 8),
-                         (8, 0.7, 0.9, None))
+                         (8, 0.7, 0.9, None, ()))
         # response_format -> grammar plumbing (draft source, never a constraint)
         opts = generation_options({"max_tokens": 4, "response_format": {"type": "json_object"}}, 8)
         self.assertIn("root ::=", opts[3])
@@ -110,6 +114,32 @@ def test_validates_generation_limits(self):
         opts = generation_options({"response_format": {"type": "gbnf", "grammar": "not a grammar ::="}}, 8)
         self.assertEqual(opts[3], "not a grammar ::=")
 
+    def test_validates_stop_sequences(self):
+        self.assertEqual(generation_options({"stop": "END"}, 8)[4], ("END",))
+        self.assertEqual(generation_options({"stop": ["ONE", "TWO"]}, 8)[4],
+                         ("ONE", "TWO"))
+        for value in ("", [], [""], ["1", "2", "3", "4", "5"], 7, ["ok", 7]):
+            with self.subTest(value=value), self.assertRaises(APIError):
+                generation_options({"stop": value}, 8)
+
+
+class StopFilterTest(unittest.TestCase):
+    def test_hides_match_split_across_chunks(self):
+        output = []
+        stop_filter = StopFilter(("STOP",), output.append)
+        for chunk in ("answer S", "TO", "Pignored"):
+            stop_filter.feed(chunk)
+        stop_filter.finish()
+        self.assertEqual("".join(output), "answer ")
+        self.assertEqual(stop_filter.matched, "STOP")
+
+    def test_flushes_partial_prefix_when_generation_finishes(self):
+        output = []
+        stop_filter = StopFilter(("STOP",), output.append)
+        stop_filter.feed("answer ST")
+        stop_filter.finish()
+        self.assertEqual("".join(output), "answer ST")
+
 
 class ProtocolTest(unittest.TestCase):
     def test_reads_payload_and_extended_status(self):
@@ -446,6 +476,30 @@ def respond(process, frame):
         self.assertEqual(output, ["x"])
         self.assertEqual(process.writes[-1].split(), [b"CANCEL", request_id])
 
+    def test_stops_generation_through_successful_done_path(self):
+        request_id = None
+
+        def respond(process, frame):
+            nonlocal request_id
+            fields = frame.split()
+            if fields[0] == b"SUBMIT":
+                request_id = fields[1]
+                process.stdout.feed(b"DATA " + request_id + b" 1\nx\n")
+            elif fields[0] == b"STOP":
+                self.assertEqual(fields[1], request_id)
+                process.stdout.feed(b"DONE " + request_id + b" STAT 1 1 0 1 2 0\n")
+
+        process = FakeProcess(respond)
+        with patch("openai_server.subprocess.Popen", return_value=process):
+            engine = Engine("glm", "model")
+        output = []
+        stats = engine.generate("hello", 8, 0.7, 0.9, output.append,
+                                stopped=lambda: output == ["x"])
+        engine.close()
+        self.assertEqual(output, ["x"])
+        self.assertEqual(stats["completion_tokens"], 1)
+        self.assertEqual(process.writes[-1].split(), [b"STOP", request_id])
+
 
 class HTTPTest(unittest.TestCase):
     @classmethod
@@ -525,6 +579,17 @@ def test_chat_completion(self):
         self.assertIn("<|user|>Hi<|assistant|><think></think>", self.engine.calls[-1][0])
         self.assertEqual(self.engine.calls[-1][4], 1)
 
+    def test_chat_completion_stops_across_engine_chunks(self):
+        before = self.engine.stop_requests
+        with self.request("/v1/chat/completions", {
+            "model": "test-model", "messages": [{"role": "user", "content": "Hi"}],
+            "stop": "éll",
+        }) as response:
+            body = json.load(response)
+        self.assertEqual(body["choices"][0]["message"]["content"], "H")
+        self.assertEqual(body["choices"][0]["finish_reason"], "stop")
+        self.assertEqual(self.engine.stop_requests, before + 1)
+
     def test_rejects_invalid_cache_slot(self):
         with self.assertRaises(HTTPError) as caught:
             self.request("/v1/chat/completions", {
@@ -545,6 +610,21 @@ def test_streaming_chat_completion(self):
         self.assertIn('\"usage\":{\"prompt_tokens\":7,\"completion_tokens\":2,\"total_tokens\":9}', stream)
         self.assertTrue(stream.endswith("data: [DONE]\n\n"))
 
+    def test_streaming_stop_never_exposes_partial_sequence(self):
+        before = self.engine.stop_requests
+        with self.request("/v1/chat/completions", {
+            "model": "test-model", "messages": [{"role": "user", "content": "Hi"}],
+            "stream": True, "stop": "éll",
+        }) as response:
+            raw = response.read().decode()
+        payloads = [json.loads(line[6:]) for line in raw.splitlines()
+                    if line.startswith("data: ") and line != "data: [DONE]"]
+        content = "".join((choice.get("delta") or {}).get("content", "")
+                          for payload in payloads for choice in payload["choices"])
+        self.assertEqual(content, "H")
+        self.assertEqual(payloads[-1]["choices"][0]["finish_reason"], "stop")
+        self.assertEqual(self.engine.stop_requests, before + 1)
+
     def test_legacy_completion(self):
         with self.request("/v1/completions", {
             "model": "test-model", "prompt": "Complete me", "temperature": 0,
diff --git a/docs/api.md b/docs/api.md
index 2b92bb6ed..f3bb16c3e 100644
--- a/docs/api.md
+++ b/docs/api.md
@@ -24,14 +24,16 @@ curl http://127.0.0.1:8000/v1/chat/completions \
 Implemented endpoints are `GET /v1/models`, `GET /v1/models/{model}`,
 `POST /v1/chat/completions`, and legacy `POST /v1/completions`. Chat and
 completion requests support JSON responses, SSE streaming, usage counts,
-`max_tokens`/`max_completion_tokens`, `temperature`, and `top_p`. The extension
+`max_tokens`/`max_completion_tokens`, `temperature`, `top_p`, and up to four
+custom `stop` sequences. Stop sequences are removed from the response and end
+generation early in both JSON and streaming modes. The extension
 `enable_thinking: true` enables GLM-5.2's reasoning block; the standard
 `reasoning_effort` field also enables it unless set to `none`.
 
 The server is deliberately text-only and serves one generation at a time: the
 744B model stays in one persistent process, so concurrent HTTP requests queue
-instead of loading duplicate model copies. Tools, image/audio input, custom
-stop sequences, log probabilities, and token penalties return an explicit error
+instead of loading duplicate model copies. Image/audio input, log probabilities,
+and token penalties return an explicit error
 rather than being silently ignored. The default bind address is localhost; set
 `COLI_API_KEY` before exposing the server beyond the machine.
 
diff --git a/docs/serve_protocol.md b/docs/serve_protocol.md
index b0eb61ec6..8b9000b47 100644
--- a/docs/serve_protocol.md
+++ b/docs/serve_protocol.md
@@ -33,6 +33,7 @@ recognize**; that is the protocol's forward-compatibility rule.
 
 ```
 SUBMIT <id> <slot> <bytes> <max_tokens> <temperature> <top_p>\n<payload>\n
+STOP <id>\n
 CANCEL <id>\n
 ```
 
@@ -44,6 +45,10 @@ CANCEL <id>\n
 - `bytes` — exact byte length of `payload` (UTF-8, may contain newlines). The engine
   reads exactly that many bytes after the header line, then one trailing `\n`.
 - `payload` — the fully rendered prompt (the server owns the chat template).
+- `STOP` ends generation through the normal successful `DONE` path. Statistics,
+  usage history, and KV state are persisted; the HTTP gateway uses it after a
+  client-provided stop sequence matches.
+- `CANCEL` aborts a request after its client disconnects and returns `CANCELLED`.
 - EOF on stdin = graceful shutdown: in-flight requests finish first.
 
 Prefill is serial; decode is continuously batched — every active slot contributes

From 37c8fac9b5bdbc64eda71e927ba049e53a442966 Mon Sep 17 00:00:00 2001
From: Bill Herald <641463+bherald@users.noreply.github.com>
Date: Wed, 22 Jul 2026 17:35:51 -0400
Subject: [PATCH 2/4] feat: support patient native stop handling

---
 c/openai_server.py            | 57 ++++++++++++++++++++++++-----------
 c/tests/test_openai_server.py | 44 +++++++++++++++++++++++++++
 docs/api.md                   |  4 +++
 3 files changed, 88 insertions(+), 17 deletions(-)

diff --git a/c/openai_server.py b/c/openai_server.py
index aa50986a3..413ab2e00 100644
--- a/c/openai_server.py
+++ b/c/openai_server.py
@@ -856,27 +856,46 @@ def parse_stop_sequences(body):
 
 class StopFilter:
     """Stream text without exposing a full or partial stop sequence."""
-    def __init__(self, sequences, emit):
+    def __init__(self, sequences, emit, ignore_leading=False):
         self.sequences = tuple(sequences)
         self.emit = emit
+        self.ignore_leading = ignore_leading
         self.pending = ""
         self.matched = None
+        self.useful_content_seen = False
+        self.leading_matches_ignored = 0
+
+    def _emit(self, text):
+        if text:
+            self.emit(text)
+            if text.strip():
+                self.useful_content_seen = True
 
     def feed(self, chunk):
         if self.matched is not None:
             return
         text = self.pending + chunk
-        match = None
-        for order, sequence in enumerate(self.sequences):
-            offset = text.find(sequence)
-            candidate = (offset, order, sequence)
-            if offset >= 0 and (match is None or candidate[:2] < match[:2]):
-                match = candidate
-        if match is not None:
-            offset, _order, self.matched = match
-            if offset:
-                self.emit(text[:offset])
-            self.pending = ""
+        self.pending = ""
+        while True:
+            match = None
+            for order, sequence in enumerate(self.sequences):
+                offset = text.find(sequence)
+                candidate = (offset, order, sequence)
+                if offset >= 0 and (match is None or candidate[:2] < match[:2]):
+                    match = candidate
+            if match is None:
+                break
+            offset, _order, sequence = match
+            prefix = text[:offset]
+            if (self.ignore_leading and not self.useful_content_seen
+                    and not prefix.strip()):
+                self.leading_matches_ignored += 1
+                text = text[offset + len(sequence):]
+                if not text:
+                    return
+                continue
+            self.matched = sequence
+            self._emit(prefix)
             return
 
         hold = 0
@@ -887,12 +906,12 @@ def feed(self, chunk):
                 hold = size
         flush = len(text) - hold
         if flush:
-            self.emit(text[:flush])
+            self._emit(text[:flush])
         self.pending = text[flush:]
 
     def finish(self):
         if self.matched is None and self.pending:
-            self.emit(self.pending)
+            self._emit(self.pending)
         self.pending = ""
 
     def stopped(self):
@@ -1521,6 +1540,10 @@ def generation(self, body, prompt, request_id, chat, tools=None, tool_choice=Non
             # grammar payload extension would desync its stdin framing.
             raise APIError(400, "`response_format` grammars are not supported by the Inkling "
                                 "engine yet.", "response_format", "unsupported_parameter")
+        ignore_leading_stop = body.get("x_colibri_ignore_leading_stop", False)
+        if not isinstance(ignore_leading_stop, bool):
+            raise APIError(400, "`x_colibri_ignore_leading_stop` must be a boolean.",
+                           "x_colibri_ignore_leading_stop", "invalid_value")
         # tools and tool_choice come from chat_completion() already processed/filtered
         if chat and tool_choice == "none":
             tools = None          # client forbade tools: never surface tool_calls
@@ -1547,7 +1570,7 @@ def generation(self, body, prompt, request_id, chat, tools=None, tool_choice=Non
             queue_headers = {"x-colibri-queue-wait-ms": str(round(queue_wait * 1000))}
             if not stream:
                 output = []
-                stop_filter = StopFilter(stop_sequences, output.append)
+                stop_filter = StopFilter(stop_sequences, output.append, ignore_leading_stop)
                 stats = self.server.engine.generate(
                     prompt, maximum, temperature, top_p, stop_filter.feed, cache_slot,
                     self.client_disconnected, grammar=grammar, stopped=stop_filter.stopped)
@@ -1671,7 +1694,7 @@ def emit_tools(chunk):
                     if flush:
                         emit(sp["buf"][:flush])
                         sp["buf"] = sp["buf"][flush:]
-                stop_filter = StopFilter(stop_sequences, emit_tools)
+                stop_filter = StopFilter(stop_sequences, emit_tools, ignore_leading_stop)
                 stats = self.server.engine.generate(
                     prompt, maximum, temperature, top_p, stop_filter.feed, cache_slot,
                     lambda: not connected, grammar=grammar, stopped=stop_filter.stopped)
@@ -1690,7 +1713,7 @@ def emit_plain(chunk):
                     if dbg_echo:
                         sys.stderr.write(chunk); sys.stderr.flush()
                     (splitter.feed if splitter else emit)(chunk)
-                stop_filter = StopFilter(stop_sequences, emit_plain)
+                stop_filter = StopFilter(stop_sequences, emit_plain, ignore_leading_stop)
                 stats = self.server.engine.generate(
                     prompt, maximum, temperature, top_p, stop_filter.feed, cache_slot,
                     lambda: not connected, grammar=grammar, stopped=stop_filter.stopped)
diff --git a/c/tests/test_openai_server.py b/c/tests/test_openai_server.py
index 24e6f47ee..eb40ce445 100644
--- a/c/tests/test_openai_server.py
+++ b/c/tests/test_openai_server.py
@@ -140,6 +140,32 @@ def test_flushes_partial_prefix_when_generation_finishes(self):
         stop_filter.finish()
         self.assertEqual("".join(output), "answer ST")
 
+    def test_optional_patient_mode_ignores_only_leading_matches(self):
+        output = []
+        stop_filter = StopFilter(("<|user|>",), output.append, ignore_leading=True)
+        for chunk in ("<|us", "er|>answer", "<|user|>ignored"):
+            stop_filter.feed(chunk)
+        stop_filter.finish()
+        self.assertEqual("".join(output), "answer")
+        self.assertEqual(stop_filter.matched, "<|user|>")
+        self.assertEqual(stop_filter.leading_matches_ignored, 1)
+
+    def test_patient_mode_preserves_remainder_after_same_chunk_leading_match(self):
+        output = []
+        stop_filter = StopFilter(("STOP",), output.append, ignore_leading=True)
+        stop_filter.feed("STOPuseful STOPdiscarded")
+        stop_filter.finish()
+        self.assertEqual("".join(output), "useful ")
+        self.assertEqual(stop_filter.matched, "STOP")
+
+    def test_strict_mode_still_stops_on_a_leading_match(self):
+        output = []
+        stop_filter = StopFilter(("STOP",), output.append)
+        stop_filter.feed("STOPignored")
+        stop_filter.finish()
+        self.assertEqual(output, [])
+        self.assertEqual(stop_filter.matched, "STOP")
+
 
 class ProtocolTest(unittest.TestCase):
     def test_reads_payload_and_extended_status(self):
@@ -590,6 +616,24 @@ def test_chat_completion_stops_across_engine_chunks(self):
         self.assertEqual(body["choices"][0]["finish_reason"], "stop")
         self.assertEqual(self.engine.stop_requests, before + 1)
 
+    def test_patient_stop_extension_ignores_a_leading_match(self):
+        before = self.engine.stop_requests
+        with self.request("/v1/chat/completions", {
+            "model": "test-model", "messages": [{"role": "user", "content": "Hi"}],
+            "stop": "H", "x_colibri_ignore_leading_stop": True,
+        }) as response:
+            body = json.load(response)
+        self.assertEqual(body["choices"][0]["message"]["content"], "éllo")
+        self.assertEqual(self.engine.stop_requests, before)
+
+    def test_patient_stop_extension_requires_a_boolean(self):
+        with self.assertRaises(HTTPError) as caught:
+            self.request("/v1/chat/completions", {
+                "model": "test-model", "messages": [{"role": "user", "content": "Hi"}],
+                "stop": "H", "x_colibri_ignore_leading_stop": "yes",
+            })
+        self.assertEqual(caught.exception.code, 400)
+
     def test_rejects_invalid_cache_slot(self):
         with self.assertRaises(HTTPError) as caught:
             self.request("/v1/chat/completions", {
diff --git a/docs/api.md b/docs/api.md
index f3bb16c3e..80ecbbe70 100644
--- a/docs/api.md
+++ b/docs/api.md
@@ -27,6 +27,10 @@ completion requests support JSON responses, SSE streaming, usage counts,
 `max_tokens`/`max_completion_tokens`, `temperature`, `top_p`, and up to four
 custom `stop` sequences. Stop sequences are removed from the response and end
 generation early in both JSON and streaming modes. The extension
+`x_colibri_ignore_leading_stop: true` discards leading stop sequences until
+the first non-whitespace response content, which is useful for local templates
+that occasionally emit a role marker before the answer; strict OpenAI stop
+behavior remains the default. The extension
 `enable_thinking: true` enables GLM-5.2's reasoning block; the standard
 `reasoning_effort` field also enables it unless set to `none`.
 

From 83fc0a71b8cb2d3a128b8b4d47ab48731b05c247 Mon Sep 17 00:00:00 2001
From: bill <bherald@wphc.us>
Date: Thu, 23 Jul 2026 06:41:21 -0400
Subject: [PATCH 3/4] fix: stop chat at implicit role boundaries

---
 c/openai_server.py            | 24 +++++++++++++++++++-----
 c/tests/test_openai_server.py | 15 +++++++++++++--
 docs/api.md                   |  6 +++++-
 3 files changed, 37 insertions(+), 8 deletions(-)

diff --git a/c/openai_server.py b/c/openai_server.py
index 413ab2e00..fa336fcdc 100644
--- a/c/openai_server.py
+++ b/c/openai_server.py
@@ -832,6 +832,8 @@ def anthropic_tools(body):
     'jws ::= ( " " | "\\t" | "\\n" | "\\r" )*\n'
 )
 
+DEFAULT_CHAT_STOP_SEQUENCES = ("<|user|>", "<|observation|>")
+
 
 def parse_stop_sequences(body):
     value = body.get("stop")
@@ -854,6 +856,21 @@ def parse_stop_sequences(body):
     return tuple(sequences)
 
 
+def stop_policy(body, chat):
+    sequences = parse_stop_sequences(body)
+    ignore_leading = body.get("x_colibri_ignore_leading_stop", False)
+    if not isinstance(ignore_leading, bool):
+        raise APIError(400, "`x_colibri_ignore_leading_stop` must be a boolean.",
+                       "x_colibri_ignore_leading_stop", "invalid_value")
+    if chat and not sequences:
+        # The chat template owns these role boundaries, so generic OpenAI
+        # clients should not need model-specific stop knowledge. Treat an
+        # occasional leading marker patiently; client-provided stops remain
+        # strict unless the extension is explicitly requested.
+        return DEFAULT_CHAT_STOP_SEQUENCES, True
+    return sequences, ignore_leading
+
+
 class StopFilter:
     """Stream text without exposing a full or partial stop sequence."""
     def __init__(self, sequences, emit, ignore_leading=False):
@@ -1533,17 +1550,14 @@ def generation(self, body, prompt, request_id, chat, tools=None, tool_choice=Non
         if dbg >= 2:
             sys.stderr.write(f"\n===== PROMPT [{request_id}] =====\n{prompt}\n===== OUTPUT [{request_id}] =====\n")
             sys.stderr.flush()
-        maximum, temperature, top_p, grammar, stop_sequences = generation_options(
+        maximum, temperature, top_p, grammar, _requested_stop_sequences = generation_options(
             body, self.server.max_tokens)
         if grammar is not None and ARCH == "inkling":
             # inkling.c's serve loop speaks the 6-field SUBMIT header only; sending the
             # grammar payload extension would desync its stdin framing.
             raise APIError(400, "`response_format` grammars are not supported by the Inkling "
                                 "engine yet.", "response_format", "unsupported_parameter")
-        ignore_leading_stop = body.get("x_colibri_ignore_leading_stop", False)
-        if not isinstance(ignore_leading_stop, bool):
-            raise APIError(400, "`x_colibri_ignore_leading_stop` must be a boolean.",
-                           "x_colibri_ignore_leading_stop", "invalid_value")
+        stop_sequences, ignore_leading_stop = stop_policy(body, chat)
         # tools and tool_choice come from chat_completion() already processed/filtered
         if chat and tool_choice == "none":
             tools = None          # client forbade tools: never surface tool_calls
diff --git a/c/tests/test_openai_server.py b/c/tests/test_openai_server.py
index eb40ce445..c3c999d18 100644
--- a/c/tests/test_openai_server.py
+++ b/c/tests/test_openai_server.py
@@ -10,9 +10,10 @@
 from urllib.request import Request, urlopen
 from pathlib import Path
 
-from openai_server import (APIError, APIHandler, APIServer, ClientCancelled, END, GenerationScheduler,
+from openai_server import (APIError, APIHandler, APIServer, ClientCancelled,
+                           DEFAULT_CHAT_STOP_SEQUENCES, END, GenerationScheduler,
                            READY, Engine, StopFilter, _engine_error, generation_options,
-                           parse_tool_calls, read_engine_turn, render_chat, serve)
+                           parse_tool_calls, read_engine_turn, render_chat, serve, stop_policy)
 
 
 class FakeEngine:
@@ -122,6 +123,16 @@ def test_validates_stop_sequences(self):
             with self.subTest(value=value), self.assertRaises(APIError):
                 generation_options({"stop": value}, 8)
 
+    def test_chat_defaults_role_stops_without_changing_client_or_completion_policy(self):
+        self.assertEqual(stop_policy({}, True), (DEFAULT_CHAT_STOP_SEQUENCES, True))
+        self.assertEqual(stop_policy({}, False), ((), False))
+        self.assertEqual(stop_policy({"stop": "END"}, True), (("END",), False))
+        self.assertEqual(stop_policy({
+            "stop": "END", "x_colibri_ignore_leading_stop": True,
+        }, True), (("END",), True))
+        with self.assertRaises(APIError):
+            stop_policy({"x_colibri_ignore_leading_stop": "yes"}, True)
+
 
 class StopFilterTest(unittest.TestCase):
     def test_hides_match_split_across_chunks(self):
diff --git a/docs/api.md b/docs/api.md
index 80ecbbe70..8522f3bef 100644
--- a/docs/api.md
+++ b/docs/api.md
@@ -30,7 +30,11 @@ generation early in both JSON and streaming modes. The extension
 `x_colibri_ignore_leading_stop: true` discards leading stop sequences until
 the first non-whitespace response content, which is useful for local templates
 that occasionally emit a role marker before the answer; strict OpenAI stop
-behavior remains the default. The extension
+behavior remains the default for client-provided sequences. Chat requests with
+no client `stop` automatically use the template's `<|user|>` and
+`<|observation|>` role markers, patiently ignoring only leading markers; this
+prevents a model-completed turn from silently generating a new user or tool
+turn. Legacy completion requests receive no implicit stop sequences. The extension
 `enable_thinking: true` enables GLM-5.2's reasoning block; the standard
 `reasoning_effort` field also enables it unless set to `none`.
 

From 74326b9d9182a0c69e7423586bbfe75f60aa48d7 Mon Sep 17 00:00:00 2001
From: bill <bherald@wphc.us>
Date: Thu, 23 Jul 2026 16:41:14 -0400
Subject: [PATCH 4/4] fix: scope implicit stops to GLM chat

---
 c/openai_server.py            |  9 +++++----
 c/tests/test_openai_server.py | 36 ++++++++++++++++++++++++++---------
 docs/api.md                   |  7 ++++---
 3 files changed, 36 insertions(+), 16 deletions(-)

diff --git a/c/openai_server.py b/c/openai_server.py
index fa336fcdc..0080bd73e 100644
--- a/c/openai_server.py
+++ b/c/openai_server.py
@@ -862,10 +862,11 @@ def stop_policy(body, chat):
     if not isinstance(ignore_leading, bool):
         raise APIError(400, "`x_colibri_ignore_leading_stop` must be a boolean.",
                        "x_colibri_ignore_leading_stop", "invalid_value")
-    if chat and not sequences:
-        # The chat template owns these role boundaries, so generic OpenAI
-        # clients should not need model-specific stop knowledge. Treat an
-        # occasional leading marker patiently; client-provided stops remain
+    if chat and ARCH == "glm" and not sequences:
+        # The GLM chat template owns these role boundaries, so generic OpenAI
+        # clients should not need model-specific stop knowledge. Inkling has a
+        # different marker family and receives no implicit GLM stops. Treat an
+        # occasional leading GLM marker patiently; client-provided stops remain
         # strict unless the extension is explicitly requested.
         return DEFAULT_CHAT_STOP_SEQUENCES, True
     return sequences, ignore_leading
diff --git a/c/tests/test_openai_server.py b/c/tests/test_openai_server.py
index c3c999d18..a9a31e69b 100644
--- a/c/tests/test_openai_server.py
+++ b/c/tests/test_openai_server.py
@@ -12,8 +12,9 @@
 
 from openai_server import (APIError, APIHandler, APIServer, ClientCancelled,
                            DEFAULT_CHAT_STOP_SEQUENCES, END, GenerationScheduler,
-                           READY, Engine, StopFilter, _engine_error, generation_options,
-                           parse_tool_calls, read_engine_turn, render_chat, serve, stop_policy)
+                           READY, Engine, InklingStreamSplit, StopFilter, _engine_error,
+                           generation_options, parse_tool_calls, read_engine_turn, render_chat,
+                           serve, stop_policy)
 
 
 class FakeEngine:
@@ -123,18 +124,35 @@ def test_validates_stop_sequences(self):
             with self.subTest(value=value), self.assertRaises(APIError):
                 generation_options({"stop": value}, 8)
 
-    def test_chat_defaults_role_stops_without_changing_client_or_completion_policy(self):
-        self.assertEqual(stop_policy({}, True), (DEFAULT_CHAT_STOP_SEQUENCES, True))
-        self.assertEqual(stop_policy({}, False), ((), False))
-        self.assertEqual(stop_policy({"stop": "END"}, True), (("END",), False))
-        self.assertEqual(stop_policy({
-            "stop": "END", "x_colibri_ignore_leading_stop": True,
-        }, True), (("END",), True))
+    def test_glm_chat_defaults_role_stops_without_changing_other_policies(self):
+        with patch("openai_server.ARCH", "glm"):
+            self.assertEqual(stop_policy({}, True), (DEFAULT_CHAT_STOP_SEQUENCES, True))
+            self.assertEqual(stop_policy({}, False), ((), False))
+            self.assertEqual(stop_policy({"stop": "END"}, True), (("END",), False))
+            self.assertEqual(stop_policy({
+                "stop": "END", "x_colibri_ignore_leading_stop": True,
+            }, True), (("END",), True))
+        with patch("openai_server.ARCH", "inkling"):
+            self.assertEqual(stop_policy({}, True), ((), False))
+            self.assertEqual(stop_policy({"stop": "END"}, True), (("END",), False))
         with self.assertRaises(APIError):
             stop_policy({"x_colibri_ignore_leading_stop": "yes"}, True)
 
 
 class StopFilterTest(unittest.TestCase):
+    def test_explicit_stop_composes_with_inkling_stream_split(self):
+        content = []
+        reasoning = []
+        splitter = InklingStreamSplit(content.append, reasoning.append)
+        stop_filter = StopFilter(("END",), splitter.feed)
+        for chunk in ("<|content_thinking|>why<|content_text|>answer EN", "Dignored"):
+            stop_filter.feed(chunk)
+        stop_filter.finish()
+        splitter.close()
+        self.assertEqual("".join(reasoning), "why")
+        self.assertEqual("".join(content), "answer ")
+        self.assertEqual(stop_filter.matched, "END")
+
     def test_hides_match_split_across_chunks(self):
         output = []
         stop_filter = StopFilter(("STOP",), output.append)
diff --git a/docs/api.md b/docs/api.md
index 8522f3bef..3f5f86b8a 100644
--- a/docs/api.md
+++ b/docs/api.md
@@ -30,11 +30,12 @@ generation early in both JSON and streaming modes. The extension
 `x_colibri_ignore_leading_stop: true` discards leading stop sequences until
 the first non-whitespace response content, which is useful for local templates
 that occasionally emit a role marker before the answer; strict OpenAI stop
-behavior remains the default for client-provided sequences. Chat requests with
-no client `stop` automatically use the template's `<|user|>` and
+behavior remains the default for client-provided sequences. GLM chat requests
+with no client `stop` automatically use the template's `<|user|>` and
 `<|observation|>` role markers, patiently ignoring only leading markers; this
 prevents a model-completed turn from silently generating a new user or tool
-turn. Legacy completion requests receive no implicit stop sequences. The extension
+turn. Inkling chat and legacy completion requests receive no implicit GLM stop
+sequences. The extension
 `enable_thinking: true` enables GLM-5.2's reasoning block; the standard
 `reasoning_effort` field also enables it unless set to `none`.