Adds ping support

lambdas/tokenizer_handler.py

@@ -43,6 +43,13 @@ def lambda_handler(event: dict, lambda_context: Context) -> dict:
 
     query_tokenizer = _get_tokenizer()
 
+    # Handle ping or health check events
+    # We add this after query_tokenizer initialization to ensure the tokenizer is
+    # initialized during cold start, even for pings
+    if "ping" in event:
+        logger.debug("Received ping event")
+        return {"status": "ok"}
+
     # Generate query tokens
     query = event.get("query", "")
 

pyproject.toml

@@ -28,6 +28,9 @@ exclude = ["tests/"]
 
 [tool.pytest.ini_options]
 log_level = "INFO"
+markers = [
+    "integration: tests that load real models/files from disk (slow)",
+]
 
 [tool.ruff]
 target-version = "py314"

tests/test_query_tokenizer.py

@@ -129,3 +129,39 @@ def test_load_idf_raises_filenotfounderror_when_idf_file_missing():
         pytest.raises(FileNotFoundError),
     ):
         QueryTokenizer()
+
+
+# ---------------------------------------------------------------------------
+# Integration tests — load the real tokenizer and IDF from disk
+# Run only integration tests: uv run pytest -m integration
+# Run all except integration tests: uv run pytest -m "not integration"
+# ---------------------------------------------------------------------------
+
+
+@pytest.mark.integration
+def test_integration_tokenize_query_returns_nonempty_dict():
+    """Real tokenizer and IDF: tokenize_query returns a non-empty dict."""
+    qt = QueryTokenizer()
+    result = qt.tokenize_query("machine learning")
+    assert isinstance(result, dict)
+    assert len(result) > 0
+
+
+@pytest.mark.integration
+def test_integration_tokenize_query_values_are_positive_floats():
+    """Real tokenizer and IDF: all weights are positive floats."""
+    qt = QueryTokenizer()
+    result = qt.tokenize_query("machine learning")
+    for token, weight in result.items():
+        assert isinstance(token, str), f"Expected str key, got {type(token)}"
+        assert isinstance(weight, float), f"Expected float weight, got {type(weight)}"
+        assert weight > 0, f"Expected positive weight for {token!r}, got {weight}"
+
+
+@pytest.mark.integration
+def test_integration_tokenize_query_deterministic():
+    """Real tokenizer and IDF: same input always produces the same output."""
+    qt = QueryTokenizer()
+    result_a = qt.tokenize_query("open access repositories")
+    result_b = qt.tokenize_query("open access repositories")
+    assert result_a == result_b

tests/test_tokenizer_handler.py

@@ -59,3 +59,35 @@ def test_returns_error_for_whitespace_only_query(mock_query_tokenizer):
     result = tokenizer_handler.lambda_handler({"query": "   "}, {})
     assert result == {"error": "Query is required in the event payload."}
     mock_query_tokenizer.tokenize_query.assert_not_called()
+
+
+def test_ping_event_returns_ok_status(mock_query_tokenizer):
+    result = tokenizer_handler.lambda_handler({"ping": True}, {})
+    assert result == {"status": "ok"}
+
+
+def test_ping_event_does_not_call_tokenize_query(mock_query_tokenizer):
+    tokenizer_handler.lambda_handler({"ping": True}, {})
+    mock_query_tokenizer.tokenize_query.assert_not_called()
+
+
+def test_non_ping_event_is_not_short_circuited(mock_query_tokenizer):
+    mock_query_tokenizer.tokenize_query.return_value = {"hello": 1.5}
+    result = tokenizer_handler.lambda_handler({"query": "hello"}, {})
+    assert "query" in result
+    mock_query_tokenizer.tokenize_query.assert_called_once()
+
+
+# ---------------------------------------------------------------------------
+# Integration tests — load the real tokenizer and IDF from disk
+# Run only integration tests: uv run pytest -m integration
+# Run all except integration tests: uv run pytest -m "not integration"
+# ---------------------------------------------------------------------------
+
+
+@pytest.mark.integration
+def test_integration_lambda_handler_returns_opensearch_query():
+    tokenizer_handler._get_tokenizer.cache_clear()  # ensure cold start
+    result = tokenizer_handler.lambda_handler({"query": "open access"}, {})
+    assert "query" in result
+    assert "bool" in result["query"]