config.yml.example

# =============================================================================
# TopicStreams configuration
# =============================================================================
# One file, three top-level sections consumed by different processes:
#
#   scraper:         scraper process behavior (cadence, engines, recycle)
#   anti_detection:  scraper process anti-bot fingerprinting / timings
#   api:             API process tuning (DB, rate limiting, feed, health, UI)
#
# Both the scraper and the API read this same file and pull their own section.
# Every key is optional — omit one to take the default shown. The file itself is
# optional too: if absent at startup it is created from this template (the
# scraper logs a warning); the API simply falls back to built-in defaults.
#
# Precedence for the `api:` section: init args > environment > .env > this file
# > built-in default. So secrets and Docker-startup vars (POSTGRES_*, API_PORT,
# HOST_PORT, TOPICSTREAMS_API_KEY, SCRAPER_PROXY) stay in .env — they win — while this file is
# the preferred surface for tunable defaults. Any `api:` key can still be set via
# the environment to override it for a single deployment.

# -----------------------------------------------------------------------------
# Scraper
# -----------------------------------------------------------------------------
scraper:
  scrape_interval: 60  # Seconds between scrape cycles (start-to-start)
  max_pages: 1  # Maximum result pages to scrape per topic
  # Search engines to scrape, in priority order. Add or remove list entries to
  # enable/disable engines. All supported engines are on by default.
  # (DuckDuckGo is not supported — it hard-blocks scraping; see
  # docs/DUCKDUCKGO_UNSUPPORTED.md.)
  engines:
    - google
    - bing
    - yahoo
    - brave
  # How the enabled engines combine each cycle:
  #   all      - scrape every enabled engine every cycle (max coverage, ~Nx requests)
  #   fallback - try in priority order, stop at the first that returns items (backup chain)
  #   rotate   - use one engine per cycle, rotating through the list (spreads load)
  engine_strategy: all
  # Recycle the Chromium context every N cycles to release accumulated memory.
  # A long-lived context grows unbounded; on a swap-less host this livelocked
  # the box (postmortem 2026-06-13). The on-disk persistent profile survives it.
  browser_recycle_cycles: 50
  # Adaptive per-engine cooldown. When an engine returns a throttle/block signal
  # (HTTP 429/403/503 or a detected block page), it is benched for an exponential
  # backoff window instead of being hammered every cycle (which re-arms the
  # rate-limit window — observed 2026-06-17 with Brave). After the window the
  # engine gets a single probe request; a clean probe resumes it, another block
  # deepens the backoff. The per-engine health label on /monitor reflects this.
  cooldown:
    enabled: true
    base_seconds: 300   # window after the first block; doubles per consecutive block
    max_seconds: 3600   # cap on the exponential window (1h)
  # Proactive per-engine pacing. Each engine runs in its own worker (its own
  # browser context/identity on the shared exit IP) and paces itself to a
  # known-safe rate — this is the primary throttle; cooldown above is only the
  # reactive backstop. Set per-engine floors instead of discovering them by
  # getting blocked: a deliberate floor keeps soft 429s from escalating into
  # CAPTCHA/IP-level blocks that would poison the IP for every engine.
  pacing:
    default_min_interval: 2.0   # min seconds between requests for one engine
    jitter_ratio: 0.25          # random extra fraction per interval (avoid regular cadence)
    per_engine:                 # overrides for engines that throttle sooner
      brave: 4.0
  # Weighted saturation signal: when to scale traffic onto another exit IP.
  # Strict engines (canaries, e.g. brave) trip first and are NOT evidence the IP
  # is saturated, so they're excluded. Saturation is flagged only when this many
  # *robust* engines are cooling at once — the cue to divide load across machines
  # with different IPs.
  saturation:
    canary_engines: [brave]
    robust_threshold: 2
  # Idle keep-alive heartbeat: warms each engine's session so an on-demand web
  # search rarely lands on a cold (CAPTCHA-prone) profile. Fires a benign
  # NEWS-vertical query (news passes cold, web doesn't) only when the worker is
  # otherwise idle — any real scrape resets the timer. OFF by default: web search
  # isn't wired to the API yet, so until then it would only add request load.
  # Turn it on when web search goes live. See docs/WEB_SEARCH_WARMUP.md.
  keepalive:
    enabled: false
    interval_seconds: 600   # idle gap before a warm-up fires (~10 min)
    jitter_ratio: 0.5       # random extra fraction per interval (avoid regular cadence)
    # queries: ["weather tomorrow", "local news", "breaking news"]  # optional override
  # On-demand web search (cross-process bridge). When enabled, each engine worker
  # drains a per-engine queue backed by the web_search_jobs table, and the API
  # (api/websearch.py) dispatches each user query to a healthy, non-cooling engine,
  # falling back to the next on a block/empty/timeout. OFF by default — turn on
  # together with keepalive above so dispatched queries land on warm sessions.
  web_search:
    enabled: false
    engines: [google]             # web-search engines, priority order; Google-only for now (add others later — each must be a running engine above)
    max_in_flight: 4              # backpressure: max queued+serving searches per engine before the API rejects with 429 (one warm session serves them sequentially)
    request_timeout_seconds: 25   # how long the API waits for one engine before falling back
    poll_interval_seconds: 0.25   # how often the API polls the job row for its result
    max_engine_attempts: 3        # max distinct healthy engines a query may fan out across
    job_ttl_seconds: 120          # purge abandoned job rows older than this

# -----------------------------------------------------------------------------
# Anti-detection (scraper)
# -----------------------------------------------------------------------------
anti_detection:
  # DISABLED (verified 2026-06-11): Google detects playwright-stealth's JS
  # patches (fake plugins, monkey-patched natives) and CAPTCHAs /search when
  # they are applied — the identical setup passes with stealth off. The two
  # signals stealth used to mask are covered without JS patching:
  # navigator.webdriver via --disable-blink-features=AutomationControlled,
  # and the UA via the runtime-detected fingerprint. Do not re-enable.
  playwright_stealth:
    enabled: false
    description: "Applies playwright-stealth to each context to mask automation"

  # Minimal arg set, verified to pass Google /search from a residential IP
  # (2026-06-11). --disable-blink-features=AutomationControlled is the one
  # anti-detection essential (keeps navigator.webdriver false; true = instant
  # CAPTCHA). The rest are container-operational flags. Avoid piling on extra
  # feature-disable switches — they make the browser LESS like stock Chrome.
  browser_args:
    enabled: true
    args:
      - "--no-sandbox"
      - "--disable-setuid-sandbox"
      - "--disable-dev-shm-usage"
      - "--disable-gpu"
      - "--window-size=1920,1080"
      - "--disable-blink-features=AutomationControlled"

  # SUPERSEDED by scraper.pacing (the per-engine worker model paces each engine
  # itself). Kept for back-compat; no longer consulted by the scraper.
  random_delays:
    enabled: true
    min_seconds: 2
    max_seconds: 5
    description: "Random delay between topics"

  # Page-interaction timings — the speed vs. block-risk trade-off. All keys are
  # optional; any omitted value falls back to the default shown.
  page_interaction:
    nav_timeout_ms: 30000       # page.goto navigation timeout
    selector_timeout_ms: 5000   # wait for the results container (non-fatal)
    settle_min_ms: 1500         # post-load settle wait range (random within [min,max])
    settle_max_ms: 3000
    # Human-simulation jitter applied after the page settles (scroll + mouse
    # motion so the session looks less robotic). Each pair is one random range;
    # scroll_back_chance is the probability (0..1) of the small upward scroll.
    human_simulation:
      scroll_steps_min: 2
      scroll_steps_max: 4
      scroll_distance_min: 80
      scroll_distance_max: 250
      scroll_wait_min: 300
      scroll_wait_max: 800
      mouse_x_min: 200
      mouse_x_max: 1700
      mouse_y_min: 200
      mouse_y_max: 800
      mouse_steps_min: 5
      mouse_steps_max: 15
      scroll_back_chance: 0.5
      scroll_back_distance_min: 30
      scroll_back_distance_max: 100
      scroll_back_wait_min: 200
      scroll_back_wait_max: 500

  # NOTE: the UA and Sec-CH-UA are derived at runtime from the installed
  # Chrome version (scraper/main.py:_detect_fingerprint). A hardcoded UA goes
  # stale as Chrome updates, and Google CAPTCHAs any /search request whose
  # claimed UA version doesn't match the real browser (verified 2026-06-11).
  browser_fingerprint:
    viewport_width: 1920
    viewport_height: 1080
    locale: "en-US"
    timezone_id: "America/Los_Angeles"
    geolocation_latitude: 37.3273
    geolocation_longitude: -121.954
    color_scheme: "light"
    permissions:
      - "geolocation"

  # The primary block signal is the redirect to google.com/sorry/ (checked in
  # code). Keywords are a fallback and must be SPECIFIC to the block page:
  # bare "captcha" false-positives because real results pages mention it in
  # Google's own inline JS.
  captcha_detection:
    enabled: true
    keywords:
      - "unusual traffic from your computer network"
      - "our systems have detected unusual traffic"
    description: "Detects CAPTCHA or blocking in response HTML"

  http_error_handling:
    enabled: true
    monitored_codes:
      - 429 # Rate limiting
      - 403 # Forbidden/blocked
      - 503 # Service unavailable
    description: "Detects and logs HTTP errors"

  # Extra headers are applied to EVERY request (documents, XHR, images), so
  # forcing per-navigation headers here (Accept, Sec-Fetch-*, Upgrade-*) is a
  # detection signal: real browsers vary them per request type. Verified
  # 2026-06-11 that dropping them (Sec-CH-UA comes from the runtime-detected
  # fingerprint in scraper/main.py) passes Google /search. Keep this empty
  # unless you have a specific reason.
  http_headers:
    enabled: true
    headers: {}

  # Proxy support (optional fallback). With a version-matched fingerprint
  # (see browser_fingerprint note above) Google /search works from a clean
  # residential IP without a proxy. Use a residential/mobile proxy if your IP
  # gets rate-limited or flagged. Datacenter proxies are detected and blocked
  # just like a flagged direct connection.
  proxy:
    enabled: false
    # Proxy URLs (scheme://[user:pass@]host:port); http, https, and socks5 are
    # supported. One endpoint is chosen per browser launch — residential
    # gateways rotate exit IPs server-side, so a single sticky endpoint is the
    # common case; a longer list varies the identity across restarts.
    #
    # Prefer the SCRAPER_PROXY env var (in .env) for credentials: this file is
    # baked into the images at build time, so .env avoids a rebuild and keeps
    # secrets out of the image. SCRAPER_PROXY overrides this list.
    #
    # IMPORTANT: set timezone_id / geolocation above to match the proxy's exit
    # country, or the mismatch itself becomes a detection signal.
    proxies: []
    # - "http://user:pass@gateway.provider.com:7777"
    # - "socks5://user:pass@gateway.provider.com:1080"

# -----------------------------------------------------------------------------
# API tuning
# -----------------------------------------------------------------------------
# Keys are flat and lowercase to match the settings field names
# (common/settings.py). Connection identity and secrets (POSTGRES_*), the ports
# the compose port-mapping / healthcheck interpolate (API_PORT / HOST_PORT), and
# the write API key stay in .env — they're needed before/aside from this file.
api:
  # Database pool & connection tuning
  db_pool_min_conn: 2
  db_pool_max_conn: 10
  db_connect_timeout: 10        # seconds to wait for a connection
  db_keepalives_idle: 30        # TCP keepalive idle (s)
  db_keepalives_interval: 10    # TCP keepalive interval (s)
  db_keepalives_count: 5        # TCP keepalive probes before giving up

  # DB retry (transient errors)
  db_retry_max_attempts: 3
  db_retry_delay_seconds: 0.1   # initial backoff; doubles each retry

  # Data retention
  news_retention_days: 30       # each cycle purges news + logs older than this

  # API authentication (Bearer token). The env var TOPICSTREAMS_API_KEY (.env) is
  # a comma-separated bootstrap key set; additional keys are managed live in the
  # api_keys table (scripts/manage_api_keys.py). This is how long the active key
  # set is cached before re-reading the DB — i.e. the delay before an add/disable
  # goes live. 0 = re-read every request (no caching).
  api_key_cache_ttl_seconds: 30

  # API rate limiting (per client IP, sliding window)
  rate_limit_calls: 120
  rate_limit_period: 60         # window length in seconds
  rate_limit_max_tracked: 10000 # client IPs tracked before the stale-IP sweep

  # Feed
  feed_engines_window_days: 7   # engine filter lists engines seen within this window
  feed_page_size: 20            # default feed page size (API + UI), 1..100

  # Scrape-health signal (/api/v1/status)
  health_log_window: 30         # recent scraper logs read for the health signal
  health_stale_min_seconds: 300 # floor for the 'stalled' threshold
  health_stale_max_seconds: 1800 # ceiling for the 'stalled' threshold
  health_stale_default_seconds: 900 # 'stalled' threshold when cadence is unknown

  # Frontend (served to the UI via GET /api/v1/config)
  status_poll_interval_ms: 30000 # UI status-strip refresh interval
  ws_reconnect_base_ms: 5000     # WebSocket reconnect backoff base
  ws_reconnect_max_ms: 30000     # WebSocket reconnect backoff cap

  # Deployment behaviour (also commonly set via .env)
  # cors_origins: "*"               # comma-separated allowed origins, or "*"
  # trusted_proxy_count: 0          # reverse proxies; >0 reads client IP from X-Forwarded-For
  # log_format: text                # 'text' (default) or 'json'