feat: flexible DB config, incremental DMM sync, logging fix

.github/workflows/cicd.yaml

@@ -1,61 +1,60 @@
-name: CI / CD for Zilean
+name: CI/CD
 
 on:
   push:
-    tags:
-      - 'v[0-9]+.[0-9]+.[0-9]+'
+    branches: [main]
+    tags: ['v[0-9]+.[0-9]+.[0-9]+']
+  pull_request:
+    branches: [main]
   workflow_dispatch:
 
 env:
-  IMAGE_NAME: ipromknight/zilean
+  REGISTRY: ghcr.io
+  IMAGE_NAME: ${{ github.repository }}
 
 jobs:
-  execution:
+  build-and-test:
     runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-dotnet@v4
+        with:
+          dotnet-version: '9.0.x'
+      - run: dotnet restore
+      - run: dotnet build --no-restore -c Release
+
+  docker:
+    needs: build-and-test
+    runs-on: ubuntu-latest
+    if: startsWith(github.ref, 'refs/tags/v')
     permissions:
       contents: read
-    name: Build Zilean Image
+      packages: write
     steps:
-      - name: Checkout code
-        uses: actions/checkout@v4.1.2
-
-      - name: Docker Setup QEMU
-        uses: docker/setup-qemu-action@v3
-        id: qemu
+      - uses: actions/checkout@v4
+      - uses: docker/setup-qemu-action@v3
         with:
           platforms: amd64,arm64
-
-      - name: Login to Docker Hub
-        uses: docker/login-action@v3
+      - uses: docker/setup-buildx-action@v3
+      - uses: docker/login-action@v3
         with:
-          username: ${{ vars.DOCKERHUB_USERNAME }}
-          password: ${{ secrets.DOCKERHUB_TOKEN }}
-
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v3.2.0
-
-      - name: Build Docker Metadata
-        id: docker-metadata
-        uses: docker/metadata-action@v5
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+      - uses: docker/metadata-action@v5
+        id: meta
         with:
-          images: ${{ env.IMAGE_NAME }}
-          flavor: |
-            latest=auto
+          images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
           tags: |
-            type=ref,event=tag
-            type=sha,commit=${{ github.sha }}
             type=semver,pattern={{version}}
+            type=sha
             type=raw,value=latest,enable={{is_default_branch}}
-
-      - name: Push Service Image to repo
-        uses: docker/build-push-action@v5
+      - uses: docker/build-push-action@v5
         with:
           context: .
-          file: ./Dockerfile
           push: true
-          provenance: mode=max
-          tags: ${{ steps.docker-metadata.outputs.tags }}
-          labels: ${{ steps.docker-metadata.outputs.labels }}
           platforms: linux/amd64,linux/arm64
-          cache-from: type=gha,scope=${{ github.workflow }}
-          cache-to: type=gha,mode=max,scope=${{ github.workflow }}
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}
+          cache-from: type=gha
+          cache-to: type=gha,mode=max

.github/workflows/release-please.yaml

@@ -16,4 +16,4 @@ jobs:
     steps:
       - uses: googleapis/release-please-action@v4
         with:
-          token: ${{ secrets.RELEASE_PLEASE_TOKEN }}
+          token: ${{ secrets.GITHUB_TOKEN }}

Directory.Packages.props

@@ -21,6 +21,7 @@
     <PackageVersion Include="Microsoft.Extensions.Hosting" Version="9.0.0" />
     <PackageVersion Include="Microsoft.Extensions.Http" Version="9.0.0" />
     <PackageVersion Include="Microsoft.IO.RecyclableMemoryStream" Version="3.0.1" />
+    <PackageVersion Include="Npgsql" Version="9.0.0" />
     <PackageVersion Include="Npgsql.EntityFrameworkCore.PostgreSQL" Version="9.0.0" />
     <PackageVersion Include="NSubstitute" Version="5.3.0" />
     <PackageVersion Include="pythonnet" Version="3.0.4" />

Dockerfile

@@ -18,7 +18,9 @@ RUN apk add --update --no-cache \
     python3=~3.11 \
     py3-pip=~23.1 \
     curl \
+    git \
     icu-libs \
+    tzdata \
     && ln -sf python3 /usr/bin/python
 ENV DOTNET_RUNNING_IN_CONTAINER=true
 ENV DOTNET_SYSTEM_GLOBALIZATION_INVARIANT=false
@@ -34,4 +36,7 @@ RUN rm -rf /app/python || true && \
     mkdir -p /app/python || true
 RUN pip3 install -r /app/requirements.txt -t /app/python
 
+HEALTHCHECK --interval=30s --timeout=5s --start-period=60s --retries=3 \
+    CMD curl -f http://localhost:8181/healthchecks/ready || exit 1
+
 ENTRYPOINT ["./zilean-api"]

README.md

@@ -1,14 +1,194 @@
-# What is Zilean
+# Zilean (Maintained Fork)
 
 <img src="docs/Writerside/images/zilean-logo.jpg" alt="zilean logo" width="300" height="300">
 
 Zilean is a service that allows you to search for [DebridMediaManager](https://github.com/debridmediamanager/debrid-media-manager) sourced content shared by users.
 This can then be configured as a Torznab indexer in your favorite content application.
 Newly added is the ability for Zilean to scrape from your running Zurg instance, and from other running Zilean instances.
 
-Documentation for zilean can be viewed at [https://ipromknight.github.io/zilean/](https://ipromknight.github.io/zilean/)
+This is an actively maintained fork of [iPromKnight/zilean](https://github.com/iPromKnight/zilean) (v3.5.0, last upstream commit May 2025).
 
----
+Upstream documentation: [https://ipromknight.github.io/zilean/](https://ipromknight.github.io/zilean/)
 
+## Requirements
 
-<a href='https://ko-fi.com/W7W616IBNG' target='_blank'><img height='36' style='border:0px;height:36px;' src='https://storage.ko-fi.com/cdn/kofi5.png?v=6' border='0' alt='Buy Me a Coffee at ko-fi.com' /></a>
+Zilean requires only **PostgreSQL 16+**. Elasticsearch is **NOT** required and was removed in v2.0.
+
+## Docker Image
+
+```
+ghcr.io/thoroslives/zilean:latest
+```
+
+## Fork Changes
+
+All changes beyond upstream v3.5.0:
+
+### v3.6.0
+- **Flexible database configuration** - supports `Zilean__Database__ConnectionString` env var (backwards compat), individual `POSTGRES_*` env vars, or sensible defaults. Uses `NpgsqlConnectionStringBuilder` for proper escaping of special characters in passwords.
+- **Incremental DMM sync** - replaces the 1.2GB zip download with `git clone --depth 1` on first run and `git pull` on subsequent runs. Supports `GITHUB_TOKEN` for authenticated requests (5,000 req/hr vs 60). Includes exponential backoff retry.
+- **Logging config preservation** - `logging.json` is only written if it doesn't exist, preserving user customizations across restarts.
+
+### v3.7.0
+- **Security hardening** - warns at startup if PostgreSQL password is empty or set to default "postgres". Docker-compose example no longer exposes Postgres ports.
+- **Database startup resilience** - retries database connection up to 5 times with 5-second delays before running migrations. Clear error messages on failure including host and database name.
+- **Filtered search fix** - `/dmm/filtered` with short query strings (e.g., "1923") combined with season/episode filters no longer returns 0 results. Similarity threshold is automatically lowered when structured filters provide precision.
+- **Scraping toggle fix** - setting `EnableScraping=false` now correctly hides the on-demand-scrape endpoint while keeping search endpoints functional.
+- **Timezone support** - set `TZ` env var (e.g., `TZ=Australia/Sydney`) to display log timestamps in your local timezone. `tzdata` package included in the image.
+- **Readiness health check** - new `/healthchecks/ready` endpoint that verifies database connectivity. Used by the Dockerfile HEALTHCHECK for orchestrator integration.
+- **HEALTHCHECK instruction** - Docker image includes a built-in health check (30s interval, 60s start period) so orchestrators can detect readiness.
+- **Graceful error handling** - database errors no longer kill the process immediately (`Process.Kill()` replaced with proper exception propagation). Search errors are logged instead of silently swallowed.
+- **Startup config validation** - validates configuration values (cron syntax, numeric ranges, required fields) at startup with clear error messages.
+- **DMM sync progress reporting** - periodic progress logs during sync showing files processed, percentage complete, and new torrents found.
+- **ISystemClock deprecation fix** - removed deprecated `ISystemClock` usage in authentication handler.
+
+## Configuration
+
+### Database Connection
+
+Three ways to configure the database connection (checked in this order):
+
+#### 1. Full Connection String (recommended for existing setups)
+
+```yaml
+environment:
+  - Zilean__Database__ConnectionString=Host=postgres;Database=zilean;Username=postgres;Password=mypass;Include Error Detail=true;Timeout=30;CommandTimeout=3600;
+```
+
+#### 2. Individual Environment Variables
+
+```yaml
+environment:
+  - POSTGRES_HOST=postgres      # default: localhost
+  - POSTGRES_PORT=5432          # default: 5432
+  - POSTGRES_DB=zilean          # default: zilean
+  - POSTGRES_USER=postgres      # default: postgres
+  - POSTGRES_PASSWORD=mypass    # default: (empty)
+```
+
+#### 3. Defaults
+
+If no database env vars are set, connects to `localhost:5432/zilean` as `postgres` with no password (suitable for trust auth).
+
+### DMM Sync
+
+Set `GITHUB_TOKEN` to avoid GitHub API rate limiting during DMM hashlist sync:
+
+```yaml
+environment:
+  - GITHUB_TOKEN=ghp_xxxxxxxxxxxx
+```
+
+The initial DMM sync is **resumable** - if interrupted, it picks up where it left off on next startup. Expected initial sync duration varies by hardware (typically 30min-2hrs for parsing, longer for IMDB matching).
+
+### Timezone
+
+Set the `TZ` environment variable to display log timestamps in your local timezone:
+
+```yaml
+environment:
+  - TZ=Australia/Sydney
+```
+
+### PostgreSQL Shared Memory
+
+PostgreSQL's default shared memory (`shm_size`) of 64MB is too small for Zilean's bulk DMM upserts. You'll get errors like:
+
+```
+could not resize shared memory segment "/PostgreSQL.xxx" to 67146560 bytes: No space left on device
+```
+
+Set `shm_size: 256m` on your PostgreSQL container to fix this. See the docker-compose example below.
+
+## Security
+
+**Never expose your PostgreSQL port to the internet.** Multiple users have been compromised with crypto miners after exposing Postgres with default credentials. Zilean will warn you at startup if your database password is empty or set to the default "postgres".
+
+Best practices:
+- Always set a strong `POSTGRES_PASSWORD`
+- Do NOT add `ports:` to your Postgres container unless you need external access
+- If you must expose Postgres, use a firewall to restrict access to trusted IPs
+- Use Docker's internal networking - Zilean connects to Postgres by container name
+
+## Resource Usage
+
+- **Initial sync:** Expect high CPU for 10-30 minutes during the first DMM sync. This is normal - Zilean is parsing ~1.2M HTML files and performing bulk database upserts. Progress is logged periodically.
+- **Subsequent syncs:** Lightweight. Only pulls new/changed files via `git pull` and processes the diff.
+- **If high usage persists** after the initial sync completes: check for security compromise (see Security section above). Persistent high CPU with unfamiliar processes is a red flag.
+- PostgreSQL requires `shm_size: 256m` for bulk operations (see PostgreSQL Shared Memory section).
+
+## Multi-Instance Deployment
+
+For high-availability or high-traffic setups, you can run multiple Zilean instances:
+
+- **1 scraper instance** (`Zilean__Dmm__EnableScraping=true`) - handles DMM sync and data ingestion
+- **N API instances** (`Zilean__Dmm__EnableScraping=false`, `Zilean__Dmm__EnableEndpoint=true`) - serve search queries only
+- All instances share the same PostgreSQL database
+- `PreventOverlapping("SyncJobs")` prevents concurrent scraping within an instance
+- PostgreSQL's default `max_connections=100` is sufficient for typical deployments
+
+## Health Checks
+
+- `/healthchecks/ping` - lightweight liveness check (always returns 200)
+- `/healthchecks/ready` - readiness check that verifies database connectivity (returns 503 if DB is unreachable)
+
+## Troubleshooting
+
+### Database not found / "does not exist"
+
+Common causes:
+- PostgreSQL hasn't finished initializing - Zilean now retries 5 times with 5-second delays
+- Wrong credentials - check `POSTGRES_PASSWORD` matches between Zilean and Postgres containers
+- Volume permissions - on Unraid/Synology, ensure the Postgres data volume has correct ownership
+
+### "could not resize shared memory segment"
+
+Set `shm_size: 256m` on your PostgreSQL container. See the docker-compose example.
+
+### Search returns 0 results
+
+- Ensure the initial DMM sync has completed (check logs for "DMM sync complete")
+- For filtered searches with short titles, the similarity threshold is automatically adjusted
+
+## Docker Compose Example
+
+```yaml
+services:
+  zilean:
+    image: ghcr.io/thoroslives/zilean:latest
+    container_name: zilean
+    restart: unless-stopped
+    ports:
+      - "8181:8181"
+    volumes:
+      - zilean-data:/app/data
+    environment:
+      - POSTGRES_HOST=postgres
+      - POSTGRES_PASSWORD=your_strong_password_here
+      - GITHUB_TOKEN=ghp_xxxxxxxxxxxx  # optional, recommended
+      - TZ=UTC                         # optional, set your timezone
+    depends_on:
+      postgres:
+        condition: service_healthy
+
+  postgres:
+    image: postgres:16-alpine
+    container_name: zilean-postgres
+    restart: unless-stopped
+    shm_size: 256m  # required - default 64m causes "No space left on device" during bulk upserts
+    # Do NOT expose ports unless you need external access - see Security section
+    volumes:
+      - zilean-pg:/var/lib/postgresql/data
+    environment:
+      - POSTGRES_DB=zilean
+      - POSTGRES_PASSWORD=your_strong_password_here
+    healthcheck:
+      test: ["CMD-SHELL", "pg_isready -U postgres -d zilean"]
+      interval: 10s
+      timeout: 5s
+      retries: 5
+
+volumes:
+  zilean-data:
+  zilean-pg:
+```

src/Zilean.ApiService/Features/Authentication/ApiKeyAuthenticationHandler.cs

@@ -4,9 +4,8 @@ public class ApiKeyAuthenticationHandler(
     IOptionsMonitor<AuthenticationSchemeOptions> options,
     ILoggerFactory logger,
     UrlEncoder encoder,
-    ISystemClock clock,
     ZileanConfiguration configuration)
-    : AuthenticationHandler<AuthenticationSchemeOptions>(options, logger, encoder, clock)
+    : AuthenticationHandler<AuthenticationSchemeOptions>(options, logger, encoder)
 {
     protected override Task<AuthenticateResult> HandleAuthenticateAsync()
     {