diff --git a/.env.example b/.env.example
index ca2e089..a632150 100644
--- a/.env.example
+++ b/.env.example
@@ -21,3 +21,12 @@ GOOGLE_CLIENT_SECRET=
# Facebook: https://developers.facebook.com/apps/
FACEBOOK_CLIENT_ID=
FACEBOOK_CLIENT_SECRET=
+
+# --- S3-compatible storage (optional) ---
+# Use with: docker compose --profile s3 up minio
+# MinIO defaults: http://localhost:9900 (S3 API), minioadmin/minioadmin
+S3_ENDPOINT=
+S3_ACCESS_KEY_ID=
+S3_SECRET_ACCESS_KEY=
+S3_BUCKET=
+S3_USE_SSL=true
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index 011518f..f0e292d 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -11,6 +11,7 @@ mails/
├── cmd/mails/ # Application entry point
├── internal/ # Private application packages
│ ├── auth/ # OAuth2 login (GitHub, Google, Facebook)
+│ ├── storage/ # Blob store (FS or S3) for user data
│ ├── user/ # User management, UUIDv7 IDs
│ ├── account/ # Per-user email account CRUD
│ ├── model/ # Shared data types
@@ -20,7 +21,7 @@ mails/
│ │ ├── gmail/ # Gmail API sync
│ │ └── pst/ # PST/OST file import (go-pst library)
│ ├── search/
-│ │ ├── eml/ # .eml file parser
+│ │ ├── eml/ # .eml file parser, CID inline image extraction
│ │ ├── index/ # DuckDB + Parquet index
│ │ └── vector/ # Qdrant similarity search
│ └── web/ # HTTP router, handlers, middleware
@@ -51,14 +52,15 @@ mails/
Packages follow a strict dependency hierarchy to avoid import cycles:
```
-cmd → internal/web → internal/sync → internal/model
- → internal/auth
+cmd → internal/web → internal/sync → internal/model
+ → internal/auth → internal/storage
→ internal/user
→ internal/account
→ internal/search
```
-- Left packages may depend on right packages
+- Left packages may depend on right packages.
+- `internal/storage` (BlobStore) is used by auth, user, account, sync, and search for FS or S3-backed user data.
- Right packages **MUST NOT** depend on left packages
- Sub-packages at the same level use interfaces to avoid circular imports
@@ -121,9 +123,18 @@ go test -race ./...
# Run specific package tests
go test ./internal/search/eml/
+
+# Run e2e tests (requires GreenMail + Qdrant + Ollama)
+docker compose --profile test up -d greenmail
+go test -tags e2e -v ./tests/e2e/
+
+# Run S3 storage integration tests (requires MinIO)
+docker compose --profile s3 up -d minio
+S3_ENDPOINT=http://localhost:9900 S3_ACCESS_KEY_ID=minioadmin S3_SECRET_ACCESS_KEY=minioadmin \
+ S3_BUCKET=mails-test S3_USE_SSL=false go test -v ./internal/storage/
```
-Use `testing.T` and table-driven tests. Mock external services (IMAP, POP3, APIs).
+Use `testing.T` and table-driven tests. Mock external services (IMAP, POP3, APIs). Integration tests skip when required services (S3, GreenMail) are unavailable.
## Templates (Gitea-style)
@@ -225,16 +236,17 @@ Based on [Google JavaScript Style Guide](https://google.github.io/styleguide/jsg
## User Data Layout
-Each user's data lives under `users/{uuidv7}/`:
+Each user's data lives under `users/{uuidv7}/`. When S3 env vars are set (`S3_ENDPOINT`, `S3_ACCESS_KEY_ID`, `S3_SECRET_ACCESS_KEY`), the following are stored in S3; otherwise on the local filesystem:
-| Path | Purpose |
-| -------------------------------- | ------------------------------------- |
-| `user.json` | User metadata (name, email, provider) |
-| `accounts.yml` | Email account configurations |
-| `sync.sqlite` | Sync jobs, UIDs, state |
-| `logs/{job-id}.jsonl` | Structured sync logs |
-| `{domain}/{local}/` | Downloaded .eml files |
-| `{domain}/{local}/index.parquet` | Search index per account |
+| Path | Purpose | Storage |
+| -------------------------------- | ------------------------------------- | ------------ |
+| `user.json` | User metadata (name, email, provider) | FS or S3 |
+| `accounts.yml` | Email account configurations | FS or S3 |
+| `sessions.json` | Session store (root of users dir) | FS or S3 |
+| `sync.sqlite` | Sync jobs, UIDs, state | Local only |
+| `logs/{job-id}.jsonl` | Structured sync logs | Local only |
+| `{domain}/{local}/*.eml` | Downloaded .eml files | FS or S3 |
+| `{domain}/{local}/index.parquet` | Search index per account | Local only |
### Email Storage
@@ -266,11 +278,19 @@ See [docs/DOCKER.md](docs/DOCKER.md) for tini, runtime dependencies, and build d
# Development
docker compose up
+# With S3 (MinIO) for user data storage
+docker compose --profile s3 up -d minio
+export S3_ENDPOINT=http://localhost:9900 S3_ACCESS_KEY_ID=minioadmin S3_SECRET_ACCESS_KEY=minioadmin
+docker compose up
+
# Production build
docker compose -f docker-compose.yml up -d
# Run tests
docker compose run --rm mails go test ./...
+
+# Run e2e tests (GreenMail for IMAP/POP3)
+docker compose --profile test up -d greenmail
```
## API Reference
@@ -337,3 +357,4 @@ All API endpoints require authentication (session cookie or `Authorization: Bear
- **IMAP connection refused:** Check host, port, and SSL settings. Gmail requires an App Password (not regular password).
- **Search returns 0 results:** Run reindex after syncing new emails.
- **SQLite busy:** Increase `_busy_timeout` or reduce concurrent sync jobs.
+- **S3/MinIO connection failed:** When using MinIO, set `S3_USE_SSL=false` and ensure `S3_ENDPOINT` includes the scheme (e.g. `http://localhost:9900`). Run MinIO with `docker compose --profile s3 up minio`.
diff --git a/README.md b/README.md
index 595ad29..e3b3926 100644
--- a/README.md
+++ b/README.md
@@ -79,6 +79,11 @@ docker pull ghcr.io/eslider/mail-archive:v1.0.1
| `QDRANT_URL` | — | Qdrant gRPC address for similarity search |
| `OLLAMA_URL` | — | Ollama API URL for embeddings |
| `EMBED_MODEL` | `all-minilm` | Embedding model name |
+| `S3_ENDPOINT` | — | S3-compatible storage endpoint (e.g. MinIO) |
+| `S3_ACCESS_KEY_ID` | — | S3 access key |
+| `S3_SECRET_ACCESS_KEY` | — | S3 secret key |
+| `S3_BUCKET` | `mails` | S3 bucket name |
+| `S3_USE_SSL` | `true` | Use HTTPS for S3 endpoint |
### OAuth Setup (Optional)
@@ -94,6 +99,8 @@ To enable OAuth login, configure one or more providers:
## User Data Layout
+When S3 env vars are set, `user.json`, `accounts.yml`, `sessions.json`, and `.eml` files are stored in S3. SQLite and Parquet stay on the local filesystem.
+
```
users/
019c56a4-a9ef-79bd-b53a-ef7a080d9c90/
@@ -117,6 +124,7 @@ users/
cmd/mails/ → Entry point, CLI (serve, fix-dates, version)
internal/
auth/ → OAuth2 (GitHub, Google, Facebook), sessions
+ storage/ → Blob store (FS or S3) for user data
user/ → User storage (users/{uuid}/)
account/ → Email account CRUD (accounts.yml)
model/ → Shared types (User, Account, SyncJob)
@@ -153,6 +161,10 @@ go test ./...
docker compose --profile test up -d greenmail
go test -tags e2e -v ./tests/e2e/
+# Run S3 storage integration tests (requires MinIO)
+docker compose --profile s3 up -d minio
+S3_ENDPOINT=http://localhost:9900 S3_ACCESS_KEY_ID=minioadmin S3_SECRET_ACCESS_KEY=minioadmin S3_BUCKET=mails-test S3_USE_SSL=false go test -v ./internal/storage/
+
# Docker dev mode (auto-rebuild on changes)
docker compose watch
```
@@ -235,11 +247,23 @@ web/static/
sw-register.js # Service worker registration
```
-## Todo
+## S3 Storage (Optional)
-- **Storage backend** — S3 (AWS/Minio) or pluggable local filesystem
+Store user data on S3 when `S3_ENDPOINT` and credentials are set:
+
+```bash
+docker compose --profile s3 up -d minio
+export S3_ENDPOINT=http://localhost:9900
+export S3_ACCESS_KEY_ID=minioadmin
+export S3_SECRET_ACCESS_KEY=minioadmin
+export S3_BUCKET=mails
+export S3_USE_SSL=false
+./mails serve
+```
+
+## Todo
-See (TODO's)[TODO.md]
+See [TODO.md](TODO.md)
## Ideas
diff --git a/cmd/mails/main.go b/cmd/mails/main.go
index 11c24d7..e285dc3 100644
--- a/cmd/mails/main.go
+++ b/cmd/mails/main.go
@@ -19,6 +19,7 @@ import (
"github.com/eslider/mails/internal/account"
"github.com/eslider/mails/internal/auth"
+ "github.com/eslider/mails/internal/storage"
"github.com/eslider/mails/internal/sync"
"github.com/eslider/mails/internal/user"
"github.com/eslider/mails/internal/web"
@@ -74,7 +75,13 @@ Environment:
QDRANT_URL Qdrant gRPC address for similarity search
OLLAMA_URL Ollama API URL for embeddings
- EMBED_MODEL Ollama embedding model (default: all-minilm)`)
+ EMBED_MODEL Ollama embedding model (default: all-minilm)
+
+ S3_ENDPOINT S3-compatible storage (e.g. MinIO)
+ S3_ACCESS_KEY_ID S3 access key
+ S3_SECRET_ACCESS_KEY S3 secret key
+ S3_BUCKET S3 bucket (default: mails)
+ S3_USE_SSL Use HTTPS for S3 (default: true)`)
}
func runServe() {
@@ -82,19 +89,24 @@ func runServe() {
dataDir := envOr("DATA_DIR", "./users")
baseURL := envOr("BASE_URL", "http://localhost:8090")
+ blobStore, err := storage.NewBlobStore(dataDir)
+ if err != nil {
+ log.Fatalf("Failed to init blob store: %v", err)
+ }
+
// Initialize stores.
- userStore, err := user.NewStore(dataDir)
+ userStore, err := user.NewStore(dataDir, blobStore)
if err != nil {
log.Fatalf("Failed to init user store: %v", err)
}
- sessionStore, err := auth.NewSessionStore(dataDir)
+ sessionStore, err := auth.NewSessionStore(dataDir, blobStore)
if err != nil {
log.Fatalf("Failed to init session store: %v", err)
}
- accountStore := account.NewStore(dataDir)
- syncService := sync.NewService(dataDir, accountStore)
+ accountStore := account.NewStore(dataDir, blobStore)
+ syncService := sync.NewService(dataDir, accountStore, blobStore)
// Configure OAuth providers.
var ghCfg, glCfg, fbCfg *auth.ProviderConfig
@@ -137,6 +149,7 @@ func runServe() {
Auth: providers,
Sync: syncService,
UsersDir: dataDir,
+ BlobStore: blobStore,
QdrantURL: envOr("QDRANT_URL", ""),
OllamaURL: envOr("OLLAMA_URL", ""),
EmbedModel: envOr("EMBED_MODEL", "all-minilm"),
diff --git a/docker-compose.yml b/docker-compose.yml
index c5216dd..1fe7f7e 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -31,6 +31,12 @@ services:
QDRANT_URL: "http://127.0.0.1:6334"
OLLAMA_URL: "http://172.17.0.1:11434"
EMBED_MODEL: "all-minilm"
+ # S3-compatible storage (optional, e.g. MinIO)
+ S3_ENDPOINT: "${S3_ENDPOINT:-}"
+ S3_ACCESS_KEY_ID: "${S3_ACCESS_KEY_ID:-}"
+ S3_SECRET_ACCESS_KEY: "${S3_SECRET_ACCESS_KEY:-}"
+ S3_BUCKET: "${S3_BUCKET:-}"
+ S3_USE_SSL: "${S3_USE_SSL:-true}"
network_mode: host
# docker compose watch — rebuild on changes (uses .dockerignore implicitly)
develop:
@@ -46,6 +52,23 @@ services:
max-file: "3"
labels: "service"
+ # S3-compatible object storage (MinIO) — optional
+ minio:
+ image: minio/minio:latest
+ container_name: minio
+ restart: unless-stopped
+ command: server /data
+ environment:
+ MINIO_ROOT_USER: "${MINIO_ROOT_USER:-minioadmin}"
+ MINIO_ROOT_PASSWORD: "${MINIO_ROOT_PASSWORD:-minioadmin}"
+ volumes:
+ - minio_data:/data
+ ports:
+ - "9900:9000" # S3 API (host 9900 to avoid conflicts)
+ - "9901:9001" # Web console
+ profiles:
+ - s3
+
# Vector DB for similarity search (optional)
qdrant:
image: qdrant/qdrant:latest
@@ -73,3 +96,4 @@ services:
volumes:
qdrant_storage:
+ minio_data:
diff --git a/go.mod b/go.mod
index f49d160..077d7eb 100644
--- a/go.mod
+++ b/go.mod
@@ -3,6 +3,9 @@ module github.com/eslider/mails
go 1.24.4
require (
+ github.com/aws/aws-sdk-go-v2 v1.41.1
+ github.com/aws/aws-sdk-go-v2/credentials v1.19.7
+ github.com/aws/aws-sdk-go-v2/service/s3 v1.96.0
github.com/emersion/go-message v0.18.2
github.com/go-chi/chi/v5 v5.2.1
github.com/google/uuid v1.6.0
@@ -20,6 +23,15 @@ require (
require (
cloud.google.com/go/compute/metadata v0.7.0 // indirect
github.com/apache/arrow-go/v18 v18.1.0 // indirect
+ github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.4 // indirect
+ github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.17 // indirect
+ github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.17 // indirect
+ github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.17 // indirect
+ github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.4 // indirect
+ github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.8 // indirect
+ github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.17 // indirect
+ github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.17 // indirect
+ github.com/aws/smithy-go v1.24.0 // indirect
github.com/go-viper/mapstructure/v2 v2.2.1 // indirect
github.com/goccy/go-json v0.10.5 // indirect
github.com/godzie44/go-uring v0.0.0-20220926161041-69611e8b13d5 // indirect
diff --git a/go.sum b/go.sum
index f46c1db..9fd476b 100644
--- a/go.sum
+++ b/go.sum
@@ -6,6 +6,30 @@ github.com/apache/arrow-go/v18 v18.1.0 h1:agLwJUiVuwXZdwPYVrlITfx7bndULJ/dggbnLF
github.com/apache/arrow-go/v18 v18.1.0/go.mod h1:tigU/sIgKNXaesf5d7Y95jBBKS5KsxTqYBKXFsvKzo0=
github.com/apache/thrift v0.21.0 h1:tdPmh/ptjE1IJnhbhrcl2++TauVjy242rkV/UzJChnE=
github.com/apache/thrift v0.21.0/go.mod h1:W1H8aR/QRtYNvrPeFXBtobyRkd0/YVhTc6i07XIAgDw=
+github.com/aws/aws-sdk-go-v2 v1.41.1 h1:ABlyEARCDLN034NhxlRUSZr4l71mh+T5KAeGh6cerhU=
+github.com/aws/aws-sdk-go-v2 v1.41.1/go.mod h1:MayyLB8y+buD9hZqkCW3kX1AKq07Y5pXxtgB+rRFhz0=
+github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.4 h1:489krEF9xIGkOaaX3CE/Be2uWjiXrkCH6gUX+bZA/BU=
+github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.4/go.mod h1:IOAPF6oT9KCsceNTvvYMNHy0+kMF8akOjeDvPENWxp4=
+github.com/aws/aws-sdk-go-v2/credentials v1.19.7 h1:tHK47VqqtJxOymRrNtUXN5SP/zUTvZKeLx4tH6PGQc8=
+github.com/aws/aws-sdk-go-v2/credentials v1.19.7/go.mod h1:qOZk8sPDrxhf+4Wf4oT2urYJrYt3RejHSzgAquYeppw=
+github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.17 h1:xOLELNKGp2vsiteLsvLPwxC+mYmO6OZ8PYgiuPJzF8U=
+github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.17/go.mod h1:5M5CI3D12dNOtH3/mk6minaRwI2/37ifCURZISxA/IQ=
+github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.17 h1:WWLqlh79iO48yLkj1v3ISRNiv+3KdQoZ6JWyfcsyQik=
+github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.17/go.mod h1:EhG22vHRrvF8oXSTYStZhJc1aUgKtnJe+aOiFEV90cM=
+github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.17 h1:JqcdRG//czea7Ppjb+g/n4o8i/R50aTBHkA7vu0lK+k=
+github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.17/go.mod h1:CO+WeGmIdj/MlPel2KwID9Gt7CNq4M65HUfBW97liM0=
+github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.4 h1:0ryTNEdJbzUCEWkVXEXoqlXV72J5keC1GvILMOuD00E=
+github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.4/go.mod h1:HQ4qwNZh32C3CBeO6iJLQlgtMzqeG17ziAA/3KDJFow=
+github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.8 h1:Z5EiPIzXKewUQK0QTMkutjiaPVeVYXX7KIqhXu/0fXs=
+github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.8/go.mod h1:FsTpJtvC4U1fyDXk7c71XoDv3HlRm8V3NiYLeYLh5YE=
+github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.17 h1:RuNSMoozM8oXlgLG/n6WLaFGoea7/CddrCfIiSA+xdY=
+github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.17/go.mod h1:F2xxQ9TZz5gDWsclCtPQscGpP0VUOc8RqgFM3vDENmU=
+github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.17 h1:bGeHBsGZx0Dvu/eJC0Lh9adJa3M1xREcndxLNZlve2U=
+github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.17/go.mod h1:dcW24lbU0CzHusTE8LLHhRLI42ejmINN8Lcr22bwh/g=
+github.com/aws/aws-sdk-go-v2/service/s3 v1.96.0 h1:oeu8VPlOre74lBA/PMhxa5vewaMIMmILM+RraSyB8KA=
+github.com/aws/aws-sdk-go-v2/service/s3 v1.96.0/go.mod h1:5jggDlZ2CLQhwJBiZJb4vfk4f0GxWdEDruWKEJ1xOdo=
+github.com/aws/smithy-go v1.24.0 h1:LpilSUItNPFr1eY85RYgTIg5eIEPtvFbskaFcmmIUnk=
+github.com/aws/smithy-go v1.24.0/go.mod h1:LEj2LM3rBRQJxPZTB4KuzZkaZYnZPnvgIhb4pu07mx0=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/emersion/go-message v0.18.2 h1:rl55SQdjd9oJcIoQNhubD2Acs1E6IzlZISRTK7x/Lpg=
diff --git a/internal/account/store.go b/internal/account/store.go
index a9788e1..06898d7 100644
--- a/internal/account/store.go
+++ b/internal/account/store.go
@@ -2,6 +2,8 @@
package account
import (
+ "context"
+ "errors"
"fmt"
"os"
"path/filepath"
@@ -9,6 +11,7 @@ import (
"sync"
"github.com/eslider/mails/internal/model"
+ "github.com/eslider/mails/internal/storage"
"gopkg.in/yaml.v3"
)
@@ -16,13 +19,14 @@ const accountsFileName = "accounts.yml"
// Store manages email account configurations per user.
type Store struct {
- mu sync.RWMutex
- usersDir string
+ mu sync.RWMutex
+ usersDir string
+ blobStore storage.BlobStore
}
-// NewStore creates an account store.
-func NewStore(usersDir string) *Store {
- return &Store{usersDir: usersDir}
+// NewStore creates an account store. blobStore may be nil to use local filesystem.
+func NewStore(usersDir string, blobStore storage.BlobStore) *Store {
+ return &Store{usersDir: usersDir, blobStore: blobStore}
}
// List returns all email accounts for a user.
@@ -69,9 +73,11 @@ func (s *Store) Create(userID string, acct model.EmailAccount) (*model.EmailAcco
return nil, err
}
- // Create the email storage directory.
- emailDir := EmailDir(s.usersDir, userID, acct)
- os.MkdirAll(emailDir, 0o755)
+ // Create the email storage directory (for local fs; S3 has no dirs).
+ if s.blobStore == nil {
+ emailDir := EmailDir(s.usersDir, userID, acct)
+ os.MkdirAll(emailDir, 0o755)
+ }
return &acct, nil
}
@@ -143,6 +149,21 @@ func (s *Store) accountsPath(userID string) string {
}
func (s *Store) load(userID string) ([]model.EmailAccount, error) {
+ key := userID + "/" + accountsFileName
+ if s.blobStore != nil {
+ data, err := s.blobStore.Read(context.Background(), key)
+ if err != nil {
+ if errors.Is(err, storage.ErrNotFound) {
+ return nil, nil
+ }
+ return nil, err
+ }
+ var file model.AccountsFile
+ if err := yaml.Unmarshal(data, &file); err != nil {
+ return nil, fmt.Errorf("parse %s: %w", key, err)
+ }
+ return file.Accounts, nil
+ }
path := s.accountsPath(userID)
data, err := os.ReadFile(path)
if err != nil {
@@ -160,16 +181,19 @@ func (s *Store) load(userID string) ([]model.EmailAccount, error) {
}
func (s *Store) save(userID string, accounts []model.EmailAccount) error {
- path := s.accountsPath(userID)
- if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
- return err
- }
-
file := model.AccountsFile{Accounts: accounts}
data, err := yaml.Marshal(file)
if err != nil {
return err
}
+ key := userID + "/" + accountsFileName
+ if s.blobStore != nil {
+ return s.blobStore.Write(context.Background(), key, data)
+ }
+ path := s.accountsPath(userID)
+ if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
+ return err
+ }
return os.WriteFile(path, data, 0o644)
}
diff --git a/internal/auth/session.go b/internal/auth/session.go
index 0231b72..df2c305 100644
--- a/internal/auth/session.go
+++ b/internal/auth/session.go
@@ -1,6 +1,7 @@
package auth
import (
+ "context"
"crypto/rand"
"encoding/hex"
"encoding/json"
@@ -12,6 +13,7 @@ import (
"time"
"github.com/eslider/mails/internal/model"
+ "github.com/eslider/mails/internal/storage"
)
const (
@@ -20,21 +22,25 @@ const (
sessionsFile = "sessions.json"
)
-// SessionStore manages user sessions backed by a JSON file.
+// SessionStore manages user sessions backed by a JSON file or S3.
type SessionStore struct {
- mu sync.RWMutex
- sessions map[string]model.Session // token -> session
- dataDir string
+ mu sync.RWMutex
+ sessions map[string]model.Session // token -> session
+ dataDir string
+ blobStore storage.BlobStore
}
-// NewSessionStore creates a session store, loading existing sessions from disk.
-func NewSessionStore(dataDir string) (*SessionStore, error) {
+// NewSessionStore creates a session store. blobStore may be nil to use local filesystem.
+func NewSessionStore(dataDir string, blobStore storage.BlobStore) (*SessionStore, error) {
s := &SessionStore{
- sessions: make(map[string]model.Session),
- dataDir: dataDir,
+ sessions: make(map[string]model.Session),
+ dataDir: dataDir,
+ blobStore: blobStore,
}
- if err := os.MkdirAll(dataDir, 0o755); err != nil {
- return nil, err
+ if blobStore == nil {
+ if err := os.MkdirAll(dataDir, 0o755); err != nil {
+ return nil, err
+ }
}
s.load()
return s, nil
@@ -133,6 +139,23 @@ func generateToken() (string, error) {
}
func (s *SessionStore) load() {
+ if s.blobStore != nil {
+ data, err := s.blobStore.Read(context.Background(), sessionsFile)
+ if err != nil {
+ return
+ }
+ var sessions []model.Session
+ if err := json.Unmarshal(data, &sessions); err != nil {
+ return
+ }
+ now := time.Now()
+ for _, sess := range sessions {
+ if now.Before(sess.ExpiresAt) {
+ s.sessions[sess.Token] = sess
+ }
+ }
+ return
+ }
path := filepath.Join(s.dataDir, sessionsFile)
data, err := os.ReadFile(path)
if err != nil {
@@ -162,6 +185,10 @@ func (s *SessionStore) save() {
if err != nil {
return
}
+ if s.blobStore != nil {
+ s.blobStore.Write(context.Background(), sessionsFile, data)
+ return
+ }
path := filepath.Join(s.dataDir, sessionsFile)
os.WriteFile(path, data, 0o600)
}
diff --git a/internal/search/eml/parser.go b/internal/search/eml/parser.go
index 7525ec6..9333f29 100644
--- a/internal/search/eml/parser.go
+++ b/internal/search/eml/parser.go
@@ -109,6 +109,35 @@ func ParseFile(path string) (Email, error) {
}, nil
}
+// ParseBytes parses .eml content from bytes. path is the logical path for the result.
+func ParseBytes(path string, data []byte) (Email, error) {
+ msg, err := mail.ReadMessage(bytes.NewReader(data))
+ if err != nil {
+ return Email{}, fmt.Errorf("parse: %w", err)
+ }
+ h := msg.Header
+ date, _ := h.Date()
+ if date.IsZero() {
+ date = parseDateFuzzy(h.Get("Date"))
+ }
+ if date.IsZero() {
+ date = parseReceivedDate(textproto.MIMEHeader(h))
+ }
+ subject := ensureUTF8(strings.TrimSpace(decodeHeader(h.Get("Subject"))))
+ from := ensureUTF8(strings.TrimSpace(decodeHeader(h.Get("From"))))
+ to := ensureUTF8(strings.TrimSpace(decodeHeader(h.Get("To"))))
+ bodyText := extractBodyText(h.Get("Content-Type"), h.Get("Content-Transfer-Encoding"), msg.Body)
+ return Email{
+ Path: path,
+ Subject: subject,
+ From: from,
+ To: to,
+ Date: date,
+ Size: int64(len(data)),
+ BodyText: bodyText,
+ }, nil
+}
+
// parseDateFuzzy tries multiple date layouts to handle non-standard Date headers
// (e.g. missing timezone, unconventional formats).
func parseDateFuzzy(raw string) time.Time {
@@ -309,6 +338,8 @@ var (
reHTMLTag = regexp.MustCompile(`<[^>]*>`)
reWhitespace = regexp.MustCompile(`[\s]+`)
reHTMLEntity = regexp.MustCompile(`&[a-zA-Z0-9#]+;`)
+ // reCID matches cid: URLs in HTML (img src, style url(), etc). Captures the CID value with/without angle brackets.
+ reCID = regexp.MustCompile(`(?i)cid:(<[^>]+>|[^"')\s\]>]+)`)
)
func stripHTML(html string) string {
@@ -325,6 +356,38 @@ func stripHTML(html string) string {
return strings.TrimSpace(text)
}
+// normalizeCID returns the Content-ID for map lookup (strips angle brackets, trims).
+func normalizeCID(cid string) string {
+ cid = strings.TrimSpace(cid)
+ if strings.HasPrefix(cid, "<") && strings.HasSuffix(cid, ">") {
+ cid = strings.TrimSpace(cid[1 : len(cid)-1])
+ }
+ return cid
+}
+
+// rewriteCIDsInHTML replaces cid: references in HTML with data: base64 URIs using the inline parts map.
+func rewriteCIDsInHTML(html string, inline map[string]inlinePart) string {
+ if len(inline) == 0 {
+ return html
+ }
+ return reCID.ReplaceAllStringFunc(html, func(match string) string {
+ subs := reCID.FindStringSubmatch(match)
+ if len(subs) < 2 {
+ return match
+ }
+ cid := normalizeCID(subs[1])
+ part, ok := inline[cid]
+ if !ok {
+ return match
+ }
+ ct := part.ContentType
+ if ct == "" {
+ ct = "application/octet-stream"
+ }
+ return "data:" + ct + ";base64," + base64.StdEncoding.EncodeToString(part.Data)
+ })
+}
+
// Attachment holds metadata about a MIME attachment.
type Attachment struct {
Filename string `json:"filename"`
@@ -332,6 +395,12 @@ type Attachment struct {
Size int `json:"size"`
}
+// inlinePart holds data for a Content-ID referenced part (e.g. inline image).
+type inlinePart struct {
+ Data []byte
+ ContentType string
+}
+
// FullEmail holds the complete parsed email for display, including HTML body.
type FullEmail struct {
Path string `json:"path"`
@@ -395,6 +464,36 @@ func ParseFileFull(path string) (FullEmail, error) {
return fe, nil
}
+// ParseFileFullFromBytes parses .eml content from bytes. path is the logical path for the result.
+func ParseFileFullFromBytes(path string, data []byte) (FullEmail, error) {
+ msg, err := mail.ReadMessage(bytes.NewReader(data))
+ if err != nil {
+ return FullEmail{}, fmt.Errorf("parse: %w", err)
+ }
+ h := msg.Header
+ date, _ := h.Date()
+ if date.IsZero() {
+ date = parseDateFuzzy(h.Get("Date"))
+ }
+ if date.IsZero() {
+ date = parseReceivedDate(textproto.MIMEHeader(h))
+ }
+ fe := FullEmail{
+ Path: path,
+ Subject: ensureUTF8(strings.TrimSpace(decodeHeader(h.Get("Subject")))),
+ From: ensureUTF8(strings.TrimSpace(decodeHeader(h.Get("From")))),
+ To: ensureUTF8(strings.TrimSpace(decodeHeader(h.Get("To")))),
+ CC: ensureUTF8(strings.TrimSpace(decodeHeader(h.Get("Cc")))),
+ ReplyTo: ensureUTF8(strings.TrimSpace(decodeHeader(h.Get("Reply-To")))),
+ Date: date,
+ Size: int64(len(data)),
+ }
+ ct := h.Get("Content-Type")
+ cte := h.Get("Content-Transfer-Encoding")
+ extractFullBody(ct, cte, msg.Body, &fe)
+ return fe, nil
+}
+
func extractFullBody(contentType, transferEncoding string, body io.Reader, fe *FullEmail) {
if contentType == "" {
contentType = "text/plain"
@@ -408,7 +507,11 @@ func extractFullBody(contentType, transferEncoding string, body io.Reader, fe *F
charset := params["charset"]
if strings.HasPrefix(mediaType, "multipart/") {
- extractFullMultipart(params["boundary"], body, fe)
+ inlineParts := make(map[string]inlinePart)
+ extractFullMultipart(params["boundary"], body, fe, inlineParts)
+ if fe.HTMLBody != "" && len(inlineParts) > 0 {
+ fe.HTMLBody = rewriteCIDsInHTML(fe.HTMLBody, inlineParts)
+ }
return
}
@@ -421,7 +524,7 @@ func extractFullBody(contentType, transferEncoding string, body io.Reader, fe *F
}
}
-func extractFullMultipart(boundary string, r io.Reader, fe *FullEmail) {
+func extractFullMultipart(boundary string, r io.Reader, fe *FullEmail, inlineParts map[string]inlinePart) {
if boundary == "" {
return
}
@@ -446,9 +549,12 @@ func extractFullMultipart(boundary string, r io.Reader, fe *FullEmail) {
charset := partParams["charset"]
- disposition := part.Header.Get("Content-Disposition")
- isAttachment := strings.HasPrefix(disposition, "attachment") ||
- (part.FileName() != "" && !strings.HasPrefix(partMedia, "text/"))
+ disposition := strings.ToLower(part.Header.Get("Content-Disposition"))
+ contentID := strings.TrimSpace(part.Header.Get("Content-ID"))
+ // Inline parts with Content-ID (cid:) are embedded images, not attachments.
+ isInlineWithCID := contentID != "" && strings.HasPrefix(disposition, "inline")
+ isAttachment := !isInlineWithCID && (strings.HasPrefix(disposition, "attachment") ||
+ (part.FileName() != "" && !strings.HasPrefix(partMedia, "text/")))
if isAttachment {
data, _ := io.ReadAll(io.LimitReader(part, 10*1024*1024))
@@ -462,7 +568,7 @@ func extractFullMultipart(boundary string, r io.Reader, fe *FullEmail) {
}
if strings.HasPrefix(partMedia, "multipart/") {
- extractFullMultipart(partParams["boundary"], part, fe)
+ extractFullMultipart(partParams["boundary"], part, fe, inlineParts)
part.Close()
continue
}
@@ -482,6 +588,93 @@ func extractFullMultipart(boundary string, r io.Reader, fe *FullEmail) {
continue
}
+ // Inline part with Content-ID (e.g. embedded image referenced by cid: in HTML).
+ if contentID != "" {
+ data, _ := io.ReadAll(io.LimitReader(decodeTransferEncoding(part, cte), 5*1024*1024))
+ cid := normalizeCID(contentID)
+ if cid != "" {
+ inlineParts[cid] = inlinePart{Data: data, ContentType: partMedia}
+ }
+ }
+
+ part.Close()
+ }
+}
+
+// ExtractPartByCID reads a MIME part by Content-ID from an .eml file.
+// Returns the raw bytes, content-type, and any error. Used for serving inline images via API.
+func ExtractPartByCID(path string, cid string) ([]byte, string, error) {
+ cid = normalizeCID(cid)
+ if cid == "" {
+ return nil, "", fmt.Errorf("empty content-id")
+ }
+ f, err := os.Open(path)
+ if err != nil {
+ return nil, "", fmt.Errorf("open %s: %w", path, err)
+ }
+ defer f.Close()
+
+ msg, err := mail.ReadMessage(bufio.NewReader(f))
+ if err != nil {
+ return nil, "", fmt.Errorf("parse %s: %w", path, err)
+ }
+
+ ct := msg.Header.Get("Content-Type")
+ mediaType, params, err := mime.ParseMediaType(ct)
+ if err != nil {
+ mediaType = "text/plain"
+ params = nil
+ }
+
+ if !strings.HasPrefix(mediaType, "multipart/") {
+ return nil, "", fmt.Errorf("email has no multipart structure")
+ }
+
+ boundary := params["boundary"]
+ if boundary == "" {
+ return nil, "", fmt.Errorf("multipart missing boundary")
+ }
+
+ var data []byte
+ var contentType string
+ extractPartByCID(multipart.NewReader(msg.Body, boundary), cid, &data, &contentType)
+ if data == nil {
+ return nil, "", fmt.Errorf("content-id %q not found", cid)
+ }
+ return data, contentType, nil
+}
+
+func extractPartByCID(mr *multipart.Reader, targetCID string, outData *[]byte, outContentType *string) {
+ for {
+ part, err := mr.NextPart()
+ if err != nil {
+ break
+ }
+ ct := part.Header.Get("Content-Type")
+ cte := part.Header.Get("Content-Transfer-Encoding")
+ if ct == "" {
+ ct = "text/plain"
+ }
+ partMedia, partParams, _ := mime.ParseMediaType(ct)
+
+ contentID := part.Header.Get("Content-ID")
+ if normalizeCID(contentID) == targetCID {
+ decoded := decodeTransferEncoding(part, cte)
+ data, _ := io.ReadAll(io.LimitReader(decoded, 10*1024*1024))
+ part.Close()
+ *outData = data
+ *outContentType = partMedia
+ return
+ }
+
+ if strings.HasPrefix(partMedia, "multipart/") && partParams["boundary"] != "" {
+ extractPartByCID(multipart.NewReader(part, partParams["boundary"]), targetCID, outData, outContentType)
+ part.Close()
+ if *outData != nil {
+ return
+ }
+ continue
+ }
part.Close()
}
}
@@ -489,15 +682,18 @@ func extractFullMultipart(boundary string, r io.Reader, fe *FullEmail) {
// ExtractAttachment reads the Nth attachment (0-based index) from an .eml file.
// Returns the raw bytes, content-type, filename, and any error.
func ExtractAttachment(path string, index int) ([]byte, string, string, error) {
- f, err := os.Open(path)
+ data, err := os.ReadFile(path)
if err != nil {
return nil, "", "", fmt.Errorf("open %s: %w", path, err)
}
- defer f.Close()
+ return ExtractAttachmentFromBytes(data, index)
+}
- msg, err := mail.ReadMessage(bufio.NewReader(f))
+// ExtractAttachmentFromBytes extracts the Nth attachment from .eml content.
+func ExtractAttachmentFromBytes(data []byte, index int) ([]byte, string, string, error) {
+ msg, err := mail.ReadMessage(bytes.NewReader(data))
if err != nil {
- return nil, "", "", fmt.Errorf("parse %s: %w", path, err)
+ return nil, "", "", fmt.Errorf("parse: %w", err)
}
ct := msg.Header.Get("Content-Type")
@@ -516,15 +712,15 @@ func ExtractAttachment(path string, index int) ([]byte, string, string, error) {
return nil, "", "", fmt.Errorf("multipart missing boundary")
}
- var data []byte
+ var outData []byte
var contentType, filename string
var found bool
var idx int
- extractPartByIndex(multipart.NewReader(msg.Body, boundary), index, &idx, &data, &contentType, &filename, &found)
+ extractPartByIndex(multipart.NewReader(msg.Body, boundary), index, &idx, &outData, &contentType, &filename, &found)
if !found {
return nil, "", "", fmt.Errorf("attachment index %d out of range", index)
}
- return data, contentType, filename, nil
+ return outData, contentType, filename, nil
}
func extractPartByIndex(mr *multipart.Reader, targetIndex int, currentIndex *int, outData *[]byte, outContentType, outFilename *string, found *bool) {
diff --git a/internal/search/eml/parser_cid_real_test.go b/internal/search/eml/parser_cid_real_test.go
new file mode 100644
index 0000000..926856b
--- /dev/null
+++ b/internal/search/eml/parser_cid_real_test.go
@@ -0,0 +1,29 @@
+package eml_test
+
+import (
+ "os"
+ "path/filepath"
+ "strings"
+ "testing"
+
+ "github.com/eslider/mails/internal/search/eml"
+)
+
+// TestParseFileFull_RealCIDInline verifies a real Gmail-style email with inline image displays correctly.
+// Run with: go test -run TestParseFileFull_RealCIDInline -v (skips if file missing)
+func TestParseFileFull_RealCIDInline(t *testing.T) {
+ path := filepath.Join("..", "..", "..", "users", "019c5708-50e9-7cf4-8acd-9a0bd39271af", "gmail.com", "eslider", "gmail", "allmail", "c54db7b808b5a65a-162582.eml")
+ if _, err := os.Stat(path); os.IsNotExist(err) {
+ t.Skip("real email file not found (users/ is gitignored)")
+ }
+ fe, err := eml.ParseFileFull(path)
+ if err != nil {
+ t.Fatalf("ParseFileFull: %v", err)
+ }
+ if strings.Contains(fe.HTMLBody, "cid:17711756476991") {
+ t.Error("cid: reference was not rewritten - inline image will not display")
+ }
+ if !strings.Contains(fe.HTMLBody, "data:image/png;base64,") {
+ t.Error("HTML should contain data URI for inline image")
+ }
+}
diff --git a/internal/search/eml/parser_test.go b/internal/search/eml/parser_test.go
index 029fbc6..4efeefc 100644
--- a/internal/search/eml/parser_test.go
+++ b/internal/search/eml/parser_test.go
@@ -251,6 +251,69 @@ func TestParseFileFull_NonExistent(t *testing.T) {
}
}
+func TestParseFileFull_CIDInlineImage(t *testing.T) {
+ dir := t.TempDir()
+ // multipart/related: HTML with cid: reference + inline PNG (1x1 red pixel)
+ // Matches Gmail/Outlook style: cid: with @domain, img src="cid:...".
+ raw := "From: a@b.com\r\nTo: c@d.com\r\nSubject: With Inline Image\r\nDate: Mon, 10 Feb 2025 12:00:00 +0000\r\n" +
+ "Content-Type: multipart/related; boundary=\"REL\"\r\n\r\n" +
+ "--REL\r\nContent-Type: text/html; charset=utf-8\r\n\r\n" +
+ `
` + "\r\n" +
+ "--REL\r\nContent-Type: image/png; name=\"logo.png\"\r\nContent-Disposition: inline\r\nContent-ID: <17711756476991fedf126cb645684293@markets-platform.com>\r\nContent-Transfer-Encoding: base64\r\n\r\n" +
+ "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z8BQHwAEBgIApD5fRAAAAABJRU5ErkJggg==\r\n" +
+ "--REL--\r\n"
+ path := writeTestEml(t, dir, "cid-inline.eml", raw)
+
+ fe, err := eml.ParseFileFull(path)
+ if err != nil {
+ t.Fatalf("unexpected error: %v", err)
+ }
+ if strings.Contains(fe.HTMLBody, "cid:17711756476991") {
+ t.Errorf("cid: was not rewritten in html_body")
+ }
+ if !strings.Contains(fe.HTMLBody, "data:image/png;base64,") {
+ t.Errorf("html_body missing data URI: %q", fe.HTMLBody)
+ }
+ if !strings.Contains(fe.HTMLBody, "iVBORw0KGgo") {
+ t.Errorf("html_body missing embedded base64 image data")
+ }
+}
+
+func TestExtractPartByCID(t *testing.T) {
+ dir := t.TempDir()
+ raw := "From: a@b.com\r\nTo: c@d.com\r\nSubject: CID Test\r\nDate: Mon, 10 Feb 2025 12:00:00 +0000\r\n" +
+ "Content-Type: multipart/mixed; boundary=\"MIX\"\r\n\r\n" +
+ "--MIX\r\nContent-Type: text/plain\r\n\r\nBody.\r\n" +
+ "--MIX\r\nContent-Type: image/gif; name=\"dot.gif\"\r\nContent-Disposition: inline\r\nContent-ID: \r\n\r\n" +
+ "GIF87a\x01\x00\x01\x00\x80\x00\x00\xff\xff\xff\x00\x00\x00!\xf9\x04\x01\x00\x00\x00\x00,\x00\x00\x00\x00\x01\x00\x01\x00\x00\x02\x02D\x01\x00;\r\n" +
+ "--MIX--\r\n"
+ path := writeTestEml(t, dir, "cid-extract.eml", raw)
+
+ data, ct, err := eml.ExtractPartByCID(path, "dot@example.com")
+ if err != nil {
+ t.Fatalf("ExtractPartByCID: %v", err)
+ }
+ if ct != "image/gif" {
+ t.Errorf("content_type = %q, want image/gif", ct)
+ }
+ if len(data) < 10 {
+ t.Errorf("data too short: %d bytes", len(data))
+ }
+ if !strings.HasPrefix(string(data), "GIF87a") {
+ t.Errorf("data should start with GIF87a, got %q", string(data)[:min(10, len(string(data)))])
+ }
+
+ _, _, err = eml.ExtractPartByCID(path, "")
+ if err != nil {
+ t.Errorf("ExtractPartByCID with angle brackets: %v", err)
+ }
+
+ _, _, err = eml.ExtractPartByCID(path, "nonexistent@example.com")
+ if err == nil {
+ t.Error("expected error for nonexistent CID")
+ }
+}
+
// --- Charset decoding tests ---
func TestParseFile_ISO88591Body(t *testing.T) {
diff --git a/internal/search/index/index.go b/internal/search/index/index.go
index 16cc0a0..e29677a 100644
--- a/internal/search/index/index.go
+++ b/internal/search/index/index.go
@@ -2,6 +2,7 @@
package index
import (
+ "context"
"database/sql"
"fmt"
"log"
@@ -15,6 +16,7 @@ import (
_ "github.com/marcboeker/go-duckdb"
"github.com/eslider/mails/internal/search/eml"
+ "github.com/eslider/mails/internal/storage"
)
// reChecksum matches the 16-hex-char checksum prefix in filenames like "a1b2c3d4e5f67890-123.eml".
@@ -23,12 +25,14 @@ var reChecksum = regexp.MustCompile(`^([0-9a-f]{16})-`)
// Index stores parsed email metadata in a DuckDB in-memory database,
// persisted to a Parquet file with zstd compression.
type Index struct {
- mu sync.RWMutex
- db *sql.DB
- buildAt time.Time
- emailDir string
- indexPath string
- total int
+ mu sync.RWMutex
+ db *sql.DB
+ buildAt time.Time
+ emailDir string
+ indexPath string
+ blobStore storage.BlobStore
+ emailKeyPref string // key prefix when using blobStore
+ total int
}
const createTableSQL = `CREATE TABLE IF NOT EXISTS emails (
@@ -43,7 +47,8 @@ const createTableSQL = `CREATE TABLE IF NOT EXISTS emails (
// New creates a new index. If indexPath points to an existing Parquet file,
// the index is loaded from it (fast startup).
-func New(emailDir, indexPath string) (*Index, error) {
+// blobStore and usersDir are optional; when set, emails are read from S3.
+func New(emailDir, indexPath string, blobStore storage.BlobStore, usersDir string) (*Index, error) {
db, err := sql.Open("duckdb", "")
if err != nil {
return nil, fmt.Errorf("open duckdb: %w", err)
@@ -54,6 +59,13 @@ func New(emailDir, indexPath string) (*Index, error) {
db: db,
emailDir: emailDir,
indexPath: indexPath,
+ blobStore: blobStore,
+ }
+ if blobStore != nil && usersDir != "" {
+ rel, err := filepath.Rel(usersDir, emailDir)
+ if err == nil {
+ idx.emailKeyPref = filepath.ToSlash(rel)
+ }
}
if indexPath != "" {
@@ -129,6 +141,54 @@ func (idx *Index) saveParquet() error {
return err
}
+func walkEmailsFromBlobStore(blob storage.BlobStore, prefix string) ([]eml.Email, int) {
+ ctx := context.Background()
+ keys, err := blob.List(ctx, prefix)
+ if err != nil {
+ log.Printf("WARN: list %s: %v", prefix, err)
+ return nil, 0
+ }
+ var parsed []eml.Email
+ var errCount int
+ seen := make(map[string]bool)
+ for _, k := range keys {
+ if !strings.HasSuffix(strings.ToLower(k), ".eml") {
+ continue
+ }
+ name := filepath.Base(k)
+ if cs := extractChecksum(name); cs != "" {
+ if seen[cs] {
+ continue
+ }
+ seen[cs] = true
+ }
+ data, err := blob.Read(ctx, k)
+ if err != nil {
+ log.Printf("WARN: read %s: %v", k, err)
+ errCount++
+ continue
+ }
+ relPath := k
+ if strings.HasPrefix(k, prefix+"/") {
+ relPath = k[len(prefix)+1:]
+ } else if strings.HasPrefix(k, prefix) {
+ relPath = k[len(prefix):]
+ if relPath != "" && relPath[0] == '/' {
+ relPath = relPath[1:]
+ }
+ }
+ e, parseErr := eml.ParseBytes(relPath, data)
+ if parseErr != nil {
+ log.Printf("WARN: parse %s: %v", k, parseErr)
+ errCount++
+ continue
+ }
+ e.Path = filepath.ToSlash(relPath)
+ parsed = append(parsed, e)
+ }
+ return parsed, errCount
+}
+
// WalkEmails walks the email directory, parses .eml files, and returns
// deduplicated emails by checksum.
func WalkEmails(emailDir string) ([]eml.Email, int) {
@@ -164,10 +224,16 @@ func WalkEmails(emailDir string) ([]eml.Email, int) {
return parsed, errCount
}
-// Build walks the email directory, parses every .eml file, stores them in
-// DuckDB and exports to Parquet with zstd.
+// Build walks the email directory (or S3 prefix), parses every .eml file,
+// stores them in DuckDB and exports to Parquet with zstd.
func (idx *Index) Build() (int, int) {
- parsed, errCount := WalkEmails(idx.emailDir)
+ var parsed []eml.Email
+ var errCount int
+ if idx.blobStore != nil && idx.emailKeyPref != "" {
+ parsed, errCount = walkEmailsFromBlobStore(idx.blobStore, idx.emailKeyPref)
+ } else {
+ parsed, errCount = WalkEmails(idx.emailDir)
+ }
idx.mu.Lock()
defer idx.mu.Unlock()
diff --git a/internal/search/index/index_test.go b/internal/search/index/index_test.go
index dd8e426..0498530 100644
--- a/internal/search/index/index_test.go
+++ b/internal/search/index/index_test.go
@@ -11,7 +11,7 @@ import (
// newTestIndex creates an in-memory index (no parquet persistence) for testing.
func newTestIndex(t *testing.T, dir string) *index.Index {
t.Helper()
- idx, err := index.New(dir, "")
+ idx, err := index.New(dir, "", nil, "")
if err != nil {
t.Fatalf("index.New: %v", err)
}
@@ -271,7 +271,7 @@ func TestParquetPersistence(t *testing.T) {
parquetPath := filepath.Join(t.TempDir(), "test.parquet")
// Build and save to parquet.
- idx1, err := index.New(dir, parquetPath)
+ idx1, err := index.New(dir, parquetPath, nil, "")
if err != nil {
t.Fatalf("index.New: %v", err)
}
@@ -292,7 +292,7 @@ func TestParquetPersistence(t *testing.T) {
t.Logf("Parquet file size: %d bytes", info.Size())
// Load from parquet — should not need to re-parse .eml files.
- idx2, err := index.New(dir, parquetPath)
+ idx2, err := index.New(dir, parquetPath, nil, "")
if err != nil {
t.Fatalf("index.New (reload): %v", err)
}
@@ -343,14 +343,14 @@ func TestSearchMultiDeduplicatesByChecksum(t *testing.T) {
parquet1 := filepath.Join(root, "idx1.parquet")
parquet2 := filepath.Join(root, "idx2.parquet")
- idx1, err := index.New(dir1, parquet1)
+ idx1, err := index.New(dir1, parquet1, nil, "")
if err != nil {
t.Fatalf("index.New 1: %v", err)
}
idx1.Build()
idx1.Close()
- idx2, err := index.New(dir2, parquet2)
+ idx2, err := index.New(dir2, parquet2, nil, "")
if err != nil {
t.Fatalf("index.New 2: %v", err)
}
@@ -389,7 +389,7 @@ func TestSearchMultiKeepsAllWhenNoChecksumInPath(t *testing.T) {
os.WriteFile(filepath.Join(inbox, "message_3.eml"), []byte("From: a@b.com\r\nTo: c@d.com\r\nSubject: Msg 3\r\nDate: Mon, 10 Feb 2025 12:00:00 +0000\r\n\r\nBody 3"), 0644)
parquetPath := filepath.Join(root, "idx.parquet")
- idx, err := index.New(dir, parquetPath)
+ idx, err := index.New(dir, parquetPath, nil, "")
if err != nil {
t.Fatalf("index.New: %v", err)
}
@@ -421,10 +421,10 @@ func TestSearchMultiDeduplicatesByContentWhenNoChecksumInPath(t *testing.T) {
os.WriteFile(filepath.Join(inbox, "message_2.eml"), []byte(eml2), 0644)
}
- idx1, _ := index.New(filepath.Join(root, "import-1"), filepath.Join(root, "idx1.parquet"))
+ idx1, _ := index.New(filepath.Join(root, "import-1"), filepath.Join(root, "idx1.parquet"), nil, "")
idx1.Build()
idx1.Close()
- idx2, _ := index.New(filepath.Join(root, "import-2"), filepath.Join(root, "idx2.parquet"))
+ idx2, _ := index.New(filepath.Join(root, "import-2"), filepath.Join(root, "idx2.parquet"), nil, "")
idx2.Build()
idx2.Close()
diff --git a/internal/storage/blobstore.go b/internal/storage/blobstore.go
new file mode 100644
index 0000000..28fdf25
--- /dev/null
+++ b/internal/storage/blobstore.go
@@ -0,0 +1,134 @@
+// Package storage provides S3-compatible object storage and a BlobStore
+// abstraction for user data (users/ directory layout).
+package storage
+
+import (
+ "context"
+ "os"
+ "path/filepath"
+ "strings"
+)
+
+// BlobStore reads and writes blobs by key. Keys use forward slashes and are
+// relative to the users directory (e.g. "uuid/user.json", "uuid/domain/local/inbox/file.eml").
+type BlobStore interface {
+ Write(ctx context.Context, key string, data []byte) error
+ Read(ctx context.Context, key string) ([]byte, error)
+ List(ctx context.Context, prefix string) ([]string, error)
+}
+
+// FSBlobStore stores blobs on the local filesystem.
+type FSBlobStore struct {
+ root string
+}
+
+// NewFSBlobStore creates a filesystem-backed blob store.
+func NewFSBlobStore(root string) *FSBlobStore {
+ return &FSBlobStore{root: filepath.Clean(root)}
+}
+
+// Write writes data to key (path relative to root).
+func (f *FSBlobStore) Write(ctx context.Context, key string, data []byte) error {
+ path := filepath.Join(f.root, filepath.FromSlash(key))
+ if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
+ return err
+ }
+ return os.WriteFile(path, data, 0o644)
+}
+
+// Read reads a blob by key.
+func (f *FSBlobStore) Read(ctx context.Context, key string) ([]byte, error) {
+ path := filepath.Join(f.root, filepath.FromSlash(key))
+ data, err := os.ReadFile(path)
+ if err != nil {
+ if os.IsNotExist(err) {
+ return nil, ErrNotFound
+ }
+ return nil, err
+ }
+ return data, nil
+}
+
+// List returns keys under prefix (non-recursive for first level, or recursive based on impl).
+// FS impl walks recursively to match S3 List behavior.
+func (f *FSBlobStore) List(ctx context.Context, prefix string) ([]string, error) {
+ dir := filepath.Join(f.root, filepath.FromSlash(prefix))
+ var keys []string
+ err := filepath.WalkDir(dir, func(path string, d os.DirEntry, err error) error {
+ if err != nil {
+ if os.IsNotExist(err) {
+ return nil
+ }
+ return err
+ }
+ if d.IsDir() {
+ return nil
+ }
+ rel, err := filepath.Rel(f.root, path)
+ if err != nil {
+ return nil
+ }
+ keys = append(keys, filepath.ToSlash(rel))
+ return nil
+ })
+ return keys, err
+}
+
+// S3BlobStore stores blobs in S3. Keys are used as S3 object keys.
+type S3BlobStore struct {
+ client *S3Client
+ prefix string
+}
+
+// NewS3BlobStore creates an S3-backed blob store with optional key prefix.
+func NewS3BlobStore(client *S3Client, prefix string) *S3BlobStore {
+ prefix = strings.Trim(prefix, "/")
+ if prefix != "" {
+ prefix += "/"
+ }
+ return &S3BlobStore{client: client, prefix: prefix}
+}
+
+// Write writes data to key.
+func (s *S3BlobStore) Write(ctx context.Context, key string, data []byte) error {
+ return s.client.PutBytes(ctx, s.prefix+key, data)
+}
+
+// Read reads a blob by key.
+func (s *S3BlobStore) Read(ctx context.Context, key string) ([]byte, error) {
+ return s.client.Get(ctx, s.prefix+key)
+}
+
+// NewBlobStore returns a BlobStore from env. If S3 env vars are set, returns S3BlobStore;
+// otherwise returns FSBlobStore rooted at dataDir.
+func NewBlobStore(dataDir string) (BlobStore, error) {
+ cfg := ConfigFromEnv()
+ if cfg != nil && cfg.AccessKeyID != "" && cfg.SecretAccessKey != "" {
+ client, err := NewS3Client(cfg)
+ if err != nil {
+ return nil, err
+ }
+ ctx := context.Background()
+ if err := client.EnsureBucket(ctx); err != nil {
+ return nil, err
+ }
+ return NewS3BlobStore(client, "users"), nil
+ }
+ return NewFSBlobStore(dataDir), nil
+}
+
+// List returns keys under prefix (relative to prefix, without store prefix).
+func (s *S3BlobStore) List(ctx context.Context, prefix string) ([]string, error) {
+ fullPrefix := s.prefix + prefix
+ all, err := s.client.List(ctx, fullPrefix)
+ if err != nil {
+ return nil, err
+ }
+ keys := make([]string, 0, len(all))
+ for _, k := range all {
+ if k != "" {
+ keys = append(keys, strings.TrimPrefix(k, s.prefix))
+ }
+ }
+ return keys, nil
+}
diff --git a/internal/storage/s3.go b/internal/storage/s3.go
new file mode 100644
index 0000000..ddce8cc
--- /dev/null
+++ b/internal/storage/s3.go
@@ -0,0 +1,194 @@
+// Package storage provides S3-compatible object storage (AWS S3, MinIO).
+package storage
+
+import (
+ "bytes"
+ "context"
+ "errors"
+ "fmt"
+ "io"
+ "os"
+ "strconv"
+ "strings"
+
+ "github.com/aws/aws-sdk-go-v2/aws"
+ "github.com/aws/aws-sdk-go-v2/credentials"
+ "github.com/aws/aws-sdk-go-v2/service/s3"
+ "github.com/aws/aws-sdk-go-v2/service/s3/types"
+)
+
+// S3Config holds S3/MinIO connection settings.
+type S3Config struct {
+ Endpoint string // e.g. http://localhost:9000
+ AccessKeyID string
+ SecretAccessKey string
+ Bucket string
+ UseSSL bool
+ Region string
+}
+
+// S3Client provides Put and Get for objects in S3-compatible storage.
+type S3Client struct {
+ client *s3.Client
+ bucket string
+}
+
+// ConfigFromEnv reads S3 config from environment variables.
+// Returns nil if S3_ENDPOINT is not set.
+func ConfigFromEnv() *S3Config {
+ endpoint := os.Getenv("S3_ENDPOINT")
+ if endpoint == "" {
+ return nil
+ }
+ useSSL := true
+ if v := os.Getenv("S3_USE_SSL"); v != "" {
+ useSSL, _ = strconv.ParseBool(v)
+ }
+ return &S3Config{
+ Endpoint: normalizeEndpoint(endpoint, useSSL),
+ AccessKeyID: os.Getenv("S3_ACCESS_KEY_ID"),
+ SecretAccessKey: os.Getenv("S3_SECRET_ACCESS_KEY"),
+ Bucket: envOr("S3_BUCKET", "mails"),
+ UseSSL: useSSL,
+ Region: envOr("AWS_REGION", "us-east-1"),
+ }
+}
+
+func normalizeEndpoint(endpoint string, useSSL bool) string {
+ endpoint = strings.TrimSpace(endpoint)
+ if endpoint == "" {
+ return ""
+ }
+ scheme := "https"
+ if !useSSL {
+ scheme = "http"
+ }
+ if !strings.HasPrefix(endpoint, "http://") && !strings.HasPrefix(endpoint, "https://") {
+ return scheme + "://" + endpoint
+ }
+ return endpoint
+}
+
+func envOr(key, fallback string) string {
+ if v := os.Getenv(key); v != "" {
+ return v
+ }
+ return fallback
+}
+
+// NewS3Client creates an S3 client from config. Returns error if config is invalid.
+func NewS3Client(cfg *S3Config) (*S3Client, error) {
+ if cfg == nil || cfg.Endpoint == "" {
+ return nil, fmt.Errorf("storage: S3 config required (endpoint)")
+ }
+ if cfg.Bucket == "" {
+ return nil, fmt.Errorf("storage: S3 bucket required")
+ }
+
+ credProvider := credentials.NewStaticCredentialsProvider(
+ cfg.AccessKeyID,
+ cfg.SecretAccessKey,
+ "",
+ )
+
+ customResolver := aws.EndpointResolverWithOptionsFunc(func(service, region string, opts ...interface{}) (aws.Endpoint, error) {
+ return aws.Endpoint{
+ URL: cfg.Endpoint,
+ HostnameImmutable: true,
+ SigningRegion: cfg.Region,
+ }, nil
+ })
+
+ client := s3.NewFromConfig(aws.Config{
+ Region: cfg.Region,
+ Credentials: credProvider,
+ EndpointResolverWithOptions: customResolver,
+ }, func(o *s3.Options) {
+ o.UsePathStyle = true // required for MinIO
+ })
+
+ return &S3Client{client: client, bucket: cfg.Bucket}, nil
+}
+
+// EnsureBucket creates the bucket if it does not exist.
+func (c *S3Client) EnsureBucket(ctx context.Context) error {
+ _, err := c.client.HeadBucket(ctx, &s3.HeadBucketInput{Bucket: aws.String(c.bucket)})
+ if err == nil {
+ return nil
+ }
+ // Try to create
+ _, err = c.client.CreateBucket(ctx, &s3.CreateBucketInput{
+ Bucket: aws.String(c.bucket),
+ })
+ if err != nil {
+ // Bucket may have been created concurrently
+ var conflict *types.BucketAlreadyOwnedByYou
+ if errors.As(err, &conflict) {
+ return nil
+ }
+ return fmt.Errorf("create bucket %s: %w", c.bucket, err)
+ }
+ return nil
+}
+
+// Put writes an object to the given key.
+func (c *S3Client) Put(ctx context.Context, key string, body io.Reader) error {
+ _, err := c.client.PutObject(ctx, &s3.PutObjectInput{
+ Bucket: aws.String(c.bucket),
+ Key: aws.String(key),
+ Body: body,
+ })
+ return err
+}
+
+// PutBytes writes bytes to the given key.
+func (c *S3Client) PutBytes(ctx context.Context, key string, data []byte) error {
+ return c.Put(ctx, key, bytes.NewReader(data))
+}
+
+// Get reads an object by key. Returns ErrNotFound if the object does not exist.
+func (c *S3Client) Get(ctx context.Context, key string) ([]byte, error) {
+ out, err := c.client.GetObject(ctx, &s3.GetObjectInput{
+ Bucket: aws.String(c.bucket),
+ Key: aws.String(key),
+ })
+ if err != nil {
+ var noSuchKey *types.NoSuchKey
+ var notFound *types.NotFound
+ if errors.As(err, &noSuchKey) || errors.As(err, ¬Found) {
+ return nil, ErrNotFound
+ }
+ return nil, err
+ }
+ defer out.Body.Close()
+ return io.ReadAll(out.Body)
+}
+
+// List lists object keys with the given prefix.
+func (c *S3Client) List(ctx context.Context, prefix string) ([]string, error) {
+ var keys []string
+ var contToken *string
+ for {
+ out, err := c.client.ListObjectsV2(ctx, &s3.ListObjectsV2Input{
+ Bucket: aws.String(c.bucket),
+ Prefix: aws.String(prefix),
+ ContinuationToken: contToken,
+ })
+ if err != nil {
+ return nil, err
+ }
+ for _, obj := range out.Contents {
+ if obj.Key != nil {
+ keys = append(keys, *obj.Key)
+ }
+ }
+ if !aws.ToBool(out.IsTruncated) {
+ break
+ }
+ contToken = out.NextContinuationToken
+ }
+ return keys, nil
+}
+
+// ErrNotFound is returned when an object does not exist.
+var ErrNotFound = errors.New("object not found")
diff --git a/internal/storage/s3_test.go b/internal/storage/s3_test.go
new file mode 100644
index 0000000..25c566e
--- /dev/null
+++ b/internal/storage/s3_test.go
@@ -0,0 +1,71 @@
+package storage
+
+import (
+ "context"
+ "errors"
+ "testing"
+)
+
+// TestS3StoreRetrieve verifies Put and Get against an S3-compatible store (e.g. MinIO).
+//
+// Run MinIO first:
+//
+// docker compose --profile s3 up -d minio
+//
+// Then set env and run:
+//
+// S3_ENDPOINT=http://localhost:9900 S3_ACCESS_KEY_ID=minioadmin S3_SECRET_ACCESS_KEY=minioadmin S3_BUCKET=mails-test S3_USE_SSL=false go test -v ./internal/storage/ -run TestS3StoreRetrieve
+func TestS3StoreRetrieve(t *testing.T) {
+ cfg := ConfigFromEnv()
+ if cfg == nil {
+ t.Skip("S3_ENDPOINT not set, skipping integration test")
+ }
+
+ client, err := NewS3Client(cfg)
+ if err != nil {
+ t.Fatalf("NewS3Client: %v", err)
+ }
+
+ ctx := context.Background()
+ if err := client.EnsureBucket(ctx); err != nil {
+ t.Fatalf("EnsureBucket: %v", err)
+ }
+
+ key := "test/integration/hello.eml"
+ content := []byte("From: test@example.com\r\nTo: you@example.com\r\nSubject: S3 Test\r\n\r\nHello from S3 storage test.\r\n")
+
+ if err := client.PutBytes(ctx, key, content); err != nil {
+ t.Fatalf("PutBytes: %v", err)
+ }
+
+ got, err := client.Get(ctx, key)
+ if err != nil {
+ t.Fatalf("Get: %v", err)
+ }
+
+ if string(got) != string(content) {
+ t.Errorf("got %q, want %q", got, content)
+ }
+}
+
+// TestS3GetNotFound verifies Get returns ErrNotFound for missing keys.
+func TestS3GetNotFound(t *testing.T) {
+ cfg := ConfigFromEnv()
+ if cfg == nil {
+ t.Skip("S3_ENDPOINT not set, skipping integration test")
+ }
+
+ client, err := NewS3Client(cfg)
+ if err != nil {
+ t.Fatalf("NewS3Client: %v", err)
+ }
+
+ ctx := context.Background()
+ _, err = client.Get(ctx, "nonexistent/key/12345.eml")
+ if err == nil {
+ t.Fatal("Get: expected error for missing key")
+ }
+ if !errors.Is(err, ErrNotFound) {
+ t.Errorf("Get: got %v, want ErrNotFound", err)
+ }
+}
diff --git a/internal/sync/gmail/gmail.go b/internal/sync/gmail/gmail.go
index 172d4a1..d34c776 100644
--- a/internal/sync/gmail/gmail.go
+++ b/internal/sync/gmail/gmail.go
@@ -16,10 +16,13 @@ type SyncState interface {
MarkUIDSynced(accountID, folder, uid string) error
}
+// SaveEmailFunc saves email data by full path. If nil, os.WriteFile is used.
+type SaveEmailFunc func(path string, data []byte) error
+
// Sync downloads new emails from a Gmail account via the Gmail API.
// Returns (newMessages, error). NEVER deletes messages from the server.
func Sync(acct model.EmailAccount, emailDir string, state SyncState) (int, error) {
- return SyncWithContext(context.Background(), acct, emailDir, state)
+ return SyncWithContext(context.Background(), acct, emailDir, state, nil)
}
// SyncWithContext downloads new emails with cancellation support.
@@ -28,7 +31,7 @@ func Sync(acct model.EmailAccount, emailDir string, state SyncState) (int, error
// and google.golang.org/api/gmail/v1. For now, this is a stub that
// returns an error indicating it needs implementation with proper
// OAuth2 credentials.
-func SyncWithContext(ctx context.Context, acct model.EmailAccount, emailDir string, state SyncState) (int, error) {
+func SyncWithContext(ctx context.Context, acct model.EmailAccount, emailDir string, state SyncState, saveFn SaveEmailFunc) (int, error) {
log.Printf("Gmail API: sync for %s (stub — not yet implemented)", acct.Email)
// Gmail API sync requires:
diff --git a/internal/sync/imap/imap.go b/internal/sync/imap/imap.go
index e35742b..bcd1f64 100644
--- a/internal/sync/imap/imap.go
+++ b/internal/sync/imap/imap.go
@@ -28,14 +28,18 @@ type SyncState interface {
// ProgressFunc is called with human-readable progress updates during sync.
type ProgressFunc func(msg string)
+// SaveEmailFunc saves email data by full path. If nil, os.WriteFile is used.
+type SaveEmailFunc func(path string, data []byte) error
+
// Sync downloads new emails from an IMAP account.
// Returns (newMessages, error). NEVER deletes or marks messages on the server.
func Sync(acct model.EmailAccount, emailDir string, state SyncState) (int, error) {
- return SyncWithContext(context.Background(), acct, emailDir, state, nil)
+ return SyncWithContext(context.Background(), acct, emailDir, state, nil, nil)
}
// SyncWithContext downloads new emails with cancellation and progress reporting.
-func SyncWithContext(ctx context.Context, acct model.EmailAccount, emailDir string, state SyncState, onProgress ProgressFunc) (int, error) {
+// saveFn optionally stores emails (e.g. to S3). If nil, uses os.WriteFile.
+func SyncWithContext(ctx context.Context, acct model.EmailAccount, emailDir string, state SyncState, onProgress ProgressFunc, saveFn SaveEmailFunc) (int, error) {
if onProgress == nil {
onProgress = func(string) {}
}
@@ -85,7 +89,7 @@ func SyncWithContext(ctx context.Context, acct model.EmailAccount, emailDir stri
}
onProgress(fmt.Sprintf("folder %d/%d: %s", fi+1, len(folders), folder))
- n, err := syncFolderWithContext(ctx, client, acct, folder, emailDir, state)
+ n, err := syncFolderWithContext(ctx, client, acct, folder, emailDir, state, saveFn)
if err != nil {
if ctx.Err() != nil {
return totalNew, ctx.Err()
@@ -102,11 +106,13 @@ func SyncWithContext(ctx context.Context, acct model.EmailAccount, emailDir stri
const fetchBatchSize = 50
-func syncFolderWithContext(ctx context.Context, client *imapClient, acct model.EmailAccount, folder, emailDir string, state SyncState) (int, error) {
+func syncFolderWithContext(ctx context.Context, client *imapClient, acct model.EmailAccount, folder, emailDir string, state SyncState, saveFn SaveEmailFunc) (int, error) {
folderPath := imapFolderToPath(folder)
dir := filepath.Join(emailDir, folderPath)
- if err := os.MkdirAll(dir, 0o755); err != nil {
- return 0, err
+ if saveFn == nil {
+ if err := os.MkdirAll(dir, 0o755); err != nil {
+ return 0, err
+ }
}
uids, err := client.selectAndSearch(folder)
@@ -153,7 +159,7 @@ func syncFolderWithContext(ctx context.Context, client *imapClient, acct model.E
log.Printf("WARN: fetch UID %d: %v", uid, err)
continue
}
- if saveEmail(dir, uid, raw, acct.ID, folder, state) {
+ if saveEmail(dir, uid, raw, acct.ID, folder, state, saveFn) {
newCount++
}
}
@@ -161,7 +167,7 @@ func syncFolderWithContext(ctx context.Context, client *imapClient, acct model.E
}
for uid, raw := range messages {
- if saveEmail(dir, uid, raw, acct.ID, folder, state) {
+ if saveEmail(dir, uid, raw, acct.ID, folder, state, saveFn) {
newCount++
}
}
@@ -170,7 +176,7 @@ func syncFolderWithContext(ctx context.Context, client *imapClient, acct model.E
return newCount, nil
}
-func saveEmail(dir string, uid int, raw []byte, accountID, folder string, state SyncState) bool {
+func saveEmail(dir string, uid int, raw []byte, accountID, folder string, state SyncState, saveFn SaveEmailFunc) bool {
if len(raw) == 0 {
return false
}
@@ -178,12 +184,19 @@ func saveEmail(dir string, uid int, raw []byte, accountID, folder string, state
filename := fmt.Sprintf("%s-%d.eml", checksum, uid)
path := filepath.Join(dir, filename)
- if err := os.WriteFile(path, raw, 0o644); err != nil {
+ if saveFn != nil {
+ if err := saveFn(path, raw); err != nil {
+ log.Printf("WARN: write %s: %v", path, err)
+ return false
+ }
+ } else if err := os.WriteFile(path, raw, 0o644); err != nil {
log.Printf("WARN: write %s: %v", path, err)
return false
}
- setFileMtime(path, raw)
+ if saveFn == nil {
+ setFileMtime(path, raw)
+ }
state.MarkUIDSynced(accountID, folder, fmt.Sprintf("%d", uid))
return true
}
diff --git a/internal/sync/pop3/pop3.go b/internal/sync/pop3/pop3.go
index 096cc34..ba211bf 100644
--- a/internal/sync/pop3/pop3.go
+++ b/internal/sync/pop3/pop3.go
@@ -25,15 +25,19 @@ type SyncState interface {
MarkUIDSynced(accountID, folder, uid string) error
}
+// SaveEmailFunc saves email data by full path. If nil, os.WriteFile is used.
+type SaveEmailFunc func(path string, data []byte) error
+
// Sync downloads new emails from a POP3 account.
// Uses SHA-256 hash deduplication since POP3 has no stable UIDs.
// Returns (newMessages, error). NEVER deletes messages from the server.
func Sync(acct model.EmailAccount, emailDir string, state SyncState) (int, error) {
- return SyncWithContext(context.Background(), acct, emailDir, state)
+ return SyncWithContext(context.Background(), acct, emailDir, state, nil)
}
// SyncWithContext downloads new emails with cancellation support.
-func SyncWithContext(ctx context.Context, acct model.EmailAccount, emailDir string, state SyncState) (int, error) {
+// saveFn optionally stores emails (e.g. to S3). If nil, uses os.WriteFile.
+func SyncWithContext(ctx context.Context, acct model.EmailAccount, emailDir string, state SyncState, saveFn SaveEmailFunc) (int, error) {
port := acct.Port
if port == 0 {
if acct.SSL {
@@ -82,7 +86,9 @@ func SyncWithContext(ctx context.Context, acct model.EmailAccount, emailDir stri
log.Printf("POP3: %d messages in mailbox", count)
inboxDir := filepath.Join(emailDir, "inbox")
- os.MkdirAll(inboxDir, 0o755)
+ if saveFn == nil {
+ os.MkdirAll(inboxDir, 0o755)
+ }
totalNew := 0
for i := 1; i <= count; i++ {
@@ -110,12 +116,19 @@ func SyncWithContext(ctx context.Context, acct model.EmailAccount, emailDir stri
filename := fmt.Sprintf("%s-%s.eml", checksum, msgHash)
path := filepath.Join(inboxDir, filename)
- if err := os.WriteFile(path, raw, 0o644); err != nil {
+ if saveFn != nil {
+ if err := saveFn(path, raw); err != nil {
+ log.Printf("WARN: write %s: %v", path, err)
+ continue
+ }
+ } else if err := os.WriteFile(path, raw, 0o644); err != nil {
log.Printf("WARN: write %s: %v", path, err)
continue
}
- setFileMtime(path, raw)
+ if saveFn == nil {
+ setFileMtime(path, raw)
+ }
state.MarkUIDSynced(acct.ID, "inbox", msgHash)
totalNew++
}
diff --git a/internal/sync/pst/pst.go b/internal/sync/pst/pst.go
index d431f0f..2ee46d5 100644
--- a/internal/sync/pst/pst.go
+++ b/internal/sync/pst/pst.go
@@ -40,11 +40,16 @@ func init() {
// ProgressFunc receives progress updates during PST import.
type ProgressFunc func(phase string, current, total int)
+// SaveEmailFunc saves extracted data by full path. If nil, os.WriteFile is used.
+type SaveEmailFunc func(path string, data []byte) error
+
// Import extracts all messages from a PST/OST file and saves them as .eml files.
// Uses go-pst first; if it panics or fails, falls back to readpst (from pst-utils)
// when available for broader OST compatibility.
+// saveFn optionally stores extracted files (e.g. to S3). If nil, uses os.WriteFile.
+// Note: readpst fallback always writes to local filesystem.
// Returns (extracted count, error count).
-func Import(pstPath, emailDir string, onProgress ProgressFunc) (int, int, error) {
+func Import(pstPath, emailDir string, onProgress ProgressFunc, saveFn SaveEmailFunc) (int, int, error) {
if onProgress == nil {
onProgress = func(string, int, int) {}
}
@@ -57,7 +62,7 @@ func Import(pstPath, emailDir string, onProgress ProgressFunc) (int, int, error)
importErr = fmt.Errorf("go-pst panic: %v", r)
}
}()
- extracted, errCount, importErr = importGoPst(pstPath, emailDir, onProgress)
+ extracted, errCount, importErr = importGoPst(pstPath, emailDir, onProgress, saveFn)
}()
if importErr == nil {
@@ -65,11 +70,12 @@ func Import(pstPath, emailDir string, onProgress ProgressFunc) (int, int, error)
}
// Fallback to readpst when go-pst fails (e.g. newer OST formats, btree bugs).
+ // readpst always writes to local filesystem.
log.Printf("INFO: go-pst failed (%v), trying readpst fallback", importErr)
return importReadpst(pstPath, emailDir, onProgress)
}
-func importGoPst(pstPath, emailDir string, onProgress ProgressFunc) (int, int, error) {
+func importGoPst(pstPath, emailDir string, onProgress ProgressFunc, saveFn SaveEmailFunc) (int, int, error) {
f, err := os.Open(pstPath)
if err != nil {
return 0, 0, fmt.Errorf("open PST: %w", err)
@@ -89,8 +95,10 @@ func importGoPst(pstPath, emailDir string, onProgress ProgressFunc) (int, int, e
if err := pstFile.WalkFolders(func(folder *pst.Folder) error {
folderPath := sanitizeFolderName(folder.Name)
dir := filepath.Join(emailDir, folderPath)
- if err := os.MkdirAll(dir, 0o755); err != nil {
- return err
+ if saveFn == nil {
+ if err := os.MkdirAll(dir, 0o755); err != nil {
+ return err
+ }
}
iter, err := folder.GetMessageIterator()
@@ -113,13 +121,19 @@ func importGoPst(pstPath, emailDir string, onProgress ProgressFunc) (int, int, e
filename := fmt.Sprintf("%s-%d.%s", checksum, extracted, ext)
path := filepath.Join(dir, filename)
- if err := os.WriteFile(path, data, 0o644); err != nil {
+ if saveFn != nil {
+ if err := saveFn(path, data); err != nil {
+ log.Printf("WARN: write %s: %v", path, err)
+ errCount++
+ continue
+ }
+ } else if err := os.WriteFile(path, data, 0o644); err != nil {
log.Printf("WARN: write %s: %v", path, err)
errCount++
continue
}
- if !date.IsZero() {
+ if saveFn == nil && !date.IsZero() {
os.Chtimes(path, date, date)
}
diff --git a/internal/sync/pst/pst_test.go b/internal/sync/pst/pst_test.go
index 4dc10a4..d2b25ff 100644
--- a/internal/sync/pst/pst_test.go
+++ b/internal/sync/pst/pst_test.go
@@ -62,7 +62,7 @@ func TestImportFromDataFiles(t *testing.T) {
progressCalls++
}
- extracted, errCount, importErr := Import(pstPath, emailDir, onProgress)
+ extracted, errCount, importErr := Import(pstPath, emailDir, onProgress, nil)
if importErr != nil {
if strings.Contains(importErr.Error(), "readpst not installed") {
t.Skipf("go-pst failed and readpst fallback unavailable: %v (install pst-utils to test OST)", importErr)
@@ -140,7 +140,7 @@ func TestImportExtractionWorks(t *testing.T) {
}
emailDir := t.TempDir()
- extracted, errCount, err := Import(pstPath, emailDir, func(phase string, current, total int) {})
+ extracted, errCount, err := Import(pstPath, emailDir, func(phase string, current, total int) {}, nil)
if err != nil {
t.Fatalf("Import: %v", err)
}
diff --git a/internal/sync/service.go b/internal/sync/service.go
index 8e0a252..4ec5864 100644
--- a/internal/sync/service.go
+++ b/internal/sync/service.go
@@ -5,12 +5,14 @@ import (
"fmt"
"log"
"os"
+ "path/filepath"
"sync"
"time"
"github.com/eslider/mails/internal/account"
"github.com/eslider/mails/internal/model"
"github.com/eslider/mails/internal/search/index"
+ "github.com/eslider/mails/internal/storage"
sync_gmail "github.com/eslider/mails/internal/sync/gmail"
sync_imap "github.com/eslider/mails/internal/sync/imap"
sync_pop3 "github.com/eslider/mails/internal/sync/pop3"
@@ -27,18 +29,20 @@ type syncEntry struct {
// Service orchestrates email sync for all accounts of a user.
type Service struct {
- mu sync.Mutex
- usersDir string
- accounts *account.Store
- running map[string]*syncEntry // accountID -> entry
+ mu sync.Mutex
+ usersDir string
+ accounts *account.Store
+ blobStore storage.BlobStore
+ running map[string]*syncEntry // accountID -> entry
}
-// NewService creates a sync service.
-func NewService(usersDir string, accounts *account.Store) *Service {
+// NewService creates a sync service. blobStore may be nil to use local filesystem only.
+func NewService(usersDir string, accounts *account.Store, blobStore storage.BlobStore) *Service {
return &Service{
- usersDir: usersDir,
- accounts: accounts,
- running: make(map[string]*syncEntry),
+ usersDir: usersDir,
+ accounts: accounts,
+ blobStore: blobStore,
+ running: make(map[string]*syncEntry),
}
}
@@ -111,7 +115,8 @@ func (s *Service) SyncAccount(userID, accountID string) error {
}()
s.setProgress(accountID, "syncing", "")
- newMsgs, syncErr := s.doSync(ctx, *acct, emailDir, stateDB, accountID)
+ saveFn := s.makeSaveEmailFunc()
+ newMsgs, syncErr := s.doSync(ctx, *acct, emailDir, stateDB, accountID, saveFn)
// Stop live indexing and wait for it to fully exit before final rebuild.
indexCancel()
@@ -233,7 +238,33 @@ func (s *Service) setProgress(accountID, progress, lastError string) {
s.mu.Unlock()
}
-func (s *Service) doSync(ctx context.Context, acct model.EmailAccount, emailDir string, stateDB *StateDB, accountID string) (int, error) {
+func (s *Service) makeSaveEmailFunc() sync_imap.SaveEmailFunc {
+ if s.blobStore == nil {
+ return nil
+ }
+ return func(path string, data []byte) error {
+ rel, err := filepath.Rel(s.usersDir, path)
+ if err != nil {
+ return err
+ }
+ return s.blobStore.Write(context.Background(), filepath.ToSlash(rel), data)
+ }
+}
+
+func (s *Service) makePstSaveFunc() sync_pst.SaveEmailFunc {
+ if s.blobStore == nil {
+ return nil
+ }
+ return sync_pst.SaveEmailFunc(func(path string, data []byte) error {
+ rel, err := filepath.Rel(s.usersDir, path)
+ if err != nil {
+ return err
+ }
+ return s.blobStore.Write(context.Background(), filepath.ToSlash(rel), data)
+ })
+}
+
+func (s *Service) doSync(ctx context.Context, acct model.EmailAccount, emailDir string, stateDB *StateDB, accountID string, saveFn sync_imap.SaveEmailFunc) (int, error) {
// Progress callback: update in-memory progress visible via API.
onProgress := func(msg string) {
s.setProgress(accountID, msg, "")
@@ -241,11 +272,11 @@ func (s *Service) doSync(ctx context.Context, acct model.EmailAccount, emailDir
switch acct.Type {
case model.AccountTypeIMAP:
- return sync_imap.SyncWithContext(ctx, acct, emailDir, stateDB, onProgress)
+ return sync_imap.SyncWithContext(ctx, acct, emailDir, stateDB, onProgress, saveFn)
case model.AccountTypePOP3:
- return sync_pop3.SyncWithContext(ctx, acct, emailDir, stateDB)
+ return sync_pop3.SyncWithContext(ctx, acct, emailDir, stateDB, sync_pop3.SaveEmailFunc(saveFn))
case model.AccountTypeGmailAPI:
- return sync_gmail.SyncWithContext(ctx, acct, emailDir, stateDB)
+ return sync_gmail.SyncWithContext(ctx, acct, emailDir, stateDB, sync_gmail.SaveEmailFunc(saveFn))
default:
return 0, fmt.Errorf("unsupported account type: %s", acct.Type)
}
@@ -268,7 +299,7 @@ func (s *Service) liveIndex(ctx context.Context, emailDir, indexPath, accountID
}
func (s *Service) rebuildIndex(emailDir, indexPath string) {
- idx, err := index.New(emailDir, indexPath)
+ idx, err := index.New(emailDir, indexPath, s.blobStore, s.usersDir)
if err != nil {
log.Printf("WARN: live index open: %v", err)
return
@@ -297,17 +328,20 @@ func (s *Service) ImportPST(userID, accountID, pstPath string, onProgress sync_p
}
emailDir := account.EmailDir(s.usersDir, userID, *acct)
- if err := os.MkdirAll(emailDir, 0o755); err != nil {
- return 0, 0, fmt.Errorf("create email dir: %w", err)
+ if s.blobStore == nil {
+ if err := os.MkdirAll(emailDir, 0o755); err != nil {
+ return 0, 0, fmt.Errorf("create email dir: %w", err)
+ }
}
- extracted, errCount, importErr := sync_pst.Import(pstPath, emailDir, onProgress)
+ saveFn := s.makePstSaveFunc()
+ extracted, errCount, importErr := sync_pst.Import(pstPath, emailDir, onProgress, saveFn)
if importErr != nil {
return extracted, errCount, fmt.Errorf("PST import: %w", importErr)
}
indexPath := account.IndexPath(s.usersDir, userID, *acct)
- idx, idxErr := index.New(emailDir, indexPath)
+ idx, idxErr := index.New(emailDir, indexPath, s.blobStore, s.usersDir)
if idxErr != nil {
return extracted, errCount, fmt.Errorf("index: %w", idxErr)
}
diff --git a/internal/user/store.go b/internal/user/store.go
index bfd4b62..3234dd0 100644
--- a/internal/user/store.go
+++ b/internal/user/store.go
@@ -2,23 +2,27 @@
package user
import (
+ "context"
"encoding/json"
"fmt"
"os"
"path/filepath"
+ "strings"
"sync"
"time"
"github.com/eslider/mails/internal/model"
+ "github.com/eslider/mails/internal/storage"
)
const userMetaFile = "user.json"
-// Store manages user data on the filesystem.
+// Store manages user data on the filesystem or S3.
// Layout: {dataDir}/{userID}/user.json
type Store struct {
- mu sync.RWMutex
- dataDir string
+ mu sync.RWMutex
+ dataDir string
+ blobStore storage.BlobStore
// In-memory index: provider:providerID -> userID
providerIndex map[string]string
// In-memory index: email -> userID (for local auth)
@@ -27,14 +31,18 @@ type Store struct {
users map[string]model.User
}
-// NewStore creates a user store, scanning existing users from disk.
-func NewStore(dataDir string) (*Store, error) {
- if err := os.MkdirAll(dataDir, 0o755); err != nil {
- return nil, fmt.Errorf("create users dir: %w", err)
+// NewStore creates a user store, scanning existing users from disk or S3.
+// blobStore may be nil to use local filesystem only.
+func NewStore(dataDir string, blobStore storage.BlobStore) (*Store, error) {
+ if blobStore == nil {
+ if err := os.MkdirAll(dataDir, 0o755); err != nil {
+ return nil, fmt.Errorf("create users dir: %w", err)
+ }
}
s := &Store{
dataDir: dataDir,
+ blobStore: blobStore,
providerIndex: make(map[string]string),
emailIndex: make(map[string]string),
users: make(map[string]model.User),
@@ -102,10 +110,12 @@ func (s *Store) CreateWithPassword(name, email, passwordHash string) (*model.Use
UpdatedAt: now,
}
- userDir := s.UserDir(user.ID)
- for _, sub := range []string{"", "logs"} {
- if err := os.MkdirAll(filepath.Join(userDir, sub), 0o755); err != nil {
- return nil, fmt.Errorf("create user dir: %w", err)
+ if s.blobStore == nil {
+ userDir := s.UserDir(user.ID)
+ for _, sub := range []string{"", "logs"} {
+ if err := os.MkdirAll(filepath.Join(userDir, sub), 0o755); err != nil {
+ return nil, fmt.Errorf("create user dir: %w", err)
+ }
}
}
@@ -139,11 +149,12 @@ func (s *Store) create(oauthUser *model.User) (*model.User, error) {
UpdatedAt: now,
}
- userDir := s.UserDir(user.ID)
- // Create user directory structure.
- for _, sub := range []string{"", "logs"} {
- if err := os.MkdirAll(filepath.Join(userDir, sub), 0o755); err != nil {
- return nil, fmt.Errorf("create user dir: %w", err)
+ if s.blobStore == nil {
+ userDir := s.UserDir(user.ID)
+ for _, sub := range []string{"", "logs"} {
+ if err := os.MkdirAll(filepath.Join(userDir, sub), 0o755); err != nil {
+ return nil, fmt.Errorf("create user dir: %w", err)
+ }
}
}
@@ -207,11 +218,52 @@ func (s *Store) saveUser(u model.User) error {
if err != nil {
return err
}
- path := filepath.Join(s.UserDir(u.ID), userMetaFile)
- return os.WriteFile(path, data, 0o644)
+ key := u.ID + "/" + userMetaFile
+ if s.blobStore != nil {
+ return s.blobStore.Write(context.Background(), key, data)
+ }
+ return os.WriteFile(filepath.Join(s.UserDir(u.ID), userMetaFile), data, 0o644)
}
func (s *Store) loadAll() error {
+ ctx := context.Background()
+
+ if s.blobStore != nil {
+ keys, err := s.blobStore.List(ctx, "")
+ if err != nil {
+ return err
+ }
+ seen := make(map[string]bool)
+ for _, k := range keys {
+ parts := strings.SplitN(k, "/", 2)
+ if len(parts) < 2 || parts[1] != userMetaFile {
+ continue
+ }
+ userID := parts[0]
+ if seen[userID] {
+ continue
+ }
+ seen[userID] = true
+ data, err := s.blobStore.Read(ctx, k)
+ if err != nil {
+ continue
+ }
+ var f userFile
+ if err := json.Unmarshal(data, &f); err != nil {
+ continue
+ }
+ u := fromUserFile(f)
+ s.users[u.ID] = u
+ if u.Provider != "" && u.ProviderID != "" {
+ s.providerIndex[providerKey(u.Provider, u.ProviderID)] = u.ID
+ }
+ if u.Email != "" {
+ s.emailIndex[u.Email] = u.ID
+ }
+ }
+ return nil
+ }
+
entries, err := os.ReadDir(s.dataDir)
if err != nil {
if os.IsNotExist(err) {
diff --git a/internal/web/handlers.go b/internal/web/handlers.go
index 967e334..0665d06 100644
--- a/internal/web/handlers.go
+++ b/internal/web/handlers.go
@@ -1,9 +1,11 @@
package web
import (
+ "context"
"crypto/rand"
"encoding/hex"
"encoding/json"
+ "errors"
"fmt"
"io"
"log"
@@ -21,6 +23,7 @@ import (
"github.com/eslider/mails/internal/model"
"github.com/eslider/mails/internal/search/eml"
"github.com/eslider/mails/internal/search/index"
+ "github.com/eslider/mails/internal/storage"
"github.com/eslider/mails/internal/sync"
sync_pst "github.com/eslider/mails/internal/sync/pst"
"github.com/eslider/mails/internal/user"
@@ -407,7 +410,7 @@ func handleSearch(cfg Config) http.HandlerFunc {
if a.ID == accountFilter {
emailDir := account.EmailDir(cfg.UsersDir, userID, a)
indexPath := account.IndexPath(cfg.UsersDir, userID, a)
- idx, err := index.New(emailDir, indexPath)
+ idx, err := index.New(emailDir, indexPath, cfg.BlobStore, cfg.UsersDir)
if err != nil {
writeError(w, http.StatusInternalServerError, "index error: "+err.Error())
return
@@ -492,16 +495,37 @@ func handleEmailDetail(cfg Config) http.HandlerFunc {
}
full := filepath.Join(emailDir, cleaned)
- fe, err := eml.ParseFileFull(full)
+ data, err := readEmailBytes(cfg, full)
+ if err != nil {
+ if errors.Is(err, storage.ErrNotFound) {
+ writeError(w, http.StatusNotFound, "email not found")
+ return
+ }
+ writeError(w, http.StatusInternalServerError, "failed to read email")
+ return
+ }
+ fe, err := eml.ParseFileFullFromBytes(cleaned, data)
if err != nil {
writeError(w, http.StatusNotFound, "email not found")
return
}
- fe.Path = cleaned
writeJSON(w, http.StatusOK, fe)
}
}
+// readEmailBytes returns email content by full path. Uses BlobStore when configured.
+func readEmailBytes(cfg Config, fullPath string) ([]byte, error) {
+ if cfg.BlobStore != nil {
+ rel, err := filepath.Rel(cfg.UsersDir, fullPath)
+ if err != nil {
+ return nil, err
+ }
+ key := filepath.ToSlash(rel)
+ return cfg.BlobStore.Read(context.Background(), key)
+ }
+ return os.ReadFile(fullPath)
+}
+
// resolveEmailPath returns the full filesystem path for an email from path + account_id query params.
func resolveEmailPath(cfg Config, r *http.Request) (string, bool) {
userID := auth.UserIDFromContext(r.Context())
@@ -539,20 +563,19 @@ func handleEmailDownload(cfg Config) http.HandlerFunc {
writeError(w, http.StatusBadRequest, "missing or invalid path")
return
}
- if _, err := os.Stat(full); err != nil {
- writeError(w, http.StatusNotFound, "email not found")
+ data, err := readEmailBytes(cfg, full)
+ if err != nil {
+ if errors.Is(err, storage.ErrNotFound) {
+ writeError(w, http.StatusNotFound, "email not found")
+ return
+ }
+ writeError(w, http.StatusInternalServerError, "failed to read email")
return
}
name := filepath.Base(full)
w.Header().Set("Content-Disposition", `attachment; filename="`+name+`"`)
w.Header().Set("Content-Type", "message/rfc822")
- f, err := os.Open(full)
- if err != nil {
- writeError(w, http.StatusInternalServerError, "failed to read email")
- return
- }
- defer f.Close()
- io.Copy(w, f)
+ w.Write(data)
}
}
@@ -563,12 +586,21 @@ func handleAttachmentDownload(cfg Config) http.HandlerFunc {
writeError(w, http.StatusBadRequest, "missing or invalid path")
return
}
- index := queryInt(r, "index", -1)
- if index < 0 {
+ emailData, err := readEmailBytes(cfg, full)
+ if err != nil {
+ if errors.Is(err, storage.ErrNotFound) {
+ writeError(w, http.StatusNotFound, "attachment not found")
+ return
+ }
+ writeError(w, http.StatusInternalServerError, "failed to read email")
+ return
+ }
+ idx := queryInt(r, "index", -1)
+ if idx < 0 {
writeError(w, http.StatusBadRequest, "missing or invalid index parameter")
return
}
- data, contentType, filename, err := eml.ExtractAttachment(full, index)
+ data, contentType, filename, err := eml.ExtractAttachmentFromBytes(emailData, idx)
if err != nil {
writeError(w, http.StatusNotFound, "attachment not found")
return
@@ -586,6 +618,33 @@ func handleAttachmentDownload(cfg Config) http.HandlerFunc {
}
}
+// handleCIDResource serves inline MIME parts by Content-ID. Protected by RequireAuth;
+// resolveEmailPath scopes access to the logged-in user's accounts only.
+func handleCIDResource(cfg Config) http.HandlerFunc {
+ return func(w http.ResponseWriter, r *http.Request) {
+ full, ok := resolveEmailPath(cfg, r)
+ if !ok {
+ writeError(w, http.StatusBadRequest, "missing or invalid path")
+ return
+ }
+ cid := strings.TrimSpace(r.URL.Query().Get("cid"))
+ if cid == "" {
+ writeError(w, http.StatusBadRequest, "missing cid parameter")
+ return
+ }
+ data, contentType, err := eml.ExtractPartByCID(full, cid)
+ if err != nil {
+ writeError(w, http.StatusNotFound, "resource not found")
+ return
+ }
+ if contentType != "" {
+ w.Header().Set("Content-Type", contentType)
+ }
+ w.Header().Set("Cache-Control", "private, max-age=3600")
+ w.Write(data)
+ }
+}
+
func handleSearchStats(cfg Config) http.HandlerFunc {
return func(w http.ResponseWriter, r *http.Request) {
userID := auth.UserIDFromContext(r.Context())
@@ -613,7 +672,7 @@ func handleReindex(cfg Config) http.HandlerFunc {
for _, acct := range accts {
emailDir := account.EmailDir(cfg.UsersDir, userID, acct)
indexPath := account.IndexPath(cfg.UsersDir, userID, acct)
- idx, err := index.New(emailDir, indexPath)
+ idx, err := index.New(emailDir, indexPath, cfg.BlobStore, cfg.UsersDir)
if err != nil {
log.Printf("WARN: reindex %s: %v", acct.Email, err)
continue
diff --git a/internal/web/router.go b/internal/web/router.go
index 9140adb..5fa1f26 100644
--- a/internal/web/router.go
+++ b/internal/web/router.go
@@ -11,6 +11,7 @@ import (
"github.com/eslider/mails/internal/account"
"github.com/eslider/mails/internal/auth"
+ "github.com/eslider/mails/internal/storage"
"github.com/eslider/mails/internal/sync"
"github.com/eslider/mails/internal/user"
)
@@ -27,12 +28,13 @@ var TemplateDir string
// Config holds dependencies for the web layer.
type Config struct {
- Users *user.Store
- Accounts *account.Store
- Sessions *auth.SessionStore
- Auth *auth.Providers
- Sync *sync.Service
- UsersDir string
+ Users *user.Store
+ Accounts *account.Store
+ Sessions *auth.SessionStore
+ Auth *auth.Providers
+ Sync *sync.Service
+ UsersDir string
+ BlobStore storage.BlobStore
// Search (optional — per-user indices are loaded on demand).
QdrantURL string
@@ -118,6 +120,7 @@ func NewRouter(cfg Config) http.Handler {
r.Get("/api/email", handleEmailDetail(cfg))
r.Get("/api/email/download", handleEmailDownload(cfg))
r.Get("/api/email/attachment", handleAttachmentDownload(cfg))
+ r.Get("/api/email/cid", handleCIDResource(cfg))
r.Get("/api/stats", handleSearchStats(cfg))
r.Post("/api/reindex", handleReindex(cfg))
})
diff --git a/internal/web/router_auth_test.go b/internal/web/router_auth_test.go
new file mode 100644
index 0000000..db5a6d1
--- /dev/null
+++ b/internal/web/router_auth_test.go
@@ -0,0 +1,55 @@
+package web
+
+import (
+ "net/http"
+ "net/http/httptest"
+ "os"
+ "testing"
+
+ "github.com/eslider/mails/internal/account"
+ "github.com/eslider/mails/internal/auth"
+ "github.com/eslider/mails/internal/user"
+)
+
+func TestProtectedRoutesRequireAuth(t *testing.T) {
+ dir := t.TempDir()
+ sessions, err := auth.NewSessionStore(dir, nil)
+ if err != nil {
+ t.Fatalf("NewSessionStore: %v", err)
+ }
+ users, err := user.NewStore(dir, nil)
+ if err != nil {
+ t.Fatalf("NewStore: %v", err)
+ }
+ os.MkdirAll(dir, 0755)
+
+ cfg := Config{
+ Users: users,
+ Accounts: account.NewStore(dir, nil),
+ Sessions: sessions,
+ UsersDir: dir,
+ BlobStore: nil,
+ }
+ handler := NewRouter(cfg)
+
+ tests := []struct {
+ path string
+ }{
+ {"/api/email"},
+ {"/api/email/download"},
+ {"/api/email/attachment"},
+ {"/api/email/cid"},
+ }
+
+ for _, tt := range tests {
+ t.Run(tt.path, func(t *testing.T) {
+ req := httptest.NewRequest(http.MethodGet, tt.path+"?path=x&cid=y", nil)
+ req.Header.Set("Accept", "application/json")
+ rec := httptest.NewRecorder()
+ handler.ServeHTTP(rec, req)
+ if rec.Code != http.StatusUnauthorized {
+ t.Errorf("GET %s without auth: status = %d, want 401", tt.path, rec.Code)
+ }
+ })
+ }
+}
diff --git a/tests/e2e/e2e_test.go b/tests/e2e/e2e_test.go
index d5ea018..eb2eb59 100644
--- a/tests/e2e/e2e_test.go
+++ b/tests/e2e/e2e_test.go
@@ -293,7 +293,7 @@ func TestIMAPSyncThenIndexAndSearch(t *testing.T) {
// Step 2: Build the search index.
indexPath := filepath.Join(emailDir, "index.parquet")
- idx, err := index.New(emailDir, indexPath)
+ idx, err := index.New(emailDir, indexPath, nil, "")
if err != nil {
t.Fatalf("create index: %v", err)
}
@@ -381,7 +381,7 @@ func TestPOP3SyncThenIndexAndSearch(t *testing.T) {
// Step 2: Build index over inbox/.
inboxDir := filepath.Join(emailDir, "inbox")
indexPath := filepath.Join(emailDir, "index.parquet")
- idx, err := index.New(inboxDir, indexPath)
+ idx, err := index.New(inboxDir, indexPath, nil, "")
if err != nil {
t.Fatalf("create index: %v", err)
}
diff --git a/tests/e2e/index_test.go b/tests/e2e/index_test.go
index 593c3b9..1811e5c 100644
--- a/tests/e2e/index_test.go
+++ b/tests/e2e/index_test.go
@@ -86,7 +86,7 @@ func TestIndex_BuildAndStats(t *testing.T) {
dir := seedSyntheticEmails(t)
indexPath := filepath.Join(t.TempDir(), "index.parquet")
- idx, err := index.New(dir, indexPath)
+ idx, err := index.New(dir, indexPath, nil, "")
if err != nil {
t.Fatalf("create index: %v", err)
}
@@ -113,7 +113,7 @@ func TestIndex_BuildAndStats(t *testing.T) {
func TestIndex_KeywordSearch(t *testing.T) {
// Case: Search by keywords present in subject or body.
dir := seedSyntheticEmails(t)
- idx, err := index.New(dir, "")
+ idx, err := index.New(dir, "", nil, "")
if err != nil {
t.Fatal(err)
}
@@ -152,7 +152,7 @@ func TestIndex_KeywordSearch(t *testing.T) {
func TestIndex_NoMatch(t *testing.T) {
// Case: Search for a term that exists in no email -> 0 hits.
dir := seedSyntheticEmails(t)
- idx, err := index.New(dir, "")
+ idx, err := index.New(dir, "", nil, "")
if err != nil {
t.Fatal(err)
}
@@ -172,7 +172,7 @@ func TestIndex_NoMatch(t *testing.T) {
func TestIndex_EmptyQueryReturnsAll(t *testing.T) {
// Case: Empty string query returns all indexed emails.
dir := seedSyntheticEmails(t)
- idx, err := index.New(dir, "")
+ idx, err := index.New(dir, "", nil, "")
if err != nil {
t.Fatal(err)
}
@@ -192,7 +192,7 @@ func TestIndex_EmptyQueryReturnsAll(t *testing.T) {
func TestIndex_Pagination(t *testing.T) {
// Case: Paginate results without overlap or omission.
dir := seedSyntheticEmails(t)
- idx, err := index.New(dir, "")
+ idx, err := index.New(dir, "", nil, "")
if err != nil {
t.Fatal(err)
}
@@ -241,7 +241,7 @@ func TestIndex_Pagination(t *testing.T) {
func TestIndex_DateOrdering(t *testing.T) {
// Case: Empty query results are ordered by date DESC (newest first).
dir := seedSyntheticEmails(t)
- idx, err := index.New(dir, "")
+ idx, err := index.New(dir, "", nil, "")
if err != nil {
t.Fatal(err)
}
@@ -265,7 +265,7 @@ func TestIndex_ParquetRoundTrip(t *testing.T) {
indexPath := filepath.Join(t.TempDir(), "test_index.parquet")
// Build and export.
- idx1, err := index.New(dir, indexPath)
+ idx1, err := index.New(dir, indexPath, nil, "")
if err != nil {
t.Fatal(err)
}
@@ -287,7 +287,7 @@ func TestIndex_ParquetRoundTrip(t *testing.T) {
t.Logf("Parquet file: %d bytes", info.Size())
// Re-open from Parquet (no rebuild needed).
- idx2, err := index.New(dir, indexPath)
+ idx2, err := index.New(dir, indexPath, nil, "")
if err != nil {
t.Fatalf("re-open from parquet: %v", err)
}
@@ -309,7 +309,7 @@ func TestIndex_ParquetRoundTrip(t *testing.T) {
func TestIndex_Rebuild(t *testing.T) {
// Case: Rebuild replaces stale index with fresh data from disk.
dir := seedSyntheticEmails(t)
- idx, err := index.New(dir, "")
+ idx, err := index.New(dir, "", nil, "")
if err != nil {
t.Fatal(err)
}
@@ -365,7 +365,7 @@ func TestIndex_Deduplication(t *testing.T) {
"Tue, 02 Jan 2024 09:00:00 +0000",
"This is a unique message.")
- idx, err := index.New(dir, "")
+ idx, err := index.New(dir, "", nil, "")
if err != nil {
t.Fatal(err)
}
@@ -382,7 +382,7 @@ func TestIndex_Deduplication(t *testing.T) {
func TestIndex_SearchWithSnippet(t *testing.T) {
// Case: Search results include a context snippet around the matched keyword.
dir := seedSyntheticEmails(t)
- idx, err := index.New(dir, "")
+ idx, err := index.New(dir, "", nil, "")
if err != nil {
t.Fatal(err)
}
diff --git a/tests/e2e/vector_test.go b/tests/e2e/vector_test.go
index 4482f95..862e4f4 100644
--- a/tests/e2e/vector_test.go
+++ b/tests/e2e/vector_test.go
@@ -407,7 +407,7 @@ func TestVector_FullPipeline_SyncIndexSearch(t *testing.T) {
// Step 2: Build keyword index (DuckDB).
indexPath := filepath.Join(emailDir, "index.parquet")
- idx, err := index.New(emailDir, indexPath)
+ idx, err := index.New(emailDir, indexPath, nil, "")
if err != nil {
t.Fatal(err)
}
@@ -430,7 +430,7 @@ func TestVector_FullPipeline_SyncIndexSearch(t *testing.T) {
}
// Step 5: Search both.
- idx2, _ := index.New(emailDir, indexPath)
+ idx2, _ := index.New(emailDir, indexPath, nil, "")
defer idx2.Close()
kwResult := idx2.Search("invoice", 0, 50)