diff --git a/README.md b/README.md index 4a6f36d..7cff4c4 100644 --- a/README.md +++ b/README.md @@ -73,6 +73,23 @@ inode config set embedding.api_key pa-xxxx These are completely optional. The Ollama path is fully featured. +### Optional: Postgres + pgvector backend + +By default inode uses SQLite + sqlite-vec — single file, zero infrastructure. + +If you're on Windows (where the sqlite-vec CGO build chain is painful) or you'd rather run on a real database, you can switch to Postgres + pgvector. The Postgres driver is pure Go, so `go install` works without any C toolchain. + +```bash +# 1. Start a pgvector-enabled Postgres (compose file in repo root) +docker compose up -d + +# 2. Point inode at it +inode config set db.backend postgres +inode config set db.dsn "postgres://postgres:password@localhost:5432/postgres?sslmode=disable" +``` + +inode will `CREATE EXTENSION vector` and create the `notes` table on first run. SQLite remains the default — Postgres is opt-in. + --- ## Commands diff --git a/docker-compose.yml b/docker-compose.yml index e0dfe63..86fe931 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -8,5 +8,22 @@ services: - ollama_data:/root/.ollama restart: unless-stopped + pgvector: + image: pgvector/pgvector:pg16 + container_name: inode-pgvector + environment: + POSTGRES_PASSWORD: password + ports: + - "5432:5432" + volumes: + - pgvector_data:/var/lib/postgresql/data + healthcheck: + test: ["CMD-SHELL", "pg_isready -U postgres"] + interval: 5s + timeout: 3s + retries: 5 + restart: unless-stopped + volumes: ollama_data: + pgvector_data: \ No newline at end of file diff --git a/go.mod b/go.mod index b2b53ce..e5a7f10 100644 --- a/go.mod +++ b/go.mod @@ -5,9 +5,13 @@ go 1.25.0 require ( github.com/anthropics/anthropic-sdk-go v1.41.0 github.com/asg017/sqlite-vec-go-bindings v0.1.6 + github.com/fatih/color v1.19.0 github.com/google/uuid v1.6.0 + github.com/jackc/pgx/v5 v5.9.2 github.com/joho/godotenv v1.5.1 + github.com/mattn/go-isatty v0.0.20 github.com/mattn/go-sqlite3 v1.14.44 + github.com/pgvector/pgvector-go v0.3.0 github.com/spf13/cobra v1.10.2 github.com/spf13/viper v1.21.0 github.com/zalando/go-keyring v0.2.8 @@ -18,15 +22,16 @@ require ( github.com/bahlo/generic-list-go v0.2.0 // indirect github.com/buger/jsonparser v1.1.2 // indirect github.com/danieljoos/wincred v1.2.3 // indirect - github.com/fatih/color v1.19.0 // indirect github.com/fsnotify/fsnotify v1.9.0 // indirect github.com/go-viper/mapstructure/v2 v2.4.0 // indirect github.com/godbus/dbus/v5 v5.2.2 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/invopop/jsonschema v0.13.0 // indirect + github.com/jackc/pgpassfile v1.0.0 // indirect + github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect + github.com/jackc/puddle/v2 v2.2.2 // indirect github.com/mailru/easyjson v0.7.7 // indirect github.com/mattn/go-colorable v0.1.14 // indirect - github.com/mattn/go-isatty v0.0.20 // indirect github.com/pelletier/go-toml/v2 v2.2.4 // indirect github.com/sagikazarmark/locafero v0.11.0 // indirect github.com/sourcegraph/conc v0.3.1-0.20240121214520-5f936abd7ae8 // indirect @@ -40,6 +45,7 @@ require ( github.com/tidwall/pretty v1.2.1 // indirect github.com/tidwall/sjson v1.2.5 // indirect github.com/wk8/go-ordered-map/v2 v2.1.8 // indirect + github.com/x448/float16 v0.8.4 // indirect go.yaml.in/yaml/v3 v3.0.4 // indirect golang.org/x/sync v0.20.0 // indirect golang.org/x/sys v0.43.0 // indirect diff --git a/go.sum b/go.sum index 16af994..b5778e7 100644 --- a/go.sum +++ b/go.sum @@ -1,3 +1,5 @@ +entgo.io/ent v0.14.3 h1:wokAV/kIlH9TeklJWGGS7AYJdVckr0DloWjIcO9iIIQ= +entgo.io/ent v0.14.3/go.mod h1:aDPE/OziPEu8+OWbzy4UlvWmD2/kbRuWfK2A40hcxJM= github.com/anthropics/anthropic-sdk-go v1.41.0 h1:D//vDxQMAmWOcOV3QtgESOneIxJOfWxGIHsmJgGGGp4= github.com/anthropics/anthropic-sdk-go v1.41.0/go.mod h1:UgnI27pSnfj2Ux2iE6Dkq9o/3MZn0jdalFdQIdWMS+Q= github.com/asg017/sqlite-vec-go-bindings v0.1.6 h1:Nx0jAzyS38XpkKznJ9xQjFXz2X9tI7KqjwVxV8RNoww= @@ -9,6 +11,7 @@ github.com/buger/jsonparser v1.1.2/go.mod h1:6RYKKt7H4d4+iWqouImQ9R2FZql3VbhNgx2 github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g= github.com/danieljoos/wincred v1.2.3 h1:v7dZC2x32Ut3nEfRH+vhoZGvN72+dQ/snVXo/vMFLdQ= github.com/danieljoos/wincred v1.2.3/go.mod h1:6qqX0WNrS4RzPZ1tnroDzq9kY3fu1KwE7MRLQK4X0bs= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/dnaeon/go-vcr v1.2.0 h1:zHCHvJYTMh1N7xnV7zf1m1GPBF9Ad0Jk/whtQ1663qI= @@ -19,18 +22,36 @@ github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHk github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= github.com/fsnotify/fsnotify v1.9.0 h1:2Ml+OJNzbYCTzsxtv8vKSFD9PbJjmhYF14k/jKC7S9k= github.com/fsnotify/fsnotify v1.9.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0= +github.com/go-pg/pg/v10 v10.11.0 h1:CMKJqLgTrfpE/aOVeLdybezR2om071Vh38OLZjsyMI0= +github.com/go-pg/pg/v10 v10.11.0/go.mod h1:4BpHRoxE61y4Onpof3x1a2SQvi9c+q1dJnrNdMjsroA= +github.com/go-pg/zerochecker v0.2.0 h1:pp7f72c3DobMWOb2ErtZsnrPaSvHd2W4o9//8HtF4mU= +github.com/go-pg/zerochecker v0.2.0/go.mod h1:NJZ4wKL0NmTtz0GKCoJ8kym6Xn/EQzXRl2OnAe7MmDo= github.com/go-viper/mapstructure/v2 v2.4.0 h1:EBsztssimR/CONLSZZ04E8qAkxNYq4Qp9LvH92wZUgs= github.com/go-viper/mapstructure/v2 v2.4.0/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM= github.com/godbus/dbus/v5 v5.2.2 h1:TUR3TgtSVDmjiXOgAAyaZbYmIeP3DPkld3jgKGV8mXQ= github.com/godbus/dbus/v5 v5.2.2/go.mod h1:3AAv2+hPq5rdnr5txxxRwiGjPXamgoIHgz9FPBfOp3c= -github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= -github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= +github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/invopop/jsonschema v0.13.0 h1:KvpoAJWEjR3uD9Kbm2HWJmqsEaHt8lBUpd0qHcIi21E= github.com/invopop/jsonschema v0.13.0/go.mod h1:ffZ5Km5SWWRAIN6wbDXItl95euhFz2uON45H2qjYt+0= +github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM= +github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= +github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 h1:iCEnooe7UlwOQYpKFhBabPMi4aNAfoODPEFNiAnClxo= +github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM= +github.com/jackc/pgx/v5 v5.9.2 h1:3ZhOzMWnR4yJ+RW1XImIPsD1aNSz4T4fyP7zlQb56hw= +github.com/jackc/pgx/v5 v5.9.2/go.mod h1:mal1tBGAFfLHvZzaYh77YS/eC6IX9OWbRV1QIIM0Jn4= +github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo= +github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= +github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD/E= +github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc= +github.com/jinzhu/now v1.1.5 h1:/o9tlHleP7gOFmsnYNz3RGnqzefHA47wQpKrrdTIwXQ= +github.com/jinzhu/now v1.1.5/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8= +github.com/jmoiron/sqlx v1.3.5 h1:vFFPA71p1o5gAeqtEAwLU4dnX2napprKtHr7PYIcN3g= +github.com/jmoiron/sqlx v1.3.5/go.mod h1:nRVWtLre0KfCLJvgxzCsLVMogSvQ1zNJtpYr2Ccp0mQ= github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0= github.com/joho/godotenv v1.5.1/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4= github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= @@ -38,6 +59,8 @@ github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw= +github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= github.com/mattn/go-colorable v0.1.14 h1:9A9LHSqF/7dyVVX6g0U9cwm9pG3kP9gSzcuIPHPsaIE= @@ -48,6 +71,8 @@ github.com/mattn/go-sqlite3 v1.14.44 h1:3VSe+xafpbzsLbdr2AWlAZk9yRHiBhTBakioXaCK github.com/mattn/go-sqlite3 v1.14.44/go.mod h1:pjEuOr8IwzLJP2MfGeTb0A35jauH+C2kbHKBr7yXKVQ= github.com/pelletier/go-toml/v2 v2.2.4 h1:mye9XuhQ6gvn5h28+VilKrrPoQVanw5PMw/TB0t5Ec4= github.com/pelletier/go-toml/v2 v2.2.4/go.mod h1:2gIqNv+qfxSVS7cM2xJQKtLSTLUE9V8t9Stt+h56mCY= +github.com/pgvector/pgvector-go v0.3.0 h1:Ij+Yt78R//uYqs3Zk35evZFvr+G0blW0OUN+Q2D1RWc= +github.com/pgvector/pgvector-go v0.3.0/go.mod h1:duFy+PXWfW7QQd5ibqutBO4GxLsUZ9RVXhFZGIBsWSA= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8= @@ -70,8 +95,11 @@ github.com/spf13/viper v1.21.0 h1:x5S+0EU27Lbphp4UKm1C+1oQO+rKx36vfCoaVebLFSU= github.com/spf13/viper v1.21.0/go.mod h1:P0lhsswPGWD/1lZJ9ny3fYnVqxiegrlNrEmgLjbTCAY= github.com/standard-webhooks/standard-webhooks/libraries v0.0.0-20260427160145-3afa6683f8b2 h1:q/QNlQMqBFYT7z9zt8vjbh0XvbcTXhN4Q+gi7aEBvkY= github.com/standard-webhooks/standard-webhooks/libraries v0.0.0-20260427160145-3afa6683f8b2/go.mod h1:L1MQhA6x4dn9r007T033lsaZMv9EmBAdXyU/+EF40fo= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8= @@ -86,8 +114,26 @@ github.com/tidwall/pretty v1.2.1 h1:qjsOFOWWQl+N3RsoF5/ssm1pHmJJwhjlSbZ51I6wMl4= github.com/tidwall/pretty v1.2.1/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= github.com/tidwall/sjson v1.2.5 h1:kLy8mja+1c9jlljvWTlSazM7cKDRfJuR/bOJhcY5NcY= github.com/tidwall/sjson v1.2.5/go.mod h1:Fvgq9kS/6ociJEDnK0Fk1cpYF4FIW6ZF7LAe+6jwd28= +github.com/tmthrgd/go-hex v0.0.0-20190904060850-447a3041c3bc h1:9lRDQMhESg+zvGYmW5DyG0UqvY96Bu5QYsTLvCHdrgo= +github.com/tmthrgd/go-hex v0.0.0-20190904060850-447a3041c3bc/go.mod h1:bciPuU6GHm1iF1pBvUfxfsH0Wmnc2VbpgvbI9ZWuIRs= +github.com/uptrace/bun v1.1.12 h1:sOjDVHxNTuM6dNGaba0wUuz7KvDE1BmNu9Gqs2gJSXQ= +github.com/uptrace/bun v1.1.12/go.mod h1:NPG6JGULBeQ9IU6yHp7YGELRa5Agmd7ATZdz4tGZ6z0= +github.com/uptrace/bun/dialect/pgdialect v1.1.12 h1:m/CM1UfOkoBTglGO5CUTKnIKKOApOYxkcP2qn0F9tJk= +github.com/uptrace/bun/dialect/pgdialect v1.1.12/go.mod h1:Ij6WIxQILxLlL2frUBxUBOZJtLElD2QQNDcu/PWDHTc= +github.com/uptrace/bun/driver/pgdriver v1.1.12 h1:3rRWB1GK0psTJrHwxzNfEij2MLibggiLdTqjTtfHc1w= +github.com/uptrace/bun/driver/pgdriver v1.1.12/go.mod h1:ssYUP+qwSEgeDDS1xm2XBip9el1y9Mi5mTAvLoiADLM= +github.com/vmihailenco/bufpool v0.1.11 h1:gOq2WmBrq0i2yW5QJ16ykccQ4wH9UyEsgLm6czKAd94= +github.com/vmihailenco/bufpool v0.1.11/go.mod h1:AFf/MOy3l2CFTKbxwt0mp2MwnqjNEs5H/UxrkA5jxTQ= +github.com/vmihailenco/msgpack/v5 v5.3.5 h1:5gO0H1iULLWGhs2H5tbAHIZTV8/cYafcFOr9znI5mJU= +github.com/vmihailenco/msgpack/v5 v5.3.5/go.mod h1:7xyJ9e+0+9SaZT0Wt1RGleJXzli6Q/V5KbhBonMG9jc= +github.com/vmihailenco/tagparser v0.1.2 h1:gnjoVuB/kljJ5wICEEOpx98oXMWPLj22G67Vbd1qPqc= +github.com/vmihailenco/tagparser v0.1.2/go.mod h1:OeAg3pn3UbLjkWt+rN9oFYB6u/cQgqMEUPoW2WPyhdI= +github.com/vmihailenco/tagparser/v2 v2.0.0 h1:y09buUbR+b5aycVFQs/g70pqKVZNBmxwAhO7/IwNM9g= +github.com/vmihailenco/tagparser/v2 v2.0.0/go.mod h1:Wri+At7QHww0WTrCBeu4J6bNtoV6mEfg5OIWRZA9qds= github.com/wk8/go-ordered-map/v2 v2.1.8 h1:5h/BUHu93oj4gIdvHHHGsScSTMijfx5PeYkE/fJgbpc= github.com/wk8/go-ordered-map/v2 v2.1.8/go.mod h1:5nJHM5DyteebpVlHnWMV0rPz6Zp7+xBAnxjb1X5vnTw= +github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM= +github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg= github.com/zalando/go-keyring v0.2.8 h1:6sD/Ucpl7jNq10rM2pgqTs0sZ9V3qMrqfIIy5YPccHs= github.com/zalando/go-keyring v0.2.8/go.mod h1:tsMo+VpRq5NGyKfxoBVjCuMrG47yj8cmakZDO5QGii0= go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc= @@ -102,9 +148,16 @@ golang.org/x/sys v0.43.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= golang.org/x/text v0.36.0 h1:JfKh3XmcRPqZPKevfXVpI1wXPTqbkE5f7JA92a55Yxg= golang.org/x/text v0.36.0/go.mod h1:NIdBknypM8iqVmPiuco0Dh6P5Jcdk8lJL0CUebqK164= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= -gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gorm.io/driver/postgres v1.5.4 h1:Iyrp9Meh3GmbSuyIAGyjkN+n9K+GHX9b9MqsTL4EJCo= +gorm.io/driver/postgres v1.5.4/go.mod h1:Bgo89+h0CRcdA33Y6frlaHHVuTdOf87pmyzwW9C/BH0= +gorm.io/gorm v1.25.5 h1:zR9lOiiYf09VNh5Q1gphfyia1JpiClIWG9hQaxB/mls= +gorm.io/gorm v1.25.5/go.mod h1:hbnx/Oo0ChWMn1BIhpy1oYozzpM15i4YPuHDmfYtwg8= +mellium.im/sasl v0.3.1 h1:wE0LW6g7U83vhvxjC1IY8DnXM+EU095yeo8XClvCdfo= +mellium.im/sasl v0.3.1/go.mod h1:xm59PUYpZHhgQ9ZqoJ5QaCqzWMi8IeS49dhp6plPCzw= diff --git a/internal/adapters/db/postgres.go b/internal/adapters/db/postgres.go new file mode 100644 index 0000000..c2a891a --- /dev/null +++ b/internal/adapters/db/postgres.go @@ -0,0 +1,297 @@ +package db + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "time" + + "github.com/google/uuid" + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgxpool" + "github.com/pgvector/pgvector-go" + pgxvec "github.com/pgvector/pgvector-go/pgx" + "github.com/shahid-io/inode/internal/model" +) + +// PostgresAdapter implements db.Adapter using PostgreSQL + pgvector. +// +// Compared to the SQLite path, this backend stores the embedding inline on +// the notes table (pgvector's vector(N) column type) rather than in a +// separate virtual table. L2 distance is computed by the `<->` operator, +// matching sqlite-vec's metric so the relevance threshold is portable. +// +// id is stored as TEXT (not UUID) so prefix-match semantics on Get/Delete +// stay identical to the SQLite implementation. +type PostgresAdapter struct { + pool *pgxpool.Pool + dimension int +} + +// NewPostgresAdapter connects to Postgres, registers pgvector types, and +// runs migrations. The DSN follows libpq format, e.g.: +// +// postgres://user:pass@host:5432/db?sslmode=disable +// +// The pgvector extension is created if missing — the connecting role +// therefore needs CREATE privilege on the database (superuser in the +// standard pgvector/pgvector Docker image). +func NewPostgresAdapter(ctx context.Context, dsn string, dimension int) (*PostgresAdapter, error) { + if dsn == "" { + return nil, errors.New("postgres dsn is empty — run: inode config set db.dsn ") + } + if dimension <= 0 { + dimension = 768 + } + + cfg, err := pgxpool.ParseConfig(dsn) + if err != nil { + return nil, fmt.Errorf("parse dsn: %w", err) + } + // Ensure the pgvector extension exists before registering its types — + // pgxvec.RegisterTypes looks the `vector` type up in pg_type and fails + // hard if the extension hasn't been installed yet. CREATE EXTENSION IF + // NOT EXISTS is idempotent and cheap on subsequent connections. + cfg.AfterConnect = func(ctx context.Context, conn *pgx.Conn) error { + if _, err := conn.Exec(ctx, `CREATE EXTENSION IF NOT EXISTS vector`); err != nil { + return fmt.Errorf("ensure vector extension: %w", err) + } + return pgxvec.RegisterTypes(ctx, conn) + } + + pool, err := pgxpool.NewWithConfig(ctx, cfg) + if err != nil { + return nil, fmt.Errorf("open postgres pool: %w", err) + } + + a := &PostgresAdapter{pool: pool, dimension: dimension} + if err := a.migrate(ctx); err != nil { + pool.Close() + return nil, fmt.Errorf("migrate: %w", err) + } + return a, nil +} + +// migrate enables pgvector and creates the notes table on first run. +// +// Dimension is baked into the schema. If embedding.dimension changes after +// a database is provisioned, the migration is a no-op (CREATE IF NOT EXISTS) +// and inserts of the new size will fail with a clear pgvector error — that +// is the same constraint the SQLite path has, just surfaced differently. +func (a *PostgresAdapter) migrate(ctx context.Context) error { + // CREATE EXTENSION is handled in AfterConnect (so type registration on + // every new pool connection can rely on it). All that's left here is + // the schema. + stmt := fmt.Sprintf(` + CREATE TABLE IF NOT EXISTS notes ( + id TEXT PRIMARY KEY, + content_enc BYTEA, + content_plain TEXT, + summary TEXT NOT NULL DEFAULT '', + category TEXT NOT NULL DEFAULT 'notes', + tags JSONB NOT NULL DEFAULT '[]'::jsonb, + is_sensitive BOOLEAN NOT NULL DEFAULT TRUE, + embedding vector(%d), + created_at TIMESTAMPTZ NOT NULL, + updated_at TIMESTAMPTZ NOT NULL + )`, a.dimension) + if _, err := a.pool.Exec(ctx, stmt); err != nil { + return fmt.Errorf("create notes table: %w", err) + } + return nil +} + +// Save persists a note and its embedding. Returns the assigned UUID. +func (a *PostgresAdapter) Save(ctx context.Context, note *model.Note) (string, error) { + if note.ID == "" { + note.ID = uuid.New().String() + } + now := time.Now().UTC() + note.CreatedAt = now + note.UpdatedAt = now + + tagsJSON, err := json.Marshal(note.Tags) + if err != nil { + return "", err + } + + var emb any + if len(note.Embedding) > 0 { + emb = pgvector.NewVector(note.Embedding) + } + + _, err = a.pool.Exec(ctx, ` + INSERT INTO notes (id, content_enc, content_plain, summary, category, tags, is_sensitive, embedding, created_at, updated_at) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10)`, + note.ID, + note.ContentEnc, + note.ContentPlain, + note.Summary, + note.Category, + string(tagsJSON), + note.IsSensitive, + emb, + now, + now, + ) + if err != nil { + return "", fmt.Errorf("insert note: %w", err) + } + return note.ID, nil +} + +// Get fetches a single note by ID prefix. +func (a *PostgresAdapter) Get(ctx context.Context, id string) (*model.Note, error) { + row := a.pool.QueryRow(ctx, ` + SELECT id, content_enc, content_plain, summary, category, tags, is_sensitive, created_at, updated_at + FROM notes + WHERE id LIKE $1 || '%' + LIMIT 1`, id) + return scanPgNote(row) +} + +// Delete removes a note by ID prefix. +func (a *PostgresAdapter) Delete(ctx context.Context, id string) error { + var fullID string + err := a.pool.QueryRow(ctx, `SELECT id FROM notes WHERE id LIKE $1 || '%' LIMIT 1`, id).Scan(&fullID) + if err != nil { + return fmt.Errorf("note not found: %w", err) + } + if _, err := a.pool.Exec(ctx, `DELETE FROM notes WHERE id = $1`, fullID); err != nil { + return fmt.Errorf("delete note: %w", err) + } + return nil +} + +// SearchSimilar returns top-K notes by L2 distance to vec, ascending. +// Each returned note has Distance populated from the pgvector `<->` operator. +// +// Category filter is pushed down; tag filter is applied in-memory after +// fetching to match the SQLite path's semantics (OR across tags, not AND). +func (a *PostgresAdapter) SearchSimilar(ctx context.Context, vec []float32, topK int, filters Filters) ([]*model.Note, error) { + args := []any{pgvector.NewVector(vec)} + query := ` + SELECT embedding <-> $1 AS distance, + id, content_enc, content_plain, summary, category, tags, is_sensitive, created_at, updated_at + FROM notes + WHERE embedding IS NOT NULL` + + if filters.Category != "" { + args = append(args, filters.Category) + query += fmt.Sprintf(` AND category = $%d`, len(args)) + } + if filters.IsSensitive != nil { + args = append(args, *filters.IsSensitive) + query += fmt.Sprintf(` AND is_sensitive = $%d`, len(args)) + } + + args = append(args, topK) + query += fmt.Sprintf(` ORDER BY distance LIMIT $%d`, len(args)) + + rows, err := a.pool.Query(ctx, query, args...) + if err != nil { + return nil, fmt.Errorf("vector search: %w", err) + } + defer rows.Close() + + return scanPgScoredNotes(rows, filters) +} + +// List returns notes matching filters with pagination. +func (a *PostgresAdapter) List(ctx context.Context, filters Filters, limit, offset int) ([]*model.Note, error) { + args := []any{} + query := ` + SELECT id, content_enc, content_plain, summary, category, tags, is_sensitive, created_at, updated_at + FROM notes + WHERE 1=1` + + if filters.Category != "" { + args = append(args, filters.Category) + query += fmt.Sprintf(` AND category = $%d`, len(args)) + } + if filters.IsSensitive != nil { + args = append(args, *filters.IsSensitive) + query += fmt.Sprintf(` AND is_sensitive = $%d`, len(args)) + } + + args = append(args, limit, offset) + query += fmt.Sprintf(` ORDER BY created_at DESC LIMIT $%d OFFSET $%d`, len(args)-1, len(args)) + + rows, err := a.pool.Query(ctx, query, args...) + if err != nil { + return nil, err + } + defer rows.Close() + + return scanPgNotes(rows, filters) +} + +// Close releases the pool. +func (a *PostgresAdapter) Close() error { + a.pool.Close() + return nil +} + +func scanPgNote(row pgx.Row) (*model.Note, error) { + var n model.Note + var tagsJSON []byte + if err := row.Scan( + &n.ID, &n.ContentEnc, &n.ContentPlain, + &n.Summary, &n.Category, &tagsJSON, + &n.IsSensitive, &n.CreatedAt, &n.UpdatedAt, + ); err != nil { + return nil, err + } + if err := json.Unmarshal(tagsJSON, &n.Tags); err != nil { + n.Tags = []string{} + } + return &n, nil +} + +func scanPgNotes(rows pgx.Rows, filters Filters) ([]*model.Note, error) { + var notes []*model.Note + for rows.Next() { + var n model.Note + var tagsJSON []byte + if err := rows.Scan( + &n.ID, &n.ContentEnc, &n.ContentPlain, + &n.Summary, &n.Category, &tagsJSON, + &n.IsSensitive, &n.CreatedAt, &n.UpdatedAt, + ); err != nil { + return nil, err + } + if err := json.Unmarshal(tagsJSON, &n.Tags); err != nil { + n.Tags = []string{} + } + if matchesTagFilter(&n, filters.Tags) { + notes = append(notes, &n) + } + } + return notes, rows.Err() +} + +func scanPgScoredNotes(rows pgx.Rows, filters Filters) ([]*model.Note, error) { + var notes []*model.Note + for rows.Next() { + var n model.Note + var distance float64 + var tagsJSON []byte + if err := rows.Scan( + &distance, + &n.ID, &n.ContentEnc, &n.ContentPlain, + &n.Summary, &n.Category, &tagsJSON, + &n.IsSensitive, &n.CreatedAt, &n.UpdatedAt, + ); err != nil { + return nil, err + } + n.Distance = float32(distance) + if err := json.Unmarshal(tagsJSON, &n.Tags); err != nil { + n.Tags = []string{} + } + if matchesTagFilter(&n, filters.Tags) { + notes = append(notes, &n) + } + } + return notes, rows.Err() +} diff --git a/internal/adapters/db/postgres_test.go b/internal/adapters/db/postgres_test.go new file mode 100644 index 0000000..1d390b9 --- /dev/null +++ b/internal/adapters/db/postgres_test.go @@ -0,0 +1,241 @@ +package db + +import ( + "context" + "os" + "testing" + + "github.com/shahid-io/inode/internal/model" +) + +// newPostgresTestAdapter connects to a real Postgres+pgvector instance via +// INODE_TEST_PG_DSN. When the env var is unset, the whole suite is skipped +// so CI stays green without docker. +// +// Local one-liner to enable these tests: +// +// docker run -d --name pgvector-test -p 5432:5432 \ +// -e POSTGRES_PASSWORD=password pgvector/pgvector:pg16 +// INODE_TEST_PG_DSN=postgres://postgres:password@localhost:5432/postgres?sslmode=disable \ +// go test ./internal/adapters/db/... +// +// Each test truncates the shared `notes` table before running, so tests must +// not be parallelised within this package. +func newPostgresTestAdapter(t *testing.T) *PostgresAdapter { + t.Helper() + dsn := os.Getenv("INODE_TEST_PG_DSN") + if dsn == "" { + t.Skip("INODE_TEST_PG_DSN not set — skipping Postgres adapter tests") + } + ctx := context.Background() + a, err := NewPostgresAdapter(ctx, dsn, 4) + if err != nil { + t.Fatalf("NewPostgresAdapter: %v", err) + } + if _, err := a.pool.Exec(ctx, "TRUNCATE notes"); err != nil { + t.Fatalf("truncate notes: %v", err) + } + t.Cleanup(func() { _ = a.Close() }) + return a +} + +func TestPostgres_SaveAndGet_FullID(t *testing.T) { + a := newPostgresTestAdapter(t) + ctx := context.Background() + + id, err := a.Save(ctx, &model.Note{ + ContentPlain: "echo hello", + Summary: "echo hello command", + Category: "commands", + Tags: []string{"bash"}, + Embedding: []float32{0.1, 0.2, 0.3, 0.4}, + }) + if err != nil { + t.Fatalf("Save: %v", err) + } + + got, err := a.Get(ctx, id) + if err != nil { + t.Fatalf("Get(%q): %v", id, err) + } + if got.Summary != "echo hello command" { + t.Errorf("Summary mismatch: %q", got.Summary) + } + if got.Category != "commands" { + t.Errorf("Category mismatch: %q", got.Category) + } + if len(got.Tags) != 1 || got.Tags[0] != "bash" { + t.Errorf("Tags mismatch: %v", got.Tags) + } +} + +func TestPostgres_GetByShortPrefix(t *testing.T) { + a := newPostgresTestAdapter(t) + ctx := context.Background() + + id, err := a.Save(ctx, &model.Note{ + ContentPlain: "x", + Summary: "n", + Category: "notes", + Embedding: []float32{0.1, 0.2, 0.3, 0.4}, + }) + if err != nil { + t.Fatalf("Save: %v", err) + } + + got, err := a.Get(ctx, id[:8]) + if err != nil { + t.Fatalf("Get short prefix: %v", err) + } + if got.ID != id { + t.Errorf("expected full ID %q, got %q", id, got.ID) + } +} + +func TestPostgres_DeleteByShortPrefix(t *testing.T) { + a := newPostgresTestAdapter(t) + ctx := context.Background() + + id, _ := a.Save(ctx, &model.Note{Summary: "x", Category: "notes", Embedding: []float32{1, 0, 0, 0}}) + + if err := a.Delete(ctx, id[:8]); err != nil { + t.Fatalf("Delete short prefix: %v", err) + } + + if _, err := a.Get(ctx, id); err == nil { + t.Fatal("expected Get to fail after delete") + } +} + +func TestPostgres_SearchSimilar_ReturnsClosestWithDistance(t *testing.T) { + a := newPostgresTestAdapter(t) + ctx := context.Background() + + mustSave := func(summary string, vec []float32) string { + id, err := a.Save(ctx, &model.Note{Summary: summary, Category: "notes", Embedding: vec}) + if err != nil { + t.Fatalf("Save %s: %v", summary, err) + } + return id + } + idA := mustSave("note A", []float32{1, 0, 0, 0}) + mustSave("note B", []float32{0, 1, 0, 0}) + idC := mustSave("note C", []float32{0.9, 0.1, 0, 0}) + + results, err := a.SearchSimilar(ctx, []float32{1, 0, 0, 0}, 3, Filters{}) + if err != nil { + t.Fatalf("SearchSimilar: %v", err) + } + if len(results) != 3 { + t.Fatalf("expected 3 results, got %d", len(results)) + } + if results[0].ID != idA { + t.Errorf("nearest should be note A, got %s", results[0].Summary) + } + if results[1].ID != idC { + t.Errorf("second-nearest should be note C, got %s", results[1].Summary) + } + if results[0].Distance > results[1].Distance || results[1].Distance > results[2].Distance { + t.Errorf("distances not ascending: %v %v %v", + results[0].Distance, results[1].Distance, results[2].Distance) + } + if results[0].Distance != 0 { + t.Errorf("identical vector should have distance 0, got %v", results[0].Distance) + } +} + +func TestPostgres_SearchSimilar_TagFilterAppliedInMemory(t *testing.T) { + a := newPostgresTestAdapter(t) + ctx := context.Background() + + mustSave := func(n *model.Note) { + t.Helper() + if _, err := a.Save(ctx, n); err != nil { + t.Fatalf("Save: %v", err) + } + } + mustSave(&model.Note{Summary: "stripe", Category: "credentials", Tags: []string{"stripe", "payment"}, Embedding: []float32{1, 0, 0, 0}}) + mustSave(&model.Note{Summary: "github", Category: "credentials", Tags: []string{"github"}, Embedding: []float32{0.95, 0.05, 0, 0}}) + mustSave(&model.Note{Summary: "aws", Category: "credentials", Tags: []string{"aws"}, Embedding: []float32{0.9, 0.1, 0, 0}}) + + results, err := a.SearchSimilar(ctx, []float32{1, 0, 0, 0}, 5, Filters{Tags: []string{"github", "aws"}}) + if err != nil { + t.Fatalf("SearchSimilar: %v", err) + } + if len(results) != 2 { + t.Fatalf("expected 2 results matching github|aws filter, got %d", len(results)) + } + for _, r := range results { + if r.Summary == "stripe" { + t.Errorf("stripe note should have been filtered out by tag filter") + } + } +} + +func TestPostgres_List_FilterByCategory(t *testing.T) { + a := newPostgresTestAdapter(t) + ctx := context.Background() + + mustSave := func(n *model.Note) { + t.Helper() + if _, err := a.Save(ctx, n); err != nil { + t.Fatalf("Save: %v", err) + } + } + mustSave(&model.Note{Summary: "key1", Category: "credentials", Embedding: []float32{1, 0, 0, 0}}) + mustSave(&model.Note{Summary: "key2", Category: "credentials", Embedding: []float32{0, 1, 0, 0}}) + mustSave(&model.Note{Summary: "ls -la", Category: "commands", Embedding: []float32{0, 0, 1, 0}}) + + results, err := a.List(ctx, Filters{Category: "credentials"}, 10, 0) + if err != nil { + t.Fatalf("List: %v", err) + } + if len(results) != 2 { + t.Fatalf("expected 2 credentials, got %d", len(results)) + } + for _, r := range results { + if r.Category != "credentials" { + t.Errorf("category leak: %q", r.Category) + } + } +} + +func TestPostgres_Save_StoresEncryptedContent(t *testing.T) { + a := newPostgresTestAdapter(t) + ctx := context.Background() + + encrypted := []byte{0xDE, 0xAD, 0xBE, 0xEF, 0x00, 0x01, 0x02} + id, err := a.Save(ctx, &model.Note{ + ContentEnc: encrypted, + Summary: "secret", + Category: "credentials", + IsSensitive: true, + Embedding: []float32{1, 0, 0, 0}, + }) + if err != nil { + t.Fatalf("Save: %v", err) + } + + got, err := a.Get(ctx, id) + if err != nil { + t.Fatalf("Get: %v", err) + } + if !got.IsSensitive { + t.Error("expected IsSensitive=true to be persisted") + } + if string(got.ContentEnc) != string(encrypted) { + t.Errorf("ContentEnc mismatch — got %x", got.ContentEnc) + } + if got.ContentPlain != "" { + t.Errorf("ContentPlain should be empty for sensitive note, got %q", got.ContentPlain) + } +} + +func TestPostgres_Get_NotFound(t *testing.T) { + a := newPostgresTestAdapter(t) + ctx := context.Background() + + if _, err := a.Get(ctx, "nonexistent"); err == nil { + t.Fatal("expected error for missing ID") + } +} diff --git a/internal/config/settings.go b/internal/config/settings.go index 470b9ab..1b6eac0 100644 --- a/internal/config/settings.go +++ b/internal/config/settings.go @@ -42,7 +42,13 @@ type EmbeddingConfig struct { } type DBConfig struct { + // Backend selects the storage engine: "sqlite" (default) or "postgres". + Backend string `mapstructure:"backend"` + // Path is the SQLite database file. Used only when Backend = sqlite. Path string `mapstructure:"path"` + // DSN is the Postgres connection string (e.g. postgres://user:pass@host:5432/db?sslmode=disable). + // Used only when Backend = postgres. + DSN string `mapstructure:"dsn"` } type DefaultsConfig struct { @@ -76,6 +82,8 @@ func Load() (*Config, error) { _ = v.BindEnv("embedding.api_key", "INODE_EMBEDDING_API_KEY") _ = v.BindEnv("embedding.base_url", "INODE_EMBEDDING_BASE_URL") _ = v.BindEnv("embedding.dimension", "INODE_EMBEDDING_DIMENSION") + _ = v.BindEnv("db.backend", "INODE_DB_BACKEND") + _ = v.BindEnv("db.dsn", "INODE_DB_DSN") _ = v.BindEnv("log.level", "INODE_LOG_LEVEL") // Config file: ~/.inode/config.toml @@ -97,7 +105,9 @@ func Load() (*Config, error) { v.SetDefault("embedding.model", "nomic-embed-text") v.SetDefault("embedding.base_url", "http://localhost:11434") v.SetDefault("embedding.dimension", 768) + v.SetDefault("db.backend", "sqlite") v.SetDefault("db.path", filepath.Join(configDir, "notes.db")) + v.SetDefault("db.dsn", "") v.SetDefault("defaults.sensitive", true) v.SetDefault("search.max_distance", 1.0) v.SetDefault("search.top_k", 5) diff --git a/internal/core/container.go b/internal/core/container.go index a6303b5..6477ee8 100644 --- a/internal/core/container.go +++ b/internal/core/container.go @@ -1,6 +1,7 @@ package core import ( + "context" "fmt" "os" @@ -29,18 +30,29 @@ func NewContainer(cfg *config.Config) (*Container, error) { return nil, fmt.Errorf("init key manager: %w", err) } - // DB adapter. - dbPath := cfg.DB.Path - if dbPath == "" { - dbPath = configDir + "/notes.db" - } - if len(dbPath) >= 2 && dbPath[:2] == "~/" { - home, _ := os.UserHomeDir() - dbPath = home + dbPath[1:] - } - dbAdapter, err := db.NewSQLiteAdapter(dbPath, cfg.Embedding.Dimension) - if err != nil { - return nil, fmt.Errorf("init db: %w", err) + // DB adapter — selected by backend config. + var dbAdapter db.Adapter + switch cfg.DB.Backend { + case "postgres": + dbAdapter, err = db.NewPostgresAdapter(context.Background(), cfg.DB.DSN, cfg.Embedding.Dimension) + if err != nil { + return nil, fmt.Errorf("init postgres: %w", err) + } + case "sqlite", "": + dbPath := cfg.DB.Path + if dbPath == "" { + dbPath = configDir + "/notes.db" + } + if len(dbPath) >= 2 && dbPath[:2] == "~/" { + home, _ := os.UserHomeDir() + dbPath = home + dbPath[1:] + } + dbAdapter, err = db.NewSQLiteAdapter(dbPath, cfg.Embedding.Dimension) + if err != nil { + return nil, fmt.Errorf("init sqlite: %w", err) + } + default: + return nil, fmt.Errorf("unknown db.backend %q (supported: sqlite, postgres)", cfg.DB.Backend) } // LLM adapter — selected by backend config.