diff --git a/README.md b/README.md index cc0d9e6..04696f4 100644 --- a/README.md +++ b/README.md @@ -224,7 +224,8 @@ Queries work on any field with no setup. For large collections, **declare indexe fields** so filters and sorts on them use a real database index (a SQLite/Postgres expression index) instead of a scan — `PUT /v1/data/:collection/indexes { fields }` is declarative, so it reconciles to exactly the set you send. It's key-gated and -never anonymous. +never anonymous. On Postgres the index is built `CONCURRENTLY`, so declaring one +on a large, live collection doesn't block writes. Collections are **schemaless by default**. Opt into validation per collection by declaring a JSON Schema; writes that violate it then fail with `422` and a list of @@ -556,34 +557,15 @@ deploy/ ## Roadmap -**Recently shipped:** pluggable backends — optional **Postgres** (data) and **S3** -(files) alongside the SQLite + disk defaults; **declared indexed fields** for fast -queries on large collections; **per-project public collections** via a non-secret -project id; and realtime **presence** (join/leave/typing) with an optional **Redis** -fan-out for multi-process scale. - -Next, split by who benefits: - -_For app developers (the API):_ - -- **Public realtime** — extend non-secret project-id access to websocket - subscriptions on public channels, so a published frontend gets live updates - without embedding a secret (closes the gap left by public collections). -- **Optional per-collection schemas** — opt-in JSON-schema validation with clear - `422` errors; zero-config collections stay schemaless by default. -- **Cursor pagination** (`?after=`) for stable iteration over large collections, - built on the new declared indexes (offset paging drifts under concurrent writes). - -_For operators (running the server):_ - -- **Redis-backed rate limiting + presence TTLs** — make per-IP limits and the - presence roster correct across the multi-process deployments Redis now enables - (today limits are per-process and a hard crash can leave a stale presence entry). -- **Observability** — a Prometheus `/metrics` endpoint and a `/ready` probe - (DB/Redis/S3 reachability) for orchestrated, scaled deployments. -- **Safe index builds** — `CREATE INDEX CONCURRENTLY` on Postgres so declaring an - index on a large live table doesn't block writes. - -_Larger, later:_ verifiable identity (JWT/OIDC) for per-user ACL + trusted presence; -outbound webhooks on data changes; per-project AI provider config; publish to npm + -GHCR (the release workflow is wired; the first tag publishes). +**Shipped:** pluggable backends — optional **Postgres** (data) and **S3** (files) +alongside the SQLite + disk defaults; **declared indexed fields** (built +`CONCURRENTLY` on Postgres) for fast queries on large collections; **per-collection +JSON Schema** validation (opt-in); **cursor pagination** for stable iteration; +**per-project public collections + live feeds** via a non-secret project id; +realtime **presence** (join/leave/typing) with an optional **Redis** fan-out, +**Redis-backed rate limiting**, and presence TTLs for multi-process scale; and +**observability** — Prometheus `/metrics` + a `/ready` probe. + +**Next (larger):** verifiable identity (JWT/OIDC) for per-user ACL + trusted +presence; outbound webhooks on data changes; per-project AI provider config; +publish to npm + GHCR (the release workflow is wired; the first tag publishes). diff --git a/public/docs.html b/public/docs.html index 8859418..f224465 100644 --- a/public/docs.html +++ b/public/docs.html @@ -193,7 +193,7 @@

REST API — Data

DELETE/v1/data/:cBulk delete by f./or. filters (or ?all=true) → { deleted }. GET/v1/dataList collections + counts. GET·PUT/v1/data/:c/aclAccess rules { read, write: public|private, hidden: [fields] } (key-gated). With API_KEY set, a read:public collection is readable anonymously; hidden fields are stripped from those reads. - GET·PUT/v1/data/:c/indexesDeclared indexed fields { fields: [...] } (key-gated). Backs hot filter/sort fields with a real DB index for large collections; declarative — PUT the full desired set. + GET·PUT/v1/data/:c/indexesDeclared indexed fields { fields: [...] } (key-gated). Backs hot filter/sort fields with a real DB index for large collections; declarative — PUT the full desired set. On Postgres, built CONCURRENTLY (no write lock). GET·PUT·DELETE/v1/data/:c/schemaOptional JSON Schema (key-gated). Schemaless until set; then writes that violate it return 422 with an errors list (PATCH checks the merged result; bulk is all-or-nothing). DELETE to go back to schemaless. diff --git a/src/db/postgres.ts b/src/db/postgres.ts index 8b1e57f..40b3fac 100644 --- a/src/db/postgres.ts +++ b/src/db/postgres.ts @@ -50,6 +50,11 @@ export const postgresDialect: Dialect = { `CASE WHEN jsonb_typeof(${asJsonb(field)}) = 'number' THEN ${asText(field)}::numeric END`, // btree over the jsonb sub-value supports the =, <, > comparisons we emit. indexExpr: (field) => `(${asJsonb(field)})`, + // CONCURRENTLY so building/dropping an index on a large live table doesn't take + // a write-blocking lock. Must run outside a transaction (the store ensures it). + createIndex: (name, expr) => + `CREATE INDEX CONCURRENTLY IF NOT EXISTS ${name} ON documents (project, collection, ${expr})`, + dropIndex: (name) => `DROP INDEX CONCURRENTLY IF EXISTS ${name}`, }; type Queryable = pg.Pool | pg.PoolClient; diff --git a/src/db/sqlite.ts b/src/db/sqlite.ts index e260cab..6037578 100644 --- a/src/db/sqlite.ts +++ b/src/db/sqlite.ts @@ -26,6 +26,8 @@ export const sqliteDialect: Dialect = { jsonCountInput: (field) => jsonPath(field), jsonNumericInput: (field) => jsonPath(field), indexExpr: (field) => jsonPath(field), + createIndex: (name, expr) => `CREATE INDEX IF NOT EXISTS ${name} ON documents (project, collection, ${expr})`, + dropIndex: (name) => `DROP INDEX IF EXISTS ${name}`, }; class SqliteDb implements Db { diff --git a/src/db/types.ts b/src/db/types.ts index b73a33e..5eacdb1 100644 --- a/src/db/types.ts +++ b/src/db/types.ts @@ -33,6 +33,10 @@ export interface Dialect { jsonNumericInput(field: string): string; /** Indexable expression for a JSON body field, used in `CREATE INDEX … (…)`. */ indexExpr(field: string): string; + /** DDL to create the documents expression index `name` over `expr` (idempotent). */ + createIndex(name: string, expr: string): string; + /** DDL to drop the documents index `name` (idempotent). */ + dropIndex(name: string): string; } /** diff --git a/src/modules/data/store.ts b/src/modules/data/store.ts index a259d0e..0bc17ad 100644 Binary files a/src/modules/data/store.ts and b/src/modules/data/store.ts differ