diff --git a/.gitignore b/.gitignore index 2d1171b3..add8286e 100644 --- a/.gitignore +++ b/.gitignore @@ -74,11 +74,16 @@ web_modules/ # dotenv environment variable files .env -.env.development.local -.env.test.local -.env.production.local +.env.development +.env.test +.env.production .env.local +config.json +config.local.json +config.development.json +config.production.json + # parcel-bundler cache (https://parceljs.org/) .cache .parcel-cache diff --git a/apps/server/.env.example b/apps/server/.env.example index 0220f218..a189968b 100644 --- a/apps/server/.env.example +++ b/apps/server/.env.example @@ -3,35 +3,15 @@ # ------------------------------------------------------------------ # Configuration now comes exclusively from config.json. Environment # variables are only read when config.json references them via pointers. -# Use: -# env://VAR_NAME → load from env (append ? to make optional) -# file://path/to/file → load file contents (relative to config.json unless absolute) -# This file is therefore just an example of the required secrets pointed to by config.json. +# Use: env://VAR_NAME → load from env # ------------------------------------------------------------------ # ─────────────────────────────────────────────────────────────── -# Required secrets referenced by config.json +# Required secrets referenced by default configurations # ─────────────────────────────────────────────────────────────── POSTGRES_URL=postgres://colanode_user:postgrespass123@localhost:5432/colanode_db REDIS_URL=redis://:your_valkey_password@localhost:6379/0 -# ─────────────────────────────────────────────────────────────── -# Optional: PostgreSQL TLS (used if config.json points to env://POSTGRES_SSL_*) -# ─────────────────────────────────────────────────────────────── -# POSTGRES_SSL_CA= -# POSTGRES_SSL_KEY= -# POSTGRES_SSL_CERT= - -# ─────────────────────────────────────────────────────────────── -# Optional: Google OAuth (account.google.* section) -# ─────────────────────────────────────────────────────────────── -# ACCOUNT_GOOGLE_CLIENT_ID= -# ACCOUNT_GOOGLE_CLIENT_SECRET= - -# ─────────────────────────────────────────────────────────────── -# Optional: Email configuration (email section) -# ─────────────────────────────────────────────────────────────── -# EMAIL_FROM= -# EMAIL_SMTP_HOST= -# EMAIL_SMTP_USER= -# EMAIL_SMTP_PASSWORD= +# Create a config.local.json (you can copy see an example in config.example.json) +# and add the following line to use it when running locally +# CONFIG=./config.local.json \ No newline at end of file diff --git a/apps/server/config.example.json b/apps/server/config.example.json new file mode 100644 index 00000000..9bc26ee5 --- /dev/null +++ b/apps/server/config.example.json @@ -0,0 +1,100 @@ +{ + "_comment": "Colanode Server Configuration - Copy this file to config.local.json for local development or use it as a template for your self-hosted server", + "_docs": "https://colanode.com/docs/self-hosting/configuration/", + + "name": "Colanode Local", + "mode": "standalone", + "cors": { + "origin": "http://localhost:4000", + "maxAge": 7200 + }, + "account": { + "verificationType": "automatic", + "otpTimeout": 600, + "google": { + "enabled": false, + "clientId": "env://ACCOUNT_GOOGLE_CLIENT_ID", + "clientSecret": "env://ACCOUNT_GOOGLE_CLIENT_SECRET" + } + }, + "postgres": { + "url": "env://POSTGRES_URL", + "ssl": { + "ca": "env://POSTGRES_SSL_CA", + "key": "env://POSTGRES_SSL_KEY", + "cert": "env://POSTGRES_SSL_CERT" + } + }, + "redis": { + "url": "env://REDIS_URL", + "db": 0, + "eventsChannel": "events" + }, + "storage": { + "tus": { + "locker": { + "type": "redis", + "prefix": "colanode:tus:lock" + }, + "cache": { + "type": "none" + } + }, + "provider": { + "type": "s3", + "endpoint": "env://S3_ENDPOINT", + "accessKey": "env://S3_ACCESS_KEY", + "secretKey": "env://S3_SECRET_KEY", + "bucket": "env://S3_BUCKET", + "region": "env://S3_REGION", + "forcePathStyle": false + } + }, + "email": { + "enabled": false, + "from": { + "email": "colanode@example.com", + "name": "Colanode" + }, + "provider": { + "type": "smtp", + "host": "env://EMAIL_SMTP_HOST", + "port": 587, + "secure": false, + "auth": { + "user": "env://EMAIL_SMTP_USER", + "password": "env://EMAIL_SMTP_PASSWORD" + } + } + }, + "jobs": { + "queue": { + "name": "jobs", + "prefix": "colanode" + }, + "nodeUpdatesMerge": { + "enabled": false, + "cron": "0 5 */2 * * *", + "batchSize": 500, + "mergeWindow": 3600, + "cutoffWindow": 7200 + }, + "documentUpdatesMerge": { + "enabled": false, + "cron": "0 5 */2 * * *", + "batchSize": 500, + "mergeWindow": 3600, + "cutoffWindow": 7200 + }, + "cleanup": { + "enabled": false, + "cron": "0 5 */2 * * *" + } + }, + "workspace": { + "maxFileSize": "524288000" + }, + "logging": { + "level": "info" + } +} diff --git a/apps/server/config.json b/apps/server/config.json deleted file mode 100644 index 91a11f66..00000000 --- a/apps/server/config.json +++ /dev/null @@ -1,114 +0,0 @@ -{ - "_comment": "Colanode Server Configuration - Copy this file to config.json and customize as needed", - "_migration_guide": { - "precedence": "config.json (plus env:// / file:// pointers) > schema defaults", - "env_syntax": "Use 'env://VAR_NAME' to reference environment variables. Add '?' suffix for optional variables (e.g., 'env://VAR_NAME?')", - "file_syntax": "Use 'file://relative/or/absolute/path' to inline file contents (e.g., PEM certificates). Add '?' to make it optional", - "backward_compatibility": "Environment variables are only read when referenced via env://" - }, - - "server": { - "_comment": "Server branding and networking configuration", - "name": "Colanode Local", - "avatar": null, - "mode": "standalone", - "pathPrefix": null, - "cors": { - "origin": "http://localhost:4000", - "maxAge": 7200 - } - }, - - "account": { - "_comment": "User account and authentication configuration", - "verificationType": "automatic", - "otpTimeout": 600, - "google": { - "enabled": false, - "_comment_when_enabled": "When enabled=true, you must provide clientId and clientSecret", - "clientId": "env://ACCOUNT_GOOGLE_CLIENT_ID?", - "clientSecret": "env://ACCOUNT_GOOGLE_CLIENT_SECRET?" - } - }, - - "postgres": { - "_comment": "PostgreSQL database connection", - "url": "env://POSTGRES_URL", - "ssl": { - "rejectUnauthorized": null, - "ca": "env://POSTGRES_SSL_CA?", - "key": "env://POSTGRES_SSL_KEY?", - "cert": "env://POSTGRES_SSL_CERT?" - } - }, - - "redis": { - "_comment": "Redis connection and namespacing", - "url": "env://REDIS_URL", - "db": 0, - "jobs": { - "name": "jobs", - "prefix": "colanode" - }, - "tus": { - "lockPrefix": "colanode:tus:lock", - "kvPrefix": "colanode:tus:kv" - }, - "eventsChannel": "events" - }, - - "storage": { - "_comment": "File storage backend configuration. Supports: 'file' (local filesystem), 's3' (S3-compatible like AWS S3, MinIO, Cloudflare R2), 'gcs' (Google Cloud Storage), 'azure' (Azure Blob Storage). For file storage: set 'directory' to the path where user files will be stored. For other backends, set STORAGE_TYPE env var and provide backend-specific credentials via env:// references", - "type": "file", - "directory": "./colanode" - }, - - "email": { - "_comment": "Email sending configuration. Supported providers: smtp", - "enabled": false, - "from": { - "email": "env://EMAIL_FROM?", - "name": "Colanode" - }, - "provider": { - "type": "smtp", - "host": "env://EMAIL_SMTP_HOST?", - "port": 587, - "secure": false, - "auth": { - "user": "env://EMAIL_SMTP_USER?", - "password": "env://EMAIL_SMTP_PASSWORD?" - } - } - }, - - "jobs": { - "_comment": "Background job configurations", - "nodeUpdatesMerge": { - "enabled": false, - "_comment_when_enabled": "Merges node update events periodically", - "cron": "0 5 */2 * * *", - "batchSize": 500, - "mergeWindow": 3600, - "cutoffWindow": 7200 - }, - "documentUpdatesMerge": { - "enabled": false, - "_comment_when_enabled": "Merges document update events periodically", - "cron": "0 5 */2 * * *", - "batchSize": 500, - "mergeWindow": 3600, - "cutoffWindow": 7200 - }, - "cleanup": { - "enabled": false, - "_comment_when_enabled": "Cleans up expired upload chunks", - "cron": "0 5 */2 * * *" - } - }, - - "logging": { - "_comment": "Logging level: trace, debug, info, warn, error, fatal, silent", - "level": "info" - } -} diff --git a/apps/server/src/api/client/plugins/cors.ts b/apps/server/src/api/client/plugins/cors.ts index 55cf0303..b6ad4368 100644 --- a/apps/server/src/api/client/plugins/cors.ts +++ b/apps/server/src/api/client/plugins/cors.ts @@ -5,15 +5,11 @@ import fp from 'fastify-plugin'; import { config } from '@colanode/server/lib/config'; const corsCallback: FastifyPluginCallback = (fastify, _, done) => { - const origin = config.server.cors.origin.includes(',') - ? config.server.cors.origin.split(',').map((o) => o.trim()) - : config.server.cors.origin; - fastify.register(cors, { - origin, + origin: config.cors.origin, methods: ['GET', 'POST', 'PUT', 'DELETE', 'OPTIONS', 'PATCH'], credentials: true, - maxAge: config.server.cors.maxAge, + maxAge: config.cors.maxAge, }); done(); diff --git a/apps/server/src/api/client/routes/workspaces/files/file-upload-tus.ts b/apps/server/src/api/client/routes/workspaces/files/file-upload-tus.ts index 46f2af34..b8c146bf 100644 --- a/apps/server/src/api/client/routes/workspaces/files/file-upload-tus.ts +++ b/apps/server/src/api/client/routes/workspaces/files/file-upload-tus.ts @@ -86,7 +86,10 @@ export const fileUploadTusRoute: FastifyPluginCallbackZod = ( const tusServer = new Server({ path: '/tus', datastore: tusStore, - locker: new RedisLocker(redis, config.redis.tus.lockPrefix), + locker: + config.storage.tus.locker.type === 'redis' + ? new RedisLocker(redis, config.storage.tus.locker.prefix) + : undefined, async onUploadCreate() { const upload = await database .selectFrom('uploads') diff --git a/apps/server/src/api/config.ts b/apps/server/src/api/config.ts index f15cd518..1f93800a 100644 --- a/apps/server/src/api/config.ts +++ b/apps/server/src/api/config.ts @@ -1,6 +1,6 @@ import { FastifyPluginCallbackZod } from 'fastify-type-provider-zod'; -import { ServerConfig, serverConfigSchema } from '@colanode/core'; +import { build, ServerConfig, serverConfigSchema } from '@colanode/core'; import { config } from '@colanode/server/lib/config'; export const configGetRoute: FastifyPluginCallbackZod = (instance, _, done) => { @@ -14,12 +14,12 @@ export const configGetRoute: FastifyPluginCallbackZod = (instance, _, done) => { }, handler: async (request) => { const output: ServerConfig = { - name: config.server.name, - avatar: config.server.avatar ?? '', - version: config.server.version, - sha: config.server.sha, + name: config.name, + avatar: config.avatar ?? '', + version: build.version, + sha: build.sha, ip: request.client.ip, - pathPrefix: config.server.pathPrefix, + pathPrefix: config.pathPrefix, account: { google: config.account.google.enabled ? { diff --git a/apps/server/src/api/home.ts b/apps/server/src/api/home.ts index cf1d3944..26e01e4b 100644 --- a/apps/server/src/api/home.ts +++ b/apps/server/src/api/home.ts @@ -1,5 +1,6 @@ import { FastifyPluginCallback } from 'fastify'; +import { build } from '@colanode/core'; import { config } from '@colanode/server/lib/config'; import { generateUrl } from '@colanode/server/lib/fastify'; import { homeTemplate } from '@colanode/server/templates'; @@ -12,10 +13,10 @@ export const homeRoute: FastifyPluginCallback = (instance, _, done) => { const configUrl = generateUrl(request, '/config'); const template = homeTemplate({ - name: config.server.name, + name: config.name, url: configUrl, - version: config.server.version, - sha: config.server.sha, + version: build.version, + sha: build.sha, }); reply.type('text/html').send(template); diff --git a/apps/server/src/api/index.ts b/apps/server/src/api/index.ts index 771d7d8c..6e6b436e 100644 --- a/apps/server/src/api/index.ts +++ b/apps/server/src/api/index.ts @@ -6,7 +6,7 @@ import { homeRoute } from '@colanode/server/api/home'; import { config } from '@colanode/server/lib/config'; export const apiRoutes: FastifyPluginCallback = (instance, _, done) => { - const prefix = config.server.pathPrefix ? `/${config.server.pathPrefix}` : ''; + const prefix = config.pathPrefix ? `/${config.pathPrefix}` : ''; instance.register(homeRoute, { prefix }); instance.register(configGetRoute, { prefix }); diff --git a/apps/server/src/app.ts b/apps/server/src/app.ts index 58b5c7dc..01ba1c69 100644 --- a/apps/server/src/app.ts +++ b/apps/server/src/app.ts @@ -36,7 +36,7 @@ export const initApp = () => { process.exit(1); } - const path = config.server.pathPrefix ? `/${config.server.pathPrefix}` : ''; + const path = config.pathPrefix ? `/${config.pathPrefix}` : ''; logger.info(`Server is running at ${address}${path}`); }); }; diff --git a/apps/server/src/lib/config/account.ts b/apps/server/src/lib/config/account.ts index 086d92fd..77bacd01 100644 --- a/apps/server/src/lib/config/account.ts +++ b/apps/server/src/lib/config/account.ts @@ -1,5 +1,7 @@ import { z } from 'zod/v4'; +import { resolveConfigReference } from './utils'; + export const accountVerificationTypeSchema = z.enum([ 'automatic', 'manual', @@ -10,27 +12,38 @@ export type AccountVerificationType = z.infer< typeof accountVerificationTypeSchema >; -export const googleConfigSchema = z.discriminatedUnion('enabled', [ - z.object({ - enabled: z.literal(true), - clientId: z.string({ - error: 'Google client ID is required when Google login is enabled.', +export const googleConfigSchema = z + .discriminatedUnion('enabled', [ + z.object({ + enabled: z.literal(true), + clientId: z + .string({ + error: 'Google client ID is required when Google login is enabled.', + }) + .transform(resolveConfigReference), + clientSecret: z + .string({ + error: + 'Google client secret is required when Google login is enabled.', + }) + .transform(resolveConfigReference), }), - clientSecret: z.string({ - error: 'Google client secret is required when Google login is enabled.', + z.object({ + enabled: z.literal(false), }), - }), - z.object({ - enabled: z.literal(false), - }), -]); - -export const accountConfigSchema = z.object({ - verificationType: accountVerificationTypeSchema.default('manual'), - otpTimeout: z.coerce.number().default(600), - google: googleConfigSchema.default({ + ]) + .prefault({ enabled: false, - }), -}); + }); + +export const accountConfigSchema = z + .object({ + verificationType: accountVerificationTypeSchema + .transform(resolveConfigReference) + .default('automatic'), + otpTimeout: z.coerce.number().default(600), + google: googleConfigSchema, + }) + .prefault({}); export type AccountConfig = z.infer; diff --git a/apps/server/src/lib/config/ai.ts b/apps/server/src/lib/config/ai.ts index 021e7d2a..e13551e9 100644 --- a/apps/server/src/lib/config/ai.ts +++ b/apps/server/src/lib/config/ai.ts @@ -41,47 +41,48 @@ export const retrievalConfigSchema = z.object({ export type RetrievalConfig = z.infer; -export const aiConfigSchema = z.discriminatedUnion('enabled', [ - z.object({ - enabled: z.literal(true), - nodeEmbeddingDelay: z.coerce.number().default(5000), - documentEmbeddingDelay: z.coerce.number().default(10000), - providers: z.object({ - openai: aiProviderConfigSchema, - google: aiProviderConfigSchema, +export const aiConfigSchema = z + .discriminatedUnion('enabled', [ + z.object({ + enabled: z.literal(true), + nodeEmbeddingDelay: z.coerce.number().default(5000), + documentEmbeddingDelay: z.coerce.number().default(10000), + providers: z.object({ + openai: aiProviderConfigSchema, + google: aiProviderConfigSchema, + }), + langfuse: z.object({ + enabled: z.boolean().default(false), + publicKey: z.string().default(''), + secretKey: z.string().default(''), + baseUrl: z.string().default('https://cloud.langfuse.com'), + }), + models: z.object({ + queryRewrite: aiModelConfigSchema, + response: aiModelConfigSchema, + rerank: aiModelConfigSchema, + summarization: aiModelConfigSchema, + contextEnhancer: aiModelConfigSchema, + noContext: aiModelConfigSchema, + intentRecognition: aiModelConfigSchema, + databaseFilter: aiModelConfigSchema, + }), + embedding: z.object({ + provider: aiProviderSchema.default('openai'), + modelName: z.string().default('text-embedding-3-large'), + dimensions: z.coerce.number().default(2000), + apiKey: z.string().default(''), + batchSize: z.coerce.number().default(50), + }), + chunking: chunkingConfigSchema, + retrieval: retrievalConfigSchema, }), - langfuse: z.object({ - enabled: z.preprocess( - (val) => val === 'true', - z.boolean().default(false) - ), - publicKey: z.string().default(''), - secretKey: z.string().default(''), - baseUrl: z.string().default('https://cloud.langfuse.com'), + z.object({ + enabled: z.literal(false), }), - models: z.object({ - queryRewrite: aiModelConfigSchema, - response: aiModelConfigSchema, - rerank: aiModelConfigSchema, - summarization: aiModelConfigSchema, - contextEnhancer: aiModelConfigSchema, - noContext: aiModelConfigSchema, - intentRecognition: aiModelConfigSchema, - databaseFilter: aiModelConfigSchema, - }), - embedding: z.object({ - provider: aiProviderSchema.default('openai'), - modelName: z.string().default('text-embedding-3-large'), - dimensions: z.coerce.number().default(2000), - apiKey: z.string().default(''), - batchSize: z.coerce.number().default(50), - }), - chunking: chunkingConfigSchema, - retrieval: retrievalConfigSchema, - }), - z.object({ - enabled: z.literal(false), - }), -]); + ]) + .prefault({ + enabled: false, + }); export type AiConfig = z.infer; diff --git a/apps/server/src/lib/config/cors.ts b/apps/server/src/lib/config/cors.ts new file mode 100644 index 00000000..b8427057 --- /dev/null +++ b/apps/server/src/lib/config/cors.ts @@ -0,0 +1,18 @@ +import { z } from 'zod/v4'; + +import { resolveConfigReference } from './utils'; + +const corsOriginSchema = z.union([ + z.string().transform(resolveConfigReference), + z.array(z.string().transform(resolveConfigReference)), + z.boolean(), +]); + +export const corsSchema = z + .object({ + origin: corsOriginSchema.default('http://localhost:4000'), + maxAge: z.coerce.number().default(7200), + }) + .prefault({}); + +export type CorsConfig = z.infer; diff --git a/apps/server/src/lib/config/email.ts b/apps/server/src/lib/config/email.ts index c287023d..ee9f8d46 100644 --- a/apps/server/src/lib/config/email.ts +++ b/apps/server/src/lib/config/email.ts @@ -1,21 +1,27 @@ import { z } from 'zod/v4'; +import { resolveConfigReference } from './utils'; + const smtpProviderConfigSchema = z.object({ type: z.literal('smtp'), host: z .string({ - error: 'EMAIL_SMTP_HOST is required when SMTP provider is used', + error: 'Email SMTP host is required when SMTP provider is used', }) - .default('smtp'), + .transform(resolveConfigReference), port: z.coerce.number().default(587), secure: z.boolean().default(false), auth: z.object({ - user: z.string({ - error: 'EMAIL_SMTP_USER is required when SMTP provider is used', - }), - password: z.string({ - error: 'EMAIL_SMTP_PASSWORD is required when SMTP provider is used', - }), + user: z + .string({ + error: 'Email SMTP user is required when SMTP provider is used', + }) + .transform(resolveConfigReference), + password: z + .string({ + error: 'Email SMTP password is required when SMTP provider is used', + }) + .transform(resolveConfigReference), }), }); @@ -25,20 +31,31 @@ export const emailProviderConfigSchema = z.discriminatedUnion('type', [ export type EmailProviderConfig = z.infer; -export const emailConfigSchema = z.discriminatedUnion('enabled', [ - z.object({ - enabled: z.literal(true), - from: z.object({ - email: z.string({ - error: 'EMAIL_FROM is required when email is enabled', +export const emailConfigSchema = z + .discriminatedUnion('enabled', [ + z.object({ + enabled: z.literal(true), + from: z.object({ + email: z + .string({ + error: 'Email from email is required when email is enabled', + }) + .transform(resolveConfigReference), + name: z + .string({ + error: 'Email from name is required when email is enabled', + }) + .default('Colanode') + .transform(resolveConfigReference), }), - name: z.string().default('Colanode'), + provider: emailProviderConfigSchema, }), - provider: emailProviderConfigSchema, - }), - z.object({ - enabled: z.literal(false), - }), -]); + z.object({ + enabled: z.literal(false), + }), + ]) + .prefault({ + enabled: false, + }); export type EmailConfig = z.infer; diff --git a/apps/server/src/lib/config/index.ts b/apps/server/src/lib/config/index.ts index da0d09ad..96634f6e 100644 --- a/apps/server/src/lib/config/index.ts +++ b/apps/server/src/lib/config/index.ts @@ -1,19 +1,32 @@ +import fs from 'fs'; + import { z } from 'zod/v4'; import { accountConfigSchema } from './account'; import { aiConfigSchema } from './ai'; +import { corsSchema } from './cors'; import { emailConfigSchema } from './email'; import { jobsConfigSchema } from './jobs'; -import { loadRawConfig } from './loader'; import { loggingConfigSchema } from './logging'; import { postgresConfigSchema } from './postgres'; import { redisConfigSchema } from './redis'; -import { serverConfigSchema } from './server'; import { storageConfigSchema } from './storage'; +import { + resolveConfigReference, + resolveOptionalConfigReference, +} from './utils'; import { workspaceConfigSchema } from './workspace'; +const serverModeSchema = z.enum(['standalone', 'cluster']); + const configSchema = z.object({ - server: serverConfigSchema, + name: z.string().default('Colanode Server').transform(resolveConfigReference), + avatar: z.string().optional().transform(resolveOptionalConfigReference), + mode: serverModeSchema + .default('standalone') + .transform(resolveConfigReference), + pathPrefix: z.string().optional().transform(resolveOptionalConfigReference), + cors: corsSchema, account: accountConfigSchema, postgres: postgresConfigSchema, redis: redisConfigSchema, @@ -27,9 +40,20 @@ const configSchema = z.object({ export type Configuration = z.infer; -const readConfigVariables = (): Configuration => { +const loadConfigFromPath = (configPath: string): Partial => { + if (!fs.existsSync(configPath)) { + throw new Error(`Configuration file not found: ${configPath}`); + } + + const raw = fs.readFileSync(configPath, 'utf-8'); + const parsed = JSON.parse(raw); + return parsed; +}; + +const buildConfig = (): Configuration => { try { - const input = loadRawConfig(); + const configPath = process.env.CONFIG; + const input = configPath ? loadConfigFromPath(configPath) : {}; return configSchema.parse(input); } catch (error) { if (error instanceof z.ZodError) { @@ -45,4 +69,4 @@ const readConfigVariables = (): Configuration => { } }; -export const config = readConfigVariables(); +export const config = buildConfig(); diff --git a/apps/server/src/lib/config/jobs.ts b/apps/server/src/lib/config/jobs.ts index fffb9553..487a7392 100644 --- a/apps/server/src/lib/config/jobs.ts +++ b/apps/server/src/lib/config/jobs.ts @@ -1,30 +1,21 @@ import ms from 'ms'; import { z } from 'zod/v4'; +import { resolveConfigReference } from './utils'; + const DEFAULT_BATCH_SIZE = 500; const DEFAULT_MERGE_WINDOW = ms('1 hour') / 1000; // in seconds const DEFAULT_CUTOFF_WINDOW = ms('2 hours') / 1000; // in seconds const DEFAULT_CRON_PATTERN = '0 5 */2 * * *'; // every 2 hours at the 5th minute -export const nodeUpdatesMergeJobConfigSchema = z.discriminatedUnion('enabled', [ - z.object({ - enabled: z.literal(true), - cron: z.string().default(DEFAULT_CRON_PATTERN), - batchSize: z.coerce.number().default(DEFAULT_BATCH_SIZE), - mergeWindow: z.coerce.number().default(DEFAULT_MERGE_WINDOW), - cutoffWindow: z.coerce.number().default(DEFAULT_CUTOFF_WINDOW), - }), - z.object({ - enabled: z.literal(false), - }), -]); - -export const documentUpdatesMergeJobConfigSchema = z.discriminatedUnion( - 'enabled', - [ +export const nodeUpdatesMergeJobConfigSchema = z + .discriminatedUnion('enabled', [ z.object({ enabled: z.literal(true), - cron: z.string().default(DEFAULT_CRON_PATTERN), + cron: z + .string() + .default(DEFAULT_CRON_PATTERN) + .transform(resolveConfigReference), batchSize: z.coerce.number().default(DEFAULT_BATCH_SIZE), mergeWindow: z.coerce.number().default(DEFAULT_MERGE_WINDOW), cutoffWindow: z.coerce.number().default(DEFAULT_CUTOFF_WINDOW), @@ -32,23 +23,62 @@ export const documentUpdatesMergeJobConfigSchema = z.discriminatedUnion( z.object({ enabled: z.literal(false), }), - ] -); + ]) + .prefault({ + enabled: false, + }); -export const cleanupJobConfigSchema = z.discriminatedUnion('enabled', [ - z.object({ - enabled: z.literal(true), - cron: z.string().default(DEFAULT_CRON_PATTERN), - }), - z.object({ - enabled: z.literal(false), - }), -]); +export const documentUpdatesMergeJobConfigSchema = z + .discriminatedUnion('enabled', [ + z.object({ + enabled: z.literal(true), + cron: z + .string() + .default(DEFAULT_CRON_PATTERN) + .transform(resolveConfigReference), + batchSize: z.coerce.number().default(DEFAULT_BATCH_SIZE), + mergeWindow: z.coerce.number().default(DEFAULT_MERGE_WINDOW), + cutoffWindow: z.coerce.number().default(DEFAULT_CUTOFF_WINDOW), + }), + z.object({ + enabled: z.literal(false), + }), + ]) + .prefault({ + enabled: false, + }); -export const jobsConfigSchema = z.object({ - nodeUpdatesMerge: nodeUpdatesMergeJobConfigSchema, - documentUpdatesMerge: documentUpdatesMergeJobConfigSchema, - cleanup: cleanupJobConfigSchema, -}); +export const cleanupJobConfigSchema = z + .discriminatedUnion('enabled', [ + z.object({ + enabled: z.literal(true), + cron: z + .string() + .default(DEFAULT_CRON_PATTERN) + .transform(resolveConfigReference), + }), + z.object({ + enabled: z.literal(false), + }), + ]) + .prefault({ + enabled: false, + }); + +export const jobsQueueSchema = z + .object({ + name: z.string().default('jobs').transform(resolveConfigReference), + prefix: z.string().default('colanode').transform(resolveConfigReference), + }) + .prefault({}); + +export const jobsConfigSchema = z + .object({ + queue: jobsQueueSchema, + nodeUpdatesMerge: nodeUpdatesMergeJobConfigSchema, + documentUpdatesMerge: documentUpdatesMergeJobConfigSchema, + cleanup: cleanupJobConfigSchema, + }) + .prefault({}); export type JobsConfig = z.infer; diff --git a/apps/server/src/lib/config/loader.ts b/apps/server/src/lib/config/loader.ts deleted file mode 100644 index 471bb11b..00000000 --- a/apps/server/src/lib/config/loader.ts +++ /dev/null @@ -1,177 +0,0 @@ -import fs from 'fs'; -import { fileURLToPath } from 'node:url'; -import path from 'path'; - -import { Configuration } from './index'; - -type ConfigSource = Partial; - -const ENV_POINTER_PATTERN = /^env:\/\/([A-Z0-9_]+)(\?)?$/; -const FILE_POINTER_PATTERN = /^file:\/\/(.+?)(\?)?$/; - -export class MissingEnvVarError extends Error { - constructor(varName: string) { - super(`Missing required environment variable: ${varName}`); - this.name = 'MissingEnvVarError'; - } -} - -export class MissingFileError extends Error { - constructor(filePath: string) { - super(`Missing required configuration file: ${filePath}`); - this.name = 'MissingFileError'; - } -} - -const isRecord = (value: unknown): value is Record => { - return typeof value === 'object' && value !== null && !Array.isArray(value); -}; - -type NormalizeContext = { - configDir: string; -}; - -const normalizeValue = (value: unknown, ctx: NormalizeContext): unknown => { - if (value === null) { - return undefined; - } - - if (typeof value === 'string') { - return resolvePointerValue(value, ctx); - } - - if (Array.isArray(value)) { - return value.map((item) => normalizeValue(item, ctx)); - } - - if (isRecord(value)) { - const normalized: Record = {}; - - for (const [key, nested] of Object.entries(value)) { - // Skip fields starting with underscore (documentation fields) - if (key.startsWith('_')) { - continue; - } - - const processed = normalizeValue(nested, ctx); - - if (processed !== undefined) { - normalized[key] = processed; - } - } - - return normalized; - } - - return value; -}; - -const candidateConfigDirectories = (): string[] => { - const cwd = process.cwd(); - const moduleDir = path.dirname(fileURLToPath(import.meta.url)); - const serverRoot = path.resolve(moduleDir, '../../..'); - const candidates = [cwd, path.join(cwd, 'apps/server'), serverRoot]; - - return Array.from(new Set(candidates.map((dir) => path.resolve(dir)))); -}; - -const findConfigFile = (filename: string): string | undefined => { - for (const dir of candidateConfigDirectories()) { - const candidate = path.join(dir, filename); - - if (fs.existsSync(candidate)) { - return candidate; - } - } - - return undefined; -}; - -const readJsonFile = (filePath: string): Record => { - try { - const raw = fs.readFileSync(filePath, 'utf-8'); - const parsed = JSON.parse(raw); - - if (!isRecord(parsed)) { - throw new Error('configuration file must contain a JSON object'); - } - - return parsed; - } catch (error) { - const reason = error instanceof Error ? error.message : String(error); - throw new Error(`Failed to read ${filePath}: ${reason}`); - } -}; - -const resolvePointerValue = (value: string, ctx: NormalizeContext): unknown => { - const envMatch = value.match(ENV_POINTER_PATTERN); - - if (envMatch) { - const [, envName, optionalFlag] = envMatch; - const envValue = process.env[envName!]; - const optional = optionalFlag === '?'; - - if (envValue === undefined) { - if (optional) { - return undefined; - } - - throw new MissingEnvVarError(envName!); - } - - return envValue; - } - - const fileMatch = value.match(FILE_POINTER_PATTERN); - - if (fileMatch) { - const [, rawPath, optionalFlag] = fileMatch; - const optional = optionalFlag === '?'; - const resolvedPath = path.isAbsolute(rawPath!) - ? rawPath! - : path.resolve(ctx.configDir, rawPath!); - - if (!fs.existsSync(resolvedPath)) { - if (optional) { - return undefined; - } - - throw new MissingFileError(resolvedPath); - } - - try { - return fs.readFileSync(resolvedPath, 'utf-8'); - } catch (error) { - const reason = error instanceof Error ? error.message : String(error); - throw new Error(`Failed to read ${resolvedPath}: ${reason}`); - } - } - - return value; -}; - -export const resolveEnvPointers = ( - value: unknown, - ctx: NormalizeContext -): unknown => { - return normalizeValue(value, ctx); -}; - -export const loadRawConfig = (): ConfigSource => { - const configPath = findConfigFile('config.json'); - - if (!configPath) { - throw new Error( - [ - 'Unable to find config.json.', - 'Copy apps/server/config.json (or mount your own) so the server has a configuration file.', - ].join(' ') - ); - } - - const jsonConfig = readJsonFile(configPath) as ConfigSource; - - return normalizeValue(jsonConfig, { - configDir: path.dirname(configPath), - }) as ConfigSource; -}; diff --git a/apps/server/src/lib/config/logging.ts b/apps/server/src/lib/config/logging.ts index 8a84b14e..810f1c02 100644 --- a/apps/server/src/lib/config/logging.ts +++ b/apps/server/src/lib/config/logging.ts @@ -1,9 +1,14 @@ import { z } from 'zod/v4'; -export const loggingConfigSchema = z.object({ - level: z - .enum(['trace', 'debug', 'info', 'warn', 'error', 'fatal', 'silent']) - .default('info'), -}); +import { resolveConfigReference } from './utils'; + +export const loggingConfigSchema = z + .object({ + level: z + .enum(['trace', 'debug', 'info', 'warn', 'error', 'fatal', 'silent']) + .default('info') + .transform(resolveConfigReference), + }) + .prefault({}); export type LoggingConfig = z.infer; diff --git a/apps/server/src/lib/config/postgres.ts b/apps/server/src/lib/config/postgres.ts index e38288f7..f92b9a01 100644 --- a/apps/server/src/lib/config/postgres.ts +++ b/apps/server/src/lib/config/postgres.ts @@ -1,19 +1,28 @@ import { z } from 'zod/v4'; -export const postgresConfigSchema = z.object({ - url: z.string({ - error: - 'POSTGRES_URL is required (e.g. postgres://postgres:postgres@localhost:5432/postgres)', - }), - ssl: z.object({ - rejectUnauthorized: z.preprocess( - (val) => (val === undefined ? undefined : val === 'true'), - z.boolean().optional() - ), - ca: z.string().optional(), - key: z.string().optional(), - cert: z.string().optional(), - }), -}); +import { + resolveConfigReference, + resolveOptionalConfigReference, +} from './utils'; + +export const postgresConfigSchema = z + .object({ + url: z + .string({ + error: + 'Postgres URL is required (e.g. postgres://postgres:postgres@localhost:5432/postgres)', + }) + .default('env://POSTGRES_URL') + .transform(resolveConfigReference), + ssl: z + .object({ + rejectUnauthorized: z.boolean().optional(), + ca: z.string().optional().transform(resolveOptionalConfigReference), + key: z.string().optional().transform(resolveOptionalConfigReference), + cert: z.string().optional().transform(resolveOptionalConfigReference), + }) + .optional(), + }) + .prefault({}); export type PostgresConfig = z.infer; diff --git a/apps/server/src/lib/config/redis.ts b/apps/server/src/lib/config/redis.ts index a820d62f..b7949eef 100644 --- a/apps/server/src/lib/config/redis.ts +++ b/apps/server/src/lib/config/redis.ts @@ -1,17 +1,16 @@ import { z } from 'zod/v4'; -export const redisConfigSchema = z.object({ - url: z.string({ error: 'REDIS_URL is required' }), - db: z.coerce.number().default(0), - jobs: z.object({ - name: z.string().optional().default('jobs'), - prefix: z.string().optional().default('colanode'), - }), - tus: z.object({ - lockPrefix: z.string().optional().default('colanode:tus:lock'), - kvPrefix: z.string().optional().default('colanode:tus:kv'), - }), - eventsChannel: z.string().optional().default('events'), -}); +import { resolveConfigReference } from './utils'; + +export const redisConfigSchema = z + .object({ + url: z + .string({ error: 'Redis URL is required' }) + .default('env://REDIS_URL') + .transform(resolveConfigReference), + db: z.coerce.number().default(0), + eventsChannel: z.string().optional().default('events'), + }) + .prefault({}); export type RedisConfig = z.infer; diff --git a/apps/server/src/lib/config/server.ts b/apps/server/src/lib/config/server.ts deleted file mode 100644 index 25b8b0de..00000000 --- a/apps/server/src/lib/config/server.ts +++ /dev/null @@ -1,21 +0,0 @@ -import { z } from 'zod/v4'; - -import { build } from '@colanode/core'; - -const serverModeSchema = z.enum(['standalone', 'cluster']); -export type ServerMode = z.infer; - -export const serverConfigSchema = z.object({ - version: z.string().default(build.version), - sha: z.string().default(build.sha), - name: z.string().default('Colanode Server'), - avatar: z.string().optional(), - mode: serverModeSchema.default('standalone'), - pathPrefix: z.string().optional(), - cors: z.object({ - origin: z.string().default('http://localhost:4000'), - maxAge: z.coerce.number().default(7200), - }), -}); - -export type ServerConfig = z.infer; diff --git a/apps/server/src/lib/config/storage.ts b/apps/server/src/lib/config/storage.ts index 68ffb274..ed073a27 100644 --- a/apps/server/src/lib/config/storage.ts +++ b/apps/server/src/lib/config/storage.ts @@ -1,45 +1,131 @@ import { z } from 'zod/v4'; -const s3StorageConfigSchema = z.object({ +import { resolveConfigReference } from './utils'; + +const s3StorageProviderConfigSchema = z.object({ type: z.literal('s3'), - endpoint: z.string({ error: 'STORAGE_S3_ENDPOINT is required' }), - accessKey: z.string({ error: 'STORAGE_S3_ACCESS_KEY is required' }), - secretKey: z.string({ error: 'STORAGE_S3_SECRET_KEY is required' }), - bucket: z.string({ error: 'STORAGE_S3_BUCKET is required' }), - region: z.string({ error: 'STORAGE_S3_REGION is required' }), + endpoint: z + .string({ error: 'Storage S3 endpoint is required' }) + .transform(resolveConfigReference), + accessKey: z + .string({ error: 'Storage S3 access key is required' }) + .transform(resolveConfigReference), + secretKey: z + .string({ error: 'Storage S3 secret key is required' }) + .transform(resolveConfigReference), + bucket: z + .string({ error: 'Storage S3 bucket is required' }) + .transform(resolveConfigReference), + region: z.string({ error: 'Storage S3 region is required' }), forcePathStyle: z.boolean().optional(), }); -const fileStorageConfigSchema = z.object({ +const fileStorageProviderConfigSchema = z.object({ type: z.literal('file'), - directory: z.string({ error: 'STORAGE_FILE_DIRECTORY is required' }), + directory: z + .string({ error: 'Storage file directory is required' }) + .default('./colanode') + .transform(resolveConfigReference), }); -const gcsStorageConfigSchema = z.object({ +const gcsStorageProviderConfigSchema = z.object({ type: z.literal('gcs'), - bucket: z.string({ error: 'STORAGE_GCS_BUCKET is required' }), - projectId: z.string({ error: 'STORAGE_GCS_PROJECT_ID is required' }), - credentials: z.string({ error: 'STORAGE_GCS_CREDENTIALS is required' }), + bucket: z + .string({ error: 'Storage GCS bucket is required' }) + .transform(resolveConfigReference), + projectId: z + .string({ error: 'Storage GCS project ID is required' }) + .transform(resolveConfigReference), + credentials: z + .string({ error: 'Storage GCS credentials is required' }) + .transform(resolveConfigReference), }); -const azureStorageConfigSchema = z.object({ +const azureStorageProviderConfigSchema = z.object({ type: z.literal('azure'), - account: z.string({ error: 'STORAGE_AZURE_ACCOUNT is required' }), - accountKey: z.string({ error: 'STORAGE_AZURE_ACCOUNT_KEY is required' }), - containerName: z.string({ - error: 'STORAGE_AZURE_CONTAINER_NAME is required', - }), + account: z + .string({ error: 'Storage Azure account is required' }) + .transform(resolveConfigReference), + accountKey: z + .string({ error: 'Storage Azure account key is required' }) + .transform(resolveConfigReference), + containerName: z + .string({ error: 'Storage Azure container name is required' }) + .transform(resolveConfigReference), }); -export const storageConfigSchema = z.discriminatedUnion('type', [ - s3StorageConfigSchema, - fileStorageConfigSchema, - gcsStorageConfigSchema, - azureStorageConfigSchema, -]); +export const storageProviderConfigSchema = z + .discriminatedUnion('type', [ + s3StorageProviderConfigSchema, + fileStorageProviderConfigSchema, + gcsStorageProviderConfigSchema, + azureStorageProviderConfigSchema, + ]) + .prefault({ + type: 'file', + }); +export const tusLockerSchema = z + .discriminatedUnion('type', [ + z.object({ + type: z.literal('redis'), + prefix: z + .string() + .default('colanode:tus:lock') + .transform(resolveConfigReference), + }), + z.object({ + type: z.literal('memory'), + }), + ]) + .prefault({ + type: 'memory', + }); + +export const tusCacheSchema = z + .discriminatedUnion('type', [ + z.object({ type: z.literal('none') }), + z.object({ + type: z.literal('redis'), + prefix: z + .string() + .default('colanode:tus:kv') + .transform(resolveConfigReference), + }), + ]) + .prefault({ + type: 'none', + }); + +export const tusConfigSchema = z + .object({ + locker: tusLockerSchema, + cache: tusCacheSchema, + }) + .prefault({}); + +export const storageConfigSchema = z + .object({ + tus: tusConfigSchema, + provider: storageProviderConfigSchema, + }) + .prefault({}); + +export type TusLockerConfig = z.infer; +export type TusCacheConfig = z.infer; +export type TusConfig = z.infer; + +export type StorageProviderConfig = z.infer; +export type S3StorageProviderConfig = z.infer< + typeof s3StorageProviderConfigSchema +>; +export type FileStorageProviderConfig = z.infer< + typeof fileStorageProviderConfigSchema +>; +export type GCSStorageProviderConfig = z.infer< + typeof gcsStorageProviderConfigSchema +>; +export type AzureStorageProviderConfig = z.infer< + typeof azureStorageProviderConfigSchema +>; export type StorageConfig = z.infer; -export type S3StorageConfig = z.infer; -export type FileStorageConfig = z.infer; -export type GCSStorageConfig = z.infer; -export type AzureStorageConfig = z.infer; diff --git a/apps/server/src/lib/config/utils.ts b/apps/server/src/lib/config/utils.ts new file mode 100644 index 00000000..682ab071 --- /dev/null +++ b/apps/server/src/lib/config/utils.ts @@ -0,0 +1,58 @@ +import fs from 'node:fs'; + +import { z } from 'zod/v4'; + +const resolveValue = ( + val: string, + ctx: z.core.$RefinementCtx +): string => { + if (val.startsWith('env://')) { + const envName = val.slice(6); + const envValue = process.env[envName]; + if (envValue === undefined) { + ctx.addIssue({ + code: 'invalid_value', + message: `Environment variable "${envName}" is not set`, + values: [envName], + }); + return z.NEVER; + } + + return envValue; + } + + if (val.startsWith('file://')) { + const filePath = val.slice(7); + try { + return fs.readFileSync(filePath, 'utf-8').trim(); + } catch (err: unknown) { + const error = err instanceof Error ? err : new Error(String(err)); + ctx.addIssue({ + code: 'invalid_value', + message: `Failed to read secret from file "${filePath}": ${error.message}`, + values: [filePath], + }); + return z.NEVER; + } + } + + return val; +}; + +export const resolveConfigReference = ( + val: string, + ctx: z.core.$RefinementCtx +): string => { + return resolveValue(val, ctx); +}; + +export const resolveOptionalConfigReference = ( + val: string | undefined, + ctx: z.core.$RefinementCtx +): string | undefined => { + if (val === undefined) { + return undefined; + } + + return resolveValue(val, ctx as z.core.$RefinementCtx); +}; diff --git a/apps/server/src/lib/config/workspace.ts b/apps/server/src/lib/config/workspace.ts index e425d462..1b2bcf94 100644 --- a/apps/server/src/lib/config/workspace.ts +++ b/apps/server/src/lib/config/workspace.ts @@ -1,7 +1,9 @@ import { z } from 'zod/v4'; -export const workspaceConfigSchema = z.object({ - maxFileSize: z.string().optional().nullable(), -}); +export const workspaceConfigSchema = z + .object({ + maxFileSize: z.string().optional(), + }) + .prefault({}); export type WorkspaceConfig = z.infer; diff --git a/apps/server/src/lib/event-bus.ts b/apps/server/src/lib/event-bus.ts index f10bf2c7..459a2d7f 100644 --- a/apps/server/src/lib/event-bus.ts +++ b/apps/server/src/lib/event-bus.ts @@ -37,7 +37,7 @@ export class EventBusService { this.initialized = true; - if (config.server.mode === 'standalone') { + if (config.mode === 'standalone') { return; } @@ -72,7 +72,7 @@ export class EventBusService { public publish(event: Event) { this.processEvent(event); - if (config.server.mode === 'standalone') { + if (config.mode === 'standalone') { return; } diff --git a/apps/server/src/lib/fastify.ts b/apps/server/src/lib/fastify.ts index 822265e4..16c05020 100644 --- a/apps/server/src/lib/fastify.ts +++ b/apps/server/src/lib/fastify.ts @@ -8,7 +8,7 @@ export const generateUrl = (request: FastifyRequest, path: string) => { ? `:${request.port}` : ''; - const prefix = config.server.pathPrefix ? `/${config.server.pathPrefix}` : ''; + const prefix = config.pathPrefix ? `/${config.pathPrefix}` : ''; return `${request.protocol}://${request.hostname}${port}${prefix}${path}`; }; diff --git a/apps/server/src/lib/storage/azure.ts b/apps/server/src/lib/storage/azure.ts index b2479328..1eafe4d5 100644 --- a/apps/server/src/lib/storage/azure.ts +++ b/apps/server/src/lib/storage/azure.ts @@ -8,38 +8,45 @@ import { import { AzureStore } from '@tus/azure-store'; import { DataStore } from '@tus/server'; -import type { AzureStorageConfig } from '@colanode/server/lib/config/storage'; +import { redis } from '@colanode/server/data/redis'; +import type { + AzureStorageProviderConfig, + TusConfig, +} from '@colanode/server/lib/config/storage'; +import { RedisKvStore } from '@colanode/server/lib/storage/tus/redis-kv'; import type { Storage } from './core'; export class AzureBlobStorage implements Storage { private readonly containerName: string; private readonly blobServiceClient: BlobServiceClient; - private readonly config: AzureStorageConfig; - private readonly azureStore: AzureStore; + private readonly store: AzureStore; - constructor(config: AzureStorageConfig) { - this.config = { ...config }; + constructor(config: AzureStorageProviderConfig, tusConfig: TusConfig) { const sharedKeyCredential = new StorageSharedKeyCredential( - this.config.account, - this.config.accountKey + config.account, + config.accountKey ); this.blobServiceClient = new BlobServiceClient( - `https://${this.config.account}.blob.core.windows.net`, + `https://${config.account}.blob.core.windows.net`, sharedKeyCredential ); - this.containerName = this.config.containerName; + this.containerName = config.containerName; - this.azureStore = new AzureStore({ - account: this.config.account, - accountKey: this.config.accountKey, + this.store = new AzureStore({ + account: config.account, + accountKey: config.accountKey, containerName: this.containerName, + cache: + tusConfig.cache.type === 'redis' + ? new RedisKvStore(redis, tusConfig.cache.prefix) + : undefined, }); } public get tusStore(): DataStore { - return this.azureStore; + return this.store; } private getBlockBlobClient(path: string): BlockBlobClient { diff --git a/apps/server/src/lib/storage/fs.ts b/apps/server/src/lib/storage/fs.ts index e3de2a45..8caf0e7a 100644 --- a/apps/server/src/lib/storage/fs.ts +++ b/apps/server/src/lib/storage/fs.ts @@ -4,17 +4,32 @@ import { Readable } from 'stream'; import { FileStore } from '@tus/file-store'; import { DataStore } from '@tus/server'; -import type { FileStorageConfig } from '@colanode/server/lib/config/storage'; +import { redis } from '@colanode/server/data/redis'; +import type { + FileStorageProviderConfig, + TusConfig, +} from '@colanode/server/lib/config/storage'; +import { RedisKvStore } from '@colanode/server/lib/storage/tus/redis-kv'; import type { Storage } from './core'; export class FileSystemStorage implements Storage { private readonly directory: string; - public readonly tusStore: DataStore; + private readonly store: DataStore; - constructor(config: FileStorageConfig) { + constructor(config: FileStorageProviderConfig, tusConfig: TusConfig) { this.directory = config.directory; - this.tusStore = new FileStore({ directory: this.directory }); + this.store = new FileStore({ + directory: this.directory, + configstore: + tusConfig.cache.type === 'redis' + ? new RedisKvStore(redis, tusConfig.cache.prefix) + : undefined, + }); + } + + public get tusStore(): DataStore { + return this.store; } async download( diff --git a/apps/server/src/lib/storage/gcs.ts b/apps/server/src/lib/storage/gcs.ts index f900ae80..c2e46ac6 100644 --- a/apps/server/src/lib/storage/gcs.ts +++ b/apps/server/src/lib/storage/gcs.ts @@ -4,7 +4,7 @@ import { Storage as GoogleStorage, Bucket, File } from '@google-cloud/storage'; import { GCSStore } from '@tus/gcs-store'; import { DataStore } from '@tus/server'; -import type { GCSStorageConfig } from '@colanode/server/lib/config/storage'; +import type { GCSStorageProviderConfig } from '@colanode/server/lib/config/storage'; import type { Storage } from './core'; @@ -12,7 +12,7 @@ export class GCSStorage implements Storage { private readonly bucket: Bucket; private readonly gcsStore: GCSStore; - constructor(config: GCSStorageConfig) { + constructor(config: GCSStorageProviderConfig) { const storage = new GoogleStorage({ projectId: config.projectId, keyFilename: config.credentials, diff --git a/apps/server/src/lib/storage/index.ts b/apps/server/src/lib/storage/index.ts index 6d921c4b..18e400a6 100644 --- a/apps/server/src/lib/storage/index.ts +++ b/apps/server/src/lib/storage/index.ts @@ -7,19 +7,19 @@ import { FileSystemStorage } from './fs'; import { GCSStorage } from './gcs'; import { S3Storage } from './s3'; -const buildStorage = (storageConfig: StorageConfig): Storage => { - switch (storageConfig.type) { +const buildStorage = (config: StorageConfig): Storage => { + switch (config.provider.type) { case 'file': - return new FileSystemStorage(storageConfig); + return new FileSystemStorage(config.provider, config.tus); case 's3': - return new S3Storage(storageConfig); + return new S3Storage(config.provider, config.tus); case 'gcs': - return new GCSStorage(storageConfig); + return new GCSStorage(config.provider); case 'azure': - return new AzureBlobStorage(storageConfig); + return new AzureBlobStorage(config.provider, config.tus); default: throw new Error( - `Unsupported storage type: ${JSON.stringify(storageConfig)}` + `Unsupported storage provider: ${JSON.stringify(config.provider)}` ); } }; diff --git a/apps/server/src/lib/storage/s3.ts b/apps/server/src/lib/storage/s3.ts index c69ae6c2..0fb33afd 100644 --- a/apps/server/src/lib/storage/s3.ts +++ b/apps/server/src/lib/storage/s3.ts @@ -11,8 +11,10 @@ import { DataStore } from '@tus/server'; import { FILE_UPLOAD_PART_SIZE } from '@colanode/core'; import { redis } from '@colanode/server/data/redis'; -import { config } from '@colanode/server/lib/config'; -import type { S3StorageConfig } from '@colanode/server/lib/config/storage'; +import type { + S3StorageProviderConfig, + TusConfig, +} from '@colanode/server/lib/config/storage'; import { RedisKvStore } from '@colanode/server/lib/storage/tus/redis-kv'; import type { Storage } from './core'; @@ -20,43 +22,44 @@ import type { Storage } from './core'; export class S3Storage implements Storage { private readonly client: S3Client; private readonly bucket: string; - private readonly s3Config: S3StorageConfig; - private readonly s3Store: S3Store; - private readonly redisKv: RedisKvStore; + private readonly store: DataStore; + private readonly cache?: RedisKvStore; - constructor(s3Config: S3StorageConfig) { - this.s3Config = { ...s3Config }; + constructor(config: S3StorageProviderConfig, tusConfig: TusConfig) { this.client = new S3Client({ - endpoint: this.s3Config.endpoint, - region: this.s3Config.region, + endpoint: config.endpoint, + region: config.region, credentials: { - accessKeyId: this.s3Config.accessKey, - secretAccessKey: this.s3Config.secretKey, + accessKeyId: config.accessKey, + secretAccessKey: config.secretKey, }, - forcePathStyle: this.s3Config.forcePathStyle, + forcePathStyle: config.forcePathStyle, }); - this.bucket = this.s3Config.bucket; + this.bucket = config.bucket; - this.redisKv = new RedisKvStore(redis, config.redis.tus.kvPrefix); - this.s3Store = new S3Store({ + if (tusConfig.cache.type === 'redis') { + this.cache = new RedisKvStore(redis, tusConfig.cache.prefix); + } + + this.store = new S3Store({ partSize: FILE_UPLOAD_PART_SIZE, - cache: this.redisKv, + cache: this.cache, s3ClientConfig: { bucket: this.bucket, - endpoint: this.s3Config.endpoint, - region: this.s3Config.region, - forcePathStyle: this.s3Config.forcePathStyle, + endpoint: config.endpoint, + region: config.region, + forcePathStyle: config.forcePathStyle, credentials: { - accessKeyId: this.s3Config.accessKey, - secretAccessKey: this.s3Config.secretKey, + accessKeyId: config.accessKey, + secretAccessKey: config.secretKey, }, }, }); } public get tusStore(): DataStore { - return this.s3Store; + return this.store; } public async download( @@ -78,10 +81,13 @@ export class S3Storage implements Storage { public async delete(path: string): Promise { const command = new DeleteObjectCommand({ Bucket: this.bucket, Key: path }); await this.client.send(command); - await this.redisKv.delete(path); - const infoPath = `${path}.info`; - await this.redisKv.delete(infoPath); + if (this.cache) { + await this.cache.delete(path); + + const infoPath = `${path}.info`; + await this.cache.delete(infoPath); + } } public async upload( diff --git a/apps/server/src/services/job-service.ts b/apps/server/src/services/job-service.ts index 4dfc6dce..6ca316b0 100644 --- a/apps/server/src/services/job-service.ts +++ b/apps/server/src/services/job-service.ts @@ -16,15 +16,15 @@ class JobService { // for more information, see: https://docs.bullmq.io/bull/patterns/redis-cluster - private readonly queueName = config.redis.jobs.name; - private readonly prefix = `{${config.redis.jobs.prefix}}`; + private readonly queueName = config.jobs.queue.name; + private readonly prefix = `{${config.jobs.queue.prefix}}`; public async initQueue(): Promise { if (this.jobQueue) { return; } - this.jobQueue = new Queue(this.queueName, { + this.jobQueue = new Queue(this.queueName!, { prefix: this.prefix, connection: { db: config.redis.db, diff --git a/hosting/docker/docker-compose.yaml b/hosting/docker/docker-compose.yaml index 6d6fde56..2364ad55 100644 --- a/hosting/docker/docker-compose.yaml +++ b/hosting/docker/docker-compose.yaml @@ -95,23 +95,20 @@ services: # ------------------------------------------------------------------ NODE_ENV: production - # Required env:// secrets (config.json marks these as env://VAR) + # Required env:// secrets (default configuration references these as env://VAR) POSTGRES_URL: postgres://colanode_user:postgrespass123@postgres:5432/colanode_db REDIS_URL: redis://:your_valkey_password@valkey:6379/0 - # Optional env://? secrets – include only the entries referenced in JSON - # Use file://path entries in config.json when you want to read the value from mounted files instead. - # ACCOUNT_GOOGLE_CLIENT_ID: '' - # ACCOUNT_GOOGLE_CLIENT_SECRET: '' - # ... + # If you want to override the default configuration mount a config.json as explained below + # and add the config.json path as an environment variable: + # CONFIG: /config.json ports: - '3000:3000' volumes: - server_storage:/var/lib/colanode/storage # Mount a config.json sitting next to this compose file to override defaults. - # If omitted, the already included config.json inside the server image is used. - - ./config.json:/app/apps/server/config.json:ro + # - ./config.json:/config.json:ro networks: - colanode_network diff --git a/hosting/kubernetes/README.md b/hosting/kubernetes/README.md index 152f3768..841d4e2c 100644 --- a/hosting/kubernetes/README.md +++ b/hosting/kubernetes/README.md @@ -75,9 +75,9 @@ helm install my-colanode ./hosting/kubernetes/chart \ ### Using config.json with Helm -- The server image already ships with a default `config.json`. Only two env vars are strictly required: `POSTGRES_URL` and `REDIS_URL` (because the JSON references them via `env://`). -- If you do not override `config.json`, the bundled file still expects those pointers. The chart wires them up automatically via `POSTGRES_URL=env://POSTGRES_URL` and `REDIS_URL=env://REDIS_URL`, so a vanilla install works without extra values. -- To supply your own JSON file, copy `apps/server/config.json`, edit it, and enable the new override: +- The server image already ships with a default configuration. Only two env vars are strictly required: `POSTGRES_URL` and `REDIS_URL` (because the default configuration references them via `env://`). +- If you do add your own `config.json`, the default configuration still expects those pointers. The chart wires them up automatically via `POSTGRES_URL=env://POSTGRES_URL` and `REDIS_URL=env://REDIS_URL`, so a vanilla install works without extra values. +- To supply your own JSON file, copy `apps/server/config.example.json`, edit it, and enable the new override: ```bash helm install my-colanode ./hosting/kubernetes/chart \ @@ -87,29 +87,30 @@ helm install my-colanode ./hosting/kubernetes/chart \ - Alternatively, create a ConfigMap yourself (`kubectl create configmap colanode-config --from-file=config.json`) and set `colanode.configFile.existingConfigMap=colanode-config`. - Environment variables no longer override config values. Only secrets referenced via `env://` (and values from files via `file://`) are read at runtime. Keep non-secret settings in your JSON, mount it with `colanode.configFile`, and surface additional env vars through `colanode.additionalEnv` when a pointer needs a value from Kubernetes secrets. -- To use `file://` pointers, mount the target files next to `config.json` (the chart stores it at `/app/apps/server/config.json`). For example, to load a PostgreSQL CA cert via `"file://secrets/postgres-ca.crt"`: - 1. Create a secret with the cert contents: +- To use `file://` pointers, mount the target files next to `config.json` (the chart stores it at `/config.json`). For example, to load a PostgreSQL CA cert via `"file://secrets/postgres-ca.crt"`: - ```bash - kubectl create secret generic postgres-ca \ - --from-file=postgres-ca.crt=./certs/rootCA.crt - ``` +1. Create a secret with the cert contents: - 2. Mount the secret and expose it inside the pod: + ```bash + kubectl create secret generic postgres-ca \ + --from-file=postgres-ca.crt=./certs/rootCA.crt + ``` - ```yaml - colanode: - extraVolumes: - - name: postgres-ca - secret: - secretName: postgres-ca - extraVolumeMounts: - - name: postgres-ca - mountPath: /app/apps/server/secrets - readOnly: true - ``` +2. Mount the secret and expose it inside the pod: - 3. Point your `config.json` field to `"file://secrets/postgres-ca.crt"`. The loader resolves the path relative to the directory containing `config.json`. + ```yaml + colanode: + extraVolumes: + - name: postgres-ca + secret: + secretName: postgres-ca + extraVolumeMounts: + - name: postgres-ca + mountPath: /config/secrets + readOnly: true + ``` + +3. Point your `config.json` field to `"file://secrets/postgres-ca.crt"`. The loader resolves the path relative to the directory containing `config.json`. ### Storage Configuration diff --git a/hosting/kubernetes/chart/Chart.yaml b/hosting/kubernetes/chart/Chart.yaml index ff5fc272..138f807d 100644 --- a/hosting/kubernetes/chart/Chart.yaml +++ b/hosting/kubernetes/chart/Chart.yaml @@ -2,7 +2,7 @@ apiVersion: v2 name: colanode description: A Helm chart for Colanode - open-source & local-first collaboration workspace type: application -version: 0.2.0 +version: 0.2.1 # appVersion is auto-updated by the release workflow appVersion: '1.0.0' diff --git a/hosting/kubernetes/chart/templates/_helpers.tpl b/hosting/kubernetes/chart/templates/_helpers.tpl index 2266942b..d5ac3960 100644 --- a/hosting/kubernetes/chart/templates/_helpers.tpl +++ b/hosting/kubernetes/chart/templates/_helpers.tpl @@ -170,6 +170,13 @@ Colanode Server Environment Variables - name: REDIS_URL value: "redis://:$(REDIS_PASSWORD)@{{ include "colanode.redis.hostname" . }}:6379/0" +{{- $configFile := .Values.colanode.configFile }} +{{- $mountConfigFile := or $configFile.enabled $configFile.existingConfigMap }} +{{- if $mountConfigFile }} +- name: CONFIG + value: "/config.json" +{{- end }} + {{- range $index, $env := .Values.colanode.additionalEnv }} - name: {{ required (printf "colanode.additionalEnv[%d].name is required" $index) $env.name }} {{- if hasKey $env "valueFrom" }} diff --git a/hosting/kubernetes/chart/templates/deployment.yaml b/hosting/kubernetes/chart/templates/deployment.yaml index dae8190f..43e01bc8 100644 --- a/hosting/kubernetes/chart/templates/deployment.yaml +++ b/hosting/kubernetes/chart/templates/deployment.yaml @@ -61,7 +61,7 @@ spec: {{- end }} {{- if $mountConfigFile }} - name: config-json - mountPath: /app/apps/server/config.json + mountPath: /config.json subPath: {{ default "config.json" $configFile.key }} readOnly: true {{- end }} diff --git a/hosting/kubernetes/chart/values.yaml b/hosting/kubernetes/chart/values.yaml index dfcedc09..8222fed1 100644 --- a/hosting/kubernetes/chart/values.yaml +++ b/hosting/kubernetes/chart/values.yaml @@ -91,7 +91,7 @@ colanode: # Example: config.json contains "ca": "file://secrets/postgres-ca.crt". # extraVolumeMounts: # - name: postgres-ca - # mountPath: /app/apps/server/secrets + # mountPath: /config/secrets # readOnly: true extraVolumeMounts: []