From 79925e1fc63dd5d05ebaf118c4ab78007a58ed6f Mon Sep 17 00:00:00 2001 From: Hakan Shehu Date: Fri, 1 Aug 2025 14:13:40 +0200 Subject: [PATCH] Improve file uploads and downloads (#164) * Track upload progress in clients * Implement client side jobs * Separate local files, downloads and uploads * Use dates instead of timestamps in jobs * Improve some recurring jobs * Local file improvements * Remove job schedules on cancel * Improve avatar handling * Fix manual download * Improve file uploads and downloads * Improve downloads * Use tus resumable uploads * Drop file states table migration * Remove some unused file system methods and types * Use Redis KV and Locker for distributed TUS deployments * Fix file name generation * Add uploads clean job * Have a dedicated endpoint for TUS uploads * Do not revert uploads state because ot TUS resumables * Use integer instead of text for job and job schedule status * Rename a query * Fix error handling for file uploads and downloads jobs * Check node sync for file uploads * Rename the temp files clean job * Minor renames for consistency * Improve uploads badge * Small refactor in server job service * Add env varaibles config for some tus stuff * Use ms package for millisecond conversions * Fix some migrations * Fix logout * Update hosting values --- apps/desktop/index.html | 2 +- apps/desktop/src/main.ts | 46 +- apps/desktop/src/main/file-system.ts | 36 +- apps/desktop/src/main/protocols.ts | 60 +- apps/server/.env.example | 13 +- apps/server/package.json | 5 +- .../accounts/email-password-reset-init.ts | 5 +- .../workspaces/files/file-upload-tus.ts | 296 ++++++ .../routes/workspaces/files/file-upload.ts | 9 +- .../client/routes/workspaces/files/index.ts | 2 + .../00031-add-created-at-index-to-uploads.ts | 12 + apps/server/src/data/migrations/index.ts | 2 + apps/server/src/data/redis.ts | 2 +- apps/server/src/data/storage.ts | 8 +- apps/server/src/index.ts | 2 +- .../server/src/jobs/document-updates-merge.ts | 15 +- apps/server/src/jobs/index.ts | 2 + apps/server/src/jobs/node-updates-merge.ts | 15 +- apps/server/src/jobs/uploads-clean.ts | 63 ++ apps/server/src/lib/accounts.ts | 5 +- apps/server/src/lib/ai/utils.ts | 4 +- apps/server/src/lib/config/jobs.ts | 40 +- apps/server/src/lib/config/redis.ts | 8 + apps/server/src/lib/config/storage.ts | 5 + apps/server/src/lib/tus/redis-kv.ts | 66 ++ apps/server/src/lib/tus/redis-locker.ts | 163 ++++ apps/server/src/services/job-service.ts | 187 ++-- apps/web/src/services/file-system.ts | 17 +- apps/web/src/workers/dedicated.ts | 44 +- hosting/docker/docker-compose.yaml | 9 +- .../kubernetes/chart/templates/_helpers.tpl | 52 +- hosting/kubernetes/chart/values.yaml | 3 + package-lock.json | 369 +++++-- packages/client/package.json | 3 +- .../migrations/00003-create-avatars-table.ts | 17 + .../src/databases/account/migrations/index.ts | 2 + .../client/src/databases/account/schema.ts | 13 + .../app/migrations/00005-create-jobs-table.ts | 44 + .../00006-create-job-schedules-table.ts | 28 + .../00007-drop-deleted-tokens-table.ts | 16 + .../00008-create-temp-files-table.ts | 21 + .../src/databases/app/migrations/index.ts | 8 + packages/client/src/databases/app/schema.ts | 66 +- .../00020-create-local-files-table.ts | 22 + .../migrations/00021-create-uploads-table.ts | 21 + .../00022-create-downloads-table.ts | 28 + .../00023-drop-file-states-table.ts | 49 + .../databases/workspace/migrations/index.ts | 8 + .../client/src/databases/workspace/schema.ts | 97 +- .../mutations/avatars/avatar-upload.ts | 8 +- .../handlers/mutations/files/file-create.ts | 2 +- .../handlers/mutations/files/file-download.ts | 84 +- .../src/handlers/mutations/files/file-save.ts | 48 - .../mutations/files/temp-file-create.ts | 40 + .../client/src/handlers/mutations/index.ts | 4 +- .../mutations/messages/message-create.ts | 11 +- .../nodes/node-interaction-opened.ts | 6 +- .../mutations/nodes/node-interaction-seen.ts | 6 +- .../handlers/queries/avatars/avatar-get.ts | 55 ++ .../queries/avatars/avatar-url-get.ts | 91 -- .../queries/emojis/emoji-category-list.ts | 4 +- .../queries/emojis/emoji-get-by-skin-id.ts | 4 +- .../src/handlers/queries/emojis/emoji-get.ts | 4 +- .../src/handlers/queries/emojis/emoji-list.ts | 4 +- .../handlers/queries/emojis/emoji-search.ts | 4 +- .../queries/files/download-list-manual.ts | 128 +++ .../handlers/queries/files/file-save-list.ts | 54 -- .../handlers/queries/files/file-state-get.ts | 133 --- .../handlers/queries/files/local-file-get.ts | 152 +++ .../handlers/queries/files/temp-file-get.ts | 63 ++ .../queries/files/upload-list-pending.ts | 157 +++ .../src/handlers/queries/files/upload-list.ts | 120 +++ .../queries/icons/icon-category-list.ts | 4 +- .../src/handlers/queries/icons/icon-list.ts | 4 +- .../src/handlers/queries/icons/icon-search.ts | 4 +- packages/client/src/handlers/queries/index.ts | 18 +- packages/client/src/jobs/account-sync.ts | 46 + packages/client/src/jobs/avatar-download.ts | 64 ++ packages/client/src/jobs/avatars-clean.ts | 46 + packages/client/src/jobs/file-download.ts | 264 +++++ packages/client/src/jobs/file-upload.ts | 288 ++++++ packages/client/src/jobs/index.ts | 91 ++ packages/client/src/jobs/mutations-sync.ts | 63 ++ packages/client/src/jobs/server-sync.ts | 46 + packages/client/src/jobs/temp-files-clean.ts | 64 ++ packages/client/src/jobs/token-delete.ts | 82 ++ .../client/src/jobs/workspace-files-clean.ts | 58 ++ packages/client/src/lib/mappers.ts | 93 +- packages/client/src/lib/nodes.ts | 13 + packages/client/src/lib/sleep-scheduler.ts | 59 ++ packages/client/src/lib/utils.ts | 2 +- .../client/src/mutations/files/file-create.ts | 4 +- .../client/src/mutations/files/file-save.ts | 20 - .../src/mutations/files/temp-file-create.ts | 25 + packages/client/src/mutations/index.ts | 3 +- .../client/src/queries/avatars/avatar-get.ts | 16 + .../src/queries/avatars/avatar-url-get.ts | 18 - .../src/queries/files/download-list-manual.ts | 18 + .../src/queries/files/file-save-list.ts | 16 - .../src/queries/files/file-state-get.ts | 17 - .../src/queries/files/local-file-get.ts | 23 + .../client/src/queries/files/temp-file-get.ts | 15 + .../src/queries/files/upload-list-pending.ts | 18 + .../client/src/queries/files/upload-list.ts | 18 + packages/client/src/queries/index.ts | 9 +- .../src/services/accounts/account-service.ts | 135 +-- .../src/services/accounts/account-socket.ts | 27 +- .../src/services/accounts/avatar-service.ts | 122 +++ packages/client/src/services/app-service.ts | 160 +-- packages/client/src/services/file-system.ts | 10 +- packages/client/src/services/job-service.ts | 489 ++++++++++ .../client/src/services/server-service.ts | 31 +- .../services/workspaces/document-service.ts | 4 +- .../src/services/workspaces/file-service.ts | 915 +++++------------- .../services/workspaces/mutation-service.ts | 35 +- .../workspaces/node-reaction-service.ts | 4 +- .../src/services/workspaces/node-service.ts | 35 +- .../src/services/workspaces/user-service.ts | 16 - .../services/workspaces/workspace-service.ts | 29 +- packages/client/src/types/avatars.ts | 8 + packages/client/src/types/events.ts | 99 +- packages/client/src/types/files.ts | 78 +- packages/client/src/types/workspaces.ts | 3 +- packages/core/src/lib/id.ts | 2 + packages/core/src/lib/utils.ts | 14 + packages/core/src/registry/nodes/file.ts | 3 +- packages/core/src/types/api.ts | 4 + packages/core/src/types/files.ts | 7 - packages/ui/package.json | 1 + .../src/components/avatars/avatar-image.tsx | 33 +- .../components/downloads/download-status.tsx | 60 -- .../downloads/downloads-breadcrumb.tsx | 22 - .../downloads/downloads-container-tab.tsx | 30 - .../downloads/downloads-container.tsx | 20 - .../components/downloads/downloads-list.tsx | 80 -- .../src/components/emojis/emoji-element.tsx | 7 +- .../ui/src/components/files/file-block.tsx | 26 +- .../ui/src/components/files/file-body.tsx | 22 +- .../files/file-download-progress.tsx | 12 +- .../ui/src/components/files/file-preview.tsx | 81 +- .../src/components/files/file-save-button.tsx | 97 +- .../ui/src/components/files/file-sidebar.tsx | 4 +- .../src/components/files/file-thumbnail.tsx | 37 +- .../files/previews/file-preview-video.tsx | 2 +- .../ui/src/components/folders/folder-body.tsx | 4 +- packages/ui/src/components/font-loader.tsx | 2 +- .../ui/src/components/icons/icon-element.tsx | 2 +- .../containers/container-tab-content.tsx | 26 +- .../containers/container-tab-trigger.tsx | 28 +- .../layouts/sidebars/sidebar-menu-icon.tsx | 21 +- .../layouts/sidebars/sidebar-menu.tsx | 33 +- .../sidebars/sidebar-settings-item.tsx | 9 + .../layouts/sidebars/sidebar-settings.tsx | 44 +- .../src/components/messages/message-list.tsx | 2 +- .../ui/src/components/ui/unread-badge.tsx | 12 +- .../downloads/workspace-download-file.tsx | 85 ++ .../downloads/workspace-download-status.tsx | 84 ++ .../downloads/workspace-downloads-tab.tsx | 10 + .../downloads/workspace-downloads.tsx | 59 ++ .../{ => storage}/storage-stats.tsx | 0 .../{ => storage}/user-storage-stats.tsx | 2 +- .../{ => storage}/workspace-storage-stats.tsx | 4 +- .../{ => storage}/workspace-storage-tab.tsx | 0 .../workspace-storage-user-table.tsx | 2 +- .../workspace-storage-user-update-dialog.tsx | 0 .../{ => storage}/workspace-storage.tsx | 4 +- .../uploads/workspace-upload-file.tsx | 64 ++ .../uploads/workspace-upload-status.tsx | 84 ++ .../uploads/workspace-uploads-tab.tsx | 10 + .../workspaces/uploads/workspace-uploads.tsx | 58 ++ packages/ui/src/editor/commands/file.tsx | 4 +- packages/ui/src/editor/extensions/file.tsx | 4 +- packages/ui/src/editor/views/temp-file.tsx | 54 +- packages/ui/src/lib/files.ts | 5 + 174 files changed, 5839 insertions(+), 2362 deletions(-) create mode 100644 apps/server/src/api/client/routes/workspaces/files/file-upload-tus.ts create mode 100644 apps/server/src/data/migrations/00031-add-created-at-index-to-uploads.ts create mode 100644 apps/server/src/jobs/uploads-clean.ts create mode 100644 apps/server/src/lib/tus/redis-kv.ts create mode 100644 apps/server/src/lib/tus/redis-locker.ts create mode 100644 packages/client/src/databases/account/migrations/00003-create-avatars-table.ts create mode 100644 packages/client/src/databases/app/migrations/00005-create-jobs-table.ts create mode 100644 packages/client/src/databases/app/migrations/00006-create-job-schedules-table.ts create mode 100644 packages/client/src/databases/app/migrations/00007-drop-deleted-tokens-table.ts create mode 100644 packages/client/src/databases/app/migrations/00008-create-temp-files-table.ts create mode 100644 packages/client/src/databases/workspace/migrations/00020-create-local-files-table.ts create mode 100644 packages/client/src/databases/workspace/migrations/00021-create-uploads-table.ts create mode 100644 packages/client/src/databases/workspace/migrations/00022-create-downloads-table.ts create mode 100644 packages/client/src/databases/workspace/migrations/00023-drop-file-states-table.ts delete mode 100644 packages/client/src/handlers/mutations/files/file-save.ts create mode 100644 packages/client/src/handlers/mutations/files/temp-file-create.ts create mode 100644 packages/client/src/handlers/queries/avatars/avatar-get.ts delete mode 100644 packages/client/src/handlers/queries/avatars/avatar-url-get.ts create mode 100644 packages/client/src/handlers/queries/files/download-list-manual.ts delete mode 100644 packages/client/src/handlers/queries/files/file-save-list.ts delete mode 100644 packages/client/src/handlers/queries/files/file-state-get.ts create mode 100644 packages/client/src/handlers/queries/files/local-file-get.ts create mode 100644 packages/client/src/handlers/queries/files/temp-file-get.ts create mode 100644 packages/client/src/handlers/queries/files/upload-list-pending.ts create mode 100644 packages/client/src/handlers/queries/files/upload-list.ts create mode 100644 packages/client/src/jobs/account-sync.ts create mode 100644 packages/client/src/jobs/avatar-download.ts create mode 100644 packages/client/src/jobs/avatars-clean.ts create mode 100644 packages/client/src/jobs/file-download.ts create mode 100644 packages/client/src/jobs/file-upload.ts create mode 100644 packages/client/src/jobs/index.ts create mode 100644 packages/client/src/jobs/mutations-sync.ts create mode 100644 packages/client/src/jobs/server-sync.ts create mode 100644 packages/client/src/jobs/temp-files-clean.ts create mode 100644 packages/client/src/jobs/token-delete.ts create mode 100644 packages/client/src/jobs/workspace-files-clean.ts create mode 100644 packages/client/src/lib/nodes.ts create mode 100644 packages/client/src/lib/sleep-scheduler.ts delete mode 100644 packages/client/src/mutations/files/file-save.ts create mode 100644 packages/client/src/mutations/files/temp-file-create.ts create mode 100644 packages/client/src/queries/avatars/avatar-get.ts delete mode 100644 packages/client/src/queries/avatars/avatar-url-get.ts create mode 100644 packages/client/src/queries/files/download-list-manual.ts delete mode 100644 packages/client/src/queries/files/file-save-list.ts delete mode 100644 packages/client/src/queries/files/file-state-get.ts create mode 100644 packages/client/src/queries/files/local-file-get.ts create mode 100644 packages/client/src/queries/files/temp-file-get.ts create mode 100644 packages/client/src/queries/files/upload-list-pending.ts create mode 100644 packages/client/src/queries/files/upload-list.ts create mode 100644 packages/client/src/services/accounts/avatar-service.ts create mode 100644 packages/client/src/services/job-service.ts create mode 100644 packages/client/src/types/avatars.ts delete mode 100644 packages/ui/src/components/downloads/download-status.tsx delete mode 100644 packages/ui/src/components/downloads/downloads-breadcrumb.tsx delete mode 100644 packages/ui/src/components/downloads/downloads-container-tab.tsx delete mode 100644 packages/ui/src/components/downloads/downloads-container.tsx delete mode 100644 packages/ui/src/components/downloads/downloads-list.tsx create mode 100644 packages/ui/src/components/workspaces/downloads/workspace-download-file.tsx create mode 100644 packages/ui/src/components/workspaces/downloads/workspace-download-status.tsx create mode 100644 packages/ui/src/components/workspaces/downloads/workspace-downloads-tab.tsx create mode 100644 packages/ui/src/components/workspaces/downloads/workspace-downloads.tsx rename packages/ui/src/components/workspaces/{ => storage}/storage-stats.tsx (100%) rename packages/ui/src/components/workspaces/{ => storage}/user-storage-stats.tsx (97%) rename packages/ui/src/components/workspaces/{ => storage}/workspace-storage-stats.tsx (97%) rename packages/ui/src/components/workspaces/{ => storage}/workspace-storage-tab.tsx (100%) rename packages/ui/src/components/workspaces/{ => storage}/workspace-storage-user-table.tsx (98%) rename packages/ui/src/components/workspaces/{ => storage}/workspace-storage-user-update-dialog.tsx (100%) rename packages/ui/src/components/workspaces/{ => storage}/workspace-storage.tsx (89%) create mode 100644 packages/ui/src/components/workspaces/uploads/workspace-upload-file.tsx create mode 100644 packages/ui/src/components/workspaces/uploads/workspace-upload-status.tsx create mode 100644 packages/ui/src/components/workspaces/uploads/workspace-uploads-tab.tsx create mode 100644 packages/ui/src/components/workspaces/uploads/workspace-uploads.tsx diff --git a/apps/desktop/index.html b/apps/desktop/index.html index 1516f859..e8e36dce 100644 --- a/apps/desktop/index.html +++ b/apps/desktop/index.html @@ -6,7 +6,7 @@ diff --git a/apps/desktop/src/main.ts b/apps/desktop/src/main.ts index 7767a802..e02bde12 100644 --- a/apps/desktop/src/main.ts +++ b/apps/desktop/src/main.ts @@ -22,10 +22,7 @@ import { IdType, } from '@colanode/core'; import { app, appBadge } from '@colanode/desktop/main/app-service'; -import { - handleAssetRequest, - handleFileRequest, -} from '@colanode/desktop/main/protocols'; +import { handleLocalRequest } from '@colanode/desktop/main/protocols'; const debug = createDebugger('desktop:main'); @@ -120,15 +117,9 @@ const createWindow = async () => { } }); - if (!protocol.isProtocolHandled('local-file')) { - protocol.handle('local-file', (request) => { - return handleFileRequest(request); - }); - } - - if (!protocol.isProtocolHandled('asset')) { - protocol.handle('asset', (request) => { - return handleAssetRequest(request); + if (!protocol.isProtocolHandled('local')) { + protocol.handle('local', (request) => { + return handleLocalRequest(request); }); } @@ -149,6 +140,10 @@ const createWindow = async () => { debug('Window created'); }; +protocol.registerSchemesAsPrivileged([ + { scheme: 'local', privileges: { standard: true, stream: true } }, +]); + // This method will be called when Electron has finished // initialization and is ready to create browser windows. // Some APIs can only be used after this event occurs. @@ -226,22 +221,35 @@ ipcMain.handle( file: { name: string; size: number; type: string; buffer: Buffer } ): Promise => { const id = generateId(IdType.TempFile); - const name = app.path.filename(file.name); const extension = app.path.extension(file.name); const mimeType = file.type; - const type = extractFileSubtype(mimeType); - const fileName = `${name}.${id}${extension}`; - const filePath = app.path.tempFile(fileName); + const subtype = extractFileSubtype(mimeType); + const filePath = app.path.tempFile(file.name); await app.fs.writeFile(filePath, file.buffer); + await app.database + .insertInto('temp_files') + .values({ + id, + name: file.name, + size: file.size, + mime_type: mimeType, + subtype, + path: filePath, + extension, + created_at: new Date().toISOString(), + opened_at: new Date().toISOString(), + }) + .execute(); + const url = await app.fs.url(filePath); return { id, - name: fileName, + name: file.name, size: file.size, mimeType, - type, + subtype, path: filePath, extension, url, diff --git a/apps/desktop/src/main/file-system.ts b/apps/desktop/src/main/file-system.ts index 361aa7cb..8a725d01 100644 --- a/apps/desktop/src/main/file-system.ts +++ b/apps/desktop/src/main/file-system.ts @@ -1,12 +1,7 @@ import fs from 'fs'; -import os from 'os'; import { Writable } from 'stream'; -import { - FileMetadata, - FileReadStream, - FileSystem, -} from '@colanode/client/services'; +import { FileReadStream, FileSystem } from '@colanode/client/services'; export class DesktopFileSystem implements FileSystem { public async makeDirectory(path: string): Promise { @@ -25,7 +20,7 @@ export class DesktopFileSystem implements FileSystem { } public async readStream(path: string): Promise { - return fs.createReadStream(path); + return fs.promises.readFile(path); } public async writeStream(path: string): Promise> { @@ -49,31 +44,8 @@ export class DesktopFileSystem implements FileSystem { await fs.promises.rm(path, { recursive: true, force: true }); } - public async metadata(filePath: string): Promise { - const stats = await fs.promises.stat(filePath); - return { - lastModified: stats.mtime.getTime(), - size: stats.size, - }; - } - public async url(path: string): Promise { - return `local-file://${DesktopFileSystem.win32PathPreUrl(path)}`; - } - - public static win32PathPreUrl(path: string): string { - if (os.platform() === 'win32') { - let urlPath = path; - let filePathPrefix = ""; - - urlPath = urlPath.replace(/\\/g, '/'); - if (/^[a-zA-Z]:/.test(urlPath)) { - filePathPrefix = '/'; - } - - return `${filePathPrefix}${urlPath}`; - } - - return path; + const base64Path = Buffer.from(path).toString('base64'); + return `local://files/${base64Path}`; } } diff --git a/apps/desktop/src/main/protocols.ts b/apps/desktop/src/main/protocols.ts index 654ad81c..4655f857 100644 --- a/apps/desktop/src/main/protocols.ts +++ b/apps/desktop/src/main/protocols.ts @@ -2,22 +2,28 @@ import { net } from 'electron'; import path from 'path'; import { app } from '@colanode/desktop/main/app-service'; -import { DesktopFileSystem } from '@colanode/desktop/main/file-system'; -export const handleAssetRequest = async ( +export const handleLocalRequest = async ( request: Request ): Promise => { - const url = request.url.replace('asset://', ''); - const [type, id] = url.split('/'); - if (!type || !id) { + const url = request.url.replace('local://', ''); + const parts = url.split('/'); + + const type = parts[0]; + if (!type) { return new Response(null, { status: 400 }); } if (type === 'emojis') { - const emoji = await app.asset.emojis + const skinId = parts[1]; + if (!skinId) { + return new Response(null, { status: 400 }); + } + + const emoji = await app.assets.emojis .selectFrom('emoji_svgs') .selectAll() - .where('skin_id', '=', id) + .where('skin_id', '=', skinId) .executeTakeFirst(); if (emoji) { @@ -30,10 +36,15 @@ export const handleAssetRequest = async ( } if (type === 'icons') { - const icon = await app.asset.icons + const iconId = parts[1]; + if (!iconId) { + return new Response(null, { status: 400 }); + } + + const icon = await app.assets.icons .selectFrom('icon_svgs') .selectAll() - .where('id', '=', id) + .where('id', '=', iconId) .executeTakeFirst(); if (icon) { @@ -46,19 +57,28 @@ export const handleAssetRequest = async ( } if (type === 'fonts') { - const filePath = path.join(app.path.assets, 'fonts', id); + const fontName = parts[1]; + if (!fontName) { + return new Response(null, { status: 400 }); + } + + const filePath = path.join(app.path.assets, 'fonts', fontName); const fileUrl = `file://${filePath}`; - return net.fetch(fileUrl); + const subRequest = new Request(fileUrl, request); + return net.fetch(subRequest); + } + + if (type === 'files') { + const base64Path = parts[1]; + if (!base64Path) { + return new Response(null, { status: 400 }); + } + + const path = Buffer.from(base64Path, 'base64').toString('utf-8'); + const fileUrl = `file://${path}`; + const subRequest = new Request(fileUrl, request); + return net.fetch(subRequest); } return new Response(null, { status: 404 }); }; - -export const handleFileRequest = async ( - request: Request -): Promise => { - return net.fetch(`file://${DesktopFileSystem.win32PathPreUrl( - request.url.replace('local-file://', '') - )}` - ); -}; diff --git a/apps/server/.env.example b/apps/server/.env.example index 7f8f06b3..1c3d13b3 100644 --- a/apps/server/.env.example +++ b/apps/server/.env.example @@ -56,10 +56,14 @@ POSTGRES_URL=postgres://colanode_user:postgrespass123@localhost:5432/colanode_db # Redis Configuration # ─────────────────────────────────────────────────────────────── REDIS_URL=redis://:your_valkey_password@localhost:6379/0 -REDIS_DB=0 -REDIS_JOBS_QUEUE_NAME=jobs -REDIS_JOBS_QUEUE_PREFIX=colanode -REDIS_EVENTS_CHANNEL=events +# Optional configurations + +# REDIS_DB=0 +# REDIS_JOBS_QUEUE_NAME=jobs +# REDIS_JOBS_QUEUE_PREFIX=colanode +# REDIS_TUS_LOCK_PREFIX=colanode:tus:lock +# REDIS_TUS_KV_PREFIX=colanode:tus:kv +# REDIS_EVENTS_CHANNEL=events # ─────────────────────────────────────────────────────────────── # S3 Storage Configuration (MinIO) @@ -70,6 +74,7 @@ STORAGE_S3_SECRET_KEY=your_minio_password STORAGE_S3_BUCKET=colanode STORAGE_S3_REGION=us-east-1 STORAGE_S3_FORCE_PATH_STYLE=true +STORAGE_S3_PART_SIZE=20971520 # 20MB # ─────────────────────────────────────────────────────────────── # SMTP Configuration diff --git a/apps/server/package.json b/apps/server/package.json index c3c2371e..7f03d8d9 100644 --- a/apps/server/package.json +++ b/apps/server/package.json @@ -38,6 +38,9 @@ "@langchain/langgraph": "^0.3.11", "@langchain/openai": "^0.6.2", "@node-rs/argon2": "^2.0.2", + "@redis/client": "^5.7.0", + "@tus/s3-store": "^2.0.0", + "@tus/server": "^2.3.0", "bullmq": "^5.56.5", "diff": "^8.0.2", "dotenv": "^17.2.0", @@ -49,10 +52,10 @@ "kysely": "^0.28.3", "langchain": "^0.3.30", "langfuse-langchain": "^3.38.4", + "ms": "^2.1.3", "nodemailer": "^7.0.5", "pg": "^8.16.3", "pino": "^9.7.0", - "redis": "^5.6.0", "sharp": "^0.34.3" } } diff --git a/apps/server/src/api/client/routes/accounts/email-password-reset-init.ts b/apps/server/src/api/client/routes/accounts/email-password-reset-init.ts index 18193603..3a1e8f7c 100644 --- a/apps/server/src/api/client/routes/accounts/email-password-reset-init.ts +++ b/apps/server/src/api/client/routes/accounts/email-password-reset-init.ts @@ -1,4 +1,5 @@ import { FastifyPluginCallbackZod } from 'fastify-type-provider-zod'; +import ms from 'ms'; import { generateId, @@ -48,7 +49,9 @@ export const emailPasswordResetInitRoute: FastifyPluginCallbackZod = ( } const id = generateId(IdType.OtpCode); - const expiresAt = new Date(Date.now() + config.account.otpTimeout * 1000); + const expiresAt = new Date( + Date.now() + ms(`${config.account.otpTimeout} seconds`) + ); const otpCode = generateOtpCode(); const account = await database diff --git a/apps/server/src/api/client/routes/workspaces/files/file-upload-tus.ts b/apps/server/src/api/client/routes/workspaces/files/file-upload-tus.ts new file mode 100644 index 00000000..4ea026f6 --- /dev/null +++ b/apps/server/src/api/client/routes/workspaces/files/file-upload-tus.ts @@ -0,0 +1,296 @@ +import { S3Store } from '@tus/s3-store'; +import { Server } from '@tus/server'; +import { FastifyPluginCallbackZod } from 'fastify-type-provider-zod'; +import { z } from 'zod/v4'; + +import { ApiErrorCode, FileStatus, generateId, IdType } from '@colanode/core'; +import { database } from '@colanode/server/data/database'; +import { redis } from '@colanode/server/data/redis'; +import { s3Config } from '@colanode/server/data/storage'; +import { config } from '@colanode/server/lib/config'; +import { fetchCounter } from '@colanode/server/lib/counters'; +import { buildFilePath, deleteFile } from '@colanode/server/lib/files'; +import { mapNode, updateNode } from '@colanode/server/lib/nodes'; +import { RedisKvStore } from '@colanode/server/lib/tus/redis-kv'; +import { RedisLocker } from '@colanode/server/lib/tus/redis-locker'; + +const s3Store = new S3Store({ + partSize: config.storage.partSize, + cache: new RedisKvStore(redis, config.redis.tus.kvPrefix), + s3ClientConfig: { + ...s3Config, + bucket: config.storage.bucket, + }, +}); + +export const fileUploadTusRoute: FastifyPluginCallbackZod = ( + instance, + _, + done +) => { + instance.addContentTypeParser( + 'application/offset+octet-stream', + (_request, _payload, done) => done(null) + ); + + instance.route({ + method: ['HEAD', 'POST', 'PATCH', 'DELETE'], + url: '/:fileId/tus', + schema: { + params: z.object({ + workspaceId: z.string(), + fileId: z.string(), + }), + }, + handler: async (request, reply) => { + const { workspaceId, fileId } = request.params; + const user = request.user; + + const workspace = await database + .selectFrom('workspaces') + .selectAll() + .where('id', '=', workspaceId) + .executeTakeFirst(); + + if (!workspace) { + return reply.code(404).send({ + code: ApiErrorCode.WorkspaceNotFound, + message: 'Workspace not found.', + }); + } + + const node = await database + .selectFrom('nodes') + .selectAll() + .where('id', '=', fileId) + .executeTakeFirst(); + + if (!node) { + return reply.code(404).send({ + code: ApiErrorCode.FileNotFound, + message: 'File not found.', + }); + } + + if (node.created_by !== user.id) { + return reply.code(403).send({ + code: ApiErrorCode.FileOwnerMismatch, + message: 'You do not have permission to upload to this file.', + }); + } + + const file = mapNode(node); + if (file.type !== 'file') { + return reply.code(400).send({ + code: ApiErrorCode.FileNotFound, + message: 'This node is not a file.', + }); + } + + const path = buildFilePath(workspaceId, fileId, file.attributes); + + const tusServer = new Server({ + path: '/tus', + datastore: s3Store, + locker: new RedisLocker(redis, config.redis.tus.lockPrefix), + async onUploadCreate() { + const upload = await database + .selectFrom('uploads') + .selectAll() + .where('file_id', '=', fileId) + .executeTakeFirst(); + + if (upload && upload.uploaded_at) { + throw { + status_code: 400, + body: JSON.stringify({ + code: ApiErrorCode.FileAlreadyUploaded, + message: 'This file is already uploaded.', + }), + }; + } + + if (file.attributes.size > BigInt(user.max_file_size)) { + throw { + status_code: 400, + body: JSON.stringify({ + code: ApiErrorCode.UserMaxFileSizeExceeded, + message: + 'The file size exceeds the maximum allowed size for your account.', + }), + }; + } + + if (workspace.max_file_size) { + if (file.attributes.size > BigInt(workspace.max_file_size)) { + throw { + status_code: 400, + body: JSON.stringify({ + code: ApiErrorCode.WorkspaceMaxFileSizeExceeded, + message: + 'The file size exceeds the maximum allowed size for this workspace.', + }), + }; + } + } + + const userStorageUsed = await fetchCounter( + database, + `${user.id}.storage.used` + ); + + if (userStorageUsed >= BigInt(user.storage_limit)) { + throw { + status_code: 400, + body: JSON.stringify({ + code: ApiErrorCode.UserStorageLimitExceeded, + message: + 'You have reached the maximum storage limit for your account.', + }), + }; + } + + if (workspace.storage_limit) { + const workspaceStorageUsed = await fetchCounter( + database, + `${workspaceId}.storage.used` + ); + + if (workspaceStorageUsed >= BigInt(workspace.storage_limit)) { + throw { + status_code: 400, + body: JSON.stringify({ + code: ApiErrorCode.WorkspaceStorageLimitExceeded, + message: + 'The workspace has reached the maximum storage limit for this workspace.', + }), + }; + } + } + + // create the upload record + const uploadId = generateId(IdType.Upload); + const createdUpload = await database + .insertInto('uploads') + .returningAll() + .values({ + file_id: fileId, + upload_id: uploadId, + workspace_id: workspaceId, + root_id: file.rootId, + mime_type: file.attributes.mimeType, + size: file.attributes.size, + path: path, + version_id: file.attributes.version, + created_at: new Date(), + created_by: request.user.id, + }) + .onConflict((oc) => + oc.columns(['file_id']).doUpdateSet({ + upload_id: uploadId, + created_at: new Date(), + created_by: request.user.id, + mime_type: file.attributes.mimeType, + size: file.attributes.size, + path: path, + version_id: file.attributes.version, + }) + ) + .executeTakeFirst(); + + if (!createdUpload) { + throw { + status_code: 500, + body: JSON.stringify({ + code: ApiErrorCode.FileUploadFailed, + message: 'Failed to create upload record.', + }), + }; + } + + return { + metadata: { + uploadId: createdUpload.upload_id, + }, + }; + }, + async onUploadFinish(_req, upload) { + const uploadId = upload.metadata?.uploadId; + if (!uploadId) { + throw { + status_code: 500, + body: JSON.stringify({ + code: ApiErrorCode.FileUploadCompleteFailed, + message: 'Failed to get upload id from metadata.', + }), + }; + } + + const updatedUpload = await database + .updateTable('uploads') + .returningAll() + .set({ + uploaded_at: new Date(), + }) + .where('file_id', '=', fileId) + .where('upload_id', '=', uploadId) + .executeTakeFirst(); + + if (!updatedUpload) { + throw { + status_code: 500, + body: JSON.stringify({ + code: ApiErrorCode.FileUploadCompleteFailed, + message: 'Failed to record file upload.', + }), + }; + } + + const result = await updateNode({ + nodeId: fileId, + userId: request.user.id, + workspaceId: workspaceId, + updater(attributes) { + if (attributes.type !== 'file') { + throw new Error('Node is not a file'); + } + attributes.status = FileStatus.Ready; + return attributes; + }, + }); + + if (result === null) { + throw { + status_code: 500, + body: JSON.stringify({ + code: ApiErrorCode.FileUploadCompleteFailed, + message: 'Failed to complete file upload.', + }), + }; + } + + const tusInfoPath = `${path}.info`; + await deleteFile(tusInfoPath); + + return { + status_code: 200, + body: JSON.stringify({ uploadId }), + }; + }, + generateUrl(_req, options) { + return `${options.proto}://${options.host}/client/v1/workspaces/${workspaceId}/files/${fileId}/tus`; + }, + getFileIdFromRequest() { + return path; + }, + namingFunction() { + return path; + }, + }); + + await tusServer.handle(request.raw, reply.raw); + }, + }); + + done(); +}; diff --git a/apps/server/src/api/client/routes/workspaces/files/file-upload.ts b/apps/server/src/api/client/routes/workspaces/files/file-upload.ts index 8b6e9368..79864547 100644 --- a/apps/server/src/api/client/routes/workspaces/files/file-upload.ts +++ b/apps/server/src/api/client/routes/workspaces/files/file-upload.ts @@ -5,7 +5,6 @@ import { z } from 'zod/v4'; import { ApiErrorCode, FileStatus, - fileUploadOutputSchema, apiErrorOutputSchema, generateId, IdType, @@ -37,7 +36,9 @@ export const fileUploadRoute: FastifyPluginCallbackZod = ( fileId: z.string(), }), response: { - 200: fileUploadOutputSchema, + 200: z.object({ + uploadId: z.string(), + }), 400: apiErrorOutputSchema, 404: apiErrorOutputSchema, }, @@ -215,7 +216,9 @@ export const fileUploadRoute: FastifyPluginCallbackZod = ( }); } - return { success: true, uploadId: upsertedUpload.upload_id }; + return { + uploadId: upsertedUpload.upload_id, + }; }, }); diff --git a/apps/server/src/api/client/routes/workspaces/files/index.ts b/apps/server/src/api/client/routes/workspaces/files/index.ts index 98e609e3..d49d9b12 100644 --- a/apps/server/src/api/client/routes/workspaces/files/index.ts +++ b/apps/server/src/api/client/routes/workspaces/files/index.ts @@ -2,9 +2,11 @@ import { FastifyPluginCallback } from 'fastify'; import { fileDownloadRoute } from './file-download'; import { fileUploadRoute } from './file-upload'; +import { fileUploadTusRoute } from './file-upload-tus'; export const fileRoutes: FastifyPluginCallback = (instance, _, done) => { instance.register(fileUploadRoute); + instance.register(fileUploadTusRoute); instance.register(fileDownloadRoute); done(); diff --git a/apps/server/src/data/migrations/00031-add-created-at-index-to-uploads.ts b/apps/server/src/data/migrations/00031-add-created-at-index-to-uploads.ts new file mode 100644 index 00000000..18dd77b8 --- /dev/null +++ b/apps/server/src/data/migrations/00031-add-created-at-index-to-uploads.ts @@ -0,0 +1,12 @@ +import { Migration, sql } from 'kysely'; + +export const addCreatedAtIndexToUploads: Migration = { + up: async (db) => { + await sql`CREATE INDEX uploads_created_at_idx ON uploads (created_at) WHERE uploaded_at IS NULL`.execute( + db + ); + }, + down: async (db) => { + await sql`DROP INDEX IF EXISTS uploads_created_at_idx`.execute(db); + }, +}; diff --git a/apps/server/src/data/migrations/index.ts b/apps/server/src/data/migrations/index.ts index 9612a48c..c4733a00 100644 --- a/apps/server/src/data/migrations/index.ts +++ b/apps/server/src/data/migrations/index.ts @@ -30,6 +30,7 @@ import { removeNodeUpdateRevisionTrigger } from './00027-remove-node-update-revi import { removeDocumentUpdateRevisionTrigger } from './00028-remove-document-update-revision-trigger'; import { addWorkspaceStorageLimitColumns } from './00029-add-workspace-storage-limit-columns'; import { addWorkspaceIndexToUploads } from './00030-add-workspace-index-to-uploads'; +import { addCreatedAtIndexToUploads } from './00031-add-created-at-index-to-uploads'; export const databaseMigrations: Record = { '00001_create_accounts_table': createAccountsTable, @@ -66,4 +67,5 @@ export const databaseMigrations: Record = { removeDocumentUpdateRevisionTrigger, '00029_add_workspace_storage_limit_columns': addWorkspaceStorageLimitColumns, '00030_add_workspace_index_to_uploads': addWorkspaceIndexToUploads, + '00031_add_created_at_index_to_uploads': addCreatedAtIndexToUploads, }; diff --git a/apps/server/src/data/redis.ts b/apps/server/src/data/redis.ts index 9f8f50c0..09caa76a 100644 --- a/apps/server/src/data/redis.ts +++ b/apps/server/src/data/redis.ts @@ -1,4 +1,4 @@ -import { createClient } from 'redis'; +import { createClient } from '@redis/client'; import { config } from '@colanode/server/lib/config'; diff --git a/apps/server/src/data/storage.ts b/apps/server/src/data/storage.ts index ba431efd..c4ccb037 100644 --- a/apps/server/src/data/storage.ts +++ b/apps/server/src/data/storage.ts @@ -1,8 +1,8 @@ -import { S3Client } from '@aws-sdk/client-s3'; +import { S3Client, S3ClientConfig } from '@aws-sdk/client-s3'; import { config } from '@colanode/server/lib/config'; -export const s3Client = new S3Client({ +export const s3Config: S3ClientConfig = { endpoint: config.storage.endpoint, region: config.storage.region, credentials: { @@ -10,4 +10,6 @@ export const s3Client = new S3Client({ secretAccessKey: config.storage.secretKey, }, forcePathStyle: config.storage.forcePathStyle, -}); +}; + +export const s3Client = new S3Client(s3Config); diff --git a/apps/server/src/index.ts b/apps/server/src/index.ts index 286fb51b..a830e91c 100644 --- a/apps/server/src/index.ts +++ b/apps/server/src/index.ts @@ -17,7 +17,7 @@ const init = async () => { initApp(); - jobService.initQueue(); + await jobService.initQueue(); await jobService.initWorker(); await eventBus.init(); diff --git a/apps/server/src/jobs/document-updates-merge.ts b/apps/server/src/jobs/document-updates-merge.ts index f64d448f..2d5d3d9f 100644 --- a/apps/server/src/jobs/document-updates-merge.ts +++ b/apps/server/src/jobs/document-updates-merge.ts @@ -1,3 +1,5 @@ +import ms from 'ms'; + import { UpdateMergeMetadata } from '@colanode/core'; import { mergeUpdates } from '@colanode/crdt'; import { database } from '@colanode/server/data/database'; @@ -32,9 +34,8 @@ export const documentUpdatesMergeHandler: JobHandler< const cursor = await fetchCounter(database, 'document.updates.merge.cursor'); - const cutoffTime = new Date(); - cutoffTime.setTime( - cutoffTime.getTime() - config.jobs.documentUpdatesMerge.cutoffWindow * 1000 + const cutoffTime = new Date( + Date.now() - ms(`${config.jobs.documentUpdatesMerge.cutoffWindow} seconds`) ); let mergedGroups = 0; @@ -75,7 +76,8 @@ export const documentUpdatesMergeHandler: JobHandler< const result = await processDocumentUpdates( documentId, documentUpdates, - config.jobs.documentUpdatesMerge.mergeWindow + config.jobs.documentUpdatesMerge.mergeWindow, + config.jobs.documentUpdatesMerge.cutoffWindow ); mergedGroups += result.mergedGroups; deletedUpdates += result.deletedUpdates; @@ -97,11 +99,12 @@ export const documentUpdatesMergeHandler: JobHandler< const processDocumentUpdates = async ( documentId: string, documentUpdates: SelectDocumentUpdate[], - mergeWindow: number + mergeWindow: number, + cutoffWindow: number ): Promise<{ mergedGroups: number; deletedUpdates: number }> => { const firstUpdate = documentUpdates[0]!; const cutoffTime = new Date( - firstUpdate.created_at.getTime() - 60 * 60 * 1000 + firstUpdate.created_at.getTime() - ms(`${cutoffWindow} seconds`) ); const previousUpdate = await database diff --git a/apps/server/src/jobs/index.ts b/apps/server/src/jobs/index.ts index c18c69d4..c16372f9 100644 --- a/apps/server/src/jobs/index.ts +++ b/apps/server/src/jobs/index.ts @@ -8,6 +8,7 @@ import { nodeCleanHandler } from '@colanode/server/jobs/node-clean'; import { nodeEmbedHandler } from '@colanode/server/jobs/node-embed'; import { nodeEmbedScanHandler } from '@colanode/server/jobs/node-embed-scan'; import { nodeUpdatesMergeHandler } from '@colanode/server/jobs/node-updates-merge'; +import { uploadsCleanHandler } from '@colanode/server/jobs/uploads-clean'; import { workspaceCleanHandler } from '@colanode/server/jobs/workspace-clean'; // eslint-disable-next-line @typescript-eslint/no-empty-object-type @@ -33,4 +34,5 @@ export const jobHandlerMap: JobHandlerMap = { 'document.embed.scan': documentEmbedScanHandler, 'node.updates.merge': nodeUpdatesMergeHandler, 'document.updates.merge': documentUpdatesMergeHandler, + 'uploads.clean': uploadsCleanHandler, }; diff --git a/apps/server/src/jobs/node-updates-merge.ts b/apps/server/src/jobs/node-updates-merge.ts index 353b4f73..6eaf1240 100644 --- a/apps/server/src/jobs/node-updates-merge.ts +++ b/apps/server/src/jobs/node-updates-merge.ts @@ -1,3 +1,5 @@ +import ms from 'ms'; + import { UpdateMergeMetadata } from '@colanode/core'; import { mergeUpdates } from '@colanode/crdt'; import { database } from '@colanode/server/data/database'; @@ -32,9 +34,8 @@ export const nodeUpdatesMergeHandler: JobHandler< const cursor = await fetchCounter(database, 'node.updates.merge.cursor'); - const cutoffTime = new Date(); - cutoffTime.setTime( - cutoffTime.getTime() - config.jobs.nodeUpdatesMerge.cutoffWindow * 1000 + const cutoffTime = new Date( + Date.now() - ms(`${config.jobs.nodeUpdatesMerge.cutoffWindow} seconds`) ); let mergedGroups = 0; @@ -75,7 +76,8 @@ export const nodeUpdatesMergeHandler: JobHandler< const result = await processNodeUpdates( nodeId, nodeUpdates, - config.jobs.nodeUpdatesMerge.mergeWindow + config.jobs.nodeUpdatesMerge.mergeWindow, + config.jobs.nodeUpdatesMerge.cutoffWindow ); mergedGroups += result.mergedGroups; deletedUpdates += result.deletedUpdates; @@ -97,11 +99,12 @@ export const nodeUpdatesMergeHandler: JobHandler< const processNodeUpdates = async ( nodeId: string, nodeUpdates: SelectNodeUpdate[], - mergeWindow: number + mergeWindow: number, + cutoffWindow: number ): Promise<{ mergedGroups: number; deletedUpdates: number }> => { const firstUpdate = nodeUpdates[0]!; const cutoffTime = new Date( - firstUpdate.created_at.getTime() - 60 * 60 * 1000 + firstUpdate.created_at.getTime() - ms(`${cutoffWindow} seconds`) ); const previousUpdate = await database diff --git a/apps/server/src/jobs/uploads-clean.ts b/apps/server/src/jobs/uploads-clean.ts new file mode 100644 index 00000000..b5c9657c --- /dev/null +++ b/apps/server/src/jobs/uploads-clean.ts @@ -0,0 +1,63 @@ +import ms from 'ms'; + +import { database } from '@colanode/server/data/database'; +import { redis } from '@colanode/server/data/redis'; +import { JobHandler } from '@colanode/server/jobs'; +import { config } from '@colanode/server/lib/config'; +import { deleteFile } from '@colanode/server/lib/files'; +import { createLogger } from '@colanode/server/lib/logger'; +import { RedisKvStore } from '@colanode/server/lib/tus/redis-kv'; + +const logger = createLogger('server:job:uploads-clean'); + +export type UploadsCleanInput = { + type: 'uploads.clean'; +}; + +declare module '@colanode/server/jobs' { + interface JobMap { + 'uploads.clean': { + input: UploadsCleanInput; + }; + } +} + +export const uploadsCleanHandler: JobHandler = async () => { + logger.debug(`Cleaning uploads`); + + try { + // Delete uploads that are older than 7 days + const sevenDaysAgo = new Date(Date.now() - ms('7 days')); + const expiredUploads = await database + .selectFrom('uploads') + .selectAll() + .where('created_at', '<', sevenDaysAgo) + .where('uploaded_at', 'is', null) + .execute(); + + if (expiredUploads.length === 0) { + logger.debug(`No expired uploads found`); + return; + } + + const redisKv = new RedisKvStore(redis, config.redis.tus.kvPrefix); + for (const upload of expiredUploads) { + await deleteFile(upload.path); + await redisKv.delete(upload.path); + + const infoPath = `${upload.path}.info`; + await deleteFile(infoPath); + + await database + .deleteFrom('uploads') + .where('file_id', '=', upload.file_id) + .where('upload_id', '=', upload.upload_id) + .execute(); + } + + logger.debug(`Deleted ${expiredUploads.length} expired uploads`); + } catch (error) { + logger.error(error, `Error cleaning workspace data`); + throw error; + } +}; diff --git a/apps/server/src/lib/accounts.ts b/apps/server/src/lib/accounts.ts index a0b102cb..efa52045 100644 --- a/apps/server/src/lib/accounts.ts +++ b/apps/server/src/lib/accounts.ts @@ -1,4 +1,5 @@ import argon2 from '@node-rs/argon2'; +import ms from 'ms'; import { IdType, @@ -143,7 +144,9 @@ export const buildLoginVerifyOutput = async ( account: SelectAccount ): Promise => { const id = generateId(IdType.OtpCode); - const expiresAt = new Date(Date.now() + config.account.otpTimeout * 1000); + const expiresAt = new Date( + Date.now() + ms(`${config.account.otpTimeout} seconds`) + ); const otpCode = generateOtpCode(); const otp: Otp = { diff --git a/apps/server/src/lib/ai/utils.ts b/apps/server/src/lib/ai/utils.ts index 9052ad2b..5aa66f5c 100644 --- a/apps/server/src/lib/ai/utils.ts +++ b/apps/server/src/lib/ai/utils.ts @@ -1,4 +1,5 @@ import { Document } from '@langchain/core/documents'; +import ms from 'ms'; import { RerankedContextItem } from '@colanode/server/types/assistant'; import { @@ -19,8 +20,7 @@ export const calculateRecencyBoost = ( ): number => { if (!createdAt) return 1; const now = new Date(); - const ageInDays = - (now.getTime() - createdAt.getTime()) / (1000 * 60 * 60 * 24); + const ageInDays = (now.getTime() - createdAt.getTime()) / ms('1 day'); return ageInDays <= halfLifeDays ? 1 + (1 - ageInDays / halfLifeDays) * boostFactor : 1; diff --git a/apps/server/src/lib/config/jobs.ts b/apps/server/src/lib/config/jobs.ts index e688c952..24959a24 100644 --- a/apps/server/src/lib/config/jobs.ts +++ b/apps/server/src/lib/config/jobs.ts @@ -1,8 +1,9 @@ +import ms from 'ms'; import { z } from 'zod/v4'; const DEFAULT_BATCH_SIZE = 500; -const DEFAULT_MERGE_WINDOW_SECONDS = 60 * 60; -const DEFAULT_CUTOFF_WINDOW_HOURS = 2 * 60 * 60; +const DEFAULT_MERGE_WINDOW = ms('1 hour') / 1000; // in seconds +const DEFAULT_CUTOFF_WINDOW = ms('2 hours') / 1000; // in seconds const DEFAULT_CRON_PATTERN = '0 5 */2 * * *'; // every 2 hours at the 5th minute export const nodeUpdatesMergeJobConfigSchema = z.discriminatedUnion('enabled', [ @@ -10,8 +11,8 @@ export const nodeUpdatesMergeJobConfigSchema = z.discriminatedUnion('enabled', [ enabled: z.literal(true), cron: z.string().default(DEFAULT_CRON_PATTERN), batchSize: z.coerce.number().default(DEFAULT_BATCH_SIZE), - mergeWindow: z.coerce.number().default(DEFAULT_MERGE_WINDOW_SECONDS), - cutoffWindow: z.coerce.number().default(DEFAULT_CUTOFF_WINDOW_HOURS), + mergeWindow: z.coerce.number().default(DEFAULT_MERGE_WINDOW), + cutoffWindow: z.coerce.number().default(DEFAULT_CUTOFF_WINDOW), }), z.object({ enabled: z.literal(false), @@ -25,8 +26,8 @@ export const documentUpdatesMergeJobConfigSchema = z.discriminatedUnion( enabled: z.literal(true), cron: z.string().default(DEFAULT_CRON_PATTERN), batchSize: z.coerce.number().default(DEFAULT_BATCH_SIZE), - mergeWindow: z.coerce.number().default(DEFAULT_MERGE_WINDOW_SECONDS), - cutoffWindow: z.coerce.number().default(DEFAULT_CUTOFF_WINDOW_HOURS), + mergeWindow: z.coerce.number().default(DEFAULT_MERGE_WINDOW), + cutoffWindow: z.coerce.number().default(DEFAULT_CUTOFF_WINDOW), }), z.object({ enabled: z.literal(false), @@ -34,9 +35,20 @@ export const documentUpdatesMergeJobConfigSchema = z.discriminatedUnion( ] ); +export const uploadsCleanJobConfigSchema = z.discriminatedUnion('enabled', [ + z.object({ + enabled: z.literal(true), + cron: z.string().default(DEFAULT_CRON_PATTERN), + }), + z.object({ + enabled: z.literal(false), + }), +]); + export const jobsConfigSchema = z.object({ nodeUpdatesMerge: nodeUpdatesMergeJobConfigSchema, documentUpdatesMerge: documentUpdatesMergeJobConfigSchema, + uploadsClean: uploadsCleanJobConfigSchema, }); export type JobsConfig = z.infer; @@ -47,19 +59,19 @@ export const readJobsConfigVariables = () => { enabled: process.env.JOBS_NODE_UPDATES_MERGE_ENABLED === 'true', cron: process.env.JOBS_NODE_UPDATES_MERGE_CRON, batchSize: process.env.JOBS_NODE_UPDATES_MERGE_BATCH_SIZE, - timeWindowMinutes: - process.env.JOBS_NODE_UPDATES_MERGE_TIME_WINDOW_MINUTES, - excludeRecentHours: - process.env.JOBS_NODE_UPDATES_MERGE_EXCLUDE_RECENT_HOURS, + mergeWindow: process.env.JOBS_NODE_UPDATES_MERGE_MERGE_WINDOW, + cutoffWindow: process.env.JOBS_NODE_UPDATES_MERGE_CUTOFF_WINDOW, }, documentUpdatesMerge: { enabled: process.env.JOBS_DOCUMENT_UPDATES_MERGE_ENABLED === 'true', cron: process.env.JOBS_DOCUMENT_UPDATES_MERGE_CRON, batchSize: process.env.JOBS_DOCUMENT_UPDATES_MERGE_BATCH_SIZE, - timeWindowMinutes: - process.env.JOBS_DOCUMENT_UPDATES_MERGE_TIME_WINDOW_MINUTES, - excludeRecentHours: - process.env.JOBS_DOCUMENT_UPDATES_MERGE_EXCLUDE_RECENT_HOURS, + mergeWindow: process.env.JOBS_DOCUMENT_UPDATES_MERGE_MERGE_WINDOW, + cutoffWindow: process.env.JOBS_DOCUMENT_UPDATES_MERGE_CUTOFF_WINDOW, + }, + uploadsClean: { + enabled: process.env.JOBS_UPLOADS_CLEAN_ENABLED === 'true', + cron: process.env.JOBS_UPLOADS_CLEAN_CRON, }, }; }; diff --git a/apps/server/src/lib/config/redis.ts b/apps/server/src/lib/config/redis.ts index c0ad5f5a..4fc2c0ff 100644 --- a/apps/server/src/lib/config/redis.ts +++ b/apps/server/src/lib/config/redis.ts @@ -7,6 +7,10 @@ export const redisConfigSchema = z.object({ name: z.string().optional().default('jobs'), prefix: z.string().optional().default('colanode'), }), + tus: z.object({ + lockPrefix: z.string().optional().default('colanode:tus:lock'), + kvPrefix: z.string().optional().default('colanode:tus:kv'), + }), eventsChannel: z.string().optional().default('events'), }); @@ -20,6 +24,10 @@ export const readRedisConfigVariables = () => { name: process.env.REDIS_JOBS_NAME, prefix: process.env.REDIS_JOBS_PREFIX, }, + tus: { + lockPrefix: process.env.REDIS_TUS_LOCK_PREFIX, + kvPrefix: process.env.REDIS_TUS_KV_PREFIX, + }, eventsChannel: process.env.REDIS_EVENTS_CHANNEL, }; }; diff --git a/apps/server/src/lib/config/storage.ts b/apps/server/src/lib/config/storage.ts index 1fcd2a4b..e79dbbac 100644 --- a/apps/server/src/lib/config/storage.ts +++ b/apps/server/src/lib/config/storage.ts @@ -7,6 +7,10 @@ export const storageConfigSchema = z.object({ secretKey: z.string({ error: 'STORAGE_S3_SECRET_KEY is required' }), bucket: z.string({ error: 'STORAGE_S3_BUCKET is required' }), region: z.string({ error: 'STORAGE_S3_REGION is required' }), + partSize: z + .number() + .optional() + .default(20 * 1024 * 1024), // 20MB forcePathStyle: z.boolean().optional(), }); @@ -20,6 +24,7 @@ export const readStorageConfigVariables = () => { secretKey: process.env.STORAGE_S3_SECRET_KEY, bucket: process.env.STORAGE_S3_BUCKET, region: process.env.STORAGE_S3_REGION, + partSize: process.env.STORAGE_S3_PART_SIZE, forcePathStyle: process.env.STORAGE_S3_FORCE_PATH_STYLE === 'true', }; }; diff --git a/apps/server/src/lib/tus/redis-kv.ts b/apps/server/src/lib/tus/redis-kv.ts new file mode 100644 index 00000000..c735cf82 --- /dev/null +++ b/apps/server/src/lib/tus/redis-kv.ts @@ -0,0 +1,66 @@ +import type { RedisClientType } from '@redis/client'; +import type { Upload } from '@tus/server'; +import type { KvStore } from '@tus/utils'; +import { sha256 } from 'js-sha256'; + +/** + * Redis based configstore. + * Based on the Tus RedisKvStore, but with a custom prefix and a sha256 hash of the key. + * + * Original source: https://github.com/tus/tus-node-server/blob/main/packages/utils/src/kvstores/RedisKvStore.ts + * Original author: Mitja Puzigaća + */ + +export class RedisKvStore implements KvStore { + private readonly redis: RedisClientType; + private readonly prefix: string; + + constructor(redis: RedisClientType, prefix: string) { + this.redis = redis; + this.prefix = prefix; + } + + public async get(key: string): Promise { + const redisKey = this.buildRedisKey(key); + const redisValue = await this.redis.get(redisKey); + return this.deserializeValue(redisValue); + } + + public async set(key: string, value: T): Promise { + const redisKey = this.buildRedisKey(key); + const redisValue = this.serializeValue(value); + await this.redis.set(redisKey, redisValue); + } + + public async delete(key: string): Promise { + const redisKey = this.buildRedisKey(key); + await this.redis.del(redisKey); + } + + public async list(): Promise> { + const keys = new Set(); + let cursor = '0'; + do { + const result = await this.redis.scan(cursor, { + MATCH: `${this.prefix}*`, + COUNT: 20, + }); + cursor = result.cursor; + for (const key of result.keys) keys.add(key); + } while (cursor !== '0'); + return Array.from(keys); + } + + private buildRedisKey(key: string): string { + const hash = sha256(key); + return `${this.prefix}:${hash}`; + } + + private serializeValue(value: T): string { + return JSON.stringify(value); + } + + private deserializeValue(buffer: string | null): T | undefined { + return buffer ? JSON.parse(buffer) : undefined; + } +} diff --git a/apps/server/src/lib/tus/redis-locker.ts b/apps/server/src/lib/tus/redis-locker.ts new file mode 100644 index 00000000..558a5957 --- /dev/null +++ b/apps/server/src/lib/tus/redis-locker.ts @@ -0,0 +1,163 @@ +import { RedisClientType } from '@redis/client'; +import { + ERRORS, + type Lock, + type Locker, + type RequestRelease, +} from '@tus/utils'; +import { sha256 } from 'js-sha256'; +import ms from 'ms'; + +/** + * RedisLocker is an implementation of the Locker interface that manages locks using Redis. + * This class is designed for distributed systems where multiple instances need to coordinate access to shared resources. + * + * Key Features: + * - Uses Redis for centralized lock management, ensuring consistency across multiple server instances. + * - Implements a polling mechanism with a timeout for lock acquisition. + * - Leverages Redis's atomic operations and TTL (Time To Live) for reliable lock handling. + * - Provides a fail-proof design by ensuring that only the lock owner can release a lock. + * + * Locking Behavior: + * - The `lock` method attempts to acquire a lock by setting a key in Redis with a unique value. + * - If the lock is already held, it polls Redis periodically until the lock is released or the timeout is reached. + * - The `unlock` method ensures that only the process that acquired the lock can release it, preventing accidental releases. + * + * Edge Case Handling: + * - **Process Crashes:** If a process crashes after acquiring a lock, Redis's TTL feature ensures that the lock is automatically released after a specified time, preventing deadlocks. + * - **Network Issues:** The implementation is designed to handle transient network issues by retrying lock acquisition. + * - **Race Conditions:** Atomic operations in Redis (SET with NX and PX options) are used to prevent race conditions during lock acquisition. + */ + +const DELAY = ms('100 milliseconds'); +const TIMEOUT = ms('30 seconds'); +const UNLOCK_SCRIPT = ` + if redis.call("get", KEYS[1]) == ARGV[1] then + return redis.call("del", KEYS[1]) + else + return 0 + end +` as const; + +export class RedisLocker implements Locker { + public readonly redis: RedisClientType; + public readonly prefix: string; + + constructor(redis: RedisClientType, prefix: string) { + this.redis = redis; + this.prefix = prefix; + } + + public newLock(id: string): Lock { + return new RedisLock(id, this); + } +} + +class RedisLock implements Lock { + private lockValue: string | null = null; + + constructor( + private readonly id: string, + private readonly locker: RedisLocker + ) {} + + public async lock( + stopSignal: AbortSignal, + requestRelease?: RequestRelease + ): Promise { + const abortController = new AbortController(); + const onAbort = () => { + abortController.abort(); + }; + stopSignal.addEventListener('abort', onAbort); + + try { + this.lockValue = crypto.randomUUID(); + const lockAcquired = await Promise.race([ + this.waitTimeout(abortController.signal), + this.acquireLock( + this.id, + this.lockValue, + requestRelease, + abortController.signal + ), + ]); + + if (!lockAcquired) { + throw ERRORS.ERR_LOCK_TIMEOUT; + } + } finally { + stopSignal.removeEventListener('abort', onAbort); + abortController.abort(); + } + } + + private async acquireLock( + id: string, + lockValue: string, + requestRelease: RequestRelease | undefined, + signal: AbortSignal + ): Promise { + if (signal.aborted) { + return false; + } + + const result = await this.locker.redis.set( + this.buildRedisKey(id), + lockValue, + { + expiration: { + type: 'EX', + value: Math.floor(TIMEOUT / 1000), + }, + condition: 'NX', + } + ); + + if (result === 'OK') { + return true; + } + + await requestRelease?.(); + + await this.wait(DELAY); + return this.acquireLock(id, lockValue, requestRelease, signal); + } + + async unlock(): Promise { + if (!this.lockValue) { + return; + } + + await this.locker.redis.eval(UNLOCK_SCRIPT, { + arguments: [this.lockValue], + keys: [this.buildRedisKey(this.id)], + }); + + this.lockValue = null; + } + + private waitTimeout(signal: AbortSignal): Promise { + return new Promise((resolve) => { + const timeout = setTimeout(() => { + resolve(false); + }, TIMEOUT); + + const abortListener = () => { + clearTimeout(timeout); + signal.removeEventListener('abort', abortListener); + resolve(false); + }; + signal.addEventListener('abort', abortListener); + }); + } + + private buildRedisKey(id: string): string { + const hash = sha256(id); + return `${this.locker.prefix}:${hash}`; + } + + private wait(ms: number): Promise { + return new Promise((resolve) => setTimeout(resolve, ms)); + } +} diff --git a/apps/server/src/services/job-service.ts b/apps/server/src/services/job-service.ts index d742dea7..c91d77aa 100644 --- a/apps/server/src/services/job-service.ts +++ b/apps/server/src/services/job-service.ts @@ -19,7 +19,7 @@ class JobService { private readonly queueName = config.redis.jobs.name; private readonly prefix = `{${config.redis.jobs.prefix}}`; - public initQueue() { + public async initQueue(): Promise { if (this.jobQueue) { return; } @@ -39,71 +39,7 @@ class JobService { logger.error(error, `Job queue error`); }); - if (config.ai.enabled) { - this.jobQueue.upsertJobScheduler( - 'node.embed.scan', - { pattern: '0 */30 * * * *' }, - { - name: 'node.embed.scan', - data: { type: 'node.embed.scan' } as JobInput, - opts: { - backoff: 3, - attempts: 5, - removeOnFail: 1000, - }, - } - ); - - this.jobQueue.upsertJobScheduler( - 'document.embed.scan', - { pattern: '0 */30 * * * *' }, - { - name: 'document.embed.scan', - data: { type: 'document.embed.scan' } as JobInput, - opts: { - backoff: 3, - attempts: 5, - removeOnFail: 1000, - }, - } - ); - } - - if (config.jobs.nodeUpdatesMerge.enabled) { - this.jobQueue.upsertJobScheduler( - 'node.updates.merge', - { pattern: config.jobs.nodeUpdatesMerge.cron }, - { - name: 'node.updates.merge', - data: { type: 'node.updates.merge' } as JobInput, - opts: { - backoff: 3, - attempts: 3, - removeOnFail: 100, - }, - } - ); - } else { - this.jobQueue.removeJobScheduler('node.updates.merge'); - } - - if (config.jobs.documentUpdatesMerge.enabled) { - this.jobQueue.upsertJobScheduler( - 'document.updates.merge', - { pattern: config.jobs.documentUpdatesMerge.cron }, - { - name: 'document.updates.merge', - data: { type: 'document.updates.merge' } as JobInput, - opts: { - backoff: 3, - attempts: 3, - removeOnFail: 100, - }, - } - ); - } else { - this.jobQueue.removeJobScheduler('document.updates.merge'); - } + await this.initRecurringJobs(); } public async initWorker() { @@ -147,6 +83,125 @@ class JobService { logger.debug(`Job ${job.id} with type ${input.type} completed.`); }; + + private async initRecurringJobs(): Promise { + await this.initNodeEmbedScanRecurringJob(); + await this.initDocumentEmbedScanRecurringJob(); + await this.initNodeUpdatesMergeRecurringJob(); + await this.initDocumentUpdatesMergeRecurringJob(); + await this.initUploadsCleanRecurringJob(); + } + + private async initNodeEmbedScanRecurringJob(): Promise { + if (!this.jobQueue) { + return; + } + + const id = 'node.embed.scan'; + if (config.ai.enabled) { + this.jobQueue.upsertJobScheduler( + id, + { pattern: '0 */30 * * * *' }, + { + name: id, + data: { type: 'node.embed.scan' } as JobInput, + opts: { + backoff: 3, + attempts: 5, + removeOnFail: 1000, + }, + } + ); + } else { + this.jobQueue.removeJobScheduler(id); + } + } + + private async initDocumentEmbedScanRecurringJob(): Promise { + if (!this.jobQueue) { + return; + } + + const id = 'document.embed.scan'; + if (config.ai.enabled) { + this.jobQueue.upsertJobScheduler( + id, + { pattern: '0 */30 * * * *' }, + { + name: id, + data: { type: 'document.embed.scan' } as JobInput, + opts: { + backoff: 3, + attempts: 5, + removeOnFail: 1000, + }, + } + ); + } else { + this.jobQueue.removeJobScheduler(id); + } + } + + private async initNodeUpdatesMergeRecurringJob(): Promise { + if (!this.jobQueue) { + return; + } + + const id = 'node.updates.merge'; + if (config.jobs.nodeUpdatesMerge.enabled) { + this.jobQueue.upsertJobScheduler( + id, + { pattern: config.jobs.nodeUpdatesMerge.cron }, + { + name: id, + data: { type: 'node.updates.merge' } as JobInput, + } + ); + return; + } else { + this.jobQueue.removeJobScheduler(id); + } + } + + private async initDocumentUpdatesMergeRecurringJob(): Promise { + if (!this.jobQueue) { + return; + } + + const id = 'document.updates.merge'; + if (config.jobs.documentUpdatesMerge.enabled) { + this.jobQueue.upsertJobScheduler( + 'document.updates.merge', + { pattern: config.jobs.documentUpdatesMerge.cron }, + { + name: 'document.updates.merge', + data: { type: 'document.updates.merge' } as JobInput, + } + ); + } else { + this.jobQueue.removeJobScheduler(id); + } + } + + private async initUploadsCleanRecurringJob(): Promise { + if (!this.jobQueue) { + return; + } + + const id = 'uploads.clean'; + if (config.jobs.uploadsClean.enabled) { + this.jobQueue.upsertJobScheduler( + id, + { pattern: config.jobs.uploadsClean.cron }, + { + name: id, + data: { type: 'uploads.clean' } as JobInput, + } + ); + } else { + this.jobQueue.removeJobScheduler(id); + } + } } export const jobService = new JobService(); diff --git a/apps/web/src/services/file-system.ts b/apps/web/src/services/file-system.ts index 6459175c..c4169147 100644 --- a/apps/web/src/services/file-system.ts +++ b/apps/web/src/services/file-system.ts @@ -1,8 +1,4 @@ -import { - FileMetadata, - FileReadStream, - FileSystem, -} from '@colanode/client/services'; +import { FileReadStream, FileSystem } from '@colanode/client/services'; export class WebFileSystem implements FileSystem { private root: FileSystemDirectoryHandle | null = null; @@ -194,17 +190,6 @@ export class WebFileSystem implements FileSystem { await writable.close(); } - public async metadata(path: string): Promise { - const { parent, name } = await this.getFileLocation(path, false); - const fileHandle = await parent.getFileHandle(name); - const file = await fileHandle.getFile(); - - return { - lastModified: file.lastModified, - size: file.size, - }; - } - public async url(path: string): Promise { const { parent, name } = await this.getFileLocation(path, false); const fileHandle = await parent.getFileHandle(name); diff --git a/apps/web/src/workers/dedicated.ts b/apps/web/src/workers/dedicated.ts index dfcafcdc..a3e09288 100644 --- a/apps/web/src/workers/dedicated.ts +++ b/apps/web/src/workers/dedicated.ts @@ -276,28 +276,58 @@ const api: ColanodeWorkerApi = { }, async saveTempFile(file) { const id = generateId(IdType.TempFile); - const name = path.filename(file.name); const extension = path.extension(file.name); const mimeType = file.type; - const type = extractFileSubtype(mimeType); - const fileName = `${name}.${id}${extension}`; + const subtype = extractFileSubtype(mimeType); + const filePath = path.tempFile(file.name); const arrayBuffer = await file.arrayBuffer(); const fileData = new Uint8Array(arrayBuffer); - const filePath = path.tempFile(fileName); await fs.writeFile(filePath, fileData); + if (app) { + await app.database + .insertInto('temp_files') + .values({ + id, + name: file.name, + size: file.size, + mime_type: mimeType, + subtype, + path: filePath, + extension, + created_at: new Date().toISOString(), + opened_at: new Date().toISOString(), + }) + .execute(); + } else { + const message: BroadcastMutationMessage = { + type: 'mutation', + mutationId: generateId(IdType.Mutation), + input: { + type: 'temp.file.create', + id, + name: file.name, + size: file.size, + mimeType, + subtype, + extension, + path: filePath, + }, + }; + + broadcastMessage(message); + } const url = await fs.url(filePath); - return { id, name: file.name, size: file.size, - type, + mimeType, + subtype, path: filePath, extension, - mimeType, url, }; }, diff --git a/hosting/docker/docker-compose.yaml b/hosting/docker/docker-compose.yaml index f3b6aee8..8b3e599e 100644 --- a/hosting/docker/docker-compose.yaml +++ b/hosting/docker/docker-compose.yaml @@ -151,9 +151,11 @@ services: REDIS_URL: 'redis://:your_valkey_password@valkey:6379/0' REDIS_DB: '0' # Optional variables: - REDIS_JOBS_QUEUE_NAME: 'jobs' - REDIS_JOBS_QUEUE_PREFIX: 'colanode' - REDIS_EVENTS_CHANNEL: 'events' + # REDIS_JOBS_QUEUE_NAME: 'jobs' + # REDIS_JOBS_QUEUE_PREFIX: 'colanode' + # REDIS_TUS_LOCK_PREFIX: 'colanode:tus:lock' + # REDIS_TUS_KV_PREFIX: 'colanode:tus:kv' + # REDIS_EVENTS_CHANNEL: 'events' # ─────────────────────────────────────────────────────────────── # S3 configuration for files. @@ -165,6 +167,7 @@ services: STORAGE_S3_BUCKET: 'colanode' STORAGE_S3_REGION: 'us-east-1' STORAGE_S3_FORCE_PATH_STYLE: 'true' + STORAGE_S3_PART_SIZE: '20971520' # 20MB # ─────────────────────────────────────────────────────────────── # SMTP configuration diff --git a/hosting/kubernetes/chart/templates/_helpers.tpl b/hosting/kubernetes/chart/templates/_helpers.tpl index 170a059d..812a23bb 100644 --- a/hosting/kubernetes/chart/templates/_helpers.tpl +++ b/hosting/kubernetes/chart/templates/_helpers.tpl @@ -202,15 +202,19 @@ Colanode Server Environment Variables value: {{ .Values.colanode.config.REDIS_JOBS_QUEUE_NAME | quote }} - name: REDIS_JOBS_QUEUE_PREFIX value: {{ .Values.colanode.config.REDIS_JOBS_QUEUE_PREFIX | quote }} +- name: REDIS_TUS_LOCK_PREFIX + value: {{ .Values.colanode.config.REDIS_TUS_LOCK_PREFIX | quote }} +- name: REDIS_TUS_KV_PREFIX + value: {{ .Values.colanode.config.REDIS_TUS_KV_PREFIX | quote }} - name: REDIS_EVENTS_CHANNEL value: {{ .Values.colanode.config.REDIS_EVENTS_CHANNEL | quote }} # ─────────────────────────────────────────────────────────────── -# S3 Configuration for Avatars +# S3 Configuration for Storage # ─────────────────────────────────────────────────────────────── -- name: S3_AVATARS_ENDPOINT +- name: STORAGE_S3_ENDPOINT value: "http://{{ include "colanode.minio.hostname" . }}:9000" -- name: S3_AVATARS_ACCESS_KEY +- name: STORAGE_S3_ACCESS_KEY {{- if .Values.minio.auth.existingSecret }} {{- include "colanode.getRequiredValueOrSecret" (dict "key" "minio.auth.rootUser" "value" (dict "value" .Values.minio.auth.rootUser "existingSecret" .Values.minio.auth.existingSecret "secretKey" .Values.minio.auth.rootUserKey )) | nindent 2 }} {{- else }} @@ -219,7 +223,7 @@ Colanode Server Environment Variables name: {{ .Release.Name }}-minio key: {{ .Values.minio.auth.rootUserKey }} {{- end }} -- name: S3_AVATARS_SECRET_KEY +- name: STORAGE_S3_SECRET_KEY {{- if .Values.minio.auth.existingSecret }} {{- include "colanode.getRequiredValueOrSecret" (dict "key" "minio.auth.rootPassword" "value" (dict "value" .Values.minio.auth.rootPassword "existingSecret" .Values.minio.auth.existingSecret "secretKey" .Values.minio.auth.rootPasswordKey )) | nindent 2 }} {{- else }} @@ -228,42 +232,14 @@ Colanode Server Environment Variables name: {{ .Release.Name }}-minio key: {{ .Values.minio.auth.rootPasswordKey }} {{- end }} -- name: S3_AVATARS_BUCKET_NAME - value: "colanode-avatars" -- name: S3_AVATARS_REGION - value: "us-east-1" # Region is often optional for MinIO but good practice -- name: S3_AVATARS_FORCE_PATH_STYLE - value: "true" - -# ─────────────────────────────────────────────────────────────── -# S3 Configuration for Files -# ─────────────────────────────────────────────────────────────── -- name: S3_FILES_ENDPOINT - value: "http://{{ include "colanode.minio.hostname" . }}:9000" -- name: S3_FILES_ACCESS_KEY - {{- if .Values.minio.auth.existingSecret }} - {{- include "colanode.getRequiredValueOrSecret" (dict "key" "minio.auth.rootUser" "value" (dict "value" .Values.minio.auth.rootUser "existingSecret" .Values.minio.auth.existingSecret "secretKey" .Values.minio.auth.rootUserKey )) | nindent 2 }} - {{- else }} - valueFrom: - secretKeyRef: - name: {{ .Release.Name }}-minio - key: {{ .Values.minio.auth.rootUserKey }} - {{- end }} -- name: S3_FILES_SECRET_KEY - {{- if .Values.minio.auth.existingSecret }} - {{- include "colanode.getRequiredValueOrSecret" (dict "key" "minio.auth.rootPassword" "value" (dict "value" .Values.minio.auth.rootPassword "existingSecret" .Values.minio.auth.existingSecret "secretKey" .Values.minio.auth.rootPasswordKey )) | nindent 2 }} - {{- else }} - valueFrom: - secretKeyRef: - name: {{ .Release.Name }}-minio - key: {{ .Values.minio.auth.rootPasswordKey }} - {{- end }} -- name: S3_FILES_BUCKET_NAME - value: "colanode-files" -- name: S3_FILES_REGION +- name: STORAGE_S3_BUCKET + value: "colanode" +- name: STORAGE_S3_REGION value: "us-east-1" -- name: S3_FILES_FORCE_PATH_STYLE +- name: STORAGE_S3_FORCE_PATH_STYLE value: "true" +- name: STORAGE_S3_PART_SIZE + value: {{ .Values.colanode.config.STORAGE_S3_PART_SIZE | quote }} # ─────────────────────────────────────────────────────────────── # SMTP configuration diff --git a/hosting/kubernetes/chart/values.yaml b/hosting/kubernetes/chart/values.yaml index d3458566..7b07cf1a 100644 --- a/hosting/kubernetes/chart/values.yaml +++ b/hosting/kubernetes/chart/values.yaml @@ -110,6 +110,8 @@ colanode: REDIS_DB: '0' REDIS_JOBS_QUEUE_NAME: 'jobs' REDIS_JOBS_QUEUE_PREFIX: 'colanode' + REDIS_TUS_LOCK_PREFIX: 'colanode:tus:lock' + REDIS_TUS_KV_PREFIX: 'colanode:tus:kv' REDIS_EVENTS_CHANNEL: 'events' # S3 storage for files @@ -119,6 +121,7 @@ colanode: STORAGE_S3_BUCKET: 'colanode' STORAGE_S3_REGION: 'us-east-1' STORAGE_S3_FORCE_PATH_STYLE: 'true' + STORAGE_S3_PART_SIZE: '20971520' # 20MB # Email configuration SMTP_ENABLED: 'false' diff --git a/package-lock.json b/package-lock.json index ba36cf26..b7f6f63b 100644 --- a/package-lock.json +++ b/package-lock.json @@ -83,6 +83,9 @@ "@langchain/langgraph": "^0.3.11", "@langchain/openai": "^0.6.2", "@node-rs/argon2": "^2.0.2", + "@redis/client": "^5.7.0", + "@tus/s3-store": "^2.0.0", + "@tus/server": "^2.3.0", "bullmq": "^5.56.5", "diff": "^8.0.2", "dotenv": "^17.2.0", @@ -94,10 +97,10 @@ "kysely": "^0.28.3", "langchain": "^0.3.30", "langfuse-langchain": "^3.38.4", + "ms": "^2.1.3", "nodemailer": "^7.0.5", "pg": "^8.16.3", "pino": "^9.7.0", - "redis": "^5.6.0", "sharp": "^0.34.3" }, "devDependencies": { @@ -7427,22 +7430,10 @@ "react-dom": ">=16.8.0" } }, - "node_modules/@redis/bloom": { - "version": "5.6.0", - "resolved": "https://registry.npmjs.org/@redis/bloom/-/bloom-5.6.0.tgz", - "integrity": "sha512-l13/d6BaZDJzogzZJEphIeZ8J0hpQpjkMiozomTm6nJiMNYkoPsNOBOOQua4QsG0fFjyPmLMDJFPAp5FBQtTXg==", - "license": "MIT", - "engines": { - "node": ">= 18" - }, - "peerDependencies": { - "@redis/client": "^5.6.0" - } - }, "node_modules/@redis/client": { - "version": "5.6.0", - "resolved": "https://registry.npmjs.org/@redis/client/-/client-5.6.0.tgz", - "integrity": "sha512-wmP9kCFElCSr4MM4+1E4VckDuN4wLtiXSM/J0rKVQppajxQhowci89RGZr2OdLualowb8SRJ/R6OjsXrn9ZNFA==", + "version": "5.7.0", + "resolved": "https://registry.npmjs.org/@redis/client/-/client-5.7.0.tgz", + "integrity": "sha512-YV3Knspdj9k6H6s4v8QRcj1WBxHt40vtPmszLKGwRUOUpUOLWSlI9oCUjprMDcQNzgSCXGXYdL/Aj6nT2+Ub0w==", "license": "MIT", "dependencies": { "cluster-key-slot": "1.1.2" @@ -7451,42 +7442,6 @@ "node": ">= 18" } }, - "node_modules/@redis/json": { - "version": "5.6.0", - "resolved": "https://registry.npmjs.org/@redis/json/-/json-5.6.0.tgz", - "integrity": "sha512-YQN9ZqaSDpdLfJqwzcF4WeuJMGru/h4WsV7GeeNtXsSeyQjHTyDxrd48xXfRRJGv7HitA7zGnzdHplNeKOgrZA==", - "license": "MIT", - "engines": { - "node": ">= 18" - }, - "peerDependencies": { - "@redis/client": "^5.6.0" - } - }, - "node_modules/@redis/search": { - "version": "5.6.0", - "resolved": "https://registry.npmjs.org/@redis/search/-/search-5.6.0.tgz", - "integrity": "sha512-sLgQl92EyMVNHtri5K8Q0j2xt9c0cO9HYurXz667Un4xeUYR+B/Dw5lLG35yqO7VvVxb9amHJo9sAWumkKZYwA==", - "license": "MIT", - "engines": { - "node": ">= 18" - }, - "peerDependencies": { - "@redis/client": "^5.6.0" - } - }, - "node_modules/@redis/time-series": { - "version": "5.6.0", - "resolved": "https://registry.npmjs.org/@redis/time-series/-/time-series-5.6.0.tgz", - "integrity": "sha512-tXABmN1vu4aTNL3WI4Iolpvx/5jgil2Bs31ozvKblT+jkUoRkk8ykmYo9Pv/Mp7Gk6/Qkr/2rMgVminrt/4BBQ==", - "license": "MIT", - "engines": { - "node": ">= 18" - }, - "peerDependencies": { - "@redis/client": "^5.6.0" - } - }, "node_modules/@remirror/core-constants": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/@remirror/core-constants/-/core-constants-3.0.0.tgz", @@ -8093,6 +8048,16 @@ "dev": true, "license": "MIT" }, + "node_modules/@shopify/semaphore": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/@shopify/semaphore/-/semaphore-3.1.0.tgz", + "integrity": "sha512-LxonkiWEu12FbZhuOMhsdocpxCqm7By8C/2U9QgNuEoXUx2iMrlXjJv3p93RwfNC6TrdlNRo17gRer1z1309VQ==", + "deprecated": "Package no longer supported. Contact Support at https://www.npmjs.com/support for more info.", + "license": "MIT", + "engines": { + "node": ">=18.12.0" + } + }, "node_modules/@sindresorhus/is": { "version": "4.6.0", "resolved": "https://registry.npmjs.org/@sindresorhus/is/-/is-4.6.0.tgz", @@ -9667,6 +9632,66 @@ "dev": true, "license": "MIT" }, + "node_modules/@tus/s3-store": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/@tus/s3-store/-/s3-store-2.0.0.tgz", + "integrity": "sha512-hhaOGRgYzIVWqwsjsWoUPhzM5OsAS097mF8BYTDEuBnnQ2QjaJW0qPIPAESea4LLvfMlX/ujShNCx1SYHQLWGg==", + "license": "MIT", + "dependencies": { + "@aws-sdk/client-s3": "^3.758.0", + "@shopify/semaphore": "^3.1.0", + "@tus/utils": "^0.6.0", + "debug": "^4.3.4", + "multistream": "^4.1.0" + }, + "engines": { + "node": ">=20.19.0" + } + }, + "node_modules/@tus/server": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/@tus/server/-/server-2.3.0.tgz", + "integrity": "sha512-7sj4Q3EPvMjS5z9JaNOZ8gvT6HZvDeg/RjEP0ebbfpAo1V095ivkzpKqV/mDIK/ioBwOKj+bOhTtNuueTjVCfw==", + "license": "MIT", + "dependencies": { + "@tus/utils": "^0.6.0", + "debug": "^4.3.4", + "lodash.throttle": "^4.1.1", + "set-cookie-parser": "^2.7.1", + "srvx": "~0.8.2" + }, + "engines": { + "node": ">=20.19.0" + }, + "optionalDependencies": { + "@redis/client": "^1.6.0", + "ioredis": "^5.4.1" + } + }, + "node_modules/@tus/server/node_modules/@redis/client": { + "version": "1.6.1", + "resolved": "https://registry.npmjs.org/@redis/client/-/client-1.6.1.tgz", + "integrity": "sha512-/KCsg3xSlR+nCK8/8ZYSknYxvXHwubJrU82F3Lm1Fp6789VQ0/3RJKfsmRXjqfaTA++23CvC3hqmqe/2GEt6Kw==", + "license": "MIT", + "optional": true, + "dependencies": { + "cluster-key-slot": "1.1.2", + "generic-pool": "3.9.0", + "yallist": "4.0.0" + }, + "engines": { + "node": ">=14" + } + }, + "node_modules/@tus/utils": { + "version": "0.6.0", + "resolved": "https://registry.npmjs.org/@tus/utils/-/utils-0.6.0.tgz", + "integrity": "sha512-GpMpAQfVdC4UDhpsZrRPjGpdXg+JW5MquqMqtObUVsORwLBV6XI67iTT5be+z98THdqb6dl3bTLIElIdgPeo2g==", + "license": "MIT", + "engines": { + "node": ">=20.19.0" + } + }, "node_modules/@tybys/wasm-util": { "version": "0.10.0", "resolved": "https://registry.npmjs.org/@tybys/wasm-util/-/wasm-util-0.10.0.tgz", @@ -11355,7 +11380,6 @@ "version": "1.1.2", "resolved": "https://registry.npmjs.org/buffer-from/-/buffer-from-1.1.2.tgz", "integrity": "sha512-E+XQCRwSbaaiChtv6k6Dwgc+bx+Bs6vuKJHHl5kox/BaKbhiXzqQOwK4cO22yElGp2OCmjwVhT3HmxgyPGnJfQ==", - "dev": true, "license": "MIT" }, "node_modules/bufferutil": { @@ -12080,6 +12104,15 @@ "integrity": "sha512-72fSenhMw2HZMTVHeCA9KCmpEIbzWiQsjN+BHcBbS9vr1mtt+vJjPdksIBNUmKAW8TFUDPJK5SUU3QhE9NEXDw==", "license": "MIT" }, + "node_modules/combine-errors": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/combine-errors/-/combine-errors-3.0.3.tgz", + "integrity": "sha512-C8ikRNRMygCwaTx+Ek3Yr+OuZzgZjduCOfSQBjbM8V3MfgcjSTeto/GXP6PAwKvJz/v15b7GHZvx5rOlczFw/Q==", + "dependencies": { + "custom-error-instance": "2.1.1", + "lodash.uniqby": "4.5.0" + } + }, "node_modules/comlink": { "version": "4.4.2", "resolved": "https://registry.npmjs.org/comlink/-/comlink-4.4.2.tgz", @@ -12187,6 +12220,12 @@ "node": ">=18" } }, + "node_modules/cookie-es": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/cookie-es/-/cookie-es-2.0.0.tgz", + "integrity": "sha512-RAj4E421UYRgqokKUmotqAwuplYw15qtdXfY+hGzgCJ/MBjCVZcSoHK/kH9kocfjRjcDME7IiDWR/1WX1TM2Pg==", + "license": "MIT" + }, "node_modules/cookie-signature": { "version": "1.0.6", "resolved": "https://registry.npmjs.org/cookie-signature/-/cookie-signature-1.0.6.tgz", @@ -12367,6 +12406,12 @@ "devOptional": true, "license": "MIT" }, + "node_modules/custom-error-instance": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/custom-error-instance/-/custom-error-instance-2.1.1.tgz", + "integrity": "sha512-p6JFxJc3M4OTD2li2qaHkDCw9SfMw82Ldr6OC9Je1aXiGfhx2W8p3GaoeaGrPJTUN9NirTM/KTxHWMUdR1rsUg==", + "license": "ISC" + }, "node_modules/data-view-buffer": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/data-view-buffer/-/data-view-buffer-1.0.2.tgz", @@ -15330,6 +15375,16 @@ "is-property": "^1.0.0" } }, + "node_modules/generic-pool": { + "version": "3.9.0", + "resolved": "https://registry.npmjs.org/generic-pool/-/generic-pool-3.9.0.tgz", + "integrity": "sha512-hymDOu5B53XvN4QT9dBmZxPX4CWhBPPLguTZ9MMFeFa/Kg0xWVfylOVNlJji/E7yTZWFd/q9GO5TxDLq156D7g==", + "license": "MIT", + "optional": true, + "engines": { + "node": ">= 4" + } + }, "node_modules/gensync": { "version": "1.0.0-beta.2", "resolved": "https://registry.npmjs.org/gensync/-/gensync-1.0.0-beta.2.tgz", @@ -15678,7 +15733,6 @@ "version": "4.2.11", "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.11.tgz", "integrity": "sha512-RbJ5/jmFcNNCcDV5o9eTnBLJ/HszWV0P73bc+Ff4nS/rJj+YaS6IGyiOL0VoBYX+l1Wrl3k63h/KrH+nhJ0XvQ==", - "dev": true, "license": "ISC" }, "node_modules/graphemer": { @@ -17703,6 +17757,52 @@ "integrity": "sha512-mKnC+QJ9pWVzv+C4/U3rRsHapFfHvQFoFB92e52xeyGMcX6/OlIl78je1u8vePzYZSkkogMPJ2yjxxsb89cxyw==", "license": "MIT" }, + "node_modules/lodash._baseiteratee": { + "version": "4.7.0", + "resolved": "https://registry.npmjs.org/lodash._baseiteratee/-/lodash._baseiteratee-4.7.0.tgz", + "integrity": "sha512-nqB9M+wITz0BX/Q2xg6fQ8mLkyfF7MU7eE+MNBNjTHFKeKaZAPEzEg+E8LWxKWf1DQVflNEn9N49yAuqKh2mWQ==", + "license": "MIT", + "dependencies": { + "lodash._stringtopath": "~4.8.0" + } + }, + "node_modules/lodash._basetostring": { + "version": "4.12.0", + "resolved": "https://registry.npmjs.org/lodash._basetostring/-/lodash._basetostring-4.12.0.tgz", + "integrity": "sha512-SwcRIbyxnN6CFEEK4K1y+zuApvWdpQdBHM/swxP962s8HIxPO3alBH5t3m/dl+f4CMUug6sJb7Pww8d13/9WSw==", + "license": "MIT" + }, + "node_modules/lodash._baseuniq": { + "version": "4.6.0", + "resolved": "https://registry.npmjs.org/lodash._baseuniq/-/lodash._baseuniq-4.6.0.tgz", + "integrity": "sha512-Ja1YevpHZctlI5beLA7oc5KNDhGcPixFhcqSiORHNsp/1QTv7amAXzw+gu4YOvErqVlMVyIJGgtzeepCnnur0A==", + "license": "MIT", + "dependencies": { + "lodash._createset": "~4.0.0", + "lodash._root": "~3.0.0" + } + }, + "node_modules/lodash._createset": { + "version": "4.0.3", + "resolved": "https://registry.npmjs.org/lodash._createset/-/lodash._createset-4.0.3.tgz", + "integrity": "sha512-GTkC6YMprrJZCYU3zcqZj+jkXkrXzq3IPBcF/fIPpNEAB4hZEtXU8zp/RwKOvZl43NUmwDbyRk3+ZTbeRdEBXA==", + "license": "MIT" + }, + "node_modules/lodash._root": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/lodash._root/-/lodash._root-3.0.1.tgz", + "integrity": "sha512-O0pWuFSK6x4EXhM1dhZ8gchNtG7JMqBtrHdoUFUWXD7dJnNSUze1GuyQr5sOs0aCvgGeI3o/OJW8f4ca7FDxmQ==", + "license": "MIT" + }, + "node_modules/lodash._stringtopath": { + "version": "4.8.0", + "resolved": "https://registry.npmjs.org/lodash._stringtopath/-/lodash._stringtopath-4.8.0.tgz", + "integrity": "sha512-SXL66C731p0xPDC5LZg4wI5H+dJo/EO4KTqOMwLYCH3+FmmfAKJEZCm6ohGpI+T1xwsDsJCfL4OnhorllvlTPQ==", + "license": "MIT", + "dependencies": { + "lodash._basetostring": "~4.12.0" + } + }, "node_modules/lodash.debounce": { "version": "4.0.8", "resolved": "https://registry.npmjs.org/lodash.debounce/-/lodash.debounce-4.0.8.tgz", @@ -17749,6 +17849,22 @@ "dev": true, "license": "MIT" }, + "node_modules/lodash.throttle": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/lodash.throttle/-/lodash.throttle-4.1.1.tgz", + "integrity": "sha512-wIkUCfVKpVsWo3JSZlc+8MB5it+2AN5W8J7YVMST30UrvcQNZ1Okbj+rbVniijTWE6FGYy4XJq/rHkas8qJMLQ==", + "license": "MIT" + }, + "node_modules/lodash.uniqby": { + "version": "4.5.0", + "resolved": "https://registry.npmjs.org/lodash.uniqby/-/lodash.uniqby-4.5.0.tgz", + "integrity": "sha512-IRt7cfTtHy6f1aRVA5n7kT8rgN3N1nH6MOWLcHfpWG2SH19E3JksLK38MktLxZDhlAjCP9jpIXkOnRXlu6oByQ==", + "license": "MIT", + "dependencies": { + "lodash._baseiteratee": "~4.7.0", + "lodash._baseuniq": "~4.6.0" + } + }, "node_modules/log-symbols": { "version": "4.1.0", "resolved": "https://registry.npmjs.org/log-symbols/-/log-symbols-4.1.0.tgz", @@ -18385,6 +18501,30 @@ "@msgpackr-extract/msgpackr-extract-win32-x64": "3.0.3" } }, + "node_modules/multistream": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/multistream/-/multistream-4.1.0.tgz", + "integrity": "sha512-J1XDiAmmNpRCBfIWJv+n0ymC4ABcf/Pl+5YvC5B/D2f/2+8PtHvCNxMPKiQcZyi922Hq69J2YOpb1pTywfifyw==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "MIT", + "dependencies": { + "once": "^1.4.0", + "readable-stream": "^3.6.0" + } + }, "node_modules/murmur-32": { "version": "0.2.0", "resolved": "https://registry.npmjs.org/murmur-32/-/murmur-32-0.2.0.tgz", @@ -19989,6 +20129,32 @@ "react-is": "^16.13.1" } }, + "node_modules/proper-lockfile": { + "version": "4.1.2", + "resolved": "https://registry.npmjs.org/proper-lockfile/-/proper-lockfile-4.1.2.tgz", + "integrity": "sha512-TjNPblN4BwAWMXU8s9AEz4JmQxnD1NNL7bNOY/AKUzyamc379FWASUhc/K1pL2noVb+XmZKLL68cjzLsiOAMaA==", + "license": "MIT", + "dependencies": { + "graceful-fs": "^4.2.4", + "retry": "^0.12.0", + "signal-exit": "^3.0.2" + } + }, + "node_modules/proper-lockfile/node_modules/retry": { + "version": "0.12.0", + "resolved": "https://registry.npmjs.org/retry/-/retry-0.12.0.tgz", + "integrity": "sha512-9LkiTwjUh6rT555DtE9rTX+BKByPfrMzEAtnlEtdEwr3Nkffwiihqe2bWADg+OQRjt9gl6ICdmB/ZFDCGAtSow==", + "license": "MIT", + "engines": { + "node": ">= 4" + } + }, + "node_modules/proper-lockfile/node_modules/signal-exit": { + "version": "3.0.7", + "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-3.0.7.tgz", + "integrity": "sha512-wnD2ZE+l+SPC/uoS0vXeE9L1+0wuaMqKlfz9AMUo38JsyLSBWSFcHR1Rri62LZc12vLr1gb3jl7iwQhgwpAbGQ==", + "license": "ISC" + }, "node_modules/prosemirror-changeset": { "version": "2.3.1", "resolved": "https://registry.npmjs.org/prosemirror-changeset/-/prosemirror-changeset-2.3.1.tgz", @@ -20260,6 +20426,12 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/querystringify": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/querystringify/-/querystringify-2.2.0.tgz", + "integrity": "sha512-FIqgj2EUvTa7R50u0rGsyTftzjYmv/a3hO345bZNrqabNqjtgiDMgmo4mkUjd+nzU5oF3dClKqFIPUKybUyqoQ==", + "license": "MIT" + }, "node_modules/queue-microtask": { "version": "1.2.3", "resolved": "https://registry.npmjs.org/queue-microtask/-/queue-microtask-1.2.3.tgz", @@ -20397,6 +20569,15 @@ "node": ">=0.10.0" } }, + "node_modules/react-circular-progressbar": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/react-circular-progressbar/-/react-circular-progressbar-2.2.0.tgz", + "integrity": "sha512-cgyqEHOzB0nWMZjKfWN3MfSa1LV3OatcDjPz68lchXQUEiBD5O1WsAtoVK4/DSL0B4USR//cTdok4zCBkq8X5g==", + "license": "MIT", + "peerDependencies": { + "react": ">=0.14.0" + } + }, "node_modules/react-day-picker": { "version": "9.8.0", "resolved": "https://registry.npmjs.org/react-day-picker/-/react-day-picker-9.8.0.tgz", @@ -20739,22 +20920,6 @@ "node": ">= 10.13.0" } }, - "node_modules/redis": { - "version": "5.6.0", - "resolved": "https://registry.npmjs.org/redis/-/redis-5.6.0.tgz", - "integrity": "sha512-0x3pM3SlYA5azdNwO8qgfMBzoOqSqr9M+sd1hojbcn0ZDM5zsmKeMM+zpTp6LIY+mbQomIc/RTTQKuBzr8QKzQ==", - "license": "MIT", - "dependencies": { - "@redis/bloom": "5.6.0", - "@redis/client": "5.6.0", - "@redis/json": "5.6.0", - "@redis/search": "5.6.0", - "@redis/time-series": "5.6.0" - }, - "engines": { - "node": ">= 18" - } - }, "node_modules/redis-errors": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/redis-errors/-/redis-errors-1.2.0.tgz", @@ -20944,6 +21109,12 @@ "node": ">=0.10.0" } }, + "node_modules/requires-port": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/requires-port/-/requires-port-1.0.0.tgz", + "integrity": "sha512-KigOCHcocU3XODJxsu8i/j8T9tzT4adHiecwORRQ0ZZFcp7ahwXuRU1m+yuO90C5ZUyGeGfocHDI14M3L3yDAQ==", + "license": "MIT" + }, "node_modules/resedit": { "version": "2.0.3", "resolved": "https://registry.npmjs.org/resedit/-/resedit-2.0.3.tgz", @@ -21944,6 +22115,18 @@ "dev": true, "license": "BSD-3-Clause" }, + "node_modules/srvx": { + "version": "0.8.2", + "resolved": "https://registry.npmjs.org/srvx/-/srvx-0.8.2.tgz", + "integrity": "sha512-anC1+7B6tryHQd4lFVSDZIfZ1QwJwqm5h1iveKwC1E40PA8nOD50hEt7+AlUoGc9jW3OdmztWBqf4yHCdCPdRQ==", + "license": "MIT", + "dependencies": { + "cookie-es": "^2.0.0" + }, + "engines": { + "node": ">=20.16.0" + } + }, "node_modules/ssri": { "version": "12.0.0", "resolved": "https://registry.npmjs.org/ssri/-/ssri-12.0.0.tgz", @@ -23428,6 +23611,36 @@ "win32" ] }, + "node_modules/tus-js-client": { + "version": "4.3.1", + "resolved": "https://registry.npmjs.org/tus-js-client/-/tus-js-client-4.3.1.tgz", + "integrity": "sha512-ZLeYmjrkaU1fUsKbIi8JML52uAocjEZtBx4DKjRrqzrZa0O4MYwT6db+oqePlspV+FxXJAyFBc/L5gwUi2OFsg==", + "license": "MIT", + "dependencies": { + "buffer-from": "^1.1.2", + "combine-errors": "^3.0.3", + "is-stream": "^2.0.0", + "js-base64": "^3.7.2", + "lodash.throttle": "^4.1.1", + "proper-lockfile": "^4.1.2", + "url-parse": "^1.5.7" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/tus-js-client/node_modules/is-stream": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/is-stream/-/is-stream-2.0.1.tgz", + "integrity": "sha512-hFoiJiTl63nn+kstHGBtewWSKnQLpyb155KHheA1l39uvtO9nWIop1p3udqPcUd/xbF1VLMO4n7OI6p7RbngDg==", + "license": "MIT", + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/tw-animate-css": { "version": "1.3.5", "resolved": "https://registry.npmjs.org/tw-animate-css/-/tw-animate-css-1.3.5.tgz", @@ -23833,6 +24046,16 @@ "punycode": "^2.1.0" } }, + "node_modules/url-parse": { + "version": "1.5.10", + "resolved": "https://registry.npmjs.org/url-parse/-/url-parse-1.5.10.tgz", + "integrity": "sha512-WypcfiRhfeUP9vvF0j6rw0J3hrWrw6iZv3+22h6iRMJ/8z1Tj6XfLP4DsUix5MhMPnXpiHDoKyoZ/bdCkwBCiQ==", + "license": "MIT", + "dependencies": { + "querystringify": "^2.1.1", + "requires-port": "^1.0.0" + } + }, "node_modules/use-callback-ref": { "version": "1.3.3", "resolved": "https://registry.npmjs.org/use-callback-ref/-/use-callback-ref-1.3.3.tgz", @@ -25042,7 +25265,7 @@ "version": "4.0.0", "resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz", "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==", - "dev": true, + "devOptional": true, "license": "ISC" }, "node_modules/yaml": { @@ -25177,7 +25400,8 @@ "ky": "^1.8.2", "kysely": "^0.28.3", "ms": "^2.1.3", - "semver": "^7.7.2" + "semver": "^7.7.2", + "tus-js-client": "^4.3.1" }, "devDependencies": { "@types/async-lock": "^1.4.2", @@ -25266,6 +25490,7 @@ "lucide-react": "^0.525.0", "re-resizable": "^6.11.2", "react": "^19.1.0", + "react-circular-progressbar": "^2.2.0", "react-day-picker": "^9.8.0", "react-dnd": "^16.0.1", "react-dnd-html5-backend": "^16.0.1", diff --git a/packages/client/package.json b/packages/client/package.json index 24a68a00..ba3334f9 100644 --- a/packages/client/package.json +++ b/packages/client/package.json @@ -33,7 +33,8 @@ "ky": "^1.8.2", "kysely": "^0.28.3", "ms": "^2.1.3", - "semver": "^7.7.2" + "semver": "^7.7.2", + "tus-js-client": "^4.3.1" }, "devDependencies": { "@types/async-lock": "^1.4.2", diff --git a/packages/client/src/databases/account/migrations/00003-create-avatars-table.ts b/packages/client/src/databases/account/migrations/00003-create-avatars-table.ts new file mode 100644 index 00000000..ba09f064 --- /dev/null +++ b/packages/client/src/databases/account/migrations/00003-create-avatars-table.ts @@ -0,0 +1,17 @@ +import { Migration } from 'kysely'; + +export const createAvatarsTable: Migration = { + up: async (db) => { + await db.schema + .createTable('avatars') + .addColumn('id', 'text', (col) => col.notNull().primaryKey()) + .addColumn('path', 'text', (col) => col.notNull()) + .addColumn('size', 'integer', (col) => col.notNull()) + .addColumn('created_at', 'text', (col) => col.notNull()) + .addColumn('opened_at', 'text', (col) => col.notNull()) + .execute(); + }, + down: async (db) => { + await db.schema.dropTable('avatars').execute(); + }, +}; diff --git a/packages/client/src/databases/account/migrations/index.ts b/packages/client/src/databases/account/migrations/index.ts index f7d20ebf..795ad16c 100644 --- a/packages/client/src/databases/account/migrations/index.ts +++ b/packages/client/src/databases/account/migrations/index.ts @@ -2,8 +2,10 @@ import { Migration } from 'kysely'; import { createWorkspacesTable } from './00001-create-workspaces-table'; import { createMetadataTable } from './00002-create-metadata-table'; +import { createAvatarsTable } from './00003-create-avatars-table'; export const accountDatabaseMigrations: Record = { '00001-create-workspaces-table': createWorkspacesTable, '00002-create-metadata-table': createMetadataTable, + '00003-create-avatars-table': createAvatarsTable, }; diff --git a/packages/client/src/databases/account/schema.ts b/packages/client/src/databases/account/schema.ts index 0c03a153..e5381439 100644 --- a/packages/client/src/databases/account/schema.ts +++ b/packages/client/src/databases/account/schema.ts @@ -30,7 +30,20 @@ export type SelectAccountMetadata = Selectable; export type CreateAccountMetadata = Insertable; export type UpdateAccountMetadata = Updateable; +interface AvatarTable { + id: ColumnType; + path: ColumnType; + size: ColumnType; + created_at: ColumnType; + opened_at: ColumnType; +} + +export type SelectAvatar = Selectable; +export type CreateAvatar = Insertable; +export type UpdateAvatar = Updateable; + export interface AccountDatabaseSchema { workspaces: WorkspaceTable; metadata: AccountMetadataTable; + avatars: AvatarTable; } diff --git a/packages/client/src/databases/app/migrations/00005-create-jobs-table.ts b/packages/client/src/databases/app/migrations/00005-create-jobs-table.ts new file mode 100644 index 00000000..9f09da78 --- /dev/null +++ b/packages/client/src/databases/app/migrations/00005-create-jobs-table.ts @@ -0,0 +1,44 @@ +import { Migration } from 'kysely'; + +export const createJobsTable: Migration = { + up: async (db) => { + await db.schema + .createTable('jobs') + .addColumn('id', 'text', (col) => col.primaryKey().notNull()) + .addColumn('queue', 'text', (col) => col.notNull()) + .addColumn('input', 'text', (col) => col.notNull()) + .addColumn('options', 'text') + .addColumn('status', 'integer', (col) => col.notNull()) + .addColumn('retries', 'integer', (col) => col.notNull().defaultTo(0)) + .addColumn('scheduled_at', 'text', (col) => col.notNull()) + .addColumn('deduplication_key', 'text') + .addColumn('concurrency_key', 'text') + .addColumn('schedule_id', 'text') + .addColumn('created_at', 'text', (col) => col.notNull()) + .addColumn('updated_at', 'text', (col) => col.notNull()) + .execute(); + + await db.schema + .createIndex('idx_jobs_queue_scheduled') + .on('jobs') + .columns(['queue', 'scheduled_at']) + .execute(); + + await db.schema + .createIndex('idx_jobs_deduplication') + .on('jobs') + .columns(['deduplication_key']) + .where('deduplication_key', 'is not', null) + .execute(); + + await db.schema + .createIndex('idx_jobs_concurrency') + .on('jobs') + .columns(['concurrency_key', 'status']) + .where('concurrency_key', 'is not', null) + .execute(); + }, + down: async (db) => { + await db.schema.dropTable('jobs').execute(); + }, +}; diff --git a/packages/client/src/databases/app/migrations/00006-create-job-schedules-table.ts b/packages/client/src/databases/app/migrations/00006-create-job-schedules-table.ts new file mode 100644 index 00000000..ba8c54ce --- /dev/null +++ b/packages/client/src/databases/app/migrations/00006-create-job-schedules-table.ts @@ -0,0 +1,28 @@ +import { Migration } from 'kysely'; + +export const createJobSchedulesTable: Migration = { + up: async (db) => { + await db.schema + .createTable('job_schedules') + .addColumn('id', 'text', (col) => col.primaryKey().notNull()) + .addColumn('queue', 'text', (col) => col.notNull()) + .addColumn('input', 'text', (col) => col.notNull()) + .addColumn('options', 'text') + .addColumn('status', 'integer', (col) => col.notNull().defaultTo(1)) + .addColumn('interval', 'integer', (col) => col.notNull()) + .addColumn('next_run_at', 'text', (col) => col.notNull()) + .addColumn('last_run_at', 'text') + .addColumn('created_at', 'text', (col) => col.notNull()) + .addColumn('updated_at', 'text', (col) => col.notNull()) + .execute(); + + await db.schema + .createIndex('idx_job_schedules_next_run') + .on('job_schedules') + .columns(['status', 'next_run_at']) + .execute(); + }, + down: async (db) => { + await db.schema.dropTable('job_schedules').execute(); + }, +}; diff --git a/packages/client/src/databases/app/migrations/00007-drop-deleted-tokens-table.ts b/packages/client/src/databases/app/migrations/00007-drop-deleted-tokens-table.ts new file mode 100644 index 00000000..abfc7779 --- /dev/null +++ b/packages/client/src/databases/app/migrations/00007-drop-deleted-tokens-table.ts @@ -0,0 +1,16 @@ +import { Migration } from 'kysely'; + +export const dropDeletedTokensTable: Migration = { + up: async (db) => { + await db.schema.dropTable('deleted_tokens').execute(); + }, + down: async (db) => { + await db.schema + .createTable('deleted_tokens') + .addColumn('account_id', 'text', (col) => col.notNull()) + .addColumn('token', 'text', (col) => col.notNull().primaryKey()) + .addColumn('server', 'text', (col) => col.notNull()) + .addColumn('created_at', 'text', (col) => col.notNull()) + .execute(); + }, +}; diff --git a/packages/client/src/databases/app/migrations/00008-create-temp-files-table.ts b/packages/client/src/databases/app/migrations/00008-create-temp-files-table.ts new file mode 100644 index 00000000..8e3c8c41 --- /dev/null +++ b/packages/client/src/databases/app/migrations/00008-create-temp-files-table.ts @@ -0,0 +1,21 @@ +import { Migration } from 'kysely'; + +export const createTempFilesTable: Migration = { + up: async (db) => { + await db.schema + .createTable('temp_files') + .addColumn('id', 'text', (col) => col.primaryKey().notNull()) + .addColumn('name', 'text', (col) => col.notNull()) + .addColumn('path', 'text', (col) => col.notNull()) + .addColumn('size', 'integer', (col) => col.notNull()) + .addColumn('subtype', 'text', (col) => col.notNull()) + .addColumn('mime_type', 'text', (col) => col.notNull()) + .addColumn('extension', 'text', (col) => col.notNull()) + .addColumn('created_at', 'text', (col) => col.notNull()) + .addColumn('opened_at', 'text', (col) => col.notNull()) + .execute(); + }, + down: async (db) => { + await db.schema.dropTable('temp_files').execute(); + }, +}; diff --git a/packages/client/src/databases/app/migrations/index.ts b/packages/client/src/databases/app/migrations/index.ts index d4cb314c..41734af2 100644 --- a/packages/client/src/databases/app/migrations/index.ts +++ b/packages/client/src/databases/app/migrations/index.ts @@ -4,10 +4,18 @@ import { createServersTable } from './00001-create-servers-table'; import { createAccountsTable } from './00002-create-accounts-table'; import { createDeletedTokensTable } from './00003-create-deleted-tokens-table'; import { createMetadataTable } from './00004-create-metadata-table'; +import { createJobsTable } from './00005-create-jobs-table'; +import { createJobSchedulesTable } from './00006-create-job-schedules-table'; +import { dropDeletedTokensTable } from './00007-drop-deleted-tokens-table'; +import { createTempFilesTable } from './00008-create-temp-files-table'; export const appDatabaseMigrations: Record = { '00001-create-servers-table': createServersTable, '00002-create-accounts-table': createAccountsTable, '00003-create-deleted-tokens-table': createDeletedTokensTable, '00004-create-metadata-table': createMetadataTable, + '00005-create-jobs-table': createJobsTable, + '00006-create-job-schedules-table': createJobSchedulesTable, + '00007-drop-deleted-tokens-table': dropDeletedTokensTable, + '00008-create-temp-files-table': createTempFilesTable, }; diff --git a/packages/client/src/databases/app/schema.ts b/packages/client/src/databases/app/schema.ts index 802f4958..e78460f5 100644 --- a/packages/client/src/databases/app/schema.ts +++ b/packages/client/src/databases/app/schema.ts @@ -1,5 +1,8 @@ import { ColumnType, Insertable, Selectable, Updateable } from 'kysely'; +import { JobScheduleStatus, JobStatus } from '@colanode/client/jobs'; +import { FileSubtype } from '@colanode/core'; + interface ServerTable { domain: ColumnType; name: ColumnType; @@ -31,13 +34,6 @@ export type SelectAccount = Selectable; export type CreateAccount = Insertable; export type UpdateAccount = Updateable; -interface DeletedTokenTable { - token: ColumnType; - account_id: ColumnType; - server: ColumnType; - created_at: ColumnType; -} - interface AppMetadataTable { key: ColumnType; value: ColumnType; @@ -49,9 +45,63 @@ export type SelectAppMetadata = Selectable; export type CreateAppMetadata = Insertable; export type UpdateAppMetadata = Updateable; +export interface JobTableSchema { + id: ColumnType; + queue: ColumnType; + input: ColumnType; + options: ColumnType; + status: ColumnType; + retries: ColumnType; + scheduled_at: ColumnType; + deduplication_key: ColumnType; + concurrency_key: ColumnType; + schedule_id: ColumnType; + created_at: ColumnType; + updated_at: ColumnType; +} + +export type SelectJob = Selectable; +export type InsertJob = Insertable; +export type UpdateJob = Updateable; + +export interface JobScheduleTableSchema { + id: ColumnType; + queue: ColumnType; + input: ColumnType; + options: ColumnType; + status: ColumnType; + interval: ColumnType; + next_run_at: ColumnType; + last_run_at: ColumnType; + created_at: ColumnType; + updated_at: ColumnType; +} + +export type SelectJobSchedule = Selectable; +export type InsertJobSchedule = Insertable; +export type UpdateJobSchedule = Updateable; + +interface TempFileTable { + id: ColumnType; + name: ColumnType; + path: ColumnType; + size: ColumnType; + subtype: ColumnType; + mime_type: ColumnType; + extension: ColumnType; + created_at: ColumnType; + opened_at: ColumnType; +} + +export type SelectTempFile = Selectable; +export type InsertTempFile = Insertable; +export type UpdateTempFile = Updateable; + export interface AppDatabaseSchema { servers: ServerTable; accounts: AccountTable; - deleted_tokens: DeletedTokenTable; metadata: AppMetadataTable; + jobs: JobTableSchema; + job_schedules: JobScheduleTableSchema; + temp_files: TempFileTable; } diff --git a/packages/client/src/databases/workspace/migrations/00020-create-local-files-table.ts b/packages/client/src/databases/workspace/migrations/00020-create-local-files-table.ts new file mode 100644 index 00000000..e1a68f02 --- /dev/null +++ b/packages/client/src/databases/workspace/migrations/00020-create-local-files-table.ts @@ -0,0 +1,22 @@ +import { Migration } from 'kysely'; + +export const createLocalFilesTable: Migration = { + up: async (db) => { + await db.schema + .createTable('local_files') + .addColumn('id', 'text', (col) => col.notNull().primaryKey()) + .addColumn('version', 'text', (col) => col.notNull()) + .addColumn('name', 'text', (col) => col.notNull()) + .addColumn('path', 'text', (col) => col.notNull()) + .addColumn('extension', 'text', (col) => col.notNull()) + .addColumn('size', 'integer', (col) => col.notNull()) + .addColumn('subtype', 'text', (col) => col.notNull()) + .addColumn('mime_type', 'text', (col) => col.notNull()) + .addColumn('created_at', 'text', (col) => col.notNull()) + .addColumn('opened_at', 'text', (col) => col.notNull()) + .execute(); + }, + down: async (db) => { + await db.schema.dropTable('local_files').execute(); + }, +}; diff --git a/packages/client/src/databases/workspace/migrations/00021-create-uploads-table.ts b/packages/client/src/databases/workspace/migrations/00021-create-uploads-table.ts new file mode 100644 index 00000000..a23c7db2 --- /dev/null +++ b/packages/client/src/databases/workspace/migrations/00021-create-uploads-table.ts @@ -0,0 +1,21 @@ +import { Migration } from 'kysely'; + +export const createUploadsTable: Migration = { + up: async (db) => { + await db.schema + .createTable('uploads') + .addColumn('file_id', 'text', (col) => col.notNull().primaryKey()) + .addColumn('status', 'integer', (col) => col.notNull()) + .addColumn('progress', 'integer', (col) => col.notNull()) + .addColumn('retries', 'integer', (col) => col.notNull().defaultTo(0)) + .addColumn('created_at', 'text', (col) => col.notNull()) + .addColumn('started_at', 'text') + .addColumn('completed_at', 'text') + .addColumn('error_code', 'text') + .addColumn('error_message', 'text') + .execute(); + }, + down: async (db) => { + await db.schema.dropTable('uploads').execute(); + }, +}; diff --git a/packages/client/src/databases/workspace/migrations/00022-create-downloads-table.ts b/packages/client/src/databases/workspace/migrations/00022-create-downloads-table.ts new file mode 100644 index 00000000..0af5c541 --- /dev/null +++ b/packages/client/src/databases/workspace/migrations/00022-create-downloads-table.ts @@ -0,0 +1,28 @@ +import { Migration } from 'kysely'; + +export const createDownloadsTable: Migration = { + up: async (db) => { + await db.schema + .createTable('downloads') + .addColumn('id', 'text', (col) => col.notNull().primaryKey()) + .addColumn('file_id', 'text', (col) => col.notNull()) + .addColumn('version', 'text', (col) => col.notNull()) + .addColumn('type', 'integer', (col) => col.notNull()) + .addColumn('name', 'text', (col) => col.notNull()) + .addColumn('path', 'text', (col) => col.notNull()) + .addColumn('size', 'integer', (col) => col.notNull()) + .addColumn('mime_type', 'text', (col) => col.notNull()) + .addColumn('status', 'integer', (col) => col.notNull()) + .addColumn('progress', 'integer', (col) => col.notNull()) + .addColumn('retries', 'integer', (col) => col.notNull().defaultTo(0)) + .addColumn('created_at', 'text', (col) => col.notNull()) + .addColumn('started_at', 'text') + .addColumn('completed_at', 'text') + .addColumn('error_code', 'text') + .addColumn('error_message', 'text') + .execute(); + }, + down: async (db) => { + await db.schema.dropTable('downloads').execute(); + }, +}; diff --git a/packages/client/src/databases/workspace/migrations/00023-drop-file-states-table.ts b/packages/client/src/databases/workspace/migrations/00023-drop-file-states-table.ts new file mode 100644 index 00000000..328a3932 --- /dev/null +++ b/packages/client/src/databases/workspace/migrations/00023-drop-file-states-table.ts @@ -0,0 +1,49 @@ +import { Migration } from 'kysely'; + +import { CreateUpload } from '@colanode/client/databases/workspace'; +import { UploadStatus } from '@colanode/client/types/files'; + +export const dropFileStatesTable: Migration = { + up: async (db) => { + const pendingUploads = await db + .selectFrom('file_states') + .select(['id']) + .where('upload_status', '=', 1) + .execute(); + + if (pendingUploads.length > 0) { + const uploadsToCreate: CreateUpload[] = pendingUploads.map((upload) => ({ + file_id: upload.id, + status: UploadStatus.Pending, + progress: 0, + retries: 0, + created_at: new Date().toISOString(), + })); + + await db + .insertInto('uploads') + .values(uploadsToCreate) + .onConflict((oc) => oc.column('file_id').doNothing()) + .execute(); + } + + await db.schema.dropTable('file_states').execute(); + }, + down: async (db) => { + await db.schema + .createTable('file_states') + .addColumn('id', 'text', (col) => col.notNull().primaryKey()) + .addColumn('version', 'text', (col) => col.notNull()) + .addColumn('download_status', 'integer') + .addColumn('download_progress', 'integer') + .addColumn('download_retries', 'integer') + .addColumn('download_started_at', 'text') + .addColumn('download_completed_at', 'text') + .addColumn('upload_status', 'integer') + .addColumn('upload_progress', 'integer') + .addColumn('upload_retries', 'integer') + .addColumn('upload_started_at', 'text') + .addColumn('upload_completed_at', 'text') + .execute(); + }, +}; diff --git a/packages/client/src/databases/workspace/migrations/index.ts b/packages/client/src/databases/workspace/migrations/index.ts index 85aa9173..8742ee03 100644 --- a/packages/client/src/databases/workspace/migrations/index.ts +++ b/packages/client/src/databases/workspace/migrations/index.ts @@ -19,6 +19,10 @@ import { createCursorsTable } from './00016-create-cursors-table'; import { createMetadataTable } from './00017-create-metadata-table'; import { createNodeReferencesTable } from './00018-create-node-references-table'; import { createNodeCountersTable } from './00019-create-node-counters-table'; +import { createLocalFilesTable } from './00020-create-local-files-table'; +import { createUploadsTable } from './00021-create-uploads-table'; +import { createDownloadsTable } from './00022-create-downloads-table'; +import { dropFileStatesTable } from './00023-drop-file-states-table'; export const workspaceDatabaseMigrations: Record = { '00001-create-users-table': createUsersTable, @@ -40,4 +44,8 @@ export const workspaceDatabaseMigrations: Record = { '00017-create-metadata-table': createMetadataTable, '00018-create-node-references-table': createNodeReferencesTable, '00019-create-node-counters-table': createNodeCountersTable, + '00020-create-local-files-table': createLocalFilesTable, + '00021-create-uploads-table': createUploadsTable, + '00022-create-downloads-table': createDownloadsTable, + '00023-drop-file-states-table': dropFileStatesTable, }; diff --git a/packages/client/src/databases/workspace/schema.ts b/packages/client/src/databases/workspace/schema.ts index 2999aff7..525da382 100644 --- a/packages/client/src/databases/workspace/schema.ts +++ b/packages/client/src/databases/workspace/schema.ts @@ -1,6 +1,10 @@ import { ColumnType, Insertable, Selectable, Updateable } from 'kysely'; -import { DownloadStatus, UploadStatus } from '@colanode/client/types/files'; +import { + DownloadStatus, + DownloadType, + UploadStatus, +} from '@colanode/client/types/files'; import { NodeCounterType } from '@colanode/client/types/nodes'; import { MutationType, @@ -8,6 +12,7 @@ import { WorkspaceRole, UserStatus, DocumentType, + FileSubtype, } from '@colanode/core'; interface UserTable { @@ -189,37 +194,6 @@ export type SelectDocumentText = Selectable; export type CreateDocumentText = Insertable; export type UpdateDocumentText = Updateable; -interface FileStateTable { - id: ColumnType; - version: ColumnType; - download_status: ColumnType< - DownloadStatus | null, - DownloadStatus | null, - DownloadStatus | null - >; - download_progress: ColumnType; - download_retries: ColumnType; - download_started_at: ColumnType; - download_completed_at: ColumnType< - string | null, - string | null, - string | null - >; - upload_status: ColumnType< - UploadStatus | null, - UploadStatus | null, - UploadStatus | null - >; - upload_progress: ColumnType; - upload_retries: ColumnType; - upload_started_at: ColumnType; - upload_completed_at: ColumnType; -} - -export type SelectFileState = Selectable; -export type CreateFileState = Insertable; -export type UpdateFileState = Updateable; - interface MutationTable { id: ColumnType; type: ColumnType; @@ -264,6 +238,61 @@ export type SelectWorkspaceMetadata = Selectable; export type CreateWorkspaceMetadata = Insertable; export type UpdateWorkspaceMetadata = Updateable; +interface LocalFileTable { + id: ColumnType; + version: ColumnType; + name: ColumnType; + path: ColumnType; + extension: ColumnType; + size: ColumnType; + subtype: ColumnType; + mime_type: ColumnType; + created_at: ColumnType; + opened_at: ColumnType; +} + +export type SelectLocalFile = Selectable; +export type CreateLocalFile = Insertable; +export type UpdateLocalFile = Updateable; + +interface UploadTable { + file_id: ColumnType; + status: ColumnType; + progress: ColumnType; + retries: ColumnType; + created_at: ColumnType; + started_at: ColumnType; + completed_at: ColumnType; + error_code: ColumnType; + error_message: ColumnType; +} + +export type SelectUpload = Selectable; +export type CreateUpload = Insertable; +export type UpdateUpload = Updateable; + +interface DownloadTable { + id: ColumnType; + file_id: ColumnType; + version: ColumnType; + type: ColumnType; + name: ColumnType; + path: ColumnType; + size: ColumnType; + mime_type: ColumnType; + status: ColumnType; + progress: ColumnType; + retries: ColumnType; + created_at: ColumnType; + started_at: ColumnType; + completed_at: ColumnType; + error_code: ColumnType; + error_message: ColumnType; +} + +export type SelectDownload = Selectable; +export type CreateDownload = Insertable; +export type UpdateDownload = Updateable; export interface WorkspaceDatabaseSchema { users: UserTable; nodes: NodeTable; @@ -279,7 +308,9 @@ export interface WorkspaceDatabaseSchema { document_updates: DocumentUpdateTable; document_texts: DocumentTextTable; collaborations: CollaborationTable; - file_states: FileStateTable; + local_files: LocalFileTable; + uploads: UploadTable; + downloads: DownloadTable; mutations: MutationTable; tombstones: TombstoneTable; cursors: CursorTable; diff --git a/packages/client/src/handlers/mutations/avatars/avatar-upload.ts b/packages/client/src/handlers/mutations/avatars/avatar-upload.ts index 8769123c..6bcc21e8 100644 --- a/packages/client/src/handlers/mutations/avatars/avatar-upload.ts +++ b/packages/client/src/handlers/mutations/avatars/avatar-upload.ts @@ -36,7 +36,10 @@ export class AvatarUploadMutationHandler const fileExists = await this.app.fs.exists(filePath); if (!fileExists) { - throw new Error(`File ${filePath} does not exist`); + throw new MutationError( + MutationErrorCode.FileNotFound, + `Avatar file does not exist` + ); } const fileStream = await this.app.fs.readStream(filePath); @@ -50,7 +53,8 @@ export class AvatarUploadMutationHandler }) .json(); - await account.downloadAvatar(response.id); + await this.app.fs.delete(filePath); + await account.avatars.downloadAvatar(response.id); return { id: response.id, diff --git a/packages/client/src/handlers/mutations/files/file-create.ts b/packages/client/src/handlers/mutations/files/file-create.ts index e966a5c0..17f65877 100644 --- a/packages/client/src/handlers/mutations/files/file-create.ts +++ b/packages/client/src/handlers/mutations/files/file-create.ts @@ -16,7 +16,7 @@ export class FileCreateMutationHandler const workspace = this.getWorkspace(input.accountId, input.workspaceId); const fileId = generateId(IdType.File); - await workspace.files.createFile(fileId, input.parentId, input.file); + await workspace.files.createFile(fileId, input.tempFileId, input.parentId); return { id: fileId, diff --git a/packages/client/src/handlers/mutations/files/file-download.ts b/packages/client/src/handlers/mutations/files/file-download.ts index 96dced39..11054993 100644 --- a/packages/client/src/handlers/mutations/files/file-download.ts +++ b/packages/client/src/handlers/mutations/files/file-download.ts @@ -1,15 +1,9 @@ import { WorkspaceMutationHandlerBase } from '@colanode/client/handlers/mutations/workspace-mutation-handler-base'; -import { eventBus } from '@colanode/client/lib/event-bus'; -import { mapFileState, mapNode } from '@colanode/client/lib/mappers'; import { MutationHandler } from '@colanode/client/lib/types'; import { - MutationError, - MutationErrorCode, FileDownloadMutationInput, FileDownloadMutationOutput, } from '@colanode/client/mutations'; -import { DownloadStatus, LocalFileNode } from '@colanode/client/types'; -import { FileStatus } from '@colanode/core'; export class FileDownloadMutationHandler extends WorkspaceMutationHandlerBase @@ -19,82 +13,22 @@ export class FileDownloadMutationHandler input: FileDownloadMutationInput ): Promise { const workspace = this.getWorkspace(input.accountId, input.workspaceId); + const path = input.path; - const node = await workspace.database - .selectFrom('nodes') - .selectAll() - .where('id', '=', input.fileId) - .executeTakeFirst(); - - if (!node) { - throw new MutationError( - MutationErrorCode.FileNotFound, - 'The file you are trying to download does not exist.' - ); - } - - const file = mapNode(node) as LocalFileNode; - if (file.attributes.status !== FileStatus.Ready) { - throw new MutationError( - MutationErrorCode.FileNotReady, - 'The file you are trying to download is not uploaded by the author yet.' - ); - } - - const fileState = await workspace.database - .selectFrom('file_states') - .selectAll() - .where('id', '=', input.fileId) - .executeTakeFirst(); - - if ( - fileState?.download_status === DownloadStatus.Completed || - fileState?.download_status === DownloadStatus.Pending - ) { + if (!path) { + const autoDownload = await workspace.files.initAutoDownload(input.fileId); return { - success: true, + success: !!autoDownload, }; } - const updatedFileState = await workspace.database - .insertInto('file_states') - .returningAll() - .values({ - id: input.fileId, - version: file.attributes.version, - download_status: DownloadStatus.Pending, - download_progress: 0, - download_retries: 0, - download_started_at: new Date().toISOString(), - }) - .onConflict((oc) => - oc.columns(['id']).doUpdateSet({ - download_status: DownloadStatus.Pending, - download_progress: 0, - download_retries: 0, - download_started_at: new Date().toISOString(), - }) - ) - .executeTakeFirst(); - - if (!updatedFileState) { - throw new MutationError( - MutationErrorCode.FileNotFound, - 'The file you are trying to download does not exist.' - ); - } - - workspace.files.triggerDownloads(); - - eventBus.publish({ - type: 'file.state.updated', - accountId: workspace.accountId, - workspaceId: workspace.id, - fileState: mapFileState(updatedFileState, null), - }); + const manualDownload = await workspace.files.initManualDownload( + input.fileId, + path + ); return { - success: true, + success: !!manualDownload, }; } } diff --git a/packages/client/src/handlers/mutations/files/file-save.ts b/packages/client/src/handlers/mutations/files/file-save.ts deleted file mode 100644 index 1f65e876..00000000 --- a/packages/client/src/handlers/mutations/files/file-save.ts +++ /dev/null @@ -1,48 +0,0 @@ -import { WorkspaceMutationHandlerBase } from '@colanode/client/handlers/mutations/workspace-mutation-handler-base'; -import { mapNode } from '@colanode/client/lib/mappers'; -import { MutationHandler } from '@colanode/client/lib/types'; -import { MutationError, MutationErrorCode } from '@colanode/client/mutations'; -import { - FileSaveMutationInput, - FileSaveMutationOutput, -} from '@colanode/client/mutations/files/file-save'; -import { LocalFileNode } from '@colanode/client/types'; -import { FileStatus } from '@colanode/core'; - -export class FileSaveMutationHandler - extends WorkspaceMutationHandlerBase - implements MutationHandler -{ - async handleMutation( - input: FileSaveMutationInput - ): Promise { - const workspace = this.getWorkspace(input.accountId, input.workspaceId); - - const node = await workspace.database - .selectFrom('nodes') - .selectAll() - .where('id', '=', input.fileId) - .executeTakeFirst(); - - if (!node) { - throw new MutationError( - MutationErrorCode.FileNotFound, - 'The file you are trying to save does not exist.' - ); - } - - const file = mapNode(node) as LocalFileNode; - if (file.attributes.status !== FileStatus.Ready) { - throw new MutationError( - MutationErrorCode.FileNotReady, - 'The file you are trying to download is not uploaded by the author yet.' - ); - } - - workspace.files.saveFile(file, input.path); - - return { - success: true, - }; - } -} diff --git a/packages/client/src/handlers/mutations/files/temp-file-create.ts b/packages/client/src/handlers/mutations/files/temp-file-create.ts new file mode 100644 index 00000000..f4eccd19 --- /dev/null +++ b/packages/client/src/handlers/mutations/files/temp-file-create.ts @@ -0,0 +1,40 @@ +import { MutationHandler } from '@colanode/client/lib/types'; +import { + TempFileCreateMutationInput, + TempFileCreateMutationOutput, +} from '@colanode/client/mutations'; +import { AppService } from '@colanode/client/services/app-service'; + +export class TempFileCreateMutationHandler + implements MutationHandler +{ + private readonly app: AppService; + + constructor(app: AppService) { + this.app = app; + } + + async handleMutation( + input: TempFileCreateMutationInput + ): Promise { + await this.app.database + .insertInto('temp_files') + .values({ + id: input.id, + name: input.name, + size: input.size, + mime_type: input.mimeType, + subtype: input.subtype, + path: input.path, + extension: input.extension, + created_at: new Date().toISOString(), + opened_at: new Date().toISOString(), + }) + .onConflict((oc) => oc.doNothing()) + .execute(); + + return { + success: true, + }; + } +} diff --git a/packages/client/src/handlers/mutations/index.ts b/packages/client/src/handlers/mutations/index.ts index 3d8de4fb..a71cb882 100644 --- a/packages/client/src/handlers/mutations/index.ts +++ b/packages/client/src/handlers/mutations/index.ts @@ -37,7 +37,7 @@ import { DocumentUpdateMutationHandler } from './documents/document-update'; import { FileCreateMutationHandler } from './files/file-create'; import { FileDeleteMutationHandler } from './files/file-delete'; import { FileDownloadMutationHandler } from './files/file-download'; -import { FileSaveMutationHandler } from './files/file-save'; +import { TempFileCreateMutationHandler } from './files/temp-file-create'; import { FolderCreateMutationHandler } from './folders/folder-create'; import { FolderDeleteMutationHandler } from './folders/folder-delete'; import { FolderUpdateMutationHandler } from './folders/folder-update'; @@ -135,7 +135,6 @@ export const buildMutationHandlerMap = ( 'folder.create': new FolderCreateMutationHandler(app), 'file.create': new FileCreateMutationHandler(app), 'file.download': new FileDownloadMutationHandler(app), - 'file.save': new FileSaveMutationHandler(app), 'space.avatar.update': new SpaceAvatarUpdateMutationHandler(app), 'space.description.update': new SpaceDescriptionUpdateMutationHandler(app), 'space.name.update': new SpaceNameUpdateMutationHandler(app), @@ -164,5 +163,6 @@ export const buildMutationHandlerMap = ( new EmailPasswordResetCompleteMutationHandler(app), 'workspace.delete': new WorkspaceDeleteMutationHandler(app), 'user.storage.update': new UserStorageUpdateMutationHandler(app), + 'temp.file.create': new TempFileCreateMutationHandler(app), }; }; diff --git a/packages/client/src/handlers/mutations/messages/message-create.ts b/packages/client/src/handlers/mutations/messages/message-create.ts index 29b6a25f..45cb10c7 100644 --- a/packages/client/src/handlers/mutations/messages/message-create.ts +++ b/packages/client/src/handlers/mutations/messages/message-create.ts @@ -7,7 +7,6 @@ import { MutationError, MutationErrorCode, } from '@colanode/client/mutations'; -import { TempFile } from '@colanode/client/types'; import { EditorNodeTypes, generateId, @@ -17,7 +16,7 @@ import { interface MessageFile { id: string; - file: TempFile; + tempFileId: string; } export class MessageCreateMutationHandler @@ -37,8 +36,8 @@ export class MessageCreateMutationHandler // check if there are nested nodes (files, pages, folders etc.) for (const block of Object.values(blocks)) { if (block.type === EditorNodeTypes.TempFile) { - const file = block.attrs?.file as TempFile; - if (!file) { + const tempFileId = block.attrs?.id; + if (!tempFileId) { throw new MutationError( MutationErrorCode.FileInvalid, 'File is invalid or could not be read.' @@ -49,7 +48,7 @@ export class MessageCreateMutationHandler filesToCreate.push({ id: fileId, - file, + tempFileId, }); block.id = fileId; @@ -73,7 +72,7 @@ export class MessageCreateMutationHandler }); for (const file of filesToCreate) { - await workspace.files.createFile(file.id, messageId, file.file); + await workspace.files.createFile(file.id, file.tempFileId, messageId); } return { diff --git a/packages/client/src/handlers/mutations/nodes/node-interaction-opened.ts b/packages/client/src/handlers/mutations/nodes/node-interaction-opened.ts index fcd22f9c..7442b5db 100644 --- a/packages/client/src/handlers/mutations/nodes/node-interaction-opened.ts +++ b/packages/client/src/handlers/mutations/nodes/node-interaction-opened.ts @@ -1,3 +1,5 @@ +import ms from 'ms'; + import { WorkspaceMutationHandlerBase } from '@colanode/client/handlers/mutations/workspace-mutation-handler-base'; import { eventBus } from '@colanode/client/lib/event-bus'; import { mapNodeInteraction } from '@colanode/client/lib/mappers'; @@ -40,7 +42,7 @@ export class NodeInteractionOpenedMutationHandler const lastOpenedAt = existingInteraction.last_opened_at; if ( lastOpenedAt && - lastOpenedAt > new Date(Date.now() - 5 * 60 * 1000).toISOString() + lastOpenedAt > new Date(Date.now() - ms('5 minutes')).toISOString() ) { return { success: true, @@ -117,7 +119,7 @@ export class NodeInteractionOpenedMutationHandler existingInteraction ); - workspace.mutations.triggerSync(); + workspace.mutations.scheduleSync(); eventBus.publish({ type: 'node.interaction.updated', diff --git a/packages/client/src/handlers/mutations/nodes/node-interaction-seen.ts b/packages/client/src/handlers/mutations/nodes/node-interaction-seen.ts index 9f3ac3b8..80cba90b 100644 --- a/packages/client/src/handlers/mutations/nodes/node-interaction-seen.ts +++ b/packages/client/src/handlers/mutations/nodes/node-interaction-seen.ts @@ -1,3 +1,5 @@ +import ms from 'ms'; + import { WorkspaceMutationHandlerBase } from '@colanode/client/handlers/mutations/workspace-mutation-handler-base'; import { eventBus } from '@colanode/client/lib/event-bus'; import { mapNodeInteraction } from '@colanode/client/lib/mappers'; @@ -41,7 +43,7 @@ export class NodeInteractionSeenMutationHandler const lastSeenAt = existingInteraction.last_seen_at; if ( lastSeenAt && - lastSeenAt > new Date(Date.now() - 5 * 60 * 1000).toISOString() + lastSeenAt > new Date(Date.now() - ms('5 minutes')).toISOString() ) { return { success: true, @@ -118,7 +120,7 @@ export class NodeInteractionSeenMutationHandler existingInteraction ); - workspace.mutations.triggerSync(); + workspace.mutations.scheduleSync(); eventBus.publish({ type: 'node.interaction.updated', diff --git a/packages/client/src/handlers/queries/avatars/avatar-get.ts b/packages/client/src/handlers/queries/avatars/avatar-get.ts new file mode 100644 index 00000000..123debd3 --- /dev/null +++ b/packages/client/src/handlers/queries/avatars/avatar-get.ts @@ -0,0 +1,55 @@ +import { ChangeCheckResult, QueryHandler } from '@colanode/client/lib/types'; +import { AvatarGetQueryInput } from '@colanode/client/queries/avatars/avatar-get'; +import { AppService } from '@colanode/client/services/app-service'; +import { Avatar } from '@colanode/client/types/avatars'; +import { Event } from '@colanode/client/types/events'; + +export class AvatarGetQueryHandler + implements QueryHandler +{ + private readonly app: AppService; + + constructor(app: AppService) { + this.app = app; + } + + public async handleQuery(input: AvatarGetQueryInput): Promise { + const account = this.app.getAccount(input.accountId); + if (!account) { + return null; + } + + return account.avatars.getAvatar(input.avatarId, true); + } + + public async checkForChanges( + event: Event, + input: AvatarGetQueryInput + ): Promise> { + if ( + event.type === 'avatar.created' && + event.accountId === input.accountId && + event.avatar.id === input.avatarId + ) { + return { + hasChanges: true, + result: event.avatar, + }; + } + + if ( + event.type === 'avatar.deleted' && + event.accountId === input.accountId && + event.avatar.id === input.avatarId + ) { + return { + hasChanges: true, + result: null, + }; + } + + return { + hasChanges: false, + }; + } +} diff --git a/packages/client/src/handlers/queries/avatars/avatar-url-get.ts b/packages/client/src/handlers/queries/avatars/avatar-url-get.ts deleted file mode 100644 index 6427aa47..00000000 --- a/packages/client/src/handlers/queries/avatars/avatar-url-get.ts +++ /dev/null @@ -1,91 +0,0 @@ -import { ChangeCheckResult, QueryHandler } from '@colanode/client/lib/types'; -import { - AvatarUrlGetQueryInput, - AvatarUrlGetQueryOutput, -} from '@colanode/client/queries/avatars/avatar-url-get'; -import { AppService } from '@colanode/client/services/app-service'; -import { Event } from '@colanode/client/types/events'; - -export class AvatarUrlGetQueryHandler - implements QueryHandler -{ - private readonly app: AppService; - - constructor(app: AppService) { - this.app = app; - } - - public async handleQuery( - input: AvatarUrlGetQueryInput - ): Promise { - const avatarPath = this.app.path.accountAvatar( - input.accountId, - input.avatarId - ); - - const avatarExists = await this.app.fs.exists(avatarPath); - if (avatarExists) { - const url = await this.app.fs.url(avatarPath); - return { - url, - }; - } - - const account = this.app.getAccount(input.accountId); - if (!account) { - return { - url: null, - }; - } - - const downloaded = await account.downloadAvatar(input.avatarId); - if (!downloaded) { - return { - url: null, - }; - } - - const url = await this.app.fs.url(avatarPath); - return { - url, - }; - } - - public async checkForChanges( - event: Event, - input: AvatarUrlGetQueryInput - ): Promise> { - if ( - event.type === 'avatar.downloaded' && - event.accountId === input.accountId && - event.avatarId === input.avatarId - ) { - const avatarPath = this.app.path.accountAvatar( - input.accountId, - input.avatarId - ); - - const avatarExists = await this.app.fs.exists(avatarPath); - if (!avatarExists) { - return { - hasChanges: true, - result: { - url: null, - }, - }; - } - - const url = await this.app.fs.url(avatarPath); - return { - hasChanges: true, - result: { - url, - }, - }; - } - - return { - hasChanges: false, - }; - } -} diff --git a/packages/client/src/handlers/queries/emojis/emoji-category-list.ts b/packages/client/src/handlers/queries/emojis/emoji-category-list.ts index 54e1b47e..0f4e8a41 100644 --- a/packages/client/src/handlers/queries/emojis/emoji-category-list.ts +++ b/packages/client/src/handlers/queries/emojis/emoji-category-list.ts @@ -16,11 +16,11 @@ export class EmojiCategoryListQueryHandler public async handleQuery( _: EmojiCategoryListQueryInput ): Promise { - if (!this.app.asset.emojis) { + if (!this.app.assets.emojis) { return []; } - const data = this.app.asset.emojis + const data = this.app.assets.emojis .selectFrom('categories') .selectAll() .execute(); diff --git a/packages/client/src/handlers/queries/emojis/emoji-get-by-skin-id.ts b/packages/client/src/handlers/queries/emojis/emoji-get-by-skin-id.ts index 7e9d8163..d2b3383b 100644 --- a/packages/client/src/handlers/queries/emojis/emoji-get-by-skin-id.ts +++ b/packages/client/src/handlers/queries/emojis/emoji-get-by-skin-id.ts @@ -17,11 +17,11 @@ export class EmojiGetBySkinIdQueryHandler public async handleQuery( input: EmojiGetBySkinIdQueryInput ): Promise { - if (!this.app.asset.emojis) { + if (!this.app.assets.emojis) { return null; } - const data = await this.app.asset.emojis + const data = await this.app.assets.emojis .selectFrom('emojis') .innerJoin('emoji_skins', 'emojis.id', 'emoji_skins.emoji_id') .selectAll('emojis') diff --git a/packages/client/src/handlers/queries/emojis/emoji-get.ts b/packages/client/src/handlers/queries/emojis/emoji-get.ts index 6be5b44f..ffe6c06e 100644 --- a/packages/client/src/handlers/queries/emojis/emoji-get.ts +++ b/packages/client/src/handlers/queries/emojis/emoji-get.ts @@ -13,11 +13,11 @@ export class EmojiGetQueryHandler implements QueryHandler { } public async handleQuery(input: EmojiGetQueryInput): Promise { - if (!this.app.asset.emojis) { + if (!this.app.assets.emojis) { return null; } - const data = await this.app.asset.emojis + const data = await this.app.assets.emojis .selectFrom('emojis') .selectAll() .where('id', '=', input.id) diff --git a/packages/client/src/handlers/queries/emojis/emoji-list.ts b/packages/client/src/handlers/queries/emojis/emoji-list.ts index 2625ca13..fa709ae5 100644 --- a/packages/client/src/handlers/queries/emojis/emoji-list.ts +++ b/packages/client/src/handlers/queries/emojis/emoji-list.ts @@ -15,12 +15,12 @@ export class EmojiListQueryHandler } public async handleQuery(input: EmojiListQueryInput): Promise { - if (!this.app.asset.emojis) { + if (!this.app.assets.emojis) { return []; } const offset = input.page * input.count; - const data = await this.app.asset.emojis + const data = await this.app.assets.emojis .selectFrom('emojis') .selectAll() .where('category_id', '=', input.category) diff --git a/packages/client/src/handlers/queries/emojis/emoji-search.ts b/packages/client/src/handlers/queries/emojis/emoji-search.ts index 1796f674..ba4d62d6 100644 --- a/packages/client/src/handlers/queries/emojis/emoji-search.ts +++ b/packages/client/src/handlers/queries/emojis/emoji-search.ts @@ -15,11 +15,11 @@ export class EmojiSearchQueryHandler } public async handleQuery(input: EmojiSearchQueryInput): Promise { - if (!this.app.asset.emojis) { + if (!this.app.assets.emojis) { return []; } - const data = await this.app.asset.emojis + const data = await this.app.assets.emojis .selectFrom('emojis') .innerJoin('emoji_search', 'emojis.id', 'emoji_search.id') .selectAll('emojis') diff --git a/packages/client/src/handlers/queries/files/download-list-manual.ts b/packages/client/src/handlers/queries/files/download-list-manual.ts new file mode 100644 index 00000000..0df2e802 --- /dev/null +++ b/packages/client/src/handlers/queries/files/download-list-manual.ts @@ -0,0 +1,128 @@ +import { WorkspaceQueryHandlerBase } from '@colanode/client/handlers/queries/workspace-query-handler-base'; +import { mapDownload } from '@colanode/client/lib'; +import { ChangeCheckResult, QueryHandler } from '@colanode/client/lib/types'; +import { DownloadListManualQueryInput } from '@colanode/client/queries/files/download-list-manual'; +import { Event } from '@colanode/client/types/events'; +import { Download, DownloadType } from '@colanode/client/types/files'; + +export class DownloadListManualQueryHandler + extends WorkspaceQueryHandlerBase + implements QueryHandler +{ + public async handleQuery( + input: DownloadListManualQueryInput + ): Promise { + return await this.fetchManualDownloads(input); + } + + public async checkForChanges( + event: Event, + input: DownloadListManualQueryInput, + output: Download[] + ): Promise> { + if ( + event.type === 'workspace.deleted' && + event.workspace.accountId === input.accountId && + event.workspace.id === input.workspaceId + ) { + return { + hasChanges: true, + result: [], + }; + } + + if ( + event.type === 'download.created' && + event.accountId === input.accountId && + event.workspaceId === input.workspaceId && + event.download.type === DownloadType.Manual + ) { + const newResult = await this.fetchManualDownloads(input); + return { + hasChanges: true, + result: newResult, + }; + } + + if ( + event.type === 'download.updated' && + event.accountId === input.accountId && + event.workspaceId === input.workspaceId && + event.download.type === DownloadType.Manual + ) { + const download = output.find( + (download) => download.id === event.download.id + ); + + if (download) { + const newResult = output.map((download) => { + if (download.id === event.download.id) { + return event.download; + } + + return download; + }); + + return { + hasChanges: true, + result: newResult, + }; + } + } + + if ( + event.type === 'download.deleted' && + event.accountId === input.accountId && + event.workspaceId === input.workspaceId && + event.download.type === DownloadType.Manual + ) { + const download = output.find( + (download) => download.id === event.download.id + ); + + if (!download) { + return { + hasChanges: false, + }; + } + + if (output.length === input.count) { + const newResult = await this.fetchManualDownloads(input); + return { + hasChanges: true, + result: newResult, + }; + } + + const newOutput = output.filter( + (download) => download.fileId !== event.download.fileId + ); + return { + hasChanges: true, + result: newOutput, + }; + } + + return { + hasChanges: false, + }; + } + + private async fetchManualDownloads( + input: DownloadListManualQueryInput + ): Promise { + const workspace = this.getWorkspace(input.accountId, input.workspaceId); + + const offset = (input.page - 1) * input.count; + const downloads = await workspace.database + .selectFrom('downloads') + .selectAll() + .where('type', '=', DownloadType.Manual) + .orderBy('id', 'desc') + .limit(input.count) + .offset(offset) + .execute(); + + return downloads.map(mapDownload); + } +} diff --git a/packages/client/src/handlers/queries/files/file-save-list.ts b/packages/client/src/handlers/queries/files/file-save-list.ts deleted file mode 100644 index 2f01aaab..00000000 --- a/packages/client/src/handlers/queries/files/file-save-list.ts +++ /dev/null @@ -1,54 +0,0 @@ -import { WorkspaceQueryHandlerBase } from '@colanode/client/handlers/queries/workspace-query-handler-base'; -import { ChangeCheckResult, QueryHandler } from '@colanode/client/lib/types'; -import { FileSaveListQueryInput } from '@colanode/client/queries/files/file-save-list'; -import { Event } from '@colanode/client/types/events'; -import { FileSaveState } from '@colanode/client/types/files'; - -export class FileSaveListQueryHandler - extends WorkspaceQueryHandlerBase - implements QueryHandler -{ - public async handleQuery( - input: FileSaveListQueryInput - ): Promise { - return this.getSaves(input); - } - - public async checkForChanges( - event: Event, - input: FileSaveListQueryInput, - _: FileSaveState[] - ): Promise> { - if ( - event.type === 'workspace.deleted' && - event.workspace.accountId === input.accountId && - event.workspace.id === input.workspaceId - ) { - return { - hasChanges: true, - result: [], - }; - } - - if ( - event.type === 'file.save.updated' && - event.accountId === input.accountId && - event.workspaceId === input.workspaceId - ) { - return { - hasChanges: true, - result: this.getSaves(input), - }; - } - - return { - hasChanges: false, - }; - } - - private getSaves(input: FileSaveListQueryInput): FileSaveState[] { - const workspace = this.getWorkspace(input.accountId, input.workspaceId); - const saves = workspace.files.getSaves(); - return saves; - } -} diff --git a/packages/client/src/handlers/queries/files/file-state-get.ts b/packages/client/src/handlers/queries/files/file-state-get.ts deleted file mode 100644 index 6fd03c08..00000000 --- a/packages/client/src/handlers/queries/files/file-state-get.ts +++ /dev/null @@ -1,133 +0,0 @@ -import { WorkspaceQueryHandlerBase } from '@colanode/client/handlers/queries/workspace-query-handler-base'; -import { mapFileState, mapNode } from '@colanode/client/lib/mappers'; -import { ChangeCheckResult, QueryHandler } from '@colanode/client/lib/types'; -import { FileStateGetQueryInput } from '@colanode/client/queries/files/file-state-get'; -import { LocalFileNode } from '@colanode/client/types'; -import { Event } from '@colanode/client/types/events'; -import { DownloadStatus, FileState } from '@colanode/client/types/files'; - -export class FileStateGetQueryHandler - extends WorkspaceQueryHandlerBase - implements QueryHandler -{ - public async handleQuery( - input: FileStateGetQueryInput - ): Promise { - return await this.fetchFileState(input); - } - - public async checkForChanges( - event: Event, - input: FileStateGetQueryInput, - _: FileState | null - ): Promise> { - if ( - event.type === 'workspace.deleted' && - event.workspace.accountId === input.accountId && - event.workspace.id === input.workspaceId - ) { - return { - hasChanges: true, - result: null, - }; - } - - if ( - event.type === 'file.state.updated' && - event.accountId === input.accountId && - event.workspaceId === input.workspaceId && - event.fileState.id === input.id - ) { - const output = await this.handleQuery(input); - return { - hasChanges: true, - result: output, - }; - } - - if ( - event.type === 'file.state.deleted' && - event.accountId === input.accountId && - event.workspaceId === input.workspaceId && - event.fileId === input.id - ) { - return { - hasChanges: true, - result: null, - }; - } - - if ( - event.type === 'node.deleted' && - event.accountId === input.accountId && - event.workspaceId === input.workspaceId && - event.node.id === input.id - ) { - return { - hasChanges: true, - result: null, - }; - } - - if ( - event.type === 'node.created' && - event.accountId === input.accountId && - event.workspaceId === input.workspaceId && - event.node.id === input.id - ) { - const newOutput = await this.handleQuery(input); - return { - hasChanges: true, - result: newOutput, - }; - } - - return { - hasChanges: false, - }; - } - - private async fetchFileState( - input: FileStateGetQueryInput - ): Promise { - const workspace = this.getWorkspace(input.accountId, input.workspaceId); - - const node = await workspace.database - .selectFrom('nodes') - .selectAll() - .where('id', '=', input.id) - .executeTakeFirst(); - - if (!node) { - return null; - } - - const file = mapNode(node) as LocalFileNode; - const fileState = await workspace.database - .selectFrom('file_states') - .selectAll() - .where('id', '=', input.id) - .executeTakeFirst(); - - if (!fileState) { - return null; - } - - let url: string | null = null; - if (fileState.download_status === DownloadStatus.Completed) { - const filePath = this.app.path.workspaceFile( - input.accountId, - input.workspaceId, - input.id, - file.attributes.extension - ); - - const exists = await this.app.fs.exists(filePath); - if (exists) { - url = await this.app.fs.url(filePath); - } - } - - return mapFileState(fileState, url); - } -} diff --git a/packages/client/src/handlers/queries/files/local-file-get.ts b/packages/client/src/handlers/queries/files/local-file-get.ts new file mode 100644 index 00000000..dd1f69e0 --- /dev/null +++ b/packages/client/src/handlers/queries/files/local-file-get.ts @@ -0,0 +1,152 @@ +import { WorkspaceQueryHandlerBase } from '@colanode/client/handlers/queries/workspace-query-handler-base'; +import { mapDownload, mapLocalFile } from '@colanode/client/lib/mappers'; +import { ChangeCheckResult, QueryHandler } from '@colanode/client/lib/types'; +import { + LocalFileGetQueryInput, + LocalFileGetQueryOutput, +} from '@colanode/client/queries'; +import { Event } from '@colanode/client/types/events'; +import { DownloadType } from '@colanode/client/types/files'; + +export class LocalFileGetQueryHandler + extends WorkspaceQueryHandlerBase + implements QueryHandler +{ + public async handleQuery( + input: LocalFileGetQueryInput + ): Promise { + return await this.fetchLocalFile(input); + } + + public async checkForChanges( + event: Event, + input: LocalFileGetQueryInput, + _: LocalFileGetQueryOutput + ): Promise> { + if ( + event.type === 'workspace.deleted' && + event.workspace.accountId === input.accountId && + event.workspace.id === input.workspaceId + ) { + return { + hasChanges: true, + result: { + localFile: null, + download: null, + }, + }; + } + + if ( + event.type === 'local.file.created' && + event.accountId === input.accountId && + event.workspaceId === input.workspaceId && + event.localFile.id === input.fileId + ) { + const output = await this.handleQuery(input); + return { + hasChanges: true, + result: output, + }; + } + + if ( + event.type === 'local.file.deleted' && + event.accountId === input.accountId && + event.workspaceId === input.workspaceId && + event.localFile.id === input.fileId + ) { + return { + hasChanges: true, + result: { + localFile: null, + download: null, + }, + }; + } + + if ( + event.type === 'node.deleted' && + event.accountId === input.accountId && + event.workspaceId === input.workspaceId && + event.node.id === input.fileId + ) { + return { + hasChanges: true, + result: { + localFile: null, + download: null, + }, + }; + } + + if ( + event.type === 'node.created' && + event.accountId === input.accountId && + event.workspaceId === input.workspaceId && + event.node.id === input.fileId + ) { + const newOutput = await this.handleQuery(input); + return { + hasChanges: true, + result: newOutput, + }; + } + + return { + hasChanges: false, + }; + } + + private async fetchLocalFile( + input: LocalFileGetQueryInput + ): Promise { + const workspace = this.getWorkspace(input.accountId, input.workspaceId); + + const localFile = await workspace.database + .updateTable('local_files') + .returningAll() + .set({ + opened_at: new Date().toISOString(), + }) + .where('id', '=', input.fileId) + .executeTakeFirst(); + + if (localFile) { + const url = await this.app.fs.url(localFile.path); + return { + localFile: mapLocalFile(localFile, url), + download: null, + }; + } + + const download = await workspace.database + .selectFrom('downloads') + .selectAll() + .where('file_id', '=', input.fileId) + .where('type', '=', DownloadType.Auto) + .orderBy('id', 'desc') + .executeTakeFirst(); + + if (download) { + return { + localFile: null, + download: mapDownload(download), + }; + } + + if (input.autoDownload) { + const download = await workspace.files.initAutoDownload(input.fileId); + + return { + localFile: null, + download: download ? mapDownload(download) : null, + }; + } + + return { + localFile: null, + download: null, + }; + } +} diff --git a/packages/client/src/handlers/queries/files/temp-file-get.ts b/packages/client/src/handlers/queries/files/temp-file-get.ts new file mode 100644 index 00000000..49f3e8c1 --- /dev/null +++ b/packages/client/src/handlers/queries/files/temp-file-get.ts @@ -0,0 +1,63 @@ +import { mapTempFile } from '@colanode/client/lib'; +import { ChangeCheckResult, QueryHandler } from '@colanode/client/lib/types'; +import { TempFileGetQueryInput } from '@colanode/client/queries'; +import { AppService } from '@colanode/client/services'; +import { Event } from '@colanode/client/types/events'; +import { TempFile } from '@colanode/client/types/files'; + +export class TempFileGetQueryHandler + implements QueryHandler +{ + private readonly app: AppService; + + public constructor(app: AppService) { + this.app = app; + } + + public async handleQuery( + input: TempFileGetQueryInput + ): Promise { + return await this.fetchTempFile(input); + } + + public async checkForChanges( + event: Event, + input: TempFileGetQueryInput, + _: TempFile | null + ): Promise> { + if (event.type === 'temp.file.created' && event.tempFile.id === input.id) { + return { + hasChanges: true, + result: event.tempFile, + }; + } + + if (event.type === 'temp.file.deleted' && event.tempFile.id === input.id) { + return { + hasChanges: true, + result: null, + }; + } + + return { + hasChanges: false, + }; + } + + private async fetchTempFile( + input: TempFileGetQueryInput + ): Promise { + const tempFile = await this.app.database + .selectFrom('temp_files') + .selectAll() + .where('id', '=', input.id) + .executeTakeFirst(); + + if (!tempFile) { + return null; + } + + const url = await this.app.fs.url(tempFile.path); + return mapTempFile(tempFile, url); + } +} diff --git a/packages/client/src/handlers/queries/files/upload-list-pending.ts b/packages/client/src/handlers/queries/files/upload-list-pending.ts new file mode 100644 index 00000000..1f9f3da2 --- /dev/null +++ b/packages/client/src/handlers/queries/files/upload-list-pending.ts @@ -0,0 +1,157 @@ +import { WorkspaceQueryHandlerBase } from '@colanode/client/handlers/queries/workspace-query-handler-base'; +import { mapUpload } from '@colanode/client/lib'; +import { ChangeCheckResult, QueryHandler } from '@colanode/client/lib/types'; +import { UploadListPendingQueryInput } from '@colanode/client/queries/files/upload-list-pending'; +import { Event } from '@colanode/client/types/events'; +import { Upload, UploadStatus } from '@colanode/client/types/files'; + +export class UploadListPendingQueryHandler + extends WorkspaceQueryHandlerBase + implements QueryHandler +{ + public async handleQuery( + input: UploadListPendingQueryInput + ): Promise { + return await this.fetchPendingUploads(input); + } + + public async checkForChanges( + event: Event, + input: UploadListPendingQueryInput, + output: Upload[] + ): Promise> { + if ( + event.type === 'workspace.deleted' && + event.workspace.accountId === input.accountId && + event.workspace.id === input.workspaceId + ) { + return { + hasChanges: true, + result: [], + }; + } + + if ( + event.type === 'upload.created' && + event.accountId === input.accountId && + event.workspaceId === input.workspaceId && + event.upload.status === UploadStatus.Pending + ) { + const newResult = await this.fetchPendingUploads(input); + return { + hasChanges: true, + result: newResult, + }; + } + + if ( + event.type === 'upload.updated' && + event.accountId === input.accountId && + event.workspaceId === input.workspaceId + ) { + const upload = output.find( + (upload) => upload.fileId === event.upload.fileId + ); + + if (!upload) { + return { + hasChanges: false, + }; + } + + if ( + upload.status === UploadStatus.Pending && + event.upload.status === UploadStatus.Uploading + ) { + const newResult = output.map((upload) => { + if (upload.fileId === event.upload.fileId) { + return event.upload; + } + + return upload; + }); + + return { + hasChanges: true, + result: newResult, + }; + } else if ( + upload.status === UploadStatus.Uploading && + event.upload.status === UploadStatus.Pending + ) { + const newResult = output.map((upload) => { + if (upload.fileId === event.upload.fileId) { + return event.upload; + } + + return upload; + }); + + return { + hasChanges: true, + result: newResult, + }; + } else { + const newResult = await this.fetchPendingUploads(input); + return { + hasChanges: true, + result: newResult, + }; + } + } + + if ( + event.type === 'upload.deleted' && + event.accountId === input.accountId && + event.workspaceId === input.workspaceId + ) { + const upload = output.find( + (upload) => upload.fileId === event.upload.fileId + ); + + if (!upload) { + return { + hasChanges: false, + }; + } + + if (output.length === input.count) { + const newResult = await this.fetchPendingUploads(input); + return { + hasChanges: true, + result: newResult, + }; + } + + const newOutput = output.filter( + (upload) => upload.fileId !== event.upload.fileId + ); + return { + hasChanges: true, + result: newOutput, + }; + } + + return { + hasChanges: false, + }; + } + + private async fetchPendingUploads( + input: UploadListPendingQueryInput + ): Promise { + const workspace = this.getWorkspace(input.accountId, input.workspaceId); + + const offset = (input.page - 1) * input.count; + const uploads = await workspace.database + .selectFrom('uploads') + .selectAll() + .where('status', 'in', [UploadStatus.Pending, UploadStatus.Uploading]) + .orderBy('file_id', 'desc') + .limit(input.count) + .offset(offset) + .execute(); + + return uploads.map(mapUpload); + } +} diff --git a/packages/client/src/handlers/queries/files/upload-list.ts b/packages/client/src/handlers/queries/files/upload-list.ts new file mode 100644 index 00000000..d7fc97ac --- /dev/null +++ b/packages/client/src/handlers/queries/files/upload-list.ts @@ -0,0 +1,120 @@ +import { WorkspaceQueryHandlerBase } from '@colanode/client/handlers/queries/workspace-query-handler-base'; +import { mapUpload } from '@colanode/client/lib'; +import { ChangeCheckResult, QueryHandler } from '@colanode/client/lib/types'; +import { UploadListQueryInput } from '@colanode/client/queries/files/upload-list'; +import { Event } from '@colanode/client/types/events'; +import { Upload } from '@colanode/client/types/files'; + +export class UploadListQueryHandler + extends WorkspaceQueryHandlerBase + implements QueryHandler +{ + public async handleQuery(input: UploadListQueryInput): Promise { + return await this.fetchUploads(input); + } + + public async checkForChanges( + event: Event, + input: UploadListQueryInput, + output: Upload[] + ): Promise> { + if ( + event.type === 'workspace.deleted' && + event.workspace.accountId === input.accountId && + event.workspace.id === input.workspaceId + ) { + return { + hasChanges: true, + result: [], + }; + } + + if ( + event.type === 'upload.created' && + event.accountId === input.accountId && + event.workspaceId === input.workspaceId + ) { + const newResult = await this.fetchUploads(input); + return { + hasChanges: true, + result: newResult, + }; + } + + if ( + event.type === 'upload.updated' && + event.accountId === input.accountId && + event.workspaceId === input.workspaceId + ) { + const upload = output.find( + (upload) => upload.fileId === event.upload.fileId + ); + + if (upload) { + const newResult = output.map((upload) => { + if (upload.fileId === event.upload.fileId) { + return event.upload; + } + + return upload; + }); + + return { + hasChanges: true, + result: newResult, + }; + } + } + + if ( + event.type === 'upload.deleted' && + event.accountId === input.accountId && + event.workspaceId === input.workspaceId + ) { + const upload = output.find( + (upload) => upload.fileId === event.upload.fileId + ); + + if (!upload) { + return { + hasChanges: false, + }; + } + + if (output.length === input.count) { + const newResult = await this.fetchUploads(input); + return { + hasChanges: true, + result: newResult, + }; + } + + const newOutput = output.filter( + (upload) => upload.fileId !== event.upload.fileId + ); + return { + hasChanges: true, + result: newOutput, + }; + } + + return { + hasChanges: false, + }; + } + + private async fetchUploads(input: UploadListQueryInput): Promise { + const workspace = this.getWorkspace(input.accountId, input.workspaceId); + + const offset = (input.page - 1) * input.count; + const uploads = await workspace.database + .selectFrom('uploads') + .selectAll() + .orderBy('file_id', 'desc') + .limit(input.count) + .offset(offset) + .execute(); + + return uploads.map(mapUpload); + } +} diff --git a/packages/client/src/handlers/queries/icons/icon-category-list.ts b/packages/client/src/handlers/queries/icons/icon-category-list.ts index 39e36044..b5a8de8b 100644 --- a/packages/client/src/handlers/queries/icons/icon-category-list.ts +++ b/packages/client/src/handlers/queries/icons/icon-category-list.ts @@ -16,11 +16,11 @@ export class IconCategoryListQueryHandler public async handleQuery( _: IconCategoryListQueryInput ): Promise { - if (!this.app.asset.icons) { + if (!this.app.assets.icons) { return []; } - const data = this.app.asset.icons + const data = this.app.assets.icons .selectFrom('categories') .selectAll() .execute(); diff --git a/packages/client/src/handlers/queries/icons/icon-list.ts b/packages/client/src/handlers/queries/icons/icon-list.ts index 1bc1a5df..98ea00bc 100644 --- a/packages/client/src/handlers/queries/icons/icon-list.ts +++ b/packages/client/src/handlers/queries/icons/icon-list.ts @@ -13,12 +13,12 @@ export class IconListQueryHandler implements QueryHandler { } public async handleQuery(input: IconListQueryInput): Promise { - if (!this.app.asset.icons) { + if (!this.app.assets.icons) { return []; } const offset = input.page * input.count; - const data = await this.app.asset.icons + const data = await this.app.assets.icons .selectFrom('icons') .selectAll() .where('category_id', '=', input.category) diff --git a/packages/client/src/handlers/queries/icons/icon-search.ts b/packages/client/src/handlers/queries/icons/icon-search.ts index 74c6105b..564a2485 100644 --- a/packages/client/src/handlers/queries/icons/icon-search.ts +++ b/packages/client/src/handlers/queries/icons/icon-search.ts @@ -15,11 +15,11 @@ export class IconSearchQueryHandler } public async handleQuery(input: IconSearchQueryInput): Promise { - if (!this.app.asset.icons) { + if (!this.app.assets.icons) { return []; } - const data = await this.app.asset.icons + const data = await this.app.assets.icons .selectFrom('icons') .innerJoin('icon_search', 'icons.id', 'icon_search.id') .selectAll('icons') diff --git a/packages/client/src/handlers/queries/index.ts b/packages/client/src/handlers/queries/index.ts index 5e3389b0..c4668abc 100644 --- a/packages/client/src/handlers/queries/index.ts +++ b/packages/client/src/handlers/queries/index.ts @@ -6,7 +6,7 @@ import { AccountGetQueryHandler } from './accounts/account-get'; import { AccountMetadataListQueryHandler } from './accounts/account-metadata-list'; import { AccountListQueryHandler } from './accounts/accounts-list'; import { AppMetadataListQueryHandler } from './apps/app-metadata-list'; -import { AvatarUrlGetQueryHandler } from './avatars/avatar-url-get'; +import { AvatarGetQueryHandler } from './avatars/avatar-get'; import { ChatListQueryHandler } from './chats/chat-list'; import { DatabaseListQueryHandler } from './databases/database-list'; import { DatabaseViewListQueryHandler } from './databases/database-view-list'; @@ -18,10 +18,13 @@ import { EmojiGetQueryHandler } from './emojis/emoji-get'; import { EmojiGetBySkinIdQueryHandler } from './emojis/emoji-get-by-skin-id'; import { EmojiListQueryHandler } from './emojis/emoji-list'; import { EmojiSearchQueryHandler } from './emojis/emoji-search'; +import { DownloadListManualQueryHandler } from './files/download-list-manual'; import { FileDownloadRequestGetQueryHandler } from './files/file-download-request-get'; import { FileListQueryHandler } from './files/file-list'; -import { FileSaveListQueryHandler } from './files/file-save-list'; -import { FileStateGetQueryHandler } from './files/file-state-get'; +import { LocalFileGetQueryHandler } from './files/local-file-get'; +import { TempFileGetQueryHandler } from './files/temp-file-get'; +import { UploadListQueryHandler } from './files/upload-list'; +import { UploadListPendingQueryHandler } from './files/upload-list-pending'; import { IconCategoryListQueryHandler } from './icons/icon-category-list'; import { IconListQueryHandler } from './icons/icon-list'; import { IconSearchQueryHandler } from './icons/icon-search'; @@ -53,7 +56,7 @@ export type QueryHandlerMap = { export const buildQueryHandlerMap = (app: AppService): QueryHandlerMap => { return { 'app.metadata.list': new AppMetadataListQueryHandler(app), - 'avatar.url.get': new AvatarUrlGetQueryHandler(app), + 'avatar.get': new AvatarGetQueryHandler(app), 'account.list': new AccountListQueryHandler(app), 'message.list': new MessageListQueryHandler(app), 'node.reaction.list': new NodeReactionsListQueryHandler(app), @@ -84,9 +87,8 @@ export const buildQueryHandlerMap = (app: AppService): QueryHandlerMap => { 'record.search': new RecordSearchQueryHandler(app), 'user.get': new UserGetQueryHandler(app), 'user.storage.get': new UserStorageGetQueryHandler(app), - 'file.state.get': new FileStateGetQueryHandler(app), + 'local.file.get': new LocalFileGetQueryHandler(app), 'file.download.request.get': new FileDownloadRequestGetQueryHandler(app), - 'file.save.list': new FileSaveListQueryHandler(app), 'chat.list': new ChatListQueryHandler(app), 'space.list': new SpaceListQueryHandler(app), 'workspace.metadata.list': new WorkspaceMetadataListQueryHandler(app), @@ -95,5 +97,9 @@ export const buildQueryHandlerMap = (app: AppService): QueryHandlerMap => { 'document.updates.list': new DocumentUpdatesListQueryHandler(app), 'account.metadata.list': new AccountMetadataListQueryHandler(app), 'workspace.storage.get': new WorkspaceStorageGetQueryHandler(app), + 'upload.list': new UploadListQueryHandler(app), + 'upload.list.pending': new UploadListPendingQueryHandler(app), + 'download.list.manual': new DownloadListManualQueryHandler(app), + 'temp.file.get': new TempFileGetQueryHandler(app), }; }; diff --git a/packages/client/src/jobs/account-sync.ts b/packages/client/src/jobs/account-sync.ts new file mode 100644 index 00000000..533942f0 --- /dev/null +++ b/packages/client/src/jobs/account-sync.ts @@ -0,0 +1,46 @@ +import { + JobHandler, + JobOutput, + JobConcurrencyConfig, +} from '@colanode/client/jobs'; +import { AppService } from '@colanode/client/services/app-service'; + +export type AccountSyncInput = { + type: 'account.sync'; + accountId: string; +}; + +declare module '@colanode/client/jobs' { + interface JobMap { + 'account.sync': { + input: AccountSyncInput; + }; + } +} + +export class AccountSyncJobHandler implements JobHandler { + private readonly app: AppService; + + constructor(app: AppService) { + this.app = app; + } + + public readonly concurrency: JobConcurrencyConfig = { + limit: 1, + key: (input: AccountSyncInput) => `account.sync.${input.accountId}`, + }; + + public async handleJob(input: AccountSyncInput): Promise { + const account = this.app.getAccount(input.accountId); + if (!account) { + return { + type: 'cancel', + }; + } + + await account.sync(); + return { + type: 'success', + }; + } +} diff --git a/packages/client/src/jobs/avatar-download.ts b/packages/client/src/jobs/avatar-download.ts new file mode 100644 index 00000000..718509eb --- /dev/null +++ b/packages/client/src/jobs/avatar-download.ts @@ -0,0 +1,64 @@ +import ms from 'ms'; + +import { + JobHandler, + JobOutput, + JobConcurrencyConfig, +} from '@colanode/client/jobs'; +import { AppService } from '@colanode/client/services/app-service'; + +export type AvatarDownloadInput = { + type: 'avatar.download'; + accountId: string; + avatar: string; +}; + +declare module '@colanode/client/jobs' { + interface JobMap { + 'avatar.download': { + input: AvatarDownloadInput; + }; + } +} + +export class AvatarDownloadJobHandler + implements JobHandler +{ + private readonly app: AppService; + + constructor(app: AppService) { + this.app = app; + } + + public readonly concurrency: JobConcurrencyConfig = { + limit: 1, + key: (input: AvatarDownloadInput) => `avatar.download.${input.avatar}`, + }; + + public async handleJob(input: AvatarDownloadInput): Promise { + const account = this.app.getAccount(input.accountId); + if (!account) { + return { + type: 'cancel', + }; + } + + const result = await account.avatars.downloadAvatar(input.avatar); + if (result === null) { + return { + type: 'retry', + delay: ms('1 minute'), + }; + } + + if (!result) { + return { + type: 'cancel', + }; + } + + return { + type: 'success', + }; + } +} diff --git a/packages/client/src/jobs/avatars-clean.ts b/packages/client/src/jobs/avatars-clean.ts new file mode 100644 index 00000000..5abbe6bc --- /dev/null +++ b/packages/client/src/jobs/avatars-clean.ts @@ -0,0 +1,46 @@ +import { + JobHandler, + JobOutput, + JobConcurrencyConfig, +} from '@colanode/client/jobs'; +import { AppService } from '@colanode/client/services/app-service'; + +export type AvatarsCleanInput = { + type: 'avatars.clean'; + accountId: string; +}; + +declare module '@colanode/client/jobs' { + interface JobMap { + 'avatars.clean': { + input: AvatarsCleanInput; + }; + } +} + +export class AvatarsCleanJobHandler implements JobHandler { + private readonly app: AppService; + + constructor(app: AppService) { + this.app = app; + } + + public readonly concurrency: JobConcurrencyConfig = { + limit: 1, + key: (input: AvatarsCleanInput) => `avatars.clean.${input.accountId}`, + }; + + public async handleJob(input: AvatarsCleanInput): Promise { + const account = this.app.getAccount(input.accountId); + if (!account) { + return { + type: 'cancel', + }; + } + + await account.avatars.cleanupAvatars(); + return { + type: 'success', + }; + } +} diff --git a/packages/client/src/jobs/file-download.ts b/packages/client/src/jobs/file-download.ts new file mode 100644 index 00000000..1640b180 --- /dev/null +++ b/packages/client/src/jobs/file-download.ts @@ -0,0 +1,264 @@ +import ms from 'ms'; + +import { SelectDownload, UpdateDownload } from '@colanode/client/databases'; +import { + JobHandler, + JobOutput, + JobConcurrencyConfig, +} from '@colanode/client/jobs'; +import { + eventBus, + mapDownload, + mapLocalFile, + mapNode, +} from '@colanode/client/lib'; +import { AppService } from '@colanode/client/services/app-service'; +import { WorkspaceService } from '@colanode/client/services/workspaces/workspace-service'; +import { DownloadStatus, LocalFileNode } from '@colanode/client/types'; +import { FileStatus } from '@colanode/core'; + +export type FileDownloadInput = { + type: 'file.download'; + accountId: string; + workspaceId: string; + downloadId: string; +}; + +declare module '@colanode/client/jobs' { + interface JobMap { + 'file.download': { + input: FileDownloadInput; + }; + } +} + +const DOWNLOAD_RETRIES_LIMIT = 10; + +export class FileDownloadJobHandler implements JobHandler { + private readonly app: AppService; + + constructor(app: AppService) { + this.app = app; + } + + public readonly concurrency: JobConcurrencyConfig = { + limit: 1, + key: (input: FileDownloadInput) => `file.download.${input.downloadId}`, + }; + + public async handleJob(input: FileDownloadInput): Promise { + const account = this.app.getAccount(input.accountId); + if (!account) { + return { + type: 'cancel', + }; + } + + if (!account.server.isAvailable) { + return { + type: 'retry', + delay: ms('5 seconds'), + }; + } + + const workspace = account.getWorkspace(input.workspaceId); + if (!workspace) { + return { + type: 'cancel', + }; + } + + const download = await this.fetchDownload(workspace, input.downloadId); + if (!download) { + return { + type: 'cancel', + }; + } + + const file = await this.fetchNode(workspace, download.file_id); + if (!file) { + return { + type: 'cancel', + }; + } + + if (file.attributes.status === FileStatus.Pending) { + return { + type: 'retry', + delay: ms('10 seconds'), + }; + } + + return this.performDownload(workspace, download, file); + } + + private async performDownload( + workspace: WorkspaceService, + download: SelectDownload, + file: LocalFileNode + ): Promise { + try { + await this.updateDownload(workspace, download.id, { + status: DownloadStatus.Downloading, + started_at: new Date().toISOString(), + }); + + const response = await workspace.account.client.get( + `v1/workspaces/${workspace.id}/files/${file.id}`, + { + onDownloadProgress: async (progress, _chunk) => { + const percent = Math.round((progress.percent || 0) * 100); + await this.updateDownload(workspace, download.id, { + progress: percent, + }); + }, + } + ); + + const writeStream = await this.app.fs.writeStream(download.path); + await response.body?.pipeTo(writeStream); + + const createdLocalFile = await workspace.database + .insertInto('local_files') + .returningAll() + .values({ + id: file.id, + version: file.attributes.version, + name: file.attributes.name, + extension: file.attributes.extension, + subtype: file.attributes.subtype, + mime_type: file.attributes.mimeType, + size: file.attributes.size, + created_at: new Date().toISOString(), + path: download.path, + opened_at: new Date().toISOString(), + }) + .onConflict((oc) => + oc.column('id').doUpdateSet({ + version: file.attributes.version, + name: file.attributes.name, + mime_type: file.attributes.mimeType, + size: file.attributes.size, + path: download.path, + }) + ) + .executeTakeFirst(); + + if (!createdLocalFile) { + await this.updateDownload(workspace, download.id, { + status: DownloadStatus.Pending, + retries: download.retries + 1, + error_code: 'file_download_failed', + error_message: 'Failed to create local file', + }); + + return { + type: 'retry', + delay: ms('10 seconds'), + }; + } + + const url = await this.app.fs.url(createdLocalFile.path); + eventBus.publish({ + type: 'local.file.created', + accountId: workspace.accountId, + workspaceId: workspace.id, + localFile: mapLocalFile(createdLocalFile, url), + }); + + await this.updateDownload(workspace, download.id, { + status: DownloadStatus.Completed, + completed_at: new Date().toISOString(), + progress: 100, + error_code: null, + error_message: null, + }); + + return { + type: 'success', + }; + } catch { + const newRetries = download.retries + 1; + + if (newRetries >= DOWNLOAD_RETRIES_LIMIT) { + await this.updateDownload(workspace, download.id, { + status: DownloadStatus.Failed, + completed_at: new Date().toISOString(), + progress: 0, + error_code: 'file_download_failed', + error_message: + 'Failed to download file after ' + newRetries + ' retries', + }); + + return { + type: 'cancel', + }; + } + + await this.updateDownload(workspace, download.id, { + status: DownloadStatus.Pending, + retries: newRetries, + started_at: new Date().toISOString(), + error_code: null, + error_message: null, + }); + + return { + type: 'retry', + delay: ms('1 minute'), + }; + } + } + + private async fetchDownload( + workspace: WorkspaceService, + downloadId: string + ): Promise { + return workspace.database + .selectFrom('downloads') + .selectAll() + .where('id', '=', downloadId) + .executeTakeFirst(); + } + + private async fetchNode( + workspace: WorkspaceService, + fileId: string + ): Promise { + const node = await workspace.database + .selectFrom('nodes') + .selectAll() + .where('id', '=', fileId) + .executeTakeFirstOrThrow(); + + if (!node) { + return undefined; + } + + return mapNode(node) as LocalFileNode; + } + + private async updateDownload( + workspace: WorkspaceService, + downloadId: string, + values: UpdateDownload + ): Promise { + const updatedDownload = await workspace.database + .updateTable('downloads') + .returningAll() + .set(values) + .where('id', '=', downloadId) + .executeTakeFirst(); + + if (!updatedDownload) { + return; + } + + eventBus.publish({ + type: 'download.updated', + accountId: workspace.accountId, + workspaceId: workspace.id, + download: mapDownload(updatedDownload), + }); + } +} diff --git a/packages/client/src/jobs/file-upload.ts b/packages/client/src/jobs/file-upload.ts new file mode 100644 index 00000000..11ff064a --- /dev/null +++ b/packages/client/src/jobs/file-upload.ts @@ -0,0 +1,288 @@ +import ms from 'ms'; +import { Upload } from 'tus-js-client'; + +import { + SelectLocalFile, + SelectUpload, + UpdateUpload, +} from '@colanode/client/databases'; +import { + JobHandler, + JobOutput, + JobConcurrencyConfig, +} from '@colanode/client/jobs'; +import { eventBus, mapNode, mapUpload } from '@colanode/client/lib'; +import { isNodeSynced } from '@colanode/client/lib/nodes'; +import { AccountService } from '@colanode/client/services/accounts/account-service'; +import { AppService } from '@colanode/client/services/app-service'; +import { WorkspaceService } from '@colanode/client/services/workspaces/workspace-service'; +import { LocalFileNode, UploadStatus } from '@colanode/client/types'; +import { ApiHeader, build, calculatePercentage } from '@colanode/core'; + +export type FileUploadInput = { + type: 'file.upload'; + accountId: string; + workspaceId: string; + fileId: string; +}; + +declare module '@colanode/client/jobs' { + interface JobMap { + 'file.upload': { + input: FileUploadInput; + }; + } +} + +const UPLOAD_RETRIES_LIMIT = 10; + +export class FileUploadJobHandler implements JobHandler { + private readonly app: AppService; + + constructor(app: AppService) { + this.app = app; + } + + public readonly concurrency: JobConcurrencyConfig = { + limit: 1, + key: (input: FileUploadInput) => `file.upload.${input.fileId}`, + }; + + public async handleJob(input: FileUploadInput): Promise { + const account = this.app.getAccount(input.accountId); + if (!account) { + return { + type: 'cancel', + }; + } + + if (!account.server.isAvailable) { + return { + type: 'retry', + delay: ms('2 seconds'), + }; + } + + const workspace = account.getWorkspace(input.workspaceId); + if (!workspace) { + return { + type: 'cancel', + }; + } + + const upload = await this.fetchUpload(workspace, input.fileId); + if (!upload) { + return { + type: 'cancel', + }; + } + + const file = await this.fetchNode(workspace, upload.file_id); + if (!file) { + return { + type: 'cancel', + }; + } + + const localFile = await this.fetchLocalFile(workspace, file.id); + if (!localFile) { + return { + type: 'cancel', + }; + } + + if (!isNodeSynced(file)) { + return { + type: 'retry', + delay: ms('2 seconds'), + }; + } + + return this.performUpload(account, workspace, upload, file, localFile); + } + + private async performUpload( + account: AccountService, + workspace: WorkspaceService, + upload: SelectUpload, + file: LocalFileNode, + localFile: SelectLocalFile + ): Promise { + try { + await this.updateUpload(workspace, upload.file_id, { + status: UploadStatus.Uploading, + started_at: new Date().toISOString(), + }); + + const updateUpload = async (values: UpdateUpload) => { + await this.updateUpload(workspace, upload.file_id, { + ...values, + }); + }; + + const fileStream = await this.app.fs.readStream(localFile.path); + await new Promise((resolve, reject) => { + const tusUpload = new Upload(fileStream, { + endpoint: `${account.server.httpBaseUrl}/v1/workspaces/${workspace.id}/files/${file.id}/tus`, + retryDelays: [ + 0, + ms('3 seconds'), + ms('5 seconds'), + ms('10 seconds'), + ms('20 seconds'), + ], + metadata: { + filename: localFile.name, + filetype: file.type, + }, + headers: { + Authorization: `Bearer ${account.token}`, + [ApiHeader.ClientType]: this.app.meta.type, + [ApiHeader.ClientPlatform]: this.app.meta.platform, + [ApiHeader.ClientVersion]: build.version, + }, + onError: function (error) { + updateUpload({ + status: UploadStatus.Failed, + completed_at: new Date().toISOString(), + progress: 0, + error_code: 'file_upload_failed', + error_message: error.message, + }); + reject(error); + }, + onProgress: function (bytesUploaded, bytesTotal) { + const percentage = calculatePercentage(bytesUploaded, bytesTotal); + updateUpload({ + progress: percentage, + }); + }, + onSuccess: function () { + updateUpload({ + status: UploadStatus.Completed, + progress: 100, + completed_at: new Date().toISOString(), + error_code: null, + error_message: null, + }); + resolve(); + }, + }); + + tusUpload.findPreviousUploads().then((previousUploads) => { + const previousUpload = previousUploads[0]; + if (previousUpload) { + tusUpload.resumeFromPreviousUpload(previousUpload); + } else { + tusUpload.start(); + } + }); + }); + + await this.updateUpload(workspace, upload.file_id, { + status: UploadStatus.Completed, + progress: 100, + completed_at: new Date().toISOString(), + error_code: null, + error_message: null, + }); + + return { + type: 'success', + }; + } catch { + const newRetries = upload.retries + 1; + + if (newRetries >= UPLOAD_RETRIES_LIMIT) { + await this.updateUpload(workspace, upload.file_id, { + status: UploadStatus.Failed, + completed_at: new Date().toISOString(), + progress: 0, + error_code: 'file_upload_failed', + error_message: + 'Failed to upload file after ' + newRetries + ' retries', + }); + + return { + type: 'cancel', + }; + } + + await this.updateUpload(workspace, upload.file_id, { + status: UploadStatus.Pending, + retries: newRetries, + started_at: new Date().toISOString(), + error_code: null, + error_message: null, + }); + + return { + type: 'retry', + delay: ms('1 minute'), + }; + } + } + + private async fetchUpload( + workspace: WorkspaceService, + fileId: string + ): Promise { + return workspace.database + .selectFrom('uploads') + .selectAll() + .where('file_id', '=', fileId) + .executeTakeFirst(); + } + + private async fetchNode( + workspace: WorkspaceService, + fileId: string + ): Promise { + const node = await workspace.database + .selectFrom('nodes') + .selectAll() + .where('id', '=', fileId) + .executeTakeFirstOrThrow(); + + if (!node) { + return undefined; + } + + return mapNode(node) as LocalFileNode; + } + + private async fetchLocalFile( + workspace: WorkspaceService, + fileId: string + ): Promise { + return workspace.database + .selectFrom('local_files') + .selectAll() + .where('id', '=', fileId) + .executeTakeFirstOrThrow(); + } + + private async updateUpload( + workspace: WorkspaceService, + fileId: string, + values: UpdateUpload + ): Promise { + const updatedUpload = await workspace.database + .updateTable('uploads') + .returningAll() + .set(values) + .where('file_id', '=', fileId) + .executeTakeFirst(); + + if (!updatedUpload) { + return; + } + + eventBus.publish({ + type: 'upload.updated', + accountId: workspace.accountId, + workspaceId: workspace.id, + upload: mapUpload(updatedUpload), + }); + } +} diff --git a/packages/client/src/jobs/index.ts b/packages/client/src/jobs/index.ts new file mode 100644 index 00000000..177c575a --- /dev/null +++ b/packages/client/src/jobs/index.ts @@ -0,0 +1,91 @@ +export enum JobStatus { + Waiting = 1, + Active = 2, +} + +export enum JobScheduleStatus { + Active = 1, + Paused = 2, +} + +export type JobDeduplicationOptions = { + key: string; + replace?: boolean; +}; + +export type JobOptions = { + retries?: number; + delay?: number; + deduplication?: JobDeduplicationOptions; +}; + +export type JobScheduleOptions = { + retries?: number; + deduplication?: JobDeduplicationOptions; +}; + +export type Job = { + id: string; + input: JobInput; + options: JobOptions; + status: JobStatus; + retries: number; + queue: string; + deduplicationKey?: string; + concurrencyKey?: string; + createdAt: string; + updatedAt: string; + scheduledAt: string; +}; + +export type JobSchedule = { + id: string; + input: JobInput; + options: JobScheduleOptions; + status: JobScheduleStatus; + interval: number; + nextRunAt: string; + lastRunAt?: string; + queue: string; + createdAt: string; + updatedAt: string; +}; + +export type JobManagerOptions = { + concurrency?: number; + interval?: number; +}; + +export type JobConcurrencyConfig = { + limit: number; + key: (input: T) => string; +}; + +// eslint-disable-next-line @typescript-eslint/no-empty-object-type +export interface JobMap {} + +export type JobInput = JobMap[keyof JobMap]['input']; + +export type JobSuccessOutput = { + type: 'success'; +}; + +export type JobRetryOutput = { + type: 'retry'; + delay: number; +}; + +export type JobCancelOutput = { + type: 'cancel'; +}; + +export type JobOutput = JobSuccessOutput | JobRetryOutput | JobCancelOutput; + +export interface JobHandler { + handleJob: (input: T) => Promise; + concurrency?: JobConcurrencyConfig; +} + +export type JobHandlerMap = { + [K in keyof JobMap]: JobHandler; +}; diff --git a/packages/client/src/jobs/mutations-sync.ts b/packages/client/src/jobs/mutations-sync.ts new file mode 100644 index 00000000..086dc14b --- /dev/null +++ b/packages/client/src/jobs/mutations-sync.ts @@ -0,0 +1,63 @@ +import ms from 'ms'; + +import { + JobHandler, + JobOutput, + JobConcurrencyConfig, +} from '@colanode/client/jobs'; +import { AppService } from '@colanode/client/services/app-service'; + +export type MutationsSyncInput = { + type: 'mutations.sync'; + accountId: string; + workspaceId: string; +}; + +declare module '@colanode/client/jobs' { + interface JobMap { + 'mutations.sync': { + input: MutationsSyncInput; + }; + } +} + +export class MutationsSyncJobHandler implements JobHandler { + private readonly app: AppService; + + constructor(app: AppService) { + this.app = app; + } + + public readonly concurrency: JobConcurrencyConfig = { + limit: 1, + key: (input: MutationsSyncInput) => `mutations.sync.${input.workspaceId}`, + }; + + public async handleJob(input: MutationsSyncInput): Promise { + const account = this.app.getAccount(input.accountId); + if (!account) { + return { + type: 'cancel', + }; + } + + if (!account.server.isAvailable) { + return { + type: 'retry', + delay: ms('1 minute'), + }; + } + + const workspace = account.getWorkspace(input.workspaceId); + if (!workspace) { + return { + type: 'cancel', + }; + } + + await workspace.mutations.sync(); + return { + type: 'success', + }; + } +} diff --git a/packages/client/src/jobs/server-sync.ts b/packages/client/src/jobs/server-sync.ts new file mode 100644 index 00000000..c6772046 --- /dev/null +++ b/packages/client/src/jobs/server-sync.ts @@ -0,0 +1,46 @@ +import { + JobHandler, + JobOutput, + JobConcurrencyConfig, +} from '@colanode/client/jobs'; +import { AppService } from '@colanode/client/services/app-service'; + +export type ServerSyncInput = { + type: 'server.sync'; + server: string; +}; + +declare module '@colanode/client/jobs' { + interface JobMap { + 'server.sync': { + input: ServerSyncInput; + }; + } +} + +export class ServerSyncJobHandler implements JobHandler { + private readonly app: AppService; + + constructor(app: AppService) { + this.app = app; + } + + public readonly concurrency: JobConcurrencyConfig = { + limit: 1, + key: (input: ServerSyncInput) => `server.sync.${input.server}`, + }; + + public async handleJob(input: ServerSyncInput): Promise { + const server = this.app.getServer(input.server); + if (!server) { + return { + type: 'cancel', + }; + } + + await server.sync(); + return { + type: 'success', + }; + } +} diff --git a/packages/client/src/jobs/temp-files-clean.ts b/packages/client/src/jobs/temp-files-clean.ts new file mode 100644 index 00000000..55cce689 --- /dev/null +++ b/packages/client/src/jobs/temp-files-clean.ts @@ -0,0 +1,64 @@ +import ms from 'ms'; + +import { + JobHandler, + JobOutput, + JobConcurrencyConfig, +} from '@colanode/client/jobs'; +import { AppService } from '@colanode/client/services/app-service'; + +export type TempFilesCleanInput = { + type: 'temp.files.clean'; +}; + +declare module '@colanode/client/jobs' { + interface JobMap { + 'temp.files.clean': { + input: TempFilesCleanInput; + }; + } +} + +export class TempFilesCleanJobHandler + implements JobHandler +{ + private readonly app: AppService; + + public readonly concurrency: JobConcurrencyConfig = { + limit: 1, + key: () => `temp.files.clean`, + }; + + constructor(app: AppService) { + this.app = app; + } + + public async handleJob(): Promise { + const exists = await this.app.fs.exists(this.app.path.temp); + if (!exists) { + return { + type: 'success', + }; + } + + const oneDayAgo = new Date(Date.now() - ms('1 day')).toISOString(); + const tempFiles = await this.app.database + .selectFrom('temp_files') + .selectAll() + .where('created_at', '<', oneDayAgo) + .execute(); + + for (const tempFile of tempFiles) { + await this.app.fs.delete(tempFile.path); + + await this.app.database + .deleteFrom('temp_files') + .where('id', '=', tempFile.id) + .execute(); + } + + return { + type: 'success', + }; + } +} diff --git a/packages/client/src/jobs/token-delete.ts b/packages/client/src/jobs/token-delete.ts new file mode 100644 index 00000000..ac2079e7 --- /dev/null +++ b/packages/client/src/jobs/token-delete.ts @@ -0,0 +1,82 @@ +import { sha256 } from 'js-sha256'; +import ms from 'ms'; + +import { + JobHandler, + JobOutput, + JobConcurrencyConfig, +} from '@colanode/client/jobs'; +import { parseApiError } from '@colanode/client/lib/ky'; +import { AppService } from '@colanode/client/services/app-service'; +import { ApiErrorCode } from '@colanode/core'; + +export type TokenDeleteInput = { + type: 'token.delete'; + token: string; + server: string; +}; + +declare module '@colanode/client/jobs' { + interface JobMap { + 'token.delete': { + input: TokenDeleteInput; + }; + } +} + +export class TokenDeleteJobHandler implements JobHandler { + private readonly app: AppService; + + public readonly concurrency: JobConcurrencyConfig = { + limit: 1, + key: (input: TokenDeleteInput) => `token.delete.${sha256(input.token)}`, + }; + + constructor(app: AppService) { + this.app = app; + } + + public async handleJob(input: TokenDeleteInput): Promise { + const server = this.app.getServer(input.server); + if (!server) { + return { + type: 'cancel', + }; + } + + if (!server.isAvailable) { + return { + type: 'retry', + delay: ms('1 minute'), + }; + } + + try { + await this.app.client.delete(`${server.httpBaseUrl}/v1/accounts/logout`, { + headers: { + Authorization: `Bearer ${input.token}`, + }, + }); + + return { + type: 'success', + }; + } catch (error) { + const parsedError = await parseApiError(error); + if ( + parsedError.code === ApiErrorCode.TokenInvalid || + parsedError.code === ApiErrorCode.AccountNotFound || + parsedError.code === ApiErrorCode.DeviceNotFound + ) { + return { + type: 'cancel', + }; + } + + return { + type: 'retry', + delay: ms('1 minute'), + }; + } + } +} diff --git a/packages/client/src/jobs/workspace-files-clean.ts b/packages/client/src/jobs/workspace-files-clean.ts new file mode 100644 index 00000000..889f981b --- /dev/null +++ b/packages/client/src/jobs/workspace-files-clean.ts @@ -0,0 +1,58 @@ +import { + JobHandler, + JobOutput, + JobConcurrencyConfig, +} from '@colanode/client/jobs'; +import { AppService } from '@colanode/client/services/app-service'; + +export type WorkspaceFilesCleanInput = { + type: 'workspace.files.clean'; + accountId: string; + workspaceId: string; +}; + +declare module '@colanode/client/jobs' { + interface JobMap { + 'workspace.files.clean': { + input: WorkspaceFilesCleanInput; + }; + } +} + +export class WorkspaceFilesCleanJobHandler + implements JobHandler +{ + private readonly app: AppService; + + constructor(app: AppService) { + this.app = app; + } + + public readonly concurrency: JobConcurrencyConfig = + { + limit: 1, + key: (input: WorkspaceFilesCleanInput) => + `workspace.files.clean.${input.accountId}.${input.workspaceId}`, + }; + + public async handleJob(input: WorkspaceFilesCleanInput): Promise { + const account = this.app.getAccount(input.accountId); + if (!account) { + return { + type: 'cancel', + }; + } + + const workspace = account.getWorkspace(input.workspaceId); + if (!workspace) { + return { + type: 'cancel', + }; + } + + await workspace.files.cleanupFiles(); + return { + type: 'success', + }; + } +} diff --git a/packages/client/src/lib/mappers.ts b/packages/client/src/lib/mappers.ts index 531da23c..ba713cbd 100644 --- a/packages/client/src/lib/mappers.ts +++ b/packages/client/src/lib/mappers.ts @@ -1,16 +1,17 @@ import { SelectAccountMetadata, + SelectAvatar, SelectWorkspace, } from '@colanode/client/databases/account'; import { SelectAccount, SelectAppMetadata, SelectServer, + SelectTempFile, } from '@colanode/client/databases/app'; import { SelectEmoji } from '@colanode/client/databases/emojis'; import { SelectIcon } from '@colanode/client/databases/icons'; import { - SelectFileState, SelectMutation, SelectNode, SelectUser, @@ -21,6 +22,9 @@ import { SelectDocumentState, SelectDocumentUpdate, SelectNodeReference, + SelectLocalFile, + SelectDownload, + SelectUpload, } from '@colanode/client/databases/workspace'; import { Account, @@ -28,13 +32,19 @@ import { AccountMetadataKey, } from '@colanode/client/types/accounts'; import { AppMetadata, AppMetadataKey } from '@colanode/client/types/apps'; +import { Avatar } from '@colanode/client/types/avatars'; import { Document, DocumentState, DocumentUpdate, } from '@colanode/client/types/documents'; import { Emoji } from '@colanode/client/types/emojis'; -import { FileState } from '@colanode/client/types/files'; +import { + LocalFile, + Download, + Upload, + TempFile, +} from '@colanode/client/types/files'; import { Icon } from '@colanode/client/types/icons'; import { LocalNode, @@ -187,27 +197,67 @@ export const mapNodeInteraction = ( }; }; -export const mapFileState = ( - row: SelectFileState, - url: string | null -): FileState => { +export const mapLocalFile = (row: SelectLocalFile, url: string): LocalFile => { return { id: row.id, version: row.version, - downloadStatus: row.download_status, - downloadProgress: row.download_progress, - downloadRetries: row.download_retries, - downloadStartedAt: row.download_started_at, - downloadCompletedAt: row.download_completed_at, - uploadStatus: row.upload_status, - uploadProgress: row.upload_progress, - uploadRetries: row.upload_retries, - uploadStartedAt: row.upload_started_at, - uploadCompletedAt: row.upload_completed_at, + name: row.name, + path: row.path, + size: row.size, + subtype: row.subtype, + openedAt: row.opened_at, + mimeType: row.mime_type, + createdAt: row.created_at, url, }; }; +export const mapTempFile = (row: SelectTempFile, url: string): TempFile => { + return { + id: row.id, + name: row.name, + path: row.path, + size: row.size, + subtype: row.subtype, + mimeType: row.mime_type, + extension: row.extension, + url, + }; +}; + +export const mapDownload = (row: SelectDownload): Download => { + return { + id: row.id, + fileId: row.file_id, + version: row.version, + type: row.type, + name: row.name, + path: row.path, + size: row.size, + mimeType: row.mime_type, + status: row.status, + progress: row.progress, + retries: row.retries, + createdAt: row.created_at, + completedAt: row.completed_at, + errorCode: row.error_code, + errorMessage: row.error_message, + }; +}; + +export const mapUpload = (row: SelectUpload): Upload => { + return { + fileId: row.file_id, + status: row.status, + progress: row.progress, + retries: row.retries, + createdAt: row.created_at, + completedAt: row.completed_at, + errorCode: row.error_code, + errorMessage: row.error_message, + }; +}; + export const mapEmoji = (row: SelectEmoji): Emoji => { return { id: row.id, @@ -269,3 +319,14 @@ export const mapNodeReference = (row: SelectNodeReference): NodeReference => { type: row.type, }; }; + +export const mapAvatar = (row: SelectAvatar, url: string): Avatar => { + return { + id: row.id, + path: row.path, + size: row.size, + createdAt: row.created_at, + openedAt: row.opened_at, + url, + }; +}; diff --git a/packages/client/src/lib/nodes.ts b/packages/client/src/lib/nodes.ts new file mode 100644 index 00000000..b4fe0949 --- /dev/null +++ b/packages/client/src/lib/nodes.ts @@ -0,0 +1,13 @@ +import { LocalNode } from '@colanode/client/types/nodes'; + +export const isNodeSynced = (node: LocalNode): boolean => { + if (typeof node.serverRevision === 'string') { + return node.serverRevision !== '0'; + } + + if (typeof node.serverRevision === 'number') { + return node.serverRevision > 0; + } + + return false; +}; diff --git a/packages/client/src/lib/sleep-scheduler.ts b/packages/client/src/lib/sleep-scheduler.ts new file mode 100644 index 00000000..09f5e514 --- /dev/null +++ b/packages/client/src/lib/sleep-scheduler.ts @@ -0,0 +1,59 @@ +interface SleepState { + date: Date; + timeout: NodeJS.Timeout; + resolve: () => void; +} + +export class SleepScheduler { + private sleepMap = new Map(); + + public sleepUntil(id: string, date: Date): Promise { + if (this.sleepMap.has(id)) { + throw new Error(`Sleep already exists for id: ${id}`); + } + + return new Promise((resolve) => { + const delay = date.getTime() - Date.now(); + const timeout = setTimeout(() => { + this.sleepMap.delete(id); + resolve(); + }, delay); + + this.sleepMap.set(id, { + date, + timeout, + resolve, + }); + }); + } + + public updateResolveTimeIfEarlier(id: string, date: Date): boolean { + const existingSleep = this.sleepMap.get(id); + if (!existingSleep) { + return false; + } + + if (date >= existingSleep.date) { + return false; + } + + clearTimeout(existingSleep.timeout); + + const delay = Math.max(0, date.getTime() - Date.now()); + if (delay === 0) { + this.sleepMap.delete(id); + existingSleep.resolve(); + return true; + } + + const newTimeout = setTimeout(() => { + this.sleepMap.delete(id); + existingSleep.resolve(); + }, delay); + + existingSleep.date = date; + existingSleep.timeout = newTimeout; + + return true; + } +} diff --git a/packages/client/src/lib/utils.ts b/packages/client/src/lib/utils.ts index 47838d9c..fe3ca110 100644 --- a/packages/client/src/lib/utils.ts +++ b/packages/client/src/lib/utils.ts @@ -119,5 +119,5 @@ export const deleteNodeRelations = async ( .where('node_id', '=', nodeId) .execute(); - await database.deleteFrom('file_states').where('id', '=', nodeId).execute(); + await database.deleteFrom('local_files').where('id', '=', nodeId).execute(); }; diff --git a/packages/client/src/mutations/files/file-create.ts b/packages/client/src/mutations/files/file-create.ts index 7fd3f013..824391f6 100644 --- a/packages/client/src/mutations/files/file-create.ts +++ b/packages/client/src/mutations/files/file-create.ts @@ -1,11 +1,9 @@ -import { TempFile } from '@colanode/client/types'; - export type FileCreateMutationInput = { type: 'file.create'; accountId: string; workspaceId: string; parentId: string; - file: TempFile; + tempFileId: string; }; export type FileCreateMutationOutput = { diff --git a/packages/client/src/mutations/files/file-save.ts b/packages/client/src/mutations/files/file-save.ts deleted file mode 100644 index 415e2ddd..00000000 --- a/packages/client/src/mutations/files/file-save.ts +++ /dev/null @@ -1,20 +0,0 @@ -export type FileSaveMutationInput = { - type: 'file.save'; - accountId: string; - workspaceId: string; - fileId: string; - path: string; -}; - -export type FileSaveMutationOutput = { - success: boolean; -}; - -declare module '@colanode/client/mutations' { - interface MutationMap { - 'file.save': { - input: FileSaveMutationInput; - output: FileSaveMutationOutput; - }; - } -} diff --git a/packages/client/src/mutations/files/temp-file-create.ts b/packages/client/src/mutations/files/temp-file-create.ts new file mode 100644 index 00000000..9d1fe9bd --- /dev/null +++ b/packages/client/src/mutations/files/temp-file-create.ts @@ -0,0 +1,25 @@ +import { FileSubtype } from '@colanode/core'; + +export type TempFileCreateMutationInput = { + type: 'temp.file.create'; + id: string; + name: string; + size: number; + mimeType: string; + subtype: FileSubtype; + extension: string; + path: string; +}; + +export type TempFileCreateMutationOutput = { + success: boolean; +}; + +declare module '@colanode/client/mutations' { + interface MutationMap { + 'temp.file.create': { + input: TempFileCreateMutationInput; + output: TempFileCreateMutationOutput; + }; + } +} diff --git a/packages/client/src/mutations/index.ts b/packages/client/src/mutations/index.ts index 58d8d01e..fb043370 100644 --- a/packages/client/src/mutations/index.ts +++ b/packages/client/src/mutations/index.ts @@ -33,7 +33,6 @@ export * from './documents/document-update'; export * from './files/file-create'; export * from './files/file-delete'; export * from './files/file-download'; -export * from './files/file-save'; export * from './folders/folder-create'; export * from './folders/folder-delete'; export * from './folders/folder-update'; @@ -71,6 +70,7 @@ export * from './workspaces/workspace-update'; export * from './users/user-role-update'; export * from './users/user-storage-update'; export * from './users/users-create'; +export * from './files/temp-file-create'; // eslint-disable-next-line @typescript-eslint/no-empty-object-type export interface MutationMap {} @@ -183,4 +183,5 @@ export enum MutationErrorCode { MessageDeleteFailed = 'message_delete_failed', MessageNotFound = 'message_not_found', NodeReactionCreateForbidden = 'node_reaction_create_forbidden', + DownloadFailed = 'download_failed', } diff --git a/packages/client/src/queries/avatars/avatar-get.ts b/packages/client/src/queries/avatars/avatar-get.ts new file mode 100644 index 00000000..c2b3e5c7 --- /dev/null +++ b/packages/client/src/queries/avatars/avatar-get.ts @@ -0,0 +1,16 @@ +import { Avatar } from '@colanode/client/types/avatars'; + +export type AvatarGetQueryInput = { + type: 'avatar.get'; + accountId: string; + avatarId: string; +}; + +declare module '@colanode/client/queries' { + interface QueryMap { + 'avatar.get': { + input: AvatarGetQueryInput; + output: Avatar | null; + }; + } +} diff --git a/packages/client/src/queries/avatars/avatar-url-get.ts b/packages/client/src/queries/avatars/avatar-url-get.ts deleted file mode 100644 index 6e9a1a68..00000000 --- a/packages/client/src/queries/avatars/avatar-url-get.ts +++ /dev/null @@ -1,18 +0,0 @@ -export type AvatarUrlGetQueryInput = { - type: 'avatar.url.get'; - accountId: string; - avatarId: string; -}; - -export type AvatarUrlGetQueryOutput = { - url: string | null; -}; - -declare module '@colanode/client/queries' { - interface QueryMap { - 'avatar.url.get': { - input: AvatarUrlGetQueryInput; - output: AvatarUrlGetQueryOutput; - }; - } -} diff --git a/packages/client/src/queries/files/download-list-manual.ts b/packages/client/src/queries/files/download-list-manual.ts new file mode 100644 index 00000000..123b5e3d --- /dev/null +++ b/packages/client/src/queries/files/download-list-manual.ts @@ -0,0 +1,18 @@ +import { Download } from '@colanode/client/types/files'; + +export type DownloadListManualQueryInput = { + type: 'download.list.manual'; + accountId: string; + workspaceId: string; + page: number; + count: number; +}; + +declare module '@colanode/client/queries' { + interface QueryMap { + 'download.list.manual': { + input: DownloadListManualQueryInput; + output: Download[]; + }; + } +} diff --git a/packages/client/src/queries/files/file-save-list.ts b/packages/client/src/queries/files/file-save-list.ts deleted file mode 100644 index ffff17ef..00000000 --- a/packages/client/src/queries/files/file-save-list.ts +++ /dev/null @@ -1,16 +0,0 @@ -import { FileSaveState } from '@colanode/client/types/files'; - -export type FileSaveListQueryInput = { - type: 'file.save.list'; - accountId: string; - workspaceId: string; -}; - -declare module '@colanode/client/queries' { - interface QueryMap { - 'file.save.list': { - input: FileSaveListQueryInput; - output: FileSaveState[]; - }; - } -} diff --git a/packages/client/src/queries/files/file-state-get.ts b/packages/client/src/queries/files/file-state-get.ts deleted file mode 100644 index cd918bca..00000000 --- a/packages/client/src/queries/files/file-state-get.ts +++ /dev/null @@ -1,17 +0,0 @@ -import { FileState } from '@colanode/client/types/files'; - -export type FileStateGetQueryInput = { - type: 'file.state.get'; - id: string; - accountId: string; - workspaceId: string; -}; - -declare module '@colanode/client/queries' { - interface QueryMap { - 'file.state.get': { - input: FileStateGetQueryInput; - output: FileState | null; - }; - } -} diff --git a/packages/client/src/queries/files/local-file-get.ts b/packages/client/src/queries/files/local-file-get.ts new file mode 100644 index 00000000..eb34ab35 --- /dev/null +++ b/packages/client/src/queries/files/local-file-get.ts @@ -0,0 +1,23 @@ +import { Download, LocalFile } from '@colanode/client/types'; + +export type LocalFileGetQueryInput = { + type: 'local.file.get'; + fileId: string; + accountId: string; + workspaceId: string; + autoDownload?: boolean; +}; + +export type LocalFileGetQueryOutput = { + localFile: LocalFile | null; + download: Download | null; +}; + +declare module '@colanode/client/queries' { + interface QueryMap { + 'local.file.get': { + input: LocalFileGetQueryInput; + output: LocalFileGetQueryOutput; + }; + } +} diff --git a/packages/client/src/queries/files/temp-file-get.ts b/packages/client/src/queries/files/temp-file-get.ts new file mode 100644 index 00000000..cd6e0be9 --- /dev/null +++ b/packages/client/src/queries/files/temp-file-get.ts @@ -0,0 +1,15 @@ +import { TempFile } from '@colanode/client/types'; + +export type TempFileGetQueryInput = { + type: 'temp.file.get'; + id: string; +}; + +declare module '@colanode/client/queries' { + interface QueryMap { + 'temp.file.get': { + input: TempFileGetQueryInput; + output: TempFile | null; + }; + } +} diff --git a/packages/client/src/queries/files/upload-list-pending.ts b/packages/client/src/queries/files/upload-list-pending.ts new file mode 100644 index 00000000..7dafbdb6 --- /dev/null +++ b/packages/client/src/queries/files/upload-list-pending.ts @@ -0,0 +1,18 @@ +import { Upload } from '@colanode/client/types/files'; + +export type UploadListPendingQueryInput = { + type: 'upload.list.pending'; + accountId: string; + workspaceId: string; + page: number; + count: number; +}; + +declare module '@colanode/client/queries' { + interface QueryMap { + 'upload.list.pending': { + input: UploadListPendingQueryInput; + output: Upload[]; + }; + } +} diff --git a/packages/client/src/queries/files/upload-list.ts b/packages/client/src/queries/files/upload-list.ts new file mode 100644 index 00000000..d93dc1dd --- /dev/null +++ b/packages/client/src/queries/files/upload-list.ts @@ -0,0 +1,18 @@ +import { Upload } from '@colanode/client/types/files'; + +export type UploadListQueryInput = { + type: 'upload.list'; + accountId: string; + workspaceId: string; + page: number; + count: number; +}; + +declare module '@colanode/client/queries' { + interface QueryMap { + 'upload.list': { + input: UploadListQueryInput; + output: Upload[]; + }; + } +} diff --git a/packages/client/src/queries/index.ts b/packages/client/src/queries/index.ts index d589bd4a..c8f4ee35 100644 --- a/packages/client/src/queries/index.ts +++ b/packages/client/src/queries/index.ts @@ -14,9 +14,8 @@ export * from './emojis/emoji-get'; export * from './emojis/emoji-list'; export * from './emojis/emoji-search'; export * from './files/file-list'; -export * from './files/file-state-get'; +export * from './files/local-file-get'; export * from './files/file-download-request-get'; -export * from './files/file-save-list'; export * from './icons/icon-category-list'; export * from './icons/icon-list'; export * from './icons/icon-search'; @@ -38,9 +37,13 @@ export * from './users/user-storage-get'; export * from './workspaces/workspace-get'; export * from './workspaces/workspace-list'; export * from './workspaces/workspace-metadata-list'; -export * from './avatars/avatar-url-get'; +export * from './avatars/avatar-get'; export * from './records/record-field-value-count'; export * from './workspaces/workspace-storage-get'; +export * from './files/upload-list'; +export * from './files/download-list-manual'; +export * from './files/temp-file-get'; +export * from './files/upload-list-pending'; // eslint-disable-next-line @typescript-eslint/no-empty-object-type export interface QueryMap {} diff --git a/packages/client/src/services/accounts/account-service.ts b/packages/client/src/services/accounts/account-service.ts index 0f1a1852..1ed8e55b 100644 --- a/packages/client/src/services/accounts/account-service.ts +++ b/packages/client/src/services/accounts/account-service.ts @@ -7,10 +7,10 @@ import { accountDatabaseMigrations, } from '@colanode/client/databases/account'; import { eventBus } from '@colanode/client/lib/event-bus'; -import { EventLoop } from '@colanode/client/lib/event-loop'; import { parseApiError } from '@colanode/client/lib/ky'; import { mapAccount, mapWorkspace } from '@colanode/client/lib/mappers'; import { AccountSocket } from '@colanode/client/services/accounts/account-socket'; +import { AvatarService } from '@colanode/client/services/accounts/avatar-service'; import { AppService } from '@colanode/client/services/app-service'; import { ServerService } from '@colanode/client/services/server-service'; import { WorkspaceService } from '@colanode/client/services/workspaces/workspace-service'; @@ -21,8 +21,6 @@ import { ApiErrorCode, ApiErrorOutput, createDebugger, - getIdType, - IdType, Message, } from '@colanode/core'; @@ -30,15 +28,16 @@ const debug = createDebugger('desktop:service:account'); export class AccountService { private readonly workspaces: Map = new Map(); - private readonly eventLoop: EventLoop; private readonly account: Account; public readonly app: AppService; public readonly server: ServerService; public readonly database: Kysely; + public readonly avatars: AvatarService; public readonly socket: AccountSocket; public readonly client: KyInstance; + private readonly accountSyncJobScheduleId: string; private readonly eventSubscriptionId: string; constructor(account: Account, server: ServerService, app: AppService) { @@ -53,6 +52,7 @@ export class AccountService { readonly: false, }); + this.avatars = new AvatarService(this); this.socket = new AccountSocket(this); this.client = this.app.client.extend({ prefixUrl: this.server.httpBaseUrl, @@ -61,24 +61,18 @@ export class AccountService { }, }); - this.eventLoop = new EventLoop( - ms('1 minute'), - ms('1 second'), - this.sync.bind(this) - ); - + this.accountSyncJobScheduleId = `account.sync.${this.account.id}`; this.eventSubscriptionId = eventBus.subscribe((event) => { if ( - event.type === 'server.availability.changed' && - event.server.domain === this.server.domain && - event.isAvailable - ) { - this.eventLoop.trigger(); - } else if ( event.type === 'account.connection.message.received' && event.accountId === this.account.id ) { this.handleMessage(event.message); + } else if ( + event.type === 'server.availability.changed' && + event.server.domain === this.server.domain + ) { + this.app.jobs.triggerJobSchedule(this.accountSyncJobScheduleId); } }); } @@ -102,13 +96,22 @@ export class AccountService { this.app.path.accountAvatars(this.account.id) ); - if (this.account.avatar) { - await this.downloadAvatar(this.account.avatar); - } + await this.app.jobs.upsertJobSchedule( + this.accountSyncJobScheduleId, + { + type: 'account.sync', + accountId: this.account.id, + }, + ms('1 minute'), + { + deduplication: { + key: this.accountSyncJobScheduleId, + replace: true, + }, + } + ); this.socket.init(); - this.eventLoop.start(); - await this.initWorkspaces(); } @@ -128,26 +131,28 @@ export class AccountService { public async logout(): Promise { try { - await this.app.database.transaction().execute(async (tx) => { - const deletedAccount = await tx - .deleteFrom('accounts') - .where('id', '=', this.account.id) - .executeTakeFirst(); + const deletedAccount = await this.app.database + .deleteFrom('accounts') + .where('id', '=', this.account.id) + .executeTakeFirst(); - if (!deletedAccount) { - throw new Error('Failed to delete account'); + if (!deletedAccount) { + throw new Error('Failed to delete account'); + } + + await this.app.jobs.addJob( + { + type: 'token.delete', + token: this.account.token, + server: this.server.domain, + }, + { + retries: 10, + delay: ms('1 second'), } + ); - await tx - .insertInto('deleted_tokens') - .values({ - account_id: this.account.id, - token: this.account.token, - server: this.server.domain, - created_at: new Date().toISOString(), - }) - .execute(); - }); + await this.app.jobs.removeJobSchedule(this.accountSyncJobScheduleId); const workspaces = this.workspaces.values(); for (const workspace of workspaces) { @@ -157,7 +162,6 @@ export class AccountService { this.database.destroy(); this.socket.close(); - this.eventLoop.stop(); eventBus.unsubscribe(this.eventSubscriptionId); const databasePath = this.app.path.accountDatabase(this.account.id); @@ -189,42 +193,6 @@ export class AccountService { await migrator.migrateToLatest(); } - public async downloadAvatar(avatar: string): Promise { - const type = getIdType(avatar); - if (type !== IdType.Avatar) { - return false; - } - - try { - const avatarPath = this.app.path.accountAvatar(this.account.id, avatar); - - const exists = await this.app.fs.exists(avatarPath); - if (exists) { - return true; - } - - const response = await this.client.get( - `v1/avatars/${avatar}` - ); - - const avatarBytes = new Uint8Array(await response.arrayBuffer()); - await this.app.fs.writeFile(avatarPath, avatarBytes); - - eventBus.publish({ - type: 'avatar.downloaded', - accountId: this.account.id, - avatarId: avatar, - }); - - return true; - } catch (err) { - console.error(err); - debug(`Error downloading avatar for account ${this.account.id}: ${err}`); - } - - return false; - } - private async initWorkspaces(): Promise { const workspaces = await this.database .selectFrom('workspaces') @@ -265,11 +233,11 @@ export class AccountService { message.type === 'user.created' || message.type === 'user.updated' ) { - this.eventLoop.trigger(); + this.app.jobs.triggerJobSchedule(this.accountSyncJobScheduleId); } } - private async sync(): Promise { + public async sync(): Promise { debug(`Syncing account ${this.account.id}`); if (!this.server.isAvailable) { @@ -307,13 +275,10 @@ export class AccountService { return; } - if (updatedAccount.avatar) { - await this.downloadAvatar(updatedAccount.avatar); - } - debug(`Updated account ${this.account.email} after sync`); const account = mapAccount(updatedAccount); this.updateAccount(account); + this.socket.checkConnection(); eventBus.publish({ type: 'account.updated', @@ -345,10 +310,6 @@ export class AccountService { continue; } - if (createdWorkspace.avatar) { - await this.downloadAvatar(createdWorkspace.avatar); - } - const mappedWorkspace = mapWorkspace(createdWorkspace); await this.initWorkspace(mappedWorkspace); @@ -375,10 +336,6 @@ export class AccountService { const mappedWorkspace = mapWorkspace(updatedWorkspace); workspaceService.updateWorkspace(mappedWorkspace); - if (updatedWorkspace.avatar) { - await this.downloadAvatar(updatedWorkspace.avatar); - } - eventBus.publish({ type: 'workspace.updated', workspace: mappedWorkspace, diff --git a/packages/client/src/services/accounts/account-socket.ts b/packages/client/src/services/accounts/account-socket.ts index fed31526..ee2c6f6b 100644 --- a/packages/client/src/services/accounts/account-socket.ts +++ b/packages/client/src/services/accounts/account-socket.ts @@ -1,9 +1,7 @@ import WebSocket from 'isomorphic-ws'; -import ms from 'ms'; import { BackoffCalculator } from '@colanode/client/lib/backoff-calculator'; import { eventBus } from '@colanode/client/lib/event-bus'; -import { EventLoop } from '@colanode/client/lib/event-loop'; import { AccountService } from '@colanode/client/services/accounts/account-service'; import { Message, SocketInitOutput, createDebugger } from '@colanode/core'; @@ -11,39 +9,19 @@ const debug = createDebugger('desktop:service:account-socket'); export class AccountSocket { private readonly account: AccountService; - private readonly eventLoop: EventLoop; private socket: WebSocket | null; private backoffCalculator: BackoffCalculator; private closingCount: number; - private eventSubscriptionId: string; - constructor(account: AccountService) { this.account = account; this.socket = null; this.backoffCalculator = new BackoffCalculator(); this.closingCount = 0; - - this.eventLoop = new EventLoop( - ms('1 minute'), - ms('1 second'), - this.checkConnection.bind(this) - ); - - this.eventSubscriptionId = eventBus.subscribe((event) => { - if ( - event.type === 'server.availability.changed' && - event.server.domain === this.account.server.domain - ) { - this.eventLoop.trigger(); - } - }); } public async init(): Promise { - this.eventLoop.start(); - if (!this.account.server.isAvailable) { return; } @@ -133,12 +111,9 @@ export class AccountSocket { this.socket.close(); this.socket = null; } - - this.eventLoop.stop(); - eventBus.unsubscribe(this.eventSubscriptionId); } - private checkConnection(): void { + public checkConnection(): void { try { debug(`Checking connection for account ${this.account.id}`); if (!this.account.server.isAvailable) { diff --git a/packages/client/src/services/accounts/avatar-service.ts b/packages/client/src/services/accounts/avatar-service.ts new file mode 100644 index 00000000..f9e4eb3d --- /dev/null +++ b/packages/client/src/services/accounts/avatar-service.ts @@ -0,0 +1,122 @@ +import ms from 'ms'; + +import { mapAvatar } from '@colanode/client/lib'; +import { eventBus } from '@colanode/client/lib/event-bus'; +import { AccountService } from '@colanode/client/services/accounts/account-service'; +import { Avatar } from '@colanode/client/types/avatars'; + +export class AvatarService { + private readonly account: AccountService; + + constructor(account: AccountService) { + this.account = account; + } + + public async getAvatar( + avatar: string, + autoDownload?: boolean + ): Promise { + const updatedAvatar = await this.account.database + .updateTable('avatars') + .returningAll() + .set({ + opened_at: new Date().toISOString(), + }) + .where('id', '=', avatar) + .executeTakeFirst(); + + if (updatedAvatar) { + const url = await this.account.app.fs.url(updatedAvatar.path); + return mapAvatar(updatedAvatar, url); + } + + if (autoDownload) { + await this.account.app.jobs.addJob( + { + type: 'avatar.download', + accountId: this.account.id, + avatar, + }, + { + deduplication: { + key: `avatar.download.${avatar}`, + }, + retries: 5, + } + ); + } + + return null; + } + + public async downloadAvatar(avatar: string): Promise { + if (!this.account.server.isAvailable) { + return null; + } + + const response = await this.account.client.get( + `v1/avatars/${avatar}` + ); + + if (response.status !== 200) { + return false; + } + + const avatarPath = this.account.app.path.accountAvatar( + this.account.id, + avatar + ); + + const avatarBytes = new Uint8Array(await response.arrayBuffer()); + await this.account.app.fs.writeFile(avatarPath, avatarBytes); + + const createdAvatar = await this.account.database + .insertInto('avatars') + .returningAll() + .values({ + id: avatar, + path: avatarPath, + size: avatarBytes.length, + created_at: new Date().toISOString(), + opened_at: new Date().toISOString(), + }) + .onConflict((oc) => + oc.columns(['id']).doUpdateSet({ + opened_at: new Date().toISOString(), + }) + ) + .executeTakeFirst(); + + if (!createdAvatar) { + return false; + } + + const url = await this.account.app.fs.url(avatarPath); + eventBus.publish({ + type: 'avatar.created', + accountId: this.account.id, + avatar: mapAvatar(createdAvatar, url), + }); + + return true; + } + + public async cleanupAvatars(): Promise { + const sevenDaysAgo = new Date(Date.now() - ms('7 days')).toISOString(); + const unopenedAvatars = await this.account.database + .deleteFrom('avatars') + .where('opened_at', '<', sevenDaysAgo) + .returningAll() + .execute(); + + for (const avatar of unopenedAvatars) { + await this.account.app.fs.delete(avatar.path); + + eventBus.publish({ + type: 'avatar.deleted', + accountId: this.account.id, + avatar: mapAvatar(avatar, ''), + }); + } + } +} diff --git a/packages/client/src/services/app-service.ts b/packages/client/src/services/app-service.ts index 01e2fc64..94343667 100644 --- a/packages/client/src/services/app-service.ts +++ b/packages/client/src/services/app-service.ts @@ -9,27 +9,25 @@ import { } from '@colanode/client/databases/app'; import { Mediator } from '@colanode/client/handlers'; import { eventBus } from '@colanode/client/lib/event-bus'; -import { EventLoop } from '@colanode/client/lib/event-loop'; -import { parseApiError } from '@colanode/client/lib/ky'; import { mapServer, mapAccount } from '@colanode/client/lib/mappers'; import { AccountService } from '@colanode/client/services/accounts/account-service'; import { AppMeta } from '@colanode/client/services/app-meta'; import { AssetService } from '@colanode/client/services/asset-service'; import { FileSystem } from '@colanode/client/services/file-system'; +import { JobService } from '@colanode/client/services/job-service'; import { KyselyService } from '@colanode/client/services/kysely-service'; import { MetadataService } from '@colanode/client/services/metadata-service'; import { PathService } from '@colanode/client/services/path-service'; import { ServerService } from '@colanode/client/services/server-service'; import { Account } from '@colanode/client/types/accounts'; import { Server, ServerAttributes } from '@colanode/client/types/servers'; -import { ApiErrorCode, ApiHeader, build, createDebugger } from '@colanode/core'; +import { ApiHeader, build, createDebugger } from '@colanode/core'; const debug = createDebugger('desktop:service:app'); export class AppService { private readonly servers: Map = new Map(); private readonly accounts: Map = new Map(); - private readonly cleanupEventLoop: EventLoop; private readonly eventSubscriptionId: string; public readonly meta: AppMeta; @@ -39,7 +37,8 @@ export class AppService { public readonly metadata: MetadataService; public readonly kysely: KyselyService; public readonly mediator: Mediator; - public readonly asset: AssetService; + public readonly assets: AssetService; + public readonly jobs: JobService; public readonly client: KyInstance; constructor( @@ -59,7 +58,8 @@ export class AppService { }); this.mediator = new Mediator(this); - this.asset = new AssetService(this); + this.assets = new AssetService(this); + this.jobs = new JobService(this); this.client = ky.create({ headers: { @@ -72,12 +72,6 @@ export class AppService { this.metadata = new MetadataService(this); - this.cleanupEventLoop = new EventLoop( - ms('10 minutes'), - ms('1 minute'), - this.cleanup.bind(this) - ); - this.eventSubscriptionId = eventBus.subscribe((event) => { if (event.type === 'account.deleted') { this.accounts.delete(event.account.id); @@ -129,8 +123,22 @@ export class AppService { await this.initServers(); await this.initAccounts(); await this.fs.makeDirectory(this.path.temp); + await this.jobs.init(); - this.cleanupEventLoop.start(); + const scheduleId = 'temp.files.clean'; + await this.jobs.upsertJobSchedule( + scheduleId, + { + type: 'temp.files.clean', + }, + ms('5 minutes'), + { + deduplication: { + key: scheduleId, + replace: true, + }, + } + ); } private async initServers(): Promise { @@ -178,8 +186,9 @@ export class AppService { } const serverService = new ServerService(this, server); - this.servers.set(server.domain, serverService); + await serverService.init(); + this.servers.set(server.domain, serverService); return serverService; } @@ -264,130 +273,11 @@ export class AppService { } } - public triggerCleanup(): void { - this.cleanupEventLoop.trigger(); - } - - private async cleanup(): Promise { - await this.syncDeletedTokens(); - await this.cleanTempFiles(); - } - - private async syncDeletedTokens(): Promise { - debug('Syncing deleted tokens'); - - const deletedTokens = await this.database - .selectFrom('deleted_tokens') - .innerJoin('servers', 'deleted_tokens.server', 'servers.domain') - .select([ - 'deleted_tokens.token', - 'deleted_tokens.account_id', - 'servers.domain', - 'servers.attributes', - ]) - .execute(); - - if (deletedTokens.length === 0) { - debug('No deleted tokens found'); - return; - } - - for (const deletedToken of deletedTokens) { - const server = this.servers.get(deletedToken.domain); - if (!server) { - debug( - `Server ${deletedToken.domain} not found, skipping token ${deletedToken.token} for account ${deletedToken.account_id}` - ); - - await this.database - .deleteFrom('deleted_tokens') - .where('token', '=', deletedToken.token) - .where('account_id', '=', deletedToken.account_id) - .execute(); - - continue; - } - - if (!server.isAvailable) { - debug( - `Server ${deletedToken.domain} is not available for logging out account ${deletedToken.account_id}` - ); - continue; - } - - try { - await this.client.delete(`${server.httpBaseUrl}/v1/accounts/logout`, { - headers: { - Authorization: `Bearer ${deletedToken.token}`, - }, - }); - - await this.database - .deleteFrom('deleted_tokens') - .where('token', '=', deletedToken.token) - .where('account_id', '=', deletedToken.account_id) - .execute(); - - debug( - `Logged out account ${deletedToken.account_id} from server ${deletedToken.domain}` - ); - } catch (error) { - const parsedError = await parseApiError(error); - if ( - parsedError.code === ApiErrorCode.TokenInvalid || - parsedError.code === ApiErrorCode.AccountNotFound || - parsedError.code === ApiErrorCode.DeviceNotFound - ) { - debug( - `Account ${deletedToken.account_id} is already logged out, skipping...` - ); - - await this.database - .deleteFrom('deleted_tokens') - .where('token', '=', deletedToken.token) - .where('account_id', '=', deletedToken.account_id) - .execute(); - - continue; - } - - debug( - `Failed to logout account ${deletedToken.account_id} from server ${deletedToken.domain}`, - error - ); - } - } - } - - private async cleanTempFiles(): Promise { - debug(`Cleaning temp files`); - - const exists = await this.fs.exists(this.path.temp); - if (!exists) { - return; - } - - const filePaths = await this.fs.listFiles(this.path.temp); - const oneDayAgo = Date.now() - 24 * 60 * 60 * 1000; // 24 hours in milliseconds - - for (const filePath of filePaths) { - const metadata = await this.fs.metadata(filePath); - - if (metadata.lastModified < oneDayAgo) { - try { - await this.fs.delete(filePath); - debug(`Deleted old temp file: ${filePath}`); - } catch (error) { - debug(`Failed to delete temp file: ${filePath}`, error); - } - } - } - } - private async deleteAllData(): Promise { await this.database.deleteFrom('accounts').execute(); await this.database.deleteFrom('metadata').execute(); - await this.database.deleteFrom('deleted_tokens').execute(); + await this.database.deleteFrom('job_schedules').execute(); + await this.database.deleteFrom('jobs').execute(); await this.fs.delete(this.path.accounts); } } diff --git a/packages/client/src/services/file-system.ts b/packages/client/src/services/file-system.ts index 27da1936..fb6c2ca9 100644 --- a/packages/client/src/services/file-system.ts +++ b/packages/client/src/services/file-system.ts @@ -1,11 +1,4 @@ -export interface FileMetadata { - lastModified: number; - size: number; -} - -// In Node.js we use the file stream which is AsyncIterable -// In the browser we use the File object which is a Blob -export type FileReadStream = AsyncIterable | File; +export type FileReadStream = Buffer | File; export interface FileSystem { makeDirectory(path: string): Promise; @@ -17,6 +10,5 @@ export interface FileSystem { listFiles(path: string): Promise; readFile(path: string): Promise; writeFile(path: string, data: Uint8Array): Promise; - metadata(path: string): Promise; url(path: string): Promise; } diff --git a/packages/client/src/services/job-service.ts b/packages/client/src/services/job-service.ts new file mode 100644 index 00000000..9e08db9a --- /dev/null +++ b/packages/client/src/services/job-service.ts @@ -0,0 +1,489 @@ +import AsyncLock from 'async-lock'; + +import { SelectJob, SelectJobSchedule } from '@colanode/client/databases/app'; +import { + Job, + JobHandler, + JobHandlerMap, + JobInput, + JobOptions, + JobScheduleOptions, + JobScheduleStatus, + JobStatus, +} from '@colanode/client/jobs'; +import { AccountSyncJobHandler } from '@colanode/client/jobs/account-sync'; +import { AvatarDownloadJobHandler } from '@colanode/client/jobs/avatar-download'; +import { AvatarsCleanJobHandler } from '@colanode/client/jobs/avatars-clean'; +import { FileDownloadJobHandler } from '@colanode/client/jobs/file-download'; +import { FileUploadJobHandler } from '@colanode/client/jobs/file-upload'; +import { MutationsSyncJobHandler } from '@colanode/client/jobs/mutations-sync'; +import { ServerSyncJobHandler } from '@colanode/client/jobs/server-sync'; +import { TempFilesCleanJobHandler } from '@colanode/client/jobs/temp-files-clean'; +import { TokenDeleteJobHandler } from '@colanode/client/jobs/token-delete'; +import { WorkspaceFilesCleanJobHandler } from '@colanode/client/jobs/workspace-files-clean'; +import { SleepScheduler } from '@colanode/client/lib/sleep-scheduler'; +import { AppService } from '@colanode/client/services/app-service'; +import { generateId, IdType } from '@colanode/core'; + +const MAX_CONCURRENCY = 5; +const JOBS_MAX_TIMEOUT = 30000; +const SCHEDULES_MAX_TIMEOUT = 30000; +const JOBS_CONCURRENCY_TIMEOUT = 50; +const CLOSE_TIMEOUT = 50; +const JOB_LOOP_ID = 'job.loop'; +const SCHEDULE_LOOP_ID = 'schedule.loop'; + +export class JobService { + private readonly app: AppService; + private readonly queue: string = 'main'; + private readonly handlerMap: JobHandlerMap; + private readonly runningConcurrency = new Map(); + private readonly lock = new AsyncLock(); + private readonly sleepScheduler = new SleepScheduler(); + + private stopped = false; + private runningJobs = 0; + + constructor(app: AppService) { + this.app = app; + + this.handlerMap = { + 'token.delete': new TokenDeleteJobHandler(app), + 'account.sync': new AccountSyncJobHandler(app), + 'server.sync': new ServerSyncJobHandler(app), + 'file.upload': new FileUploadJobHandler(app), + 'file.download': new FileDownloadJobHandler(app), + 'mutations.sync': new MutationsSyncJobHandler(app), + 'temp.files.clean': new TempFilesCleanJobHandler(app), + 'workspace.files.clean': new WorkspaceFilesCleanJobHandler(app), + 'avatar.download': new AvatarDownloadJobHandler(app), + 'avatars.clean': new AvatarsCleanJobHandler(app), + }; + } + + public async init(): Promise { + await this.app.database + .updateTable('jobs') + .set({ status: JobStatus.Waiting }) + .execute(); + + this.jobLoop().catch((err) => console.error('Job loop error:', err)); + this.scheduleLoop().catch((err) => + console.error('Schedule loop error:', err) + ); + } + + public async addJob( + input: JobInput, + opts?: JobOptions, + scheduleId?: string + ): Promise { + const handler = this.handlerMap[input.type] as JobHandler; + if (!handler) { + return null; + } + + const now = new Date(); + const scheduledAt = + opts?.delay && opts.delay > 0 + ? new Date(now.getTime() + opts.delay) + : now; + + const concurrencyKey = handler.concurrency?.key?.(input); + + if (opts?.deduplication) { + const deduplication = opts.deduplication; + + const result = await this.lock.acquire(deduplication.key, async () => { + const existing = await this.app.database + .selectFrom('jobs') + .selectAll() + .where('deduplication_key', '=', deduplication.key) + .where('status', '=', JobStatus.Waiting) + .executeTakeFirst(); + + if (!existing) { + return null; + } + + if (!deduplication.replace) { + return existing; + } + + const updatedJob = await this.app.database + .updateTable('jobs') + .set({ + input: this.toJSON(input), + options: this.toJSON(opts), + scheduled_at: scheduledAt.toISOString(), + concurrency_key: concurrencyKey || null, + updated_at: now.toISOString(), + }) + .where('id', '=', existing.id) + .where('status', '=', JobStatus.Waiting) + .returningAll() + .executeTakeFirst(); + + return updatedJob ?? null; + }); + + if (result) { + const date = new Date(result.scheduled_at); + this.sleepScheduler.updateResolveTimeIfEarlier(JOB_LOOP_ID, date); + return result; + } + } + + const id = generateId(IdType.Job); + const job = await this.app.database + .insertInto('jobs') + .returningAll() + .values({ + id, + queue: this.queue, + input: this.toJSON(input), + options: this.toJSON(opts), + status: JobStatus.Waiting, + retries: 0, + scheduled_at: scheduledAt.toISOString(), + deduplication_key: opts?.deduplication?.key || null, + concurrency_key: concurrencyKey || null, + schedule_id: scheduleId || null, + created_at: now.toISOString(), + updated_at: now.toISOString(), + }) + .executeTakeFirst(); + + if (job) { + const date = new Date(job.scheduled_at); + this.sleepScheduler.updateResolveTimeIfEarlier(JOB_LOOP_ID, date); + } + + return job ?? null; + } + + public async upsertJobSchedule( + id: string, + input: JobInput, + interval: number, + opts?: JobScheduleOptions + ): Promise { + const now = new Date().toISOString(); + const nextRunAt = new Date(Date.parse(now) + interval).toISOString(); + + const schedule = await this.app.database + .insertInto('job_schedules') + .returningAll() + .values({ + id, + queue: this.queue, + input: this.toJSON(input), + options: this.toJSON(opts), + status: JobScheduleStatus.Active, + interval: interval, + next_run_at: nextRunAt, + created_at: now, + updated_at: now, + }) + .onConflict((oc) => + oc.columns(['id']).doUpdateSet({ + input: this.toJSON(input), + options: this.toJSON(opts), + interval: interval, + next_run_at: nextRunAt, + updated_at: now, + status: JobScheduleStatus.Active, + }) + ) + .executeTakeFirst(); + + if (schedule) { + const date = new Date(schedule.next_run_at); + this.sleepScheduler.updateResolveTimeIfEarlier(SCHEDULE_LOOP_ID, date); + } + + return schedule ?? null; + } + + public async removeJobSchedule(id: string): Promise { + await this.app.database + .deleteFrom('job_schedules') + .where('id', '=', id) + .execute(); + } + + public async triggerJobSchedule(id: string): Promise { + const schedule = await this.app.database + .selectFrom('job_schedules') + .selectAll() + .where('id', '=', id) + .where('status', '=', JobScheduleStatus.Active) + .executeTakeFirst(); + + if (!schedule) { + return null; + } + + return this.addJobFromSchedule(schedule); + } + + public async close() { + this.stopped = true; + while (this.runningJobs > 0) { + await new Promise((r) => setTimeout(r, CLOSE_TIMEOUT)); + } + } + + private async jobLoop() { + while (!this.stopped) { + if (this.runningJobs >= MAX_CONCURRENCY) { + const date = new Date(Date.now() + JOBS_CONCURRENCY_TIMEOUT); + await this.sleepScheduler.sleepUntil(JOB_LOOP_ID, date); + continue; + } + + const limitReachedTypes = new Set(); + for (const [type, handler] of Object.entries(this.handlerMap)) { + if (handler.concurrency) { + const currentCount = this.runningConcurrency.get(type) || 0; + if (currentCount >= handler.concurrency.limit) { + limitReachedTypes.add(type); + } + } + } + + const now = new Date(); + const jobRow = await this.app.database + .updateTable('jobs') + .set({ + status: JobStatus.Active, + updated_at: now.toISOString(), + }) + .where('id', 'in', (qb) => + qb + .selectFrom('jobs') + .select('id') + .where('queue', '=', this.queue) + .where('status', '=', JobStatus.Waiting) + .where('scheduled_at', '<=', now.toISOString()) + .where('concurrency_key', 'not in', Array.from(limitReachedTypes)) + .orderBy('scheduled_at', 'asc') + .limit(1) + ) + .returningAll() + .executeTakeFirst(); + + if (!jobRow) { + const nextScheduledJob = await this.app.database + .selectFrom('jobs') + .select('scheduled_at') + .where('queue', '=', this.queue) + .where('status', '=', JobStatus.Waiting) + .orderBy('scheduled_at', 'asc') + .limit(1) + .executeTakeFirst(); + + if (nextScheduledJob) { + const date = new Date(nextScheduledJob.scheduled_at); + await this.sleepScheduler.sleepUntil(JOB_LOOP_ID, date); + } else { + const date = new Date(now.getTime() + JOBS_MAX_TIMEOUT); + await this.sleepScheduler.sleepUntil(JOB_LOOP_ID, date); + } + + continue; + } + + const input = this.fromJSON(jobRow.input) as JobInput; + const handler = this.handlerMap[input.type] as JobHandler; + + if (handler?.concurrency) { + const concurrencyKey = handler.concurrency.key(input); + const currentCount = this.runningConcurrency.get(concurrencyKey) || 0; + + if (currentCount >= handler.concurrency.limit) { + await this.app.database + .updateTable('jobs') + .set({ status: JobStatus.Waiting, updated_at: now.toISOString() }) + .where('id', '=', jobRow.id) + .execute(); + continue; + } + } + + this.handleJob(jobRow).catch((err) => + console.error('Job handling error:', err) + ); + } + } + + private async scheduleLoop() { + while (!this.stopped) { + const now = new Date(); + + const schedules = await this.app.database + .selectFrom('job_schedules') + .selectAll() + .where('status', '=', JobScheduleStatus.Active) + .where('next_run_at', '<=', now.toISOString()) + .execute(); + + for (const schedule of schedules) { + try { + await this.addJobFromSchedule(schedule); + + const nextRunAt = new Date(now.getTime() + schedule.interval); + await this.app.database + .updateTable('job_schedules') + .set({ + next_run_at: nextRunAt.toISOString(), + last_run_at: now.toISOString(), + updated_at: now.toISOString(), + }) + .where('id', '=', schedule.id) + .execute(); + } catch (error) { + console.error(`Error processing schedule ${schedule.id}:`, error); + } + } + + const nextSchedule = await this.app.database + .selectFrom('job_schedules') + .select('next_run_at') + .where('status', '=', JobScheduleStatus.Active) + .orderBy('next_run_at', 'asc') + .limit(1) + .executeTakeFirst(); + + if (nextSchedule) { + const date = new Date(nextSchedule.next_run_at); + await this.sleepScheduler.sleepUntil(SCHEDULE_LOOP_ID, date); + } else { + const date = new Date(now.getTime() + SCHEDULES_MAX_TIMEOUT); + await this.sleepScheduler.sleepUntil(SCHEDULE_LOOP_ID, date); + } + } + } + + private async addJobFromSchedule(schedule: SelectJobSchedule) { + const input = this.fromJSON(schedule.input) as JobInput; + const options = this.fromJSON(schedule.options); + + return this.addJob( + input, + { + retries: options?.retries, + deduplication: options?.deduplication, + }, + schedule.id + ); + } + + private async handleJob(jobRow: SelectJob) { + const options = this.fromJSON(jobRow.options) ?? {}; + const input = this.fromJSON(jobRow.input) as JobInput; + + const job: Job = { + id: jobRow.id, + queue: jobRow.queue, + input, + options, + status: jobRow.status, + retries: jobRow.retries, + deduplicationKey: jobRow.deduplication_key ?? undefined, + concurrencyKey: jobRow.concurrency_key ?? undefined, + createdAt: jobRow.created_at, + updatedAt: jobRow.updated_at, + scheduledAt: jobRow.scheduled_at, + }; + + try { + this.runningJobs++; + + if (jobRow.concurrency_key) { + const current = this.runningConcurrency.get(input.type) || 0; + this.runningConcurrency.set(input.type, current + 1); + } + + const handler = this.handlerMap[input.type] as JobHandler; + if (!handler) { + await this.app.database + .deleteFrom('jobs') + .where('id', '=', jobRow.id) + .execute(); + return; + } + + const output = await handler.handleJob(job.input); + + if (output.type === 'retry') { + const retryAt = new Date(Date.now() + output.delay); + await this.app.database + .updateTable('jobs') + .set({ + status: JobStatus.Waiting, + scheduled_at: retryAt.toISOString(), + updated_at: new Date().toISOString(), + }) + .where('id', '=', jobRow.id) + .where('status', '=', JobStatus.Active) + .execute(); + } else if (output.type === 'cancel') { + if (jobRow.schedule_id) { + await this.removeJobSchedule(jobRow.schedule_id); + } + + await this.app.database + .deleteFrom('jobs') + .where('id', '=', jobRow.id) + .execute(); + } else if (output.type === 'success') { + await this.app.database + .deleteFrom('jobs') + .where('id', '=', jobRow.id) + .execute(); + } + } catch (error) { + console.error(`Job ${jobRow.id} failed:`, error); + + const retries = jobRow.retries + 1; + if (options.retries && retries < options.retries) { + const retryDelay = Math.min(1000 * Math.pow(2, retries), 60000); + const retryAt = new Date(Date.now() + retryDelay); + await this.app.database + .updateTable('jobs') + .set({ + status: JobStatus.Waiting, + retries, + scheduled_at: retryAt.toISOString(), + updated_at: new Date().toISOString(), + }) + .where('id', '=', jobRow.id) + .execute(); + } else { + await this.app.database + .deleteFrom('jobs') + .where('id', '=', jobRow.id) + .execute(); + } + } finally { + this.runningJobs--; + + if (jobRow.concurrency_key) { + const current = this.runningConcurrency.get(input.type) || 0; + if (current > 0) { + this.runningConcurrency.set(input.type, current - 1); + } + } + } + } + + private toJSON(value: unknown) { + return JSON.stringify(value ?? null); + } + + private fromJSON(txt: string | null): T | null { + if (txt == null) return null; + return JSON.parse(txt) as T; + } +} diff --git a/packages/client/src/services/server-service.ts b/packages/client/src/services/server-service.ts index 73f1fe93..3fe435a4 100644 --- a/packages/client/src/services/server-service.ts +++ b/packages/client/src/services/server-service.ts @@ -2,7 +2,6 @@ import ky from 'ky'; import ms from 'ms'; import { eventBus } from '@colanode/client/lib/event-bus'; -import { EventLoop } from '@colanode/client/lib/event-loop'; import { mapServer } from '@colanode/client/lib/mappers'; import { isServerOutdated } from '@colanode/client/lib/servers'; import { AppService } from '@colanode/client/services/app-service'; @@ -17,7 +16,6 @@ const debug = createDebugger('desktop:service:server'); export class ServerService { private readonly app: AppService; - private readonly eventLoop: EventLoop; public state: ServerState | null = null; public isOutdated: boolean; @@ -34,13 +32,6 @@ export class ServerService { this.socketBaseUrl = this.buildSocketBaseUrl(); this.httpBaseUrl = this.buildHttpBaseUrl(); this.isOutdated = isServerOutdated(server.version); - - this.eventLoop = new EventLoop( - ms('1 minute'), - ms('1 second'), - this.sync.bind(this) - ); - this.eventLoop.start(); } public get isAvailable() { @@ -55,7 +46,27 @@ export class ServerService { return this.server.version; } - private async sync() { + public async init(): Promise { + const scheduleId = `server.sync.${this.domain}`; + await this.app.jobs.upsertJobSchedule( + scheduleId, + { + type: 'server.sync', + server: this.domain, + }, + ms('1 minute'), + { + deduplication: { + key: scheduleId, + replace: true, + }, + } + ); + + await this.app.jobs.triggerJobSchedule(scheduleId); + } + + public async sync() { const config = await ServerService.fetchServerConfig(this.configUrl); const existingState = this.state; diff --git a/packages/client/src/services/workspaces/document-service.ts b/packages/client/src/services/workspaces/document-service.ts index 25a7f87c..126af6bd 100644 --- a/packages/client/src/services/workspaces/document-service.ts +++ b/packages/client/src/services/workspaces/document-service.ts @@ -245,7 +245,7 @@ export class DocumentService { } if (createdMutation) { - this.workspace.mutations.triggerSync(); + this.workspace.mutations.scheduleSync(); } return true; @@ -420,7 +420,7 @@ export class DocumentService { } if (createdMutation) { - this.workspace.mutations.triggerSync(); + this.workspace.mutations.scheduleSync(); } return true; diff --git a/packages/client/src/services/workspaces/file-service.ts b/packages/client/src/services/workspaces/file-service.ts index 8f0c239e..4e7ba0da 100644 --- a/packages/client/src/services/workspaces/file-service.ts +++ b/packages/client/src/services/workspaces/file-service.ts @@ -1,22 +1,24 @@ -import { cloneDeep } from 'lodash-es'; +import AsyncLock from 'async-lock'; import ms from 'ms'; import { - SelectFileState, + SelectDownload, SelectNode, } from '@colanode/client/databases/workspace'; import { eventBus } from '@colanode/client/lib/event-bus'; -import { EventLoop } from '@colanode/client/lib/event-loop'; -import { mapFileState, mapNode } from '@colanode/client/lib/mappers'; +import { + mapDownload, + mapLocalFile, + mapNode, + mapUpload, +} from '@colanode/client/lib/mappers'; import { fetchNode, fetchUserStorageUsed } from '@colanode/client/lib/utils'; import { MutationError, MutationErrorCode } from '@colanode/client/mutations'; import { AppService } from '@colanode/client/services/app-service'; import { WorkspaceService } from '@colanode/client/services/workspaces/workspace-service'; import { DownloadStatus, - FileSaveState, - SaveStatus, - TempFile, + DownloadType, UploadStatus, } from '@colanode/client/types/files'; import { LocalFileNode } from '@colanode/client/types/nodes'; @@ -30,20 +32,13 @@ import { formatBytes, } from '@colanode/core'; -const UPLOAD_RETRIES_LIMIT = 10; -const DOWNLOAD_RETRIES_LIMIT = 10; - const debug = createDebugger('desktop:service:file'); export class FileService { private readonly app: AppService; private readonly workspace: WorkspaceService; private readonly filesDir: string; - private readonly saves: FileSaveState[] = []; - - private readonly uploadsEventLoop: EventLoop; - private readonly downloadsEventLoop: EventLoop; - private readonly cleanupEventLoop: EventLoop; + private readonly lock = new AsyncLock(); constructor(workspace: WorkspaceService) { this.app = workspace.account.app; @@ -54,36 +49,42 @@ export class FileService { ); this.app.fs.makeDirectory(this.filesDir); + } - this.uploadsEventLoop = new EventLoop( - ms('1 minute'), - ms('1 second'), - this.uploadFiles.bind(this) - ); - - this.downloadsEventLoop = new EventLoop( - ms('1 minute'), - ms('1 second'), - this.downloadFiles.bind(this) - ); - - this.cleanupEventLoop = new EventLoop( - ms('5 minutes'), - ms('1 minute'), - this.cleanupFiles.bind(this) - ); - - this.uploadsEventLoop.start(); - this.downloadsEventLoop.start(); - this.cleanupEventLoop.start(); + public async init(): Promise { + // if the download was interrupted, we need to reset the status on app start + await this.workspace.database + .updateTable('downloads') + .set({ + status: DownloadStatus.Pending, + started_at: null, + completed_at: null, + error_code: null, + error_message: null, + }) + .where('status', '=', DownloadStatus.Downloading) + .execute(); } public async createFile( - id: string, - parentId: string, - file: TempFile + fileId: string, + tempFileId: string, + parentId: string ): Promise { - const fileSize = BigInt(file.size); + const tempFile = await this.app.database + .selectFrom('temp_files') + .selectAll() + .where('id', '=', tempFileId) + .executeTakeFirst(); + + if (!tempFile) { + throw new MutationError( + MutationErrorCode.FileNotFound, + 'The file you are trying to upload does not exist.' + ); + } + + const fileSize = BigInt(tempFile.size); const maxFileSize = BigInt(this.workspace.maxFileSize); if (fileSize > maxFileSize) { throw new MutationError( @@ -119,93 +120,104 @@ export class FileService { ); } - const destinationFilePath = this.buildFilePath(id, file.extension); + const destinationFilePath = this.buildFilePath(fileId, tempFile.extension); await this.app.fs.makeDirectory(this.filesDir); - await this.app.fs.copy(file.path, destinationFilePath); - await this.app.fs.delete(file.path); + await this.app.fs.copy(tempFile.path, destinationFilePath); + await this.app.fs.delete(tempFile.path); const attributes: FileAttributes = { type: 'file', - subtype: extractFileSubtype(file.mimeType), + subtype: extractFileSubtype(tempFile.mime_type), parentId: parentId, - name: file.name, - originalName: file.name, - extension: file.extension, - mimeType: file.mimeType, - size: file.size, + name: tempFile.name, + originalName: tempFile.name, + extension: tempFile.extension, + mimeType: tempFile.mime_type, + size: tempFile.size, status: FileStatus.Pending, version: generateId(IdType.Version), }; - await this.workspace.nodes.createNode({ - id: id, + const createdNode = await this.workspace.nodes.createNode({ + id: fileId, attributes: attributes, parentId: parentId, }); - const createdFileState = await this.workspace.database - .insertInto('file_states') + const createdLocalFile = await this.workspace.database + .insertInto('local_files') .returningAll() .values({ - id: id, - version: attributes.version, - download_progress: 100, - download_status: DownloadStatus.Completed, - download_completed_at: new Date().toISOString(), - upload_progress: 0, - upload_status: UploadStatus.Pending, - upload_retries: 0, - upload_started_at: new Date().toISOString(), + id: fileId, + version: generateId(IdType.Version), + name: tempFile.name, + extension: tempFile.extension, + subtype: tempFile.subtype, + mime_type: tempFile.mime_type, + size: tempFile.size, + created_at: new Date().toISOString(), + path: this.buildFilePath(fileId, tempFile.extension), + opened_at: new Date().toISOString(), }) .executeTakeFirst(); - if (!createdFileState) { + if (!createdLocalFile) { throw new MutationError( MutationErrorCode.FileCreateFailed, 'Failed to create file state' ); } - const url = await this.app.fs.url(destinationFilePath); + const createdUpload = await this.workspace.database + .insertInto('uploads') + .returningAll() + .values({ + file_id: fileId, + status: UploadStatus.Pending, + retries: 0, + created_at: createdNode.created_at, + progress: 0, + }) + .executeTakeFirst(); + + if (!createdUpload) { + throw new MutationError( + MutationErrorCode.FileCreateFailed, + 'Failed to create upload' + ); + } + + await this.app.database + .deleteFrom('temp_files') + .where('id', '=', tempFileId) + .execute(); + + const url = await this.app.fs.url(createdLocalFile.path); eventBus.publish({ - type: 'file.state.updated', + type: 'local.file.created', accountId: this.workspace.accountId, workspaceId: this.workspace.id, - fileState: mapFileState(createdFileState, url), + localFile: mapLocalFile(createdLocalFile, url), }); - this.triggerUploads(); - } - - public saveFile(file: LocalFileNode, path: string): void { - const id = generateId(IdType.Save); - const state: FileSaveState = { - id, - file, - status: SaveStatus.Active, - startedAt: new Date().toISOString(), - completedAt: null, - path, - progress: 0, - }; - - this.saves.push(state); - this.processSaveAsync(state); - eventBus.publish({ - type: 'file.save.updated', + type: 'upload.created', accountId: this.workspace.accountId, workspaceId: this.workspace.id, - fileSave: state, + upload: mapUpload(createdUpload), }); - } - public getSaves(): FileSaveState[] { - const clonedSaves = cloneDeep(this.saves); - return clonedSaves.sort((a, b) => { - // latest first - return -a.id.localeCompare(b.id); - }); + this.app.jobs.addJob( + { + type: 'file.upload', + accountId: this.workspace.accountId, + workspaceId: this.workspace.id, + fileId: fileId, + }, + { + delay: ms('2 seconds'), + } + ); } public async deleteFile(node: SelectNode): Promise { @@ -219,449 +231,170 @@ export class FileService { await this.app.fs.delete(filePath); } - public triggerUploads(): void { - this.uploadsEventLoop.trigger(); - } - - public triggerDownloads(): void { - this.downloadsEventLoop.trigger(); - } - - public destroy(): void { - this.uploadsEventLoop.stop(); - this.downloadsEventLoop.stop(); - this.cleanupEventLoop.stop(); - } - - private async uploadFiles(): Promise { - if (!this.workspace.account.server.isAvailable) { - return; - } - - debug(`Uploading files for workspace ${this.workspace.id}`); - - const uploads = await this.workspace.database - .selectFrom('file_states') - .selectAll() - .where('upload_status', '=', UploadStatus.Pending) - .execute(); - - if (uploads.length === 0) { - return; - } - - for (const upload of uploads) { - await this.uploadFile(upload); - } - } - - private async uploadFile(state: SelectFileState): Promise { - const node = await this.workspace.database - .selectFrom('nodes') - .selectAll() - .where('id', '=', state.id) - .executeTakeFirst(); - - if (!node) { - return; - } - - if (node.server_revision === '0') { - // file is not synced with the server, we need to wait for the sync to complete - return; - } - - const file = mapNode(node) as LocalFileNode; - const filePath = this.buildFilePath(file.id, file.attributes.extension); - const exists = await this.app.fs.exists(filePath); - if (!exists) { - debug(`File ${file.id} not found on disk`); - - await this.workspace.database - .deleteFrom('file_states') - .returningAll() - .where('id', '=', state.id) - .executeTakeFirst(); - - return; - } - - const url = await this.app.fs.url(filePath); - if (state.upload_retries && state.upload_retries >= UPLOAD_RETRIES_LIMIT) { - debug(`File ${state.id} upload retries limit reached, marking as failed`); - - const updatedFileState = await this.workspace.database - .updateTable('file_states') - .returningAll() - .set({ - upload_status: UploadStatus.Failed, - upload_retries: state.upload_retries + 1, - }) - .where('id', '=', state.id) - .executeTakeFirst(); - - if (updatedFileState) { - eventBus.publish({ - type: 'file.state.updated', - accountId: this.workspace.accountId, - workspaceId: this.workspace.id, - fileState: mapFileState(updatedFileState, url), - }); - } - - return; - } - - if (file.attributes.status === FileStatus.Ready) { - const updatedFileState = await this.workspace.database - .updateTable('file_states') - .returningAll() - .set({ - upload_status: UploadStatus.Completed, - upload_progress: 100, - upload_completed_at: new Date().toISOString(), - }) - .where('id', '=', file.id) - .executeTakeFirst(); - - if (updatedFileState) { - eventBus.publish({ - type: 'file.state.updated', - accountId: this.workspace.accountId, - workspaceId: this.workspace.id, - fileState: mapFileState(updatedFileState, url), - }); - } - - return; - } - - try { - const fileStream = await this.app.fs.readStream(filePath); - - await this.workspace.account.client.put( - `v1/workspaces/${this.workspace.id}/files/${file.id}`, - { - body: fileStream, - headers: { - 'Content-Type': file.attributes.mimeType, - 'Content-Length': file.attributes.size.toString(), - }, - } - ); - - const finalFileState = await this.workspace.database - .updateTable('file_states') - .returningAll() - .set({ - upload_status: UploadStatus.Completed, - upload_progress: 100, - upload_completed_at: new Date().toISOString(), - }) - .where('id', '=', file.id) - .executeTakeFirst(); - - if (finalFileState) { - eventBus.publish({ - type: 'file.state.updated', - accountId: this.workspace.accountId, - workspaceId: this.workspace.id, - fileState: mapFileState(finalFileState, url), - }); - } - - debug(`File ${file.id} uploaded successfully`); - } catch (error) { - debug(`Error uploading file ${file.id}: ${error}`); - - const updatedFileState = await this.workspace.database - .updateTable('file_states') - .returningAll() - .set((eb) => ({ upload_retries: eb('upload_retries', '+', 1) })) - .where('id', '=', file.id) - .executeTakeFirst(); - - if (updatedFileState) { - eventBus.publish({ - type: 'file.state.updated', - accountId: this.workspace.accountId, - workspaceId: this.workspace.id, - fileState: mapFileState(updatedFileState, url), - }); - } - } - } - - public async downloadFiles(): Promise { - if (!this.workspace.account.server.isAvailable) { - return; - } - - debug(`Downloading files for workspace ${this.workspace.id}`); - - const downloads = await this.workspace.database - .selectFrom('file_states') - .selectAll() - .where('download_status', '=', DownloadStatus.Pending) - .execute(); - - if (downloads.length === 0) { - return; - } - - for (const download of downloads) { - await this.downloadFile(download); - } - } - - private async downloadFile(fileState: SelectFileState): Promise { - if ( - fileState.download_retries && - fileState.download_retries >= DOWNLOAD_RETRIES_LIMIT - ) { - debug( - `File ${fileState.id} download retries limit reached, marking as failed` - ); - - const updatedFileState = await this.workspace.database - .updateTable('file_states') - .returningAll() - .set({ - download_status: DownloadStatus.Failed, - download_retries: fileState.download_retries + 1, - }) - .where('id', '=', fileState.id) - .executeTakeFirst(); - - if (updatedFileState) { - eventBus.publish({ - type: 'file.state.updated', - accountId: this.workspace.accountId, - workspaceId: this.workspace.id, - fileState: mapFileState(updatedFileState, null), - }); - } - - return; - } + public async initAutoDownload( + fileId: string + ): Promise { + const lockKey = `download.auto.${fileId}`; const node = await this.workspace.database .selectFrom('nodes') .selectAll() - .where('id', '=', fileState.id) + .where('id', '=', fileId) .executeTakeFirst(); if (!node) { - return; + throw new MutationError( + MutationErrorCode.FileNotFound, + 'The file you are trying to download does not exist.' + ); } const file = mapNode(node) as LocalFileNode; - - if (node.server_revision === '0') { - // file is not synced with the server, we need to wait for the sync to complete - return; - } - - const filePath = this.buildFilePath(file.id, file.attributes.extension); - const exists = await this.app.fs.exists(filePath); - if (exists) { - const url = await this.app.fs.url(filePath); - - const updatedFileState = await this.workspace.database - .updateTable('file_states') - .returningAll() - .set({ - download_status: DownloadStatus.Completed, - download_progress: 100, - download_completed_at: new Date().toISOString(), - }) - .where('id', '=', fileState.id) - .executeTakeFirst(); - - if (updatedFileState) { - eventBus.publish({ - type: 'file.state.updated', - accountId: this.workspace.accountId, - workspaceId: this.workspace.id, - fileState: mapFileState(updatedFileState, url), - }); - } - - return; - } - - try { - const response = await this.workspace.account.client.get( - `v1/workspaces/${this.workspace.id}/files/${file.id}`, - { - onDownloadProgress: async (progress, _chunk) => { - const percent = Math.round((progress.percent || 0) * 100); - - const updatedFileState = await this.workspace.database - .updateTable('file_states') - .returningAll() - .set({ - download_progress: percent, - }) - .where('id', '=', file.id) - .executeTakeFirst(); - - if (!updatedFileState) { - return; - } - - eventBus.publish({ - type: 'file.state.updated', - accountId: this.workspace.accountId, - workspaceId: this.workspace.id, - fileState: mapFileState(updatedFileState, null), - }); - }, - } + if (file.attributes.status !== FileStatus.Ready) { + throw new MutationError( + MutationErrorCode.FileNotReady, + 'The file you are trying to download is not uploaded by the author yet.' ); - - const writeStream = await this.app.fs.writeStream(filePath); - await response.body?.pipeTo(writeStream); - const url = await this.app.fs.url(filePath); - - const updatedFileState = await this.workspace.database - .updateTable('file_states') - .returningAll() - .set({ - download_status: DownloadStatus.Completed, - download_progress: 100, - download_completed_at: new Date().toISOString(), - }) - .where('id', '=', fileState.id) - .executeTakeFirst(); - - if (updatedFileState) { - eventBus.publish({ - type: 'file.state.updated', - accountId: this.workspace.accountId, - workspaceId: this.workspace.id, - fileState: mapFileState(updatedFileState, url), - }); - } - } catch { - const updatedFileState = await this.workspace.database - .updateTable('file_states') - .returningAll() - .set((eb) => ({ download_retries: eb('download_retries', '+', 1) })) - .where('id', '=', fileState.id) - .executeTakeFirst(); - - if (updatedFileState) { - eventBus.publish({ - type: 'file.state.updated', - accountId: this.workspace.accountId, - workspaceId: this.workspace.id, - fileState: mapFileState(updatedFileState, null), - }); - } } + + const result = await this.lock.acquire(lockKey, async () => { + const existingDownload = await this.workspace.database + .selectFrom('downloads') + .selectAll() + .where('file_id', '=', fileId) + .where('type', '=', DownloadType.Auto) + .executeTakeFirst(); + + if (existingDownload) { + return { existingDownload }; + } + + const createdDownload = await this.workspace.database + .insertInto('downloads') + .returningAll() + .values({ + id: generateId(IdType.Download), + file_id: fileId, + version: file.attributes.version, + type: DownloadType.Auto, + name: file.attributes.name, + path: this.buildFilePath(fileId, file.attributes.extension), + size: file.attributes.size, + mime_type: file.attributes.mimeType, + status: DownloadStatus.Pending, + progress: 0, + retries: 0, + created_at: new Date().toISOString(), + }) + .executeTakeFirst(); + + if (!createdDownload) { + return null; + } + + return { createdDownload }; + }); + + if (!result) { + return null; + } + + if (result.existingDownload) { + return result.existingDownload; + } + + if (result.createdDownload) { + await this.app.jobs.addJob({ + type: 'file.download', + accountId: this.workspace.accountId, + workspaceId: this.workspace.id, + downloadId: result.createdDownload.id, + }); + + eventBus.publish({ + type: 'download.created', + accountId: this.workspace.accountId, + workspaceId: this.workspace.id, + download: mapDownload(result.createdDownload), + }); + } + + return result.createdDownload; + } + + public async initManualDownload( + fileId: string, + path: string + ): Promise { + const node = await this.workspace.database + .selectFrom('nodes') + .selectAll() + .where('id', '=', fileId) + .executeTakeFirst(); + + if (!node) { + throw new MutationError( + MutationErrorCode.FileNotFound, + 'The file you are trying to download does not exist.' + ); + } + + const file = mapNode(node) as LocalFileNode; + if (file.attributes.status !== FileStatus.Ready) { + throw new MutationError( + MutationErrorCode.FileNotReady, + 'The file you are trying to download is not uploaded by the author yet.' + ); + } + + const name = this.app.path.filename(path); + const createdDownload = await this.workspace.database + .insertInto('downloads') + .returningAll() + .values({ + id: generateId(IdType.Download), + file_id: fileId, + version: file.attributes.version, + type: DownloadType.Manual, + name: name, + path: path, + size: file.attributes.size, + mime_type: file.attributes.mimeType, + status: DownloadStatus.Pending, + progress: 0, + retries: 0, + created_at: new Date().toISOString(), + }) + .executeTakeFirst(); + + if (!createdDownload) { + return null; + } + + await this.app.jobs.addJob({ + type: 'file.download', + accountId: this.workspace.accountId, + workspaceId: this.workspace.id, + downloadId: createdDownload.id, + }); + + eventBus.publish({ + type: 'download.created', + accountId: this.workspace.accountId, + workspaceId: this.workspace.id, + download: mapDownload(createdDownload), + }); + + return createdDownload; } private buildFilePath(id: string, extension: string): string { return this.app.path.join(this.filesDir, `${id}${extension}`); } - private async processSaveAsync(save: FileSaveState): Promise { - if (this.app.meta.type !== 'desktop') { - return; - } - - try { - const fileState = await this.workspace.database - .selectFrom('file_states') - .selectAll() - .where('id', '=', save.file.id) - .executeTakeFirst(); - - // if file is already downloaded, copy it to the save path - if (fileState && fileState.download_progress === 100) { - const sourceFilePath = this.buildFilePath( - save.file.id, - save.file.attributes.extension - ); - - await this.app.fs.copy(sourceFilePath, save.path); - save.status = SaveStatus.Completed; - save.completedAt = new Date().toISOString(); - save.progress = 100; - - eventBus.publish({ - type: 'file.save.updated', - accountId: this.workspace.accountId, - workspaceId: this.workspace.id, - fileSave: save, - }); - - return; - } - - // if file is not downloaded, download it - try { - const response = await this.workspace.account.client.get( - `v1/workspaces/${this.workspace.id}/files/${save.file.id}`, - { - onDownloadProgress: async (progress, _chunk) => { - const percent = Math.round((progress.percent || 0) * 100); - save.progress = percent; - - eventBus.publish({ - type: 'file.save.updated', - accountId: this.workspace.accountId, - workspaceId: this.workspace.id, - fileSave: save, - }); - }, - } - ); - - const writeStream = await this.app.fs.writeStream(save.path); - await response.body?.pipeTo(writeStream); - - save.status = SaveStatus.Completed; - save.completedAt = new Date().toISOString(); - save.progress = 100; - - eventBus.publish({ - type: 'file.save.updated', - accountId: this.workspace.accountId, - workspaceId: this.workspace.id, - fileSave: save, - }); - } catch { - save.status = SaveStatus.Failed; - save.completedAt = new Date().toISOString(); - save.progress = 0; - - eventBus.publish({ - type: 'file.save.updated', - accountId: this.workspace.accountId, - workspaceId: this.workspace.id, - fileSave: save, - }); - } - } catch (error) { - debug(`Error saving file ${save.file.id}: ${error}`); - save.status = SaveStatus.Failed; - save.completedAt = new Date().toISOString(); - save.progress = 0; - - eventBus.publish({ - type: 'file.save.updated', - accountId: this.workspace.accountId, - workspaceId: this.workspace.id, - fileSave: save, - }); - } - } - - private async cleanupFiles(): Promise { + public async cleanupFiles(): Promise { await this.cleanDeletedFiles(); - await this.cleanOldDownloadedFiles(); + await this.cleanUnopenedFiles(); } private async cleanDeletedFiles(): Promise { @@ -678,15 +411,15 @@ export class FileService { } const fileIds = Object.keys(fileIdMap); - const fileStates = await this.workspace.database - .selectFrom('file_states') + const localFiles = await this.workspace.database + .selectFrom('local_files') .select(['id']) .where('id', 'in', fileIds) .execute(); for (const fileId of fileIds) { - const fileState = fileStates.find((f) => f.id === fileId); - if (fileState) { + const localFile = localFiles.find((lf) => lf.id === fileId); + if (localFile) { continue; } @@ -696,135 +429,25 @@ export class FileService { } } - private async cleanOldDownloadedFiles(): Promise { - debug(`Cleaning old downloaded files for workspace ${this.workspace.id}`); + private async cleanUnopenedFiles(): Promise { + debug(`Cleaning unopened files for workspace ${this.workspace.id}`); const sevenDaysAgo = new Date(Date.now() - ms('7 days')).toISOString(); - let lastId = ''; - const batchSize = 100; + const unopenedFiles = await this.workspace.database + .deleteFrom('local_files') + .where('opened_at', '<', sevenDaysAgo) + .returningAll() + .execute(); - let hasMoreFiles = true; - while (hasMoreFiles) { - let query = this.workspace.database - .selectFrom('file_states') - .select(['id', 'upload_status', 'download_completed_at']) - .where('download_status', '=', DownloadStatus.Completed) - .where('download_progress', '=', 100) - .where('download_completed_at', '<', sevenDaysAgo) - .orderBy('id', 'asc') - .limit(batchSize); + for (const localFile of unopenedFiles) { + await this.app.fs.delete(localFile.path); - if (lastId) { - query = query.where('id', '>', lastId); - } - - const fileStates = await query.execute(); - - if (fileStates.length === 0) { - hasMoreFiles = false; - continue; - } - - const fileIds = fileStates.map((f) => f.id); - - const fileInteractions = await this.workspace.database - .selectFrom('node_interactions') - .select(['node_id', 'last_opened_at']) - .where('node_id', 'in', fileIds) - .where('collaborator_id', '=', this.workspace.userId) - .execute(); - - const nodes = await this.workspace.database - .selectFrom('nodes') - .select(['id', 'attributes']) - .where('id', 'in', fileIds) - .execute(); - - const interactionMap = new Map( - fileInteractions.map((fi) => [fi.node_id, fi.last_opened_at]) - ); - const nodeMap = new Map(nodes.map((n) => [n.id, n.attributes])); - - for (const fileState of fileStates) { - try { - const lastOpenedAt = interactionMap.get(fileState.id); - const shouldDelete = !lastOpenedAt || lastOpenedAt < sevenDaysAgo; - - if (!shouldDelete) { - continue; - } - - const nodeAttributes = nodeMap.get(fileState.id); - if (!nodeAttributes) { - continue; - } - - const attributes = JSON.parse(nodeAttributes); - if (attributes.type !== 'file' || !attributes.extension) { - continue; - } - - const filePath = this.buildFilePath( - fileState.id, - attributes.extension - ); - - const exists = await this.app.fs.exists(filePath); - if (!exists) { - continue; - } - - debug(`Deleting old downloaded file: ${fileState.id}`); - await this.app.fs.delete(filePath); - - if ( - fileState.upload_status !== null && - fileState.upload_status !== UploadStatus.None - ) { - const updatedFileState = await this.workspace.database - .updateTable('file_states') - .returningAll() - .set({ - download_status: DownloadStatus.None, - download_progress: 0, - download_completed_at: null, - }) - .where('id', '=', fileState.id) - .executeTakeFirst(); - - if (updatedFileState) { - eventBus.publish({ - type: 'file.state.updated', - accountId: this.workspace.accountId, - workspaceId: this.workspace.id, - fileState: mapFileState(updatedFileState, null), - }); - } - } else { - const deleted = await this.workspace.database - .deleteFrom('file_states') - .returningAll() - .where('id', '=', fileState.id) - .executeTakeFirst(); - - if (deleted) { - eventBus.publish({ - type: 'file.state.deleted', - accountId: this.workspace.accountId, - workspaceId: this.workspace.id, - fileId: fileState.id, - }); - } - } - } catch { - continue; - } - } - - lastId = fileStates[fileStates.length - 1]!.id; - if (fileStates.length < batchSize) { - hasMoreFiles = false; - } + eventBus.publish({ + type: 'local.file.deleted', + accountId: this.workspace.accountId, + workspaceId: this.workspace.id, + localFile: mapLocalFile(localFile, ''), + }); } } } diff --git a/packages/client/src/services/workspaces/mutation-service.ts b/packages/client/src/services/workspaces/mutation-service.ts index c63b6527..3f557e08 100644 --- a/packages/client/src/services/workspaces/mutation-service.ts +++ b/packages/client/src/services/workspaces/mutation-service.ts @@ -1,6 +1,3 @@ -import ms from 'ms'; - -import { EventLoop } from '@colanode/client/lib/event-loop'; import { mapMutation } from '@colanode/client/lib/mappers'; import { WorkspaceService } from '@colanode/client/services/workspaces/workspace-service'; import { @@ -18,24 +15,29 @@ const debug = createDebugger('desktop:service:mutation'); export class MutationService { private readonly workspace: WorkspaceService; - private readonly eventLoop: EventLoop; constructor(workspaceService: WorkspaceService) { this.workspace = workspaceService; - - this.eventLoop = new EventLoop(ms('1 minute'), 100, this.sync.bind(this)); - this.eventLoop.start(); } - public destroy(): void { - this.eventLoop.stop(); + public async scheduleSync(): Promise { + await this.workspace.account.app.jobs.addJob( + { + type: 'mutations.sync', + accountId: this.workspace.accountId, + workspaceId: this.workspace.id, + }, + { + deduplication: { + key: `mutations.sync.${this.workspace.accountId}.${this.workspace.id}`, + replace: true, + }, + delay: 500, + } + ); } - public triggerSync(): void { - this.eventLoop.trigger(); - } - - private async sync(): Promise { + public async sync(): Promise { try { let hasMutations = true; @@ -105,7 +107,10 @@ export class MutationService { const unsyncedMutationIds: string[] = []; for (const result of response.results) { - if (result.status === MutationStatus.OK) { + if ( + result.status === MutationStatus.OK || + result.status === MutationStatus.CREATED + ) { syncedMutationIds.push(result.id); } else { unsyncedMutationIds.push(result.id); diff --git a/packages/client/src/services/workspaces/node-reaction-service.ts b/packages/client/src/services/workspaces/node-reaction-service.ts index 740383b2..66c5e5c1 100644 --- a/packages/client/src/services/workspaces/node-reaction-service.ts +++ b/packages/client/src/services/workspaces/node-reaction-service.ts @@ -136,7 +136,7 @@ export class NodeReactionService { throw new Error('Failed to create node reaction'); } - this.workspace.mutations.triggerSync(); + this.workspace.mutations.scheduleSync(); eventBus.publish({ type: 'node.reaction.created', @@ -223,7 +223,7 @@ export class NodeReactionService { throw new Error('Failed to delete node reaction'); } - this.workspace.mutations.triggerSync(); + this.workspace.mutations.scheduleSync(); eventBus.publish({ type: 'node.reaction.deleted', diff --git a/packages/client/src/services/workspaces/node-service.ts b/packages/client/src/services/workspaces/node-service.ts index 17153826..2ac96ab3 100644 --- a/packages/client/src/services/workspaces/node-service.ts +++ b/packages/client/src/services/workspaces/node-service.ts @@ -25,7 +25,6 @@ import { CanCreateNodeContext, CanUpdateAttributesContext, CanDeleteNodeContext, - extractNodeAvatar, } from '@colanode/core'; import { decodeState, encodeState, YDoc } from '@colanode/crdt'; @@ -194,14 +193,11 @@ export class NodeService { debug(`Created node ${createdNode.id} with type ${createdNode.type}`); - const node = mapNode(createdNode); - await this.downloadNodeAvatar(node.attributes); - eventBus.publish({ type: 'node.created', accountId: this.workspace.accountId, workspaceId: this.workspace.id, - node, + node: mapNode(createdNode), }); for (const createdNodeReference of createdNodeReferences) { @@ -213,7 +209,7 @@ export class NodeService { }); } - this.workspace.mutations.triggerSync(); + this.workspace.mutations.scheduleSync(); return createdNode; } @@ -387,21 +383,18 @@ export class NodeService { if (updatedNode) { debug(`Updated node ${updatedNode.id} with type ${updatedNode.type}`); - const node = mapNode(updatedNode); - await this.downloadNodeAvatar(node.attributes); - eventBus.publish({ type: 'node.updated', accountId: this.workspace.accountId, workspaceId: this.workspace.id, - node, + node: mapNode(updatedNode), }); } else { debug(`Failed to update node ${nodeId}`); } if (createdMutation) { - this.workspace.mutations.triggerSync(); + this.workspace.mutations.scheduleSync(); } for (const createdNodeReference of createdNodeReferences) { @@ -509,7 +502,7 @@ export class NodeService { } if (createdMutation) { - this.workspace.mutations.triggerSync(); + this.workspace.mutations.scheduleSync(); } } @@ -618,14 +611,11 @@ export class NodeService { debug(`Created node ${createdNode.id} with type ${createdNode.type}`); - const node = mapNode(createdNode); - await this.downloadNodeAvatar(node.attributes); - eventBus.publish({ type: 'node.created', accountId: this.workspace.accountId, workspaceId: this.workspace.id, - node, + node: mapNode(createdNode), }); for (const createdNodeReference of createdNodeReferences) { @@ -771,14 +761,11 @@ export class NodeService { debug(`Updated node ${updatedNode.id} with type ${updatedNode.type}`); - const node = mapNode(updatedNode); - await this.downloadNodeAvatar(node.attributes); - eventBus.publish({ type: 'node.updated', accountId: this.workspace.accountId, workspaceId: this.workspace.id, - node, + node: mapNode(updatedNode), }); for (const createdNodeReference of createdNodeReferences) { @@ -1120,12 +1107,4 @@ export class NodeService { return false; } - - private async downloadNodeAvatar(attributes: NodeAttributes) { - const avatar = extractNodeAvatar(attributes); - - if (avatar) { - await this.workspace.account.downloadAvatar(avatar); - } - } } diff --git a/packages/client/src/services/workspaces/user-service.ts b/packages/client/src/services/workspaces/user-service.ts index 11c3052d..70b24756 100644 --- a/packages/client/src/services/workspaces/user-service.ts +++ b/packages/client/src/services/workspaces/user-service.ts @@ -48,14 +48,6 @@ export class UserService { ) .executeTakeFirst(); - if (createdUser?.avatar) { - await this.workspace.account.downloadAvatar(createdUser.avatar); - } - - if (createdUser?.custom_avatar) { - await this.workspace.account.downloadAvatar(createdUser.custom_avatar); - } - if (createdUser) { eventBus.publish({ type: 'user.created', @@ -102,14 +94,6 @@ export class UserService { ) .executeTakeFirst(); - if (createdUser?.avatar) { - await this.workspace.account.downloadAvatar(createdUser.avatar); - } - - if (createdUser?.custom_avatar) { - await this.workspace.account.downloadAvatar(createdUser.custom_avatar); - } - if (createdUser) { eventBus.publish({ type: 'user.created', diff --git a/packages/client/src/services/workspaces/workspace-service.ts b/packages/client/src/services/workspaces/workspace-service.ts index 5b41368d..9e9b1382 100644 --- a/packages/client/src/services/workspaces/workspace-service.ts +++ b/packages/client/src/services/workspaces/workspace-service.ts @@ -1,4 +1,5 @@ import { Kysely, Migration, Migrator } from 'kysely'; +import ms from 'ms'; import { WorkspaceDatabaseSchema, @@ -39,6 +40,8 @@ export class WorkspaceService { public readonly radar: RadarService; public readonly nodeCounters: NodeCountersService; + private readonly workspaceFilesCleanJobScheduleId: string; + constructor(workspace: Workspace, account: AccountService) { debug(`Initializing workspace service ${workspace.id}`); @@ -64,6 +67,8 @@ export class WorkspaceService { this.synchronizer = new SyncService(this); this.radar = new RadarService(this); this.nodeCounters = new NodeCountersService(this); + + this.workspaceFilesCleanJobScheduleId = `workspace.files.clean.${this.account.id}.${this.workspace.id}`; } public get id(): string { @@ -102,6 +107,23 @@ export class WorkspaceService { await this.collaborations.init(); await this.synchronizer.init(); await this.radar.init(); + await this.files.init(); + + await this.account.app.jobs.upsertJobSchedule( + this.workspaceFilesCleanJobScheduleId, + { + type: 'workspace.files.clean', + accountId: this.account.id, + workspaceId: this.workspace.id, + }, + ms('1 minute'), + { + deduplication: { + key: this.workspaceFilesCleanJobScheduleId, + replace: true, + }, + } + ); } private async migrate(): Promise { @@ -122,10 +144,7 @@ export class WorkspaceService { public async delete(): Promise { try { this.database.destroy(); - this.mutations.destroy(); this.synchronizer.destroy(); - this.files.destroy(); - this.mutations.destroy(); this.radar.destroy(); const databasePath = this.account.app.path.workspaceDatabase( @@ -147,6 +166,10 @@ export class WorkspaceService { .where('id', '=', this.workspace.id) .execute(); + await this.account.app.jobs.removeJobSchedule( + this.workspaceFilesCleanJobScheduleId + ); + eventBus.publish({ type: 'workspace.deleted', workspace: this.workspace, diff --git a/packages/client/src/types/avatars.ts b/packages/client/src/types/avatars.ts new file mode 100644 index 00000000..4548ea42 --- /dev/null +++ b/packages/client/src/types/avatars.ts @@ -0,0 +1,8 @@ +export type Avatar = { + id: string; + path: string; + size: number; + createdAt: string; + openedAt: string; + url: string; +}; diff --git a/packages/client/src/types/events.ts b/packages/client/src/types/events.ts index 91a70344..4c9856aa 100644 --- a/packages/client/src/types/events.ts +++ b/packages/client/src/types/events.ts @@ -1,11 +1,17 @@ import { Account, AccountMetadata } from '@colanode/client/types/accounts'; import { AppMetadata } from '@colanode/client/types/apps'; +import { Avatar } from '@colanode/client/types/avatars'; import { Document, DocumentState, DocumentUpdate, } from '@colanode/client/types/documents'; -import { FileSaveState, FileState } from '@colanode/client/types/files'; +import { + LocalFile, + Upload, + Download, + TempFile, +} from '@colanode/client/types/files'; import { LocalNode, NodeCounter, @@ -84,25 +90,60 @@ export type NodeReactionDeletedEvent = { nodeReaction: NodeReaction; }; -export type FileStateUpdatedEvent = { - type: 'file.state.updated'; +export type LocalFileCreatedEvent = { + type: 'local.file.created'; accountId: string; workspaceId: string; - fileState: FileState; + localFile: LocalFile; }; -export type FileStateDeletedEvent = { - type: 'file.state.deleted'; +export type LocalFileDeletedEvent = { + type: 'local.file.deleted'; accountId: string; workspaceId: string; - fileId: string; + localFile: LocalFile; }; -export type FileSaveUpdatedEvent = { - type: 'file.save.updated'; +export type UploadCreatedEvent = { + type: 'upload.created'; accountId: string; workspaceId: string; - fileSave: FileSaveState; + upload: Upload; +}; + +export type UploadUpdatedEvent = { + type: 'upload.updated'; + accountId: string; + workspaceId: string; + upload: Upload; +}; + +export type UploadDeletedEvent = { + type: 'upload.deleted'; + accountId: string; + workspaceId: string; + upload: Upload; +}; + +export type DownloadCreatedEvent = { + type: 'download.created'; + accountId: string; + workspaceId: string; + download: Download; +}; + +export type DownloadUpdatedEvent = { + type: 'download.updated'; + accountId: string; + workspaceId: string; + download: Download; +}; + +export type DownloadDeletedEvent = { + type: 'download.deleted'; + accountId: string; + workspaceId: string; + download: Download; }; export type AccountCreatedEvent = { @@ -296,10 +337,26 @@ export type NodeCounterDeletedEvent = { counter: NodeCounter; }; -export type AvatarDownloadedEvent = { - type: 'avatar.downloaded'; +export type AvatarCreatedEvent = { + type: 'avatar.created'; accountId: string; - avatarId: string; + avatar: Avatar; +}; + +export type AvatarDeletedEvent = { + type: 'avatar.deleted'; + accountId: string; + avatar: Avatar; +}; + +export type TempFileCreatedEvent = { + type: 'temp.file.created'; + tempFile: TempFile; +}; + +export type TempFileDeletedEvent = { + type: 'temp.file.deleted'; + tempFile: TempFile; }; export type Event = @@ -322,9 +379,14 @@ export type Event = | ServerUpdatedEvent | ServerDeletedEvent | ServerAvailabilityChangedEvent - | FileStateUpdatedEvent - | FileStateDeletedEvent - | FileSaveUpdatedEvent + | LocalFileCreatedEvent + | LocalFileDeletedEvent + | UploadCreatedEvent + | UploadUpdatedEvent + | UploadDeletedEvent + | DownloadCreatedEvent + | DownloadUpdatedEvent + | DownloadDeletedEvent | QueryResultUpdatedEvent | RadarDataUpdatedEvent | CollaborationCreatedEvent @@ -347,4 +409,7 @@ export type Event = | NodeReferenceDeletedEvent | NodeCounterUpdatedEvent | NodeCounterDeletedEvent - | AvatarDownloadedEvent; + | AvatarCreatedEvent + | AvatarDeletedEvent + | TempFileCreatedEvent + | TempFileDeletedEvent; diff --git a/packages/client/src/types/files.ts b/packages/client/src/types/files.ts index 1afce169..a9718076 100644 --- a/packages/client/src/types/files.ts +++ b/packages/client/src/types/files.ts @@ -1,4 +1,3 @@ -import { LocalFileNode } from '@colanode/client/types'; import { FileSubtype } from '@colanode/core'; export type OpenFileDialogOptions = { @@ -11,54 +10,69 @@ export type TempFile = { name: string; path: string; size: number; - type: FileSubtype; + subtype: FileSubtype; mimeType: string; extension: string; url: string; }; -export type FileState = { +export type LocalFile = { id: string; version: string; - downloadStatus: DownloadStatus | null; - downloadProgress: number | null; - downloadRetries: number | null; - downloadStartedAt: string | null; - downloadCompletedAt: string | null; - uploadStatus: UploadStatus | null; - uploadProgress: number | null; - uploadRetries: number | null; - uploadStartedAt: string | null; - uploadCompletedAt: string | null; - url: string | null; + name: string; + path: string; + size: number; + subtype: FileSubtype; + mimeType: string; + createdAt: string; + openedAt: string; + url: string; +}; + +export type Upload = { + fileId: string; + status: UploadStatus; + progress: number; + retries: number; + createdAt: string; + completedAt: string | null; + errorCode: string | null; + errorMessage: string | null; +}; + +export type Download = { + id: string; + fileId: string; + version: string; + type: DownloadType; + name: string; + path: string; + size: number; + mimeType: string; + status: DownloadStatus; + progress: number; + retries: number; + createdAt: string; + completedAt: string | null; + errorCode: string | null; + errorMessage: string | null; }; export enum DownloadStatus { - None = 0, - Pending = 1, + Pending = 0, + Downloading = 1, Completed = 2, Failed = 3, } export enum UploadStatus { - None = 0, - Pending = 1, + Pending = 0, + Uploading = 1, Completed = 2, Failed = 3, } -export enum SaveStatus { - Active = 1, - Completed = 2, - Failed = 3, +export enum DownloadType { + Auto = 0, + Manual = 1, } - -export type FileSaveState = { - id: string; - file: LocalFileNode; - status: SaveStatus; - startedAt: string; - completedAt: string | null; - path: string; - progress: number; -}; diff --git a/packages/client/src/types/workspaces.ts b/packages/client/src/types/workspaces.ts index fbd91f23..cf8bdb11 100644 --- a/packages/client/src/types/workspaces.ts +++ b/packages/client/src/types/workspaces.ts @@ -65,10 +65,11 @@ export type WorkspaceMetadataMap = { }; export enum SpecialContainerTabPath { - Downloads = 'downloads', WorkspaceSettings = 'workspace/settings', WorkspaceStorage = 'workspace/storage', WorkspaceUsers = 'workspace/users', + WorkspaceUploads = 'workspace/uploads', + WorkspaceDownloads = 'workspace/downloads', WorkspaceDelete = 'workspace/delete', AccountSettings = 'account/settings', AccountLogout = 'account/logout', diff --git a/packages/core/src/lib/id.ts b/packages/core/src/lib/id.ts index fb2029c7..9c93c513 100644 --- a/packages/core/src/lib/id.ts +++ b/packages/core/src/lib/id.ts @@ -41,6 +41,8 @@ export enum IdType { TempFile = 'tf', Socket = 'sk', Save = 'sv', + Job = 'jb', + Download = 'dl', } export const SpecialId = { diff --git a/packages/core/src/lib/utils.ts b/packages/core/src/lib/utils.ts index 2dbe7b21..3bc9753c 100644 --- a/packages/core/src/lib/utils.ts +++ b/packages/core/src/lib/utils.ts @@ -216,3 +216,17 @@ export const trimString = (str: string, maxLength: number) => { return str.slice(0, maxLength - 3) + '...'; }; + +export const calculatePercentage = ( + partial: number, + total: number, + digits: number = 2 +) => { + if (total === 0) { + return 0; + } + + const raw = (partial / total) * 100; + const factor = 10 ** digits; + return Math.round(raw * factor) / factor; +}; diff --git a/packages/core/src/registry/nodes/file.ts b/packages/core/src/registry/nodes/file.ts index 06872955..6eeb8472 100644 --- a/packages/core/src/registry/nodes/file.ts +++ b/packages/core/src/registry/nodes/file.ts @@ -3,10 +3,11 @@ import { z } from 'zod/v4'; import { extractNodeRole } from '@colanode/core/lib/nodes'; import { hasNodeRole } from '@colanode/core/lib/permissions'; import { NodeModel } from '@colanode/core/registry/nodes/core'; +import { fileSubtypeSchema } from '@colanode/core/types/files'; export const fileAttributesSchema = z.object({ type: z.literal('file'), - subtype: z.string(), + subtype: fileSubtypeSchema, parentId: z.string(), index: z.string().optional(), name: z.string(), diff --git a/packages/core/src/types/api.ts b/packages/core/src/types/api.ts index f6fc7d3a..209ff05c 100644 --- a/packages/core/src/types/api.ts +++ b/packages/core/src/types/api.ts @@ -34,6 +34,10 @@ export enum ApiErrorCode { FileAlreadyUploaded = 'file_already_uploaded', FileUploadInitFailed = 'file_upload_init_failed', FileUploadFailed = 'file_upload_failed', + UserMaxFileSizeExceeded = 'user_max_file_size_exceeded', + WorkspaceMaxFileSizeExceeded = 'workspace_max_file_size_exceeded', + UserStorageLimitExceeded = 'user_storage_limit_exceeded', + WorkspaceStorageLimitExceeded = 'workspace_storage_limit_exceeded', WorkspaceMismatch = 'workspace_mismatch', FileError = 'file_error', FileSizeMismatch = 'file_size_mismatch', diff --git a/packages/core/src/types/files.ts b/packages/core/src/types/files.ts index 6a987fb6..72e24ae9 100644 --- a/packages/core/src/types/files.ts +++ b/packages/core/src/types/files.ts @@ -1,12 +1,5 @@ import { z } from 'zod/v4'; -export const fileUploadOutputSchema = z.object({ - success: z.boolean(), - uploadId: z.string(), -}); - -export type FileUploadOutput = z.infer; - export const fileSubtypeSchema = z.enum([ 'image', 'video', diff --git a/packages/ui/package.json b/packages/ui/package.json index b6e1aa43..48e9c5e5 100644 --- a/packages/ui/package.json +++ b/packages/ui/package.json @@ -71,6 +71,7 @@ "lucide-react": "^0.525.0", "re-resizable": "^6.11.2", "react": "^19.1.0", + "react-circular-progressbar": "^2.2.0", "react-day-picker": "^9.8.0", "react-dnd": "^16.0.1", "react-dnd-html5-backend": "^16.0.1", diff --git a/packages/ui/src/components/avatars/avatar-image.tsx b/packages/ui/src/components/avatars/avatar-image.tsx index 3e754eb0..d79473c8 100644 --- a/packages/ui/src/components/avatars/avatar-image.tsx +++ b/packages/ui/src/components/avatars/avatar-image.tsx @@ -10,25 +10,32 @@ export const AvatarImage = (props: AvatarProps) => { const account = useAccount(); const [failed, setFailed] = useState(false); - const { data, isPending } = useLiveQuery( - { - type: 'avatar.url.get', - accountId: account.id, - avatarId: props.avatar!, - }, - { - enabled: !!props.avatar, - } - ); + const avatarQuery = useLiveQuery({ + type: 'avatar.get', + accountId: account.id, + avatarId: props.avatar!, + }); - const url = data?.url; - if (failed || !url || isPending) { + if (avatarQuery.isPending) { + return ( +
+ ); + } + + const avatar = avatarQuery.data; + if (!avatar || failed) { return ; } return ( { - switch (status) { - case SaveStatus.Active: - return ( - - -
- -
- - - Downloading ... {progress}% - - - ); - case SaveStatus.Completed: - return ( - - -
- -
- - - Downloaded - - - ); - case SaveStatus.Failed: - return ( - - -
- -
- - - Download failed - - - ); - default: - return null; - } -}; diff --git a/packages/ui/src/components/downloads/downloads-breadcrumb.tsx b/packages/ui/src/components/downloads/downloads-breadcrumb.tsx deleted file mode 100644 index 6e6ac93a..00000000 --- a/packages/ui/src/components/downloads/downloads-breadcrumb.tsx +++ /dev/null @@ -1,22 +0,0 @@ -import { Download } from 'lucide-react'; - -import { - Breadcrumb, - BreadcrumbItem, - BreadcrumbList, -} from '@colanode/ui/components/ui/breadcrumb'; - -export const DownloadsBreadcrumb = () => { - return ( - - - -
- - Downloads -
- - - - ); -}; diff --git a/packages/ui/src/components/downloads/downloads-container-tab.tsx b/packages/ui/src/components/downloads/downloads-container-tab.tsx deleted file mode 100644 index 65f12213..00000000 --- a/packages/ui/src/components/downloads/downloads-container-tab.tsx +++ /dev/null @@ -1,30 +0,0 @@ -import { Download } from 'lucide-react'; - -import { SaveStatus } from '@colanode/client/types'; -import { useWorkspace } from '@colanode/ui/contexts/workspace'; -import { useLiveQuery } from '@colanode/ui/hooks/use-live-query'; - -export const DownloadsContainerTab = () => { - const workspace = useWorkspace(); - - const fileSaveListQuery = useLiveQuery({ - type: 'file.save.list', - accountId: workspace.accountId, - workspaceId: workspace.id, - }); - - if (fileSaveListQuery.isPending) { - return

Loading...

; - } - - const activeSaves = - fileSaveListQuery.data?.filter((save) => save.status === SaveStatus.Active) - ?.length ?? 0; - - return ( -
- - Downloads {activeSaves > 0 ? `(${activeSaves})` : ''} -
- ); -}; diff --git a/packages/ui/src/components/downloads/downloads-container.tsx b/packages/ui/src/components/downloads/downloads-container.tsx deleted file mode 100644 index 93631e55..00000000 --- a/packages/ui/src/components/downloads/downloads-container.tsx +++ /dev/null @@ -1,20 +0,0 @@ -import { DownloadsBreadcrumb } from '@colanode/ui/components/downloads/downloads-breadcrumb'; -import { DownloadsList } from '@colanode/ui/components/downloads/downloads-list'; -import { - Container, - ContainerBody, - ContainerHeader, -} from '@colanode/ui/components/ui/container'; - -export const DownloadsContainer = () => { - return ( - - - - - - - - - ); -}; diff --git a/packages/ui/src/components/downloads/downloads-list.tsx b/packages/ui/src/components/downloads/downloads-list.tsx deleted file mode 100644 index c3235f03..00000000 --- a/packages/ui/src/components/downloads/downloads-list.tsx +++ /dev/null @@ -1,80 +0,0 @@ -import { Folder } from 'lucide-react'; - -import { SaveStatus } from '@colanode/client/types'; -import { DownloadStatus } from '@colanode/ui/components/downloads/download-status'; -import { FileThumbnail } from '@colanode/ui/components/files/file-thumbnail'; -import { Button } from '@colanode/ui/components/ui/button'; -import { - Tooltip, - TooltipContent, - TooltipTrigger, -} from '@colanode/ui/components/ui/tooltip'; -import { useWorkspace } from '@colanode/ui/contexts/workspace'; -import { useLiveQuery } from '@colanode/ui/hooks/use-live-query'; - -export const DownloadsList = () => { - const workspace = useWorkspace(); - - const fileSaveListQuery = useLiveQuery({ - type: 'file.save.list', - accountId: workspace.accountId, - workspaceId: workspace.id, - }); - - const saves = fileSaveListQuery.data || []; - - const handleOpenDirectory = (path: string) => { - window.colanode.showItemInFolder(path); - }; - - if (saves.length === 0) { - return ( -
- No downloads yet -
- ); - } - - return ( -
- {saves.map((save) => ( -
- - -
-

- {save.file.attributes.name} -

-

- {save.path} -

-
-
- {save.status === SaveStatus.Completed && ( - - - - - Show in folder - - )} - -
-
- ))} -
- ); -}; diff --git a/packages/ui/src/components/emojis/emoji-element.tsx b/packages/ui/src/components/emojis/emoji-element.tsx index cb414ba7..06914765 100644 --- a/packages/ui/src/components/emojis/emoji-element.tsx +++ b/packages/ui/src/components/emojis/emoji-element.tsx @@ -18,6 +18,11 @@ export const EmojiElement = ({ id, className, onClick }: EmojiElementProps) => { } return ( - + {id} ); }; diff --git a/packages/ui/src/components/files/file-block.tsx b/packages/ui/src/components/files/file-block.tsx index 873efb30..599ccd6c 100644 --- a/packages/ui/src/components/files/file-block.tsx +++ b/packages/ui/src/components/files/file-block.tsx @@ -1,9 +1,10 @@ import { LocalFileNode } from '@colanode/client/types'; +import { FileIcon } from '@colanode/ui/components/files/file-icon'; import { FilePreview } from '@colanode/ui/components/files/file-preview'; import { useLayout } from '@colanode/ui/contexts/layout'; import { useWorkspace } from '@colanode/ui/contexts/workspace'; import { useLiveQuery } from '@colanode/ui/hooks/use-live-query'; -import { useNodeRadar } from '@colanode/ui/hooks/use-node-radar'; +import { canPreviewFile } from '@colanode/ui/lib/files'; interface FileBlockProps { id: string; @@ -19,22 +20,39 @@ export const FileBlock = ({ id }: FileBlockProps) => { accountId: workspace.accountId, workspaceId: workspace.id, }); - useNodeRadar(nodeGetQuery.data); if (nodeGetQuery.isPending || !nodeGetQuery.data) { return null; } const file = nodeGetQuery.data as LocalFileNode; + const canPreview = canPreviewFile(file.attributes.subtype); + + if (canPreview) { + return ( +
{ + layout.previewLeft(id, true); + }} + > + +
+ ); + } return (
{ layout.previewLeft(id, true); }} > - + +
+
{file.attributes.name}
+
{file.attributes.mimeType}
+
); }; diff --git a/packages/ui/src/components/files/file-body.tsx b/packages/ui/src/components/files/file-body.tsx index d985901f..bb2e7a53 100644 --- a/packages/ui/src/components/files/file-body.tsx +++ b/packages/ui/src/components/files/file-body.tsx @@ -3,29 +3,29 @@ import { FileNoPreview } from '@colanode/ui/components/files/file-no-preview'; import { FilePreview } from '@colanode/ui/components/files/file-preview'; import { FileSaveButton } from '@colanode/ui/components/files/file-save-button'; import { FileSidebar } from '@colanode/ui/components/files/file-sidebar'; +import { canPreviewFile } from '@colanode/ui/lib/files'; interface FileBodyProps { file: LocalFileNode; } export const FileBody = ({ file }: FileBodyProps) => { - const canPreview = - file.attributes.subtype === 'image' || - file.attributes.subtype === 'video' || - file.attributes.subtype === 'audio'; + const canPreview = canPreviewFile(file.attributes.subtype); return (
-
-
+
+
- {canPreview ? ( - - ) : ( - - )} +
+ {canPreview ? ( + + ) : ( + + )} +
diff --git a/packages/ui/src/components/files/file-download-progress.tsx b/packages/ui/src/components/files/file-download-progress.tsx index c9f3950c..58cca507 100644 --- a/packages/ui/src/components/files/file-download-progress.tsx +++ b/packages/ui/src/components/files/file-download-progress.tsx @@ -1,14 +1,14 @@ -import { Download } from 'lucide-react'; +import { DownloadIcon } from 'lucide-react'; -import { FileState } from '@colanode/client/types'; import { Spinner } from '@colanode/ui/components/ui/spinner'; interface FileDownloadProgressProps { - state: FileState | null | undefined; + progress: number; } -export const FileDownloadProgress = ({ state }: FileDownloadProgressProps) => { - const progress = state?.downloadProgress || 0; +export const FileDownloadProgress = ({ + progress, +}: FileDownloadProgressProps) => { const showProgress = progress > 0; return ( @@ -17,7 +17,7 @@ export const FileDownloadProgress = ({ state }: FileDownloadProgressProps) => {
- +
diff --git a/packages/ui/src/components/files/file-preview.tsx b/packages/ui/src/components/files/file-preview.tsx index 9379333c..222f6c17 100644 --- a/packages/ui/src/components/files/file-preview.tsx +++ b/packages/ui/src/components/files/file-preview.tsx @@ -1,5 +1,3 @@ -import { useEffect } from 'react'; - import { DownloadStatus, LocalFileNode } from '@colanode/client/types'; import { FileDownloadProgress } from '@colanode/ui/components/files/file-download-progress'; import { FileNoPreview } from '@colanode/ui/components/files/file-no-preview'; @@ -8,7 +6,6 @@ import { FilePreviewImage } from '@colanode/ui/components/files/previews/file-pr import { FilePreviewVideo } from '@colanode/ui/components/files/previews/file-preview-video'; import { useWorkspace } from '@colanode/ui/contexts/workspace'; import { useLiveQuery } from '@colanode/ui/hooks/use-live-query'; -import { useMutation } from '@colanode/ui/hooks/use-mutation'; interface FilePreviewProps { file: LocalFileNode; @@ -16,68 +13,40 @@ interface FilePreviewProps { export const FilePreview = ({ file }: FilePreviewProps) => { const workspace = useWorkspace(); - const mutation = useMutation(); - - const fileStateQuery = useLiveQuery({ - type: 'file.state.get', - id: file.id, + const localFileQuery = useLiveQuery({ + type: 'local.file.get', + fileId: file.id, accountId: workspace.accountId, workspaceId: workspace.id, + autoDownload: true, }); - const isDownloading = - fileStateQuery.data?.downloadStatus === DownloadStatus.Pending; - const isDownloaded = - fileStateQuery.data?.downloadStatus === DownloadStatus.Completed; - - useEffect(() => { - if (!fileStateQuery.isPending && !isDownloaded && !isDownloading) { - mutation.mutate({ - input: { - type: 'file.download', - accountId: workspace.accountId, - workspaceId: workspace.id, - fileId: file.id, - path: null, - }, - onError: (error) => { - console.error('Failed to start file download:', error.message); - }, - }); - } - }, [ - fileStateQuery.isPending, - isDownloaded, - isDownloading, - mutation, - workspace.accountId, - workspace.id, - file.id, - ]); - - if (fileStateQuery.isPending) { + if (localFileQuery.isPending) { return null; } - if (isDownloading) { - return ; + const localFile = localFileQuery.data?.localFile; + if (localFile) { + if (file.attributes.subtype === 'image') { + return ( + + ); + } + + if (file.attributes.subtype === 'video') { + return ; + } + + if (file.attributes.subtype === 'audio') { + return ( + + ); + } } - const url = fileStateQuery.data?.url; - if (!url) { - return ; - } - - if (file.attributes.subtype === 'image') { - return ; - } - - if (file.attributes.subtype === 'video') { - return ; - } - - if (file.attributes.subtype === 'audio') { - return ; + const download = localFileQuery.data?.download; + if (download && download.status !== DownloadStatus.Completed) { + return ; } return ; diff --git a/packages/ui/src/components/files/file-save-button.tsx b/packages/ui/src/components/files/file-save-button.tsx index db60a908..1b6531df 100644 --- a/packages/ui/src/components/files/file-save-button.tsx +++ b/packages/ui/src/components/files/file-save-button.tsx @@ -2,13 +2,12 @@ import { Download } from 'lucide-react'; import { useState } from 'react'; import { toast } from 'sonner'; -import { LocalFileNode } from '@colanode/client/types'; +import { LocalFileNode, SpecialContainerTabPath } from '@colanode/client/types'; import { Button } from '@colanode/ui/components/ui/button'; import { Spinner } from '@colanode/ui/components/ui/spinner'; import { useApp } from '@colanode/ui/contexts/app'; import { useLayout } from '@colanode/ui/contexts/layout'; import { useWorkspace } from '@colanode/ui/contexts/workspace'; -import { useLiveQuery } from '@colanode/ui/hooks/use-live-query'; import { useMutation } from '@colanode/ui/hooks/use-mutation'; interface FileSaveButtonProps { @@ -22,13 +21,6 @@ export const FileSaveButton = ({ file }: FileSaveButtonProps) => { const layout = useLayout(); const [isSaving, setIsSaving] = useState(false); - const fileStateQuery = useLiveQuery({ - type: 'file.state.get', - id: file.id, - accountId: workspace.accountId, - workspaceId: workspace.id, - }); - const handleDownloadDesktop = async () => { const path = await window.colanode.showFileSaveDialog({ name: file.attributes.name, @@ -40,14 +32,14 @@ export const FileSaveButton = ({ file }: FileSaveButtonProps) => { mutation.mutate({ input: { - type: 'file.save', + type: 'file.download', accountId: workspace.accountId, workspaceId: workspace.id, fileId: file.id, path, }, onSuccess: () => { - layout.open('downloads'); + layout.open(SpecialContainerTabPath.WorkspaceDownloads); }, onError: () => { toast.error('Failed to save file'); @@ -56,61 +48,64 @@ export const FileSaveButton = ({ file }: FileSaveButtonProps) => { }; const handleDownloadWeb = async () => { - if (fileStateQuery.isPending) { - return; - } - setIsSaving(true); try { - const url = fileStateQuery.data?.url; - if (url) { + const localFileQuery = await window.colanode.executeQuery({ + type: 'local.file.get', + fileId: file.id, + accountId: workspace.accountId, + workspaceId: workspace.id, + }); + + if (localFileQuery.localFile) { // the file is already downloaded locally, so we can just trigger a download const link = document.createElement('a'); - link.href = url; + link.href = localFileQuery.localFile.url; link.download = file.attributes.name; link.style.display = 'none'; document.body.appendChild(link); link.click(); document.body.removeChild(link); - } else { - // the file is not downloaded locally, so we need to download it - const request = await window.colanode.executeQuery({ - type: 'file.download.request.get', - id: file.id, - accountId: workspace.accountId, - workspaceId: workspace.id, - }); + return; + } - if (!request) { - toast.error('Failed to save file'); - return; - } + // the file is not downloaded locally, so we need to download it + const request = await window.colanode.executeQuery({ + type: 'file.download.request.get', + id: file.id, + accountId: workspace.accountId, + workspaceId: workspace.id, + }); - const response = await fetch(request.url, { - method: 'GET', - headers: request.headers, - }); + if (!request) { + toast.error('Failed to save file'); + return; + } - if (!response.ok) { - toast.error('Failed to save file'); - return; - } + const response = await fetch(request.url, { + method: 'GET', + headers: request.headers, + }); - const blob = await response.blob(); - const blobUrl = URL.createObjectURL(blob); + if (!response.ok) { + toast.error('Failed to save file'); + return; + } - const link = document.createElement('a'); - link.href = blobUrl; - link.download = file.attributes.name; - link.style.display = 'none'; - document.body.appendChild(link); - link.click(); - document.body.removeChild(link); + const downloadBlob = await response.blob(); + const downloadBlobUrl = URL.createObjectURL(downloadBlob); - if (blobUrl) { - URL.revokeObjectURL(blobUrl); - } + const link = document.createElement('a'); + link.href = downloadBlobUrl; + link.download = file.attributes.name; + link.style.display = 'none'; + document.body.appendChild(link); + link.click(); + document.body.removeChild(link); + + if (downloadBlobUrl) { + URL.revokeObjectURL(downloadBlobUrl); } } catch { toast.error('Failed to save file'); @@ -131,7 +126,7 @@ export const FileSaveButton = ({ file }: FileSaveButtonProps) => {