From 80dd1c60c6b65601d4268f6330e88faa39ad37bb Mon Sep 17 00:00:00 2001 From: Hakan Shehu Date: Fri, 1 Aug 2025 23:07:21 +0200 Subject: [PATCH] Define chunk and file part size constant (#170) --- apps/server/.env.example | 1 - .../client/routes/workspaces/files/file-upload-tus.ts | 11 +++++++++-- apps/server/src/lib/config/storage.ts | 5 ----- hosting/docker/docker-compose.yaml | 1 - hosting/kubernetes/chart/templates/_helpers.tpl | 3 +-- hosting/kubernetes/chart/values.yaml | 1 - packages/client/src/jobs/file-upload.ts | 10 ++++++++-- packages/core/src/types/files.ts | 2 ++ .../downloads/workspace-download-status.tsx | 6 ++++-- .../workspaces/uploads/workspace-upload-status.tsx | 6 ++++-- 10 files changed, 28 insertions(+), 18 deletions(-) diff --git a/apps/server/.env.example b/apps/server/.env.example index 1c3d13b3..c9cdaf71 100644 --- a/apps/server/.env.example +++ b/apps/server/.env.example @@ -74,7 +74,6 @@ STORAGE_S3_SECRET_KEY=your_minio_password STORAGE_S3_BUCKET=colanode STORAGE_S3_REGION=us-east-1 STORAGE_S3_FORCE_PATH_STYLE=true -STORAGE_S3_PART_SIZE=20971520 # 20MB # ─────────────────────────────────────────────────────────────── # SMTP Configuration diff --git a/apps/server/src/api/client/routes/workspaces/files/file-upload-tus.ts b/apps/server/src/api/client/routes/workspaces/files/file-upload-tus.ts index 4ea026f6..8f53aa40 100644 --- a/apps/server/src/api/client/routes/workspaces/files/file-upload-tus.ts +++ b/apps/server/src/api/client/routes/workspaces/files/file-upload-tus.ts @@ -3,7 +3,13 @@ import { Server } from '@tus/server'; import { FastifyPluginCallbackZod } from 'fastify-type-provider-zod'; import { z } from 'zod/v4'; -import { ApiErrorCode, FileStatus, generateId, IdType } from '@colanode/core'; +import { + ApiErrorCode, + FILE_UPLOAD_PART_SIZE, + FileStatus, + generateId, + IdType, +} from '@colanode/core'; import { database } from '@colanode/server/data/database'; import { redis } from '@colanode/server/data/redis'; import { s3Config } from '@colanode/server/data/storage'; @@ -15,7 +21,7 @@ import { RedisKvStore } from '@colanode/server/lib/tus/redis-kv'; import { RedisLocker } from '@colanode/server/lib/tus/redis-locker'; const s3Store = new S3Store({ - partSize: config.storage.partSize, + partSize: FILE_UPLOAD_PART_SIZE, cache: new RedisKvStore(redis, config.redis.tus.kvPrefix), s3ClientConfig: { ...s3Config, @@ -211,6 +217,7 @@ export const fileUploadTusRoute: FastifyPluginCallbackZod = ( return { metadata: { uploadId: createdUpload.upload_id, + contentType: file.attributes.mimeType, }, }; }, diff --git a/apps/server/src/lib/config/storage.ts b/apps/server/src/lib/config/storage.ts index e79dbbac..1fcd2a4b 100644 --- a/apps/server/src/lib/config/storage.ts +++ b/apps/server/src/lib/config/storage.ts @@ -7,10 +7,6 @@ export const storageConfigSchema = z.object({ secretKey: z.string({ error: 'STORAGE_S3_SECRET_KEY is required' }), bucket: z.string({ error: 'STORAGE_S3_BUCKET is required' }), region: z.string({ error: 'STORAGE_S3_REGION is required' }), - partSize: z - .number() - .optional() - .default(20 * 1024 * 1024), // 20MB forcePathStyle: z.boolean().optional(), }); @@ -24,7 +20,6 @@ export const readStorageConfigVariables = () => { secretKey: process.env.STORAGE_S3_SECRET_KEY, bucket: process.env.STORAGE_S3_BUCKET, region: process.env.STORAGE_S3_REGION, - partSize: process.env.STORAGE_S3_PART_SIZE, forcePathStyle: process.env.STORAGE_S3_FORCE_PATH_STYLE === 'true', }; }; diff --git a/hosting/docker/docker-compose.yaml b/hosting/docker/docker-compose.yaml index 8b3e599e..93c3e7a0 100644 --- a/hosting/docker/docker-compose.yaml +++ b/hosting/docker/docker-compose.yaml @@ -167,7 +167,6 @@ services: STORAGE_S3_BUCKET: 'colanode' STORAGE_S3_REGION: 'us-east-1' STORAGE_S3_FORCE_PATH_STYLE: 'true' - STORAGE_S3_PART_SIZE: '20971520' # 20MB # ─────────────────────────────────────────────────────────────── # SMTP configuration diff --git a/hosting/kubernetes/chart/templates/_helpers.tpl b/hosting/kubernetes/chart/templates/_helpers.tpl index 812a23bb..aafcbc60 100644 --- a/hosting/kubernetes/chart/templates/_helpers.tpl +++ b/hosting/kubernetes/chart/templates/_helpers.tpl @@ -238,8 +238,6 @@ Colanode Server Environment Variables value: "us-east-1" - name: STORAGE_S3_FORCE_PATH_STYLE value: "true" -- name: STORAGE_S3_PART_SIZE - value: {{ .Values.colanode.config.STORAGE_S3_PART_SIZE | quote }} # ─────────────────────────────────────────────────────────────── # SMTP configuration @@ -260,3 +258,4 @@ Colanode Server Environment Variables - name: SMTP_EMAIL_FROM_NAME value: {{ .Values.colanode.config.SMTP_EMAIL_FROM_NAME | quote }} {{- end }} +{{- end }} \ No newline at end of file diff --git a/hosting/kubernetes/chart/values.yaml b/hosting/kubernetes/chart/values.yaml index 7b07cf1a..70b96a6d 100644 --- a/hosting/kubernetes/chart/values.yaml +++ b/hosting/kubernetes/chart/values.yaml @@ -121,7 +121,6 @@ colanode: STORAGE_S3_BUCKET: 'colanode' STORAGE_S3_REGION: 'us-east-1' STORAGE_S3_FORCE_PATH_STYLE: 'true' - STORAGE_S3_PART_SIZE: '20971520' # 20MB # Email configuration SMTP_ENABLED: 'false' diff --git a/packages/client/src/jobs/file-upload.ts b/packages/client/src/jobs/file-upload.ts index b110d26a..a3678f1e 100644 --- a/packages/client/src/jobs/file-upload.ts +++ b/packages/client/src/jobs/file-upload.ts @@ -17,7 +17,12 @@ import { AccountService } from '@colanode/client/services/accounts/account-servi import { AppService } from '@colanode/client/services/app-service'; import { WorkspaceService } from '@colanode/client/services/workspaces/workspace-service'; import { LocalFileNode, UploadStatus } from '@colanode/client/types'; -import { ApiHeader, build, calculatePercentage } from '@colanode/core'; +import { + ApiHeader, + build, + calculatePercentage, + FILE_UPLOAD_PART_SIZE, +} from '@colanode/core'; export type FileUploadInput = { type: 'file.upload'; @@ -156,6 +161,7 @@ export class FileUploadJobHandler implements JobHandler { await new Promise((resolve, reject) => { const tusUpload = new Upload(fileStream, { endpoint: `${account.server.httpBaseUrl}/v1/workspaces/${workspace.id}/files/${file.id}/tus`, + chunkSize: FILE_UPLOAD_PART_SIZE, retryDelays: [ 0, ms('3 seconds'), @@ -165,7 +171,7 @@ export class FileUploadJobHandler implements JobHandler { ], metadata: { filename: localFile.name, - filetype: file.type, + contentType: file.attributes.mimeType, }, headers: { Authorization: `Bearer ${account.token}`, diff --git a/packages/core/src/types/files.ts b/packages/core/src/types/files.ts index 72e24ae9..7c0fc3a9 100644 --- a/packages/core/src/types/files.ts +++ b/packages/core/src/types/files.ts @@ -15,3 +15,5 @@ export enum FileStatus { Ready = 1, Error = 2, } + +export const FILE_UPLOAD_PART_SIZE = 20 * 1024 * 1024; // 20MB diff --git a/packages/ui/src/components/workspaces/downloads/workspace-download-status.tsx b/packages/ui/src/components/workspaces/downloads/workspace-download-status.tsx index 9a0053b8..2810e692 100644 --- a/packages/ui/src/components/workspaces/downloads/workspace-download-status.tsx +++ b/packages/ui/src/components/workspaces/downloads/workspace-download-status.tsx @@ -1,6 +1,6 @@ import 'react-circular-progressbar/dist/styles.css'; import { Check, Clock, X } from 'lucide-react'; -import { CircularProgressbar } from 'react-circular-progressbar'; +import { buildStyles, CircularProgressbar } from 'react-circular-progressbar'; import { DownloadStatus } from '@colanode/client/types'; import { @@ -39,7 +39,9 @@ export const WorkspaceDownloadStatus = ({ diff --git a/packages/ui/src/components/workspaces/uploads/workspace-upload-status.tsx b/packages/ui/src/components/workspaces/uploads/workspace-upload-status.tsx index ab93351c..12238860 100644 --- a/packages/ui/src/components/workspaces/uploads/workspace-upload-status.tsx +++ b/packages/ui/src/components/workspaces/uploads/workspace-upload-status.tsx @@ -1,6 +1,6 @@ import 'react-circular-progressbar/dist/styles.css'; import { Check, X, Clock } from 'lucide-react'; -import { CircularProgressbar } from 'react-circular-progressbar'; +import { buildStyles, CircularProgressbar } from 'react-circular-progressbar'; import { UploadStatus } from '@colanode/client/types'; import { @@ -39,7 +39,9 @@ export const WorkspaceUploadStatus = ({