Define chunk and file part size constant (#170)

This commit is contained in:
Hakan Shehu
2025-08-01 23:07:21 +02:00
committed by GitHub
parent 85ad4e7551
commit 80dd1c60c6
10 changed files with 28 additions and 18 deletions

View File

@@ -74,7 +74,6 @@ STORAGE_S3_SECRET_KEY=your_minio_password
STORAGE_S3_BUCKET=colanode
STORAGE_S3_REGION=us-east-1
STORAGE_S3_FORCE_PATH_STYLE=true
STORAGE_S3_PART_SIZE=20971520 # 20MB
# ───────────────────────────────────────────────────────────────
# SMTP Configuration

View File

@@ -3,7 +3,13 @@ import { Server } from '@tus/server';
import { FastifyPluginCallbackZod } from 'fastify-type-provider-zod';
import { z } from 'zod/v4';
import { ApiErrorCode, FileStatus, generateId, IdType } from '@colanode/core';
import {
ApiErrorCode,
FILE_UPLOAD_PART_SIZE,
FileStatus,
generateId,
IdType,
} from '@colanode/core';
import { database } from '@colanode/server/data/database';
import { redis } from '@colanode/server/data/redis';
import { s3Config } from '@colanode/server/data/storage';
@@ -15,7 +21,7 @@ import { RedisKvStore } from '@colanode/server/lib/tus/redis-kv';
import { RedisLocker } from '@colanode/server/lib/tus/redis-locker';
const s3Store = new S3Store({
partSize: config.storage.partSize,
partSize: FILE_UPLOAD_PART_SIZE,
cache: new RedisKvStore(redis, config.redis.tus.kvPrefix),
s3ClientConfig: {
...s3Config,
@@ -211,6 +217,7 @@ export const fileUploadTusRoute: FastifyPluginCallbackZod = (
return {
metadata: {
uploadId: createdUpload.upload_id,
contentType: file.attributes.mimeType,
},
};
},

View File

@@ -7,10 +7,6 @@ export const storageConfigSchema = z.object({
secretKey: z.string({ error: 'STORAGE_S3_SECRET_KEY is required' }),
bucket: z.string({ error: 'STORAGE_S3_BUCKET is required' }),
region: z.string({ error: 'STORAGE_S3_REGION is required' }),
partSize: z
.number()
.optional()
.default(20 * 1024 * 1024), // 20MB
forcePathStyle: z.boolean().optional(),
});
@@ -24,7 +20,6 @@ export const readStorageConfigVariables = () => {
secretKey: process.env.STORAGE_S3_SECRET_KEY,
bucket: process.env.STORAGE_S3_BUCKET,
region: process.env.STORAGE_S3_REGION,
partSize: process.env.STORAGE_S3_PART_SIZE,
forcePathStyle: process.env.STORAGE_S3_FORCE_PATH_STYLE === 'true',
};
};

View File

@@ -167,7 +167,6 @@ services:
STORAGE_S3_BUCKET: 'colanode'
STORAGE_S3_REGION: 'us-east-1'
STORAGE_S3_FORCE_PATH_STYLE: 'true'
STORAGE_S3_PART_SIZE: '20971520' # 20MB
# ───────────────────────────────────────────────────────────────
# SMTP configuration

View File

@@ -238,8 +238,6 @@ Colanode Server Environment Variables
value: "us-east-1"
- name: STORAGE_S3_FORCE_PATH_STYLE
value: "true"
- name: STORAGE_S3_PART_SIZE
value: {{ .Values.colanode.config.STORAGE_S3_PART_SIZE | quote }}
# ───────────────────────────────────────────────────────────────
# SMTP configuration
@@ -260,3 +258,4 @@ Colanode Server Environment Variables
- name: SMTP_EMAIL_FROM_NAME
value: {{ .Values.colanode.config.SMTP_EMAIL_FROM_NAME | quote }}
{{- end }}
{{- end }}

View File

@@ -121,7 +121,6 @@ colanode:
STORAGE_S3_BUCKET: 'colanode'
STORAGE_S3_REGION: 'us-east-1'
STORAGE_S3_FORCE_PATH_STYLE: 'true'
STORAGE_S3_PART_SIZE: '20971520' # 20MB
# Email configuration
SMTP_ENABLED: 'false'

View File

@@ -17,7 +17,12 @@ import { AccountService } from '@colanode/client/services/accounts/account-servi
import { AppService } from '@colanode/client/services/app-service';
import { WorkspaceService } from '@colanode/client/services/workspaces/workspace-service';
import { LocalFileNode, UploadStatus } from '@colanode/client/types';
import { ApiHeader, build, calculatePercentage } from '@colanode/core';
import {
ApiHeader,
build,
calculatePercentage,
FILE_UPLOAD_PART_SIZE,
} from '@colanode/core';
export type FileUploadInput = {
type: 'file.upload';
@@ -156,6 +161,7 @@ export class FileUploadJobHandler implements JobHandler<FileUploadInput> {
await new Promise<void>((resolve, reject) => {
const tusUpload = new Upload(fileStream, {
endpoint: `${account.server.httpBaseUrl}/v1/workspaces/${workspace.id}/files/${file.id}/tus`,
chunkSize: FILE_UPLOAD_PART_SIZE,
retryDelays: [
0,
ms('3 seconds'),
@@ -165,7 +171,7 @@ export class FileUploadJobHandler implements JobHandler<FileUploadInput> {
],
metadata: {
filename: localFile.name,
filetype: file.type,
contentType: file.attributes.mimeType,
},
headers: {
Authorization: `Bearer ${account.token}`,

View File

@@ -15,3 +15,5 @@ export enum FileStatus {
Ready = 1,
Error = 2,
}
export const FILE_UPLOAD_PART_SIZE = 20 * 1024 * 1024; // 20MB

View File

@@ -1,6 +1,6 @@
import 'react-circular-progressbar/dist/styles.css';
import { Check, Clock, X } from 'lucide-react';
import { CircularProgressbar } from 'react-circular-progressbar';
import { buildStyles, CircularProgressbar } from 'react-circular-progressbar';
import { DownloadStatus } from '@colanode/client/types';
import {
@@ -39,7 +39,9 @@ export const WorkspaceDownloadStatus = ({
<CircularProgressbar
value={progress}
strokeWidth={8}
className="text-xs text-muted-foreground"
styles={buildStyles({
pathColor: 'var(--color-blue-500)',
})}
/>
</div>
<span className="text-xs text-muted-foreground font-medium">

View File

@@ -1,6 +1,6 @@
import 'react-circular-progressbar/dist/styles.css';
import { Check, X, Clock } from 'lucide-react';
import { CircularProgressbar } from 'react-circular-progressbar';
import { buildStyles, CircularProgressbar } from 'react-circular-progressbar';
import { UploadStatus } from '@colanode/client/types';
import {
@@ -39,7 +39,9 @@ export const WorkspaceUploadStatus = ({
<CircularProgressbar
value={progress}
strokeWidth={8}
className="text-xs text-muted-foreground"
styles={buildStyles({
pathColor: 'var(--color-blue-500)',
})}
/>
</div>
<span className="text-xs text-muted-foreground font-medium">