mirror of
https://github.com/colanode/colanode.git
synced 2025-12-16 19:57:46 +01:00
Define chunk and file part size constant (#170)
This commit is contained in:
@@ -74,7 +74,6 @@ STORAGE_S3_SECRET_KEY=your_minio_password
|
|||||||
STORAGE_S3_BUCKET=colanode
|
STORAGE_S3_BUCKET=colanode
|
||||||
STORAGE_S3_REGION=us-east-1
|
STORAGE_S3_REGION=us-east-1
|
||||||
STORAGE_S3_FORCE_PATH_STYLE=true
|
STORAGE_S3_FORCE_PATH_STYLE=true
|
||||||
STORAGE_S3_PART_SIZE=20971520 # 20MB
|
|
||||||
|
|
||||||
# ───────────────────────────────────────────────────────────────
|
# ───────────────────────────────────────────────────────────────
|
||||||
# SMTP Configuration
|
# SMTP Configuration
|
||||||
|
|||||||
@@ -3,7 +3,13 @@ import { Server } from '@tus/server';
|
|||||||
import { FastifyPluginCallbackZod } from 'fastify-type-provider-zod';
|
import { FastifyPluginCallbackZod } from 'fastify-type-provider-zod';
|
||||||
import { z } from 'zod/v4';
|
import { z } from 'zod/v4';
|
||||||
|
|
||||||
import { ApiErrorCode, FileStatus, generateId, IdType } from '@colanode/core';
|
import {
|
||||||
|
ApiErrorCode,
|
||||||
|
FILE_UPLOAD_PART_SIZE,
|
||||||
|
FileStatus,
|
||||||
|
generateId,
|
||||||
|
IdType,
|
||||||
|
} from '@colanode/core';
|
||||||
import { database } from '@colanode/server/data/database';
|
import { database } from '@colanode/server/data/database';
|
||||||
import { redis } from '@colanode/server/data/redis';
|
import { redis } from '@colanode/server/data/redis';
|
||||||
import { s3Config } from '@colanode/server/data/storage';
|
import { s3Config } from '@colanode/server/data/storage';
|
||||||
@@ -15,7 +21,7 @@ import { RedisKvStore } from '@colanode/server/lib/tus/redis-kv';
|
|||||||
import { RedisLocker } from '@colanode/server/lib/tus/redis-locker';
|
import { RedisLocker } from '@colanode/server/lib/tus/redis-locker';
|
||||||
|
|
||||||
const s3Store = new S3Store({
|
const s3Store = new S3Store({
|
||||||
partSize: config.storage.partSize,
|
partSize: FILE_UPLOAD_PART_SIZE,
|
||||||
cache: new RedisKvStore(redis, config.redis.tus.kvPrefix),
|
cache: new RedisKvStore(redis, config.redis.tus.kvPrefix),
|
||||||
s3ClientConfig: {
|
s3ClientConfig: {
|
||||||
...s3Config,
|
...s3Config,
|
||||||
@@ -211,6 +217,7 @@ export const fileUploadTusRoute: FastifyPluginCallbackZod = (
|
|||||||
return {
|
return {
|
||||||
metadata: {
|
metadata: {
|
||||||
uploadId: createdUpload.upload_id,
|
uploadId: createdUpload.upload_id,
|
||||||
|
contentType: file.attributes.mimeType,
|
||||||
},
|
},
|
||||||
};
|
};
|
||||||
},
|
},
|
||||||
|
|||||||
@@ -7,10 +7,6 @@ export const storageConfigSchema = z.object({
|
|||||||
secretKey: z.string({ error: 'STORAGE_S3_SECRET_KEY is required' }),
|
secretKey: z.string({ error: 'STORAGE_S3_SECRET_KEY is required' }),
|
||||||
bucket: z.string({ error: 'STORAGE_S3_BUCKET is required' }),
|
bucket: z.string({ error: 'STORAGE_S3_BUCKET is required' }),
|
||||||
region: z.string({ error: 'STORAGE_S3_REGION is required' }),
|
region: z.string({ error: 'STORAGE_S3_REGION is required' }),
|
||||||
partSize: z
|
|
||||||
.number()
|
|
||||||
.optional()
|
|
||||||
.default(20 * 1024 * 1024), // 20MB
|
|
||||||
forcePathStyle: z.boolean().optional(),
|
forcePathStyle: z.boolean().optional(),
|
||||||
});
|
});
|
||||||
|
|
||||||
@@ -24,7 +20,6 @@ export const readStorageConfigVariables = () => {
|
|||||||
secretKey: process.env.STORAGE_S3_SECRET_KEY,
|
secretKey: process.env.STORAGE_S3_SECRET_KEY,
|
||||||
bucket: process.env.STORAGE_S3_BUCKET,
|
bucket: process.env.STORAGE_S3_BUCKET,
|
||||||
region: process.env.STORAGE_S3_REGION,
|
region: process.env.STORAGE_S3_REGION,
|
||||||
partSize: process.env.STORAGE_S3_PART_SIZE,
|
|
||||||
forcePathStyle: process.env.STORAGE_S3_FORCE_PATH_STYLE === 'true',
|
forcePathStyle: process.env.STORAGE_S3_FORCE_PATH_STYLE === 'true',
|
||||||
};
|
};
|
||||||
};
|
};
|
||||||
|
|||||||
@@ -167,7 +167,6 @@ services:
|
|||||||
STORAGE_S3_BUCKET: 'colanode'
|
STORAGE_S3_BUCKET: 'colanode'
|
||||||
STORAGE_S3_REGION: 'us-east-1'
|
STORAGE_S3_REGION: 'us-east-1'
|
||||||
STORAGE_S3_FORCE_PATH_STYLE: 'true'
|
STORAGE_S3_FORCE_PATH_STYLE: 'true'
|
||||||
STORAGE_S3_PART_SIZE: '20971520' # 20MB
|
|
||||||
|
|
||||||
# ───────────────────────────────────────────────────────────────
|
# ───────────────────────────────────────────────────────────────
|
||||||
# SMTP configuration
|
# SMTP configuration
|
||||||
|
|||||||
@@ -238,8 +238,6 @@ Colanode Server Environment Variables
|
|||||||
value: "us-east-1"
|
value: "us-east-1"
|
||||||
- name: STORAGE_S3_FORCE_PATH_STYLE
|
- name: STORAGE_S3_FORCE_PATH_STYLE
|
||||||
value: "true"
|
value: "true"
|
||||||
- name: STORAGE_S3_PART_SIZE
|
|
||||||
value: {{ .Values.colanode.config.STORAGE_S3_PART_SIZE | quote }}
|
|
||||||
|
|
||||||
# ───────────────────────────────────────────────────────────────
|
# ───────────────────────────────────────────────────────────────
|
||||||
# SMTP configuration
|
# SMTP configuration
|
||||||
@@ -260,3 +258,4 @@ Colanode Server Environment Variables
|
|||||||
- name: SMTP_EMAIL_FROM_NAME
|
- name: SMTP_EMAIL_FROM_NAME
|
||||||
value: {{ .Values.colanode.config.SMTP_EMAIL_FROM_NAME | quote }}
|
value: {{ .Values.colanode.config.SMTP_EMAIL_FROM_NAME | quote }}
|
||||||
{{- end }}
|
{{- end }}
|
||||||
|
{{- end }}
|
||||||
@@ -121,7 +121,6 @@ colanode:
|
|||||||
STORAGE_S3_BUCKET: 'colanode'
|
STORAGE_S3_BUCKET: 'colanode'
|
||||||
STORAGE_S3_REGION: 'us-east-1'
|
STORAGE_S3_REGION: 'us-east-1'
|
||||||
STORAGE_S3_FORCE_PATH_STYLE: 'true'
|
STORAGE_S3_FORCE_PATH_STYLE: 'true'
|
||||||
STORAGE_S3_PART_SIZE: '20971520' # 20MB
|
|
||||||
|
|
||||||
# Email configuration
|
# Email configuration
|
||||||
SMTP_ENABLED: 'false'
|
SMTP_ENABLED: 'false'
|
||||||
|
|||||||
@@ -17,7 +17,12 @@ import { AccountService } from '@colanode/client/services/accounts/account-servi
|
|||||||
import { AppService } from '@colanode/client/services/app-service';
|
import { AppService } from '@colanode/client/services/app-service';
|
||||||
import { WorkspaceService } from '@colanode/client/services/workspaces/workspace-service';
|
import { WorkspaceService } from '@colanode/client/services/workspaces/workspace-service';
|
||||||
import { LocalFileNode, UploadStatus } from '@colanode/client/types';
|
import { LocalFileNode, UploadStatus } from '@colanode/client/types';
|
||||||
import { ApiHeader, build, calculatePercentage } from '@colanode/core';
|
import {
|
||||||
|
ApiHeader,
|
||||||
|
build,
|
||||||
|
calculatePercentage,
|
||||||
|
FILE_UPLOAD_PART_SIZE,
|
||||||
|
} from '@colanode/core';
|
||||||
|
|
||||||
export type FileUploadInput = {
|
export type FileUploadInput = {
|
||||||
type: 'file.upload';
|
type: 'file.upload';
|
||||||
@@ -156,6 +161,7 @@ export class FileUploadJobHandler implements JobHandler<FileUploadInput> {
|
|||||||
await new Promise<void>((resolve, reject) => {
|
await new Promise<void>((resolve, reject) => {
|
||||||
const tusUpload = new Upload(fileStream, {
|
const tusUpload = new Upload(fileStream, {
|
||||||
endpoint: `${account.server.httpBaseUrl}/v1/workspaces/${workspace.id}/files/${file.id}/tus`,
|
endpoint: `${account.server.httpBaseUrl}/v1/workspaces/${workspace.id}/files/${file.id}/tus`,
|
||||||
|
chunkSize: FILE_UPLOAD_PART_SIZE,
|
||||||
retryDelays: [
|
retryDelays: [
|
||||||
0,
|
0,
|
||||||
ms('3 seconds'),
|
ms('3 seconds'),
|
||||||
@@ -165,7 +171,7 @@ export class FileUploadJobHandler implements JobHandler<FileUploadInput> {
|
|||||||
],
|
],
|
||||||
metadata: {
|
metadata: {
|
||||||
filename: localFile.name,
|
filename: localFile.name,
|
||||||
filetype: file.type,
|
contentType: file.attributes.mimeType,
|
||||||
},
|
},
|
||||||
headers: {
|
headers: {
|
||||||
Authorization: `Bearer ${account.token}`,
|
Authorization: `Bearer ${account.token}`,
|
||||||
|
|||||||
@@ -15,3 +15,5 @@ export enum FileStatus {
|
|||||||
Ready = 1,
|
Ready = 1,
|
||||||
Error = 2,
|
Error = 2,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export const FILE_UPLOAD_PART_SIZE = 20 * 1024 * 1024; // 20MB
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
import 'react-circular-progressbar/dist/styles.css';
|
import 'react-circular-progressbar/dist/styles.css';
|
||||||
import { Check, Clock, X } from 'lucide-react';
|
import { Check, Clock, X } from 'lucide-react';
|
||||||
import { CircularProgressbar } from 'react-circular-progressbar';
|
import { buildStyles, CircularProgressbar } from 'react-circular-progressbar';
|
||||||
|
|
||||||
import { DownloadStatus } from '@colanode/client/types';
|
import { DownloadStatus } from '@colanode/client/types';
|
||||||
import {
|
import {
|
||||||
@@ -39,7 +39,9 @@ export const WorkspaceDownloadStatus = ({
|
|||||||
<CircularProgressbar
|
<CircularProgressbar
|
||||||
value={progress}
|
value={progress}
|
||||||
strokeWidth={8}
|
strokeWidth={8}
|
||||||
className="text-xs text-muted-foreground"
|
styles={buildStyles({
|
||||||
|
pathColor: 'var(--color-blue-500)',
|
||||||
|
})}
|
||||||
/>
|
/>
|
||||||
</div>
|
</div>
|
||||||
<span className="text-xs text-muted-foreground font-medium">
|
<span className="text-xs text-muted-foreground font-medium">
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
import 'react-circular-progressbar/dist/styles.css';
|
import 'react-circular-progressbar/dist/styles.css';
|
||||||
import { Check, X, Clock } from 'lucide-react';
|
import { Check, X, Clock } from 'lucide-react';
|
||||||
import { CircularProgressbar } from 'react-circular-progressbar';
|
import { buildStyles, CircularProgressbar } from 'react-circular-progressbar';
|
||||||
|
|
||||||
import { UploadStatus } from '@colanode/client/types';
|
import { UploadStatus } from '@colanode/client/types';
|
||||||
import {
|
import {
|
||||||
@@ -39,7 +39,9 @@ export const WorkspaceUploadStatus = ({
|
|||||||
<CircularProgressbar
|
<CircularProgressbar
|
||||||
value={progress}
|
value={progress}
|
||||||
strokeWidth={8}
|
strokeWidth={8}
|
||||||
className="text-xs text-muted-foreground"
|
styles={buildStyles({
|
||||||
|
pathColor: 'var(--color-blue-500)',
|
||||||
|
})}
|
||||||
/>
|
/>
|
||||||
</div>
|
</div>
|
||||||
<span className="text-xs text-muted-foreground font-medium">
|
<span className="text-xs text-muted-foreground font-medium">
|
||||||
|
|||||||
Reference in New Issue
Block a user