Add total storage limit and max file size limit for files

This commit is contained in:
Hakan Shehu
2025-01-12 18:25:04 +01:00
parent b39b90b71a
commit 75132d5b60
17 changed files with 161 additions and 5 deletions

View File

@@ -11,6 +11,8 @@ const createUsersTable: Migration = {
.addColumn('custom_name', 'text')
.addColumn('custom_avatar', 'text')
.addColumn('role', 'text', (col) => col.notNull())
.addColumn('storage_limit', 'integer', (col) => col.notNull())
.addColumn('max_file_size', 'integer', (col) => col.notNull())
.addColumn('status', 'text', (col) => col.notNull())
.addColumn('created_at', 'text', (col) => col.notNull())
.addColumn('updated_at', 'text')

View File

@@ -19,6 +19,8 @@ interface UserTable {
custom_name: ColumnType<string | null, string | null, string | null>;
custom_avatar: ColumnType<string | null, string | null, string | null>;
role: ColumnType<WorkspaceRole, WorkspaceRole, WorkspaceRole>;
storage_limit: ColumnType<bigint, bigint, bigint>;
max_file_size: ColumnType<bigint, bigint, bigint>;
status: ColumnType<string, string, never>;
created_at: ColumnType<string, string, never>;
updated_at: ColumnType<string | null, string | null, string | null>;

View File

@@ -15,7 +15,14 @@ import {
import { MutationError, MutationErrorCode } from '@/shared/mutations';
import { databaseService } from '@/main/data/database-service';
import { eventBus } from '@/shared/lib/event-bus';
import { fetchEntry, fetchUser, mapEntry, mapFile } from '@/main/utils';
import {
fetchEntry,
fetchUser,
fetchUserStorageUsed,
mapEntry,
mapFile,
} from '@/main/utils';
import { formatBytes } from '@/shared/lib/files';
export class FileCreateMutationHandler
implements MutationHandler<FileCreateMutationInput>
@@ -43,6 +50,31 @@ export class FileCreateMutationHandler
);
}
if (metadata.size > user.max_file_size) {
throw new MutationError(
MutationErrorCode.FileTooLarge,
'The file you are trying to upload is too large. The maximum file size is ' +
formatBytes(user.max_file_size)
);
}
const storageUsed = await fetchUserStorageUsed(
workspaceDatabase,
input.userId
);
if (storageUsed + BigInt(metadata.size) > user.storage_limit) {
throw new MutationError(
MutationErrorCode.StorageLimitExceeded,
'You have reached your storage limit. You have used ' +
formatBytes(storageUsed) +
' and you are trying to upload a file of size ' +
formatBytes(metadata.size) +
'. Your storage limit is ' +
formatBytes(user.storage_limit)
);
}
const entry = await fetchEntry(workspaceDatabase, input.entryId);
if (!entry) {
throw new MutationError(

View File

@@ -29,11 +29,13 @@ import { eventBus } from '@/shared/lib/event-bus';
import {
fetchEntry,
fetchUser,
fetchUserStorageUsed,
mapEntry,
mapFile,
mapFileState,
mapMessage,
} from '@/main/utils';
import { formatBytes } from '@/shared/lib/files';
export class MessageCreateMutationHandler
implements MutationHandler<MessageCreateMutationInput>
@@ -106,6 +108,14 @@ export class MessageCreateMutationHandler
);
}
if (metadata.size > user.max_file_size) {
throw new MutationError(
MutationErrorCode.FileTooLarge,
'The file you are trying to upload is too large. The maximum file size is ' +
formatBytes(user.max_file_size)
);
}
const fileId = generateId(IdType.File);
block.id = fileId;
block.type = 'file';
@@ -171,6 +181,33 @@ export class MessageCreateMutationHandler
}
}
if (files.length > 0) {
const storageUsed = await fetchUserStorageUsed(
workspaceDatabase,
input.userId
);
const fileSizeSum = BigInt(
files.reduce((sum, file) => sum + file.size, 0)
);
if (storageUsed + fileSizeSum > user.storage_limit) {
for (const file of files) {
fileService.deleteFile(input.userId, file.id, file.extension);
}
throw new MutationError(
MutationErrorCode.StorageLimitExceeded,
'You have reached your storage limit. You have used ' +
formatBytes(storageUsed) +
' and you are trying to upload files of size ' +
formatBytes(fileSizeSum) +
'. Your storage limit is ' +
formatBytes(user.storage_limit)
);
}
}
const blocksRecord = blocks.reduce(
(acc, block) => {
acc[block.id] = block;

View File

@@ -18,6 +18,8 @@ class UserService {
name: user.name,
avatar: user.avatar,
role: user.role,
storage_limit: BigInt(user.storageLimit),
max_file_size: BigInt(user.maxFileSize),
version: BigInt(user.version),
created_at: user.createdAt,
updated_at: user.updatedAt,
@@ -34,6 +36,8 @@ class UserService {
custom_name: user.customName,
custom_avatar: user.customAvatar,
role: user.role,
storage_limit: BigInt(user.storageLimit),
max_file_size: BigInt(user.maxFileSize),
version: BigInt(user.version),
updated_at: user.updatedAt,
})

View File

@@ -149,6 +149,21 @@ export const fetchUser = (
.executeTakeFirst();
};
export const fetchUserStorageUsed = async (
database:
| Kysely<WorkspaceDatabaseSchema>
| Transaction<WorkspaceDatabaseSchema>,
userId: string
): Promise<bigint> => {
const storageUsedRow = await database
.selectFrom('files')
.select(({ fn }) => [fn.sum('size').as('storage_used')])
.where('created_by', '=', userId)
.executeTakeFirst();
return BigInt(storageUsedRow?.storage_used ?? 0);
};
export const fetchWorkspaceCredentials = async (
userId: string
): Promise<WorkspaceCredentials | null> => {

View File

@@ -1,13 +1,29 @@
export const formatBytes = (bytes: number, decimals?: number): string => {
export const formatBytes = (
bytes: number | bigint,
decimals?: number
): string => {
if (bytes === 0) {
return '0 Bytes';
}
const k = 1024;
const bytesBigInt = BigInt(bytes);
const k = BigInt(1024);
const dm = decimals || 2;
const sizes = ['Bytes', 'KB', 'MB', 'GB', 'TB', 'PB', 'EB', 'ZB', 'YB'];
const i = Math.floor(Math.log(bytes) / Math.log(k));
return `${parseFloat((bytes / k ** i).toFixed(dm))} ${sizes[i]}`;
// Find the largest power of k that's smaller than bytes
let i = 0;
let reducedBytes = bytesBigInt;
while (reducedBytes >= k && i < sizes.length - 1) {
reducedBytes = reducedBytes / k;
i++;
}
// Convert to decimal representation with proper precision
const factor = Math.pow(10, dm);
const value = Number((reducedBytes * BigInt(factor)) / BigInt(factor));
return `${value.toFixed(dm)} ${sizes[i]}`;
};
export const getFileUrl = (

View File

@@ -99,6 +99,8 @@ export enum MutationErrorCode {
FileDeleteForbidden = 'file_delete_forbidden',
FileDeleteFailed = 'file_delete_failed',
FolderUpdateForbidden = 'folder_update_forbidden',
StorageLimitExceeded = 'storage_limit_exceeded',
FileTooLarge = 'file_too_large',
FolderUpdateFailed = 'folder_update_failed',
MessageCreateForbidden = 'message_create_forbidden',
MessageCreateFailed = 'message_create_failed',

View File

@@ -14,6 +14,7 @@ import { getNameFromEmail } from '@/lib/utils';
import { SelectAccount, SelectUser } from '@/data/schema';
import { eventBus } from '@/lib/event-bus';
import { ResponseBuilder } from '@/lib/response-builder';
import { configuration } from '@/lib/configuration';
export const userCreateHandler = async (
req: Request,
@@ -75,6 +76,8 @@ export const userCreateHandler = async (
name: account.name,
email: account.email,
avatar: account.avatar,
storage_limit: configuration.user.storageLimit,
max_file_size: configuration.user.maxFileSize,
created_at: new Date(),
created_by: res.locals.account.id,
status: UserStatus.Active,

View File

@@ -94,6 +94,8 @@ const createUsersTable: Migration = {
.addColumn('custom_name', 'varchar(256)')
.addColumn('custom_avatar', 'varchar(256)')
.addColumn('role', 'varchar(30)', (col) => col.notNull())
.addColumn('storage_limit', 'bigint', (col) => col.notNull())
.addColumn('max_file_size', 'bigint', (col) => col.notNull())
.addColumn('created_at', 'timestamptz', (col) => col.notNull())
.addColumn('created_by', 'varchar(30)', (col) => col.notNull())
.addColumn('updated_at', 'timestamptz')

View File

@@ -80,6 +80,8 @@ interface UserTable {
avatar: ColumnType<string | null, string | null, string | null>;
custom_name: ColumnType<string | null, string | null, string | null>;
custom_avatar: ColumnType<string | null, string | null, string | null>;
storage_limit: ColumnType<bigint, bigint, bigint>;
max_file_size: ColumnType<bigint, bigint, bigint>;
created_at: ColumnType<Date, Date, never>;
created_by: ColumnType<string, string, never>;
updated_at: ColumnType<Date | null, Date | null, Date>;

View File

@@ -1,6 +1,7 @@
export interface Configuration {
server: ServerConfiguration;
account: AccountConfiguration;
user: UserConfiguration;
postgres: PostgresConfiguration;
redis: RedisConfiguration;
avatarS3: S3Configuration;
@@ -21,6 +22,11 @@ export interface AccountConfiguration {
allowGoogleLogin: boolean;
}
export interface UserConfiguration {
storageLimit: bigint;
maxFileSize: bigint;
}
export interface PostgresConfiguration {
url: string;
}
@@ -102,6 +108,10 @@ export const configuration: Configuration = {
otpTimeout: parseInt(getOptionalEnv('ACCOUNT_OTP_TIMEOUT') || '600'),
allowGoogleLogin: getOptionalEnv('ACCOUNT_ALLOW_GOOGLE_LOGIN') === 'true',
},
user: {
storageLimit: BigInt(getOptionalEnv('USER_STORAGE_LIMIT') || '10737418240'),
maxFileSize: BigInt(getOptionalEnv('USER_MAX_FILE_SIZE') || '104857600'),
},
postgres: {
url: getRequiredEnv('POSTGRES_URL'),
},

View File

@@ -59,6 +59,22 @@ class FileService {
return false;
}
if (mutation.data.size > user.max_file_size) {
return false;
}
const storageUsedRow = await database
.selectFrom('files')
.select(({ fn }) => [fn.sum('size').as('storage_used')])
.where('created_by', '=', user.id)
.executeTakeFirst();
const storageUsed = BigInt(storageUsedRow?.storage_used ?? 0);
if (storageUsed + BigInt(mutation.data.size) > user.storage_limit) {
return false;
}
const createdFile = await database
.insertInto('files')
.returningAll()

View File

@@ -12,6 +12,7 @@ import { SelectAccount } from '@/data/schema';
import { mapEntry } from '@/lib/entries';
import { entryService } from '@/services/entry-service';
import { eventBus } from '@/lib/event-bus';
import { configuration } from '@/lib/configuration';
class WorkspaceService {
public async createWorkspace(
@@ -51,6 +52,8 @@ class WorkspaceService {
name: account.name,
email: account.email,
avatar: account.avatar,
storage_limit: configuration.user.storageLimit,
max_file_size: configuration.user.maxFileSize,
created_at: date,
created_by: account.id,
status: UserStatus.Active,

View File

@@ -65,6 +65,8 @@ export class UserSynchronizer extends BaseSynchronizer<SyncUsersInput> {
avatar: user.avatar,
customName: user.custom_name,
customAvatar: user.custom_avatar,
storageLimit: user.storage_limit.toString(),
maxFileSize: user.max_file_size.toString(),
createdAt: user.created_at.toISOString(),
updatedAt: user.updated_at?.toISOString() ?? null,
version: user.version.toString(),