Mirrors/colanode
1
0
Fork 0
mirror of https://github.com/colanode/colanode.git synced 2025-12-22 22:49:31 +01:00
Code Issues Packages Projects Releases Wiki Activity

Merge files with nodes

Browse Source
This commit is contained in:
Hakan Shehu
2025-02-12 21:45:02 +01:00
parent a9e028af52
commit f4ad5756f3
53 changed files with 598 additions and 1014 deletions
Download Patch File Download Diff File Expand all files Collapse all files

0
apps/desktop/src/main/databases/workspace/migrations/00011-create-documents-table.ts → apps/desktop/src/main/databases/workspace/migrations/00008-create-documents-table.ts
View File

39
apps/desktop/src/main/databases/workspace/migrations/00008-create-files-table.ts
View File

@@ -1,39 +0,0 @@
import { Migration } from 'kysely';
export const createFilesTable: Migration = {
up: async (db) => {
await db.schema
.createTable('files')
.addColumn('id', 'text', (col) => col.notNull().primaryKey())
.addColumn('type', 'text', (col) => col.notNull())
.addColumn('parent_id', 'text', (col) => col.notNull())
.addColumn('root_id', 'text', (col) => col.notNull())
.addColumn('revision', 'integer', (col) => col.notNull())
.addColumn('name', 'text', (col) => col.notNull())
.addColumn('original_name', 'text', (col) => col.notNull())
.addColumn('mime_type', 'text', (col) => col.notNull())
.addColumn('extension', 'text', (col) => col.notNull())
.addColumn('size', 'integer', (col) => col.notNull())
.addColumn('created_at', 'text', (col) => col.notNull())
.addColumn('created_by', 'text', (col) => col.notNull())
.addColumn('updated_at', 'text')
.addColumn('updated_by', 'text')
.addColumn('download_status', 'integer', (col) => col.notNull())
.addColumn('download_progress', 'integer', (col) => col.notNull())
.addColumn('download_retries', 'integer', (col) => col.notNull())
.addColumn('upload_status', 'integer', (col) => col.notNull())
.addColumn('upload_progress', 'integer', (col) => col.notNull())
.addColumn('upload_retries', 'integer', (col) => col.notNull())
.addColumn('status', 'integer', (col) => col.notNull())
.execute();
await db.schema
.createIndex('files_parent_id_index')
.on('files')
.columns(['parent_id'])
.execute();
},
down: async (db) => {
await db.schema.dropTable('files').execute();
},
};

0
apps/desktop/src/main/databases/workspace/migrations/00012-create-document-updates-table.ts → apps/desktop/src/main/databases/workspace/migrations/00009-create-document-updates-table.ts
View File

24
apps/desktop/src/main/databases/workspace/migrations/00010-create-file-states-table.ts Normal file
View File

@@ -0,0 +1,24 @@
import { Migration } from 'kysely';
export const createFileStatesTable: Migration = {
up: async (db) => {
await db.schema
.createTable('file_states')
.addColumn('id', 'text', (col) => col.notNull().primaryKey())
.addColumn('version', 'text', (col) => col.notNull())
.addColumn('download_status', 'integer')
.addColumn('download_progress', 'integer')
.addColumn('download_retries', 'integer')
.addColumn('download_started_at', 'text')
.addColumn('download_completed_at', 'text')
.addColumn('upload_status', 'integer')
.addColumn('upload_progress', 'integer')
.addColumn('upload_retries', 'integer')
.addColumn('upload_started_at', 'text')
.addColumn('upload_completed_at', 'text')
.execute();
},
down: async (db) => {
await db.schema.dropTable('file_states').execute();
},
};

0
apps/desktop/src/main/databases/workspace/migrations/00009-create-mutations-table.ts → apps/desktop/src/main/databases/workspace/migrations/00011-create-mutations-table.ts
View File

0
apps/desktop/src/main/databases/workspace/migrations/00010-create-tombstones-table.ts → apps/desktop/src/main/databases/workspace/migrations/00012-create-tombstones-table.ts
View File

20
apps/desktop/src/main/databases/workspace/migrations/index.ts
View File

@@ -7,11 +7,11 @@ import { createNodeUpdatesTable } from './00004-create-node-updates-table';
import { createNodeInteractionsTable } from './00005-create-node-interactions-table';
import { createNodeReactionsTable } from './00006-create-node-reactions-table';
import { createCollaborationsTable } from './00007-create-collaborations-table';
import { createFilesTable } from './00008-create-files-table';
import { createMutationsTable } from './00009-create-mutations-table';
import { createTombstonesTable } from './00010-create-tombstones-table';
import { createDocumentsTable } from './00011-create-documents-table';
import { createDocumentUpdatesTable } from './00012-create-document-updates-table';
import { createDocumentsTable } from './00008-create-documents-table';
import { createDocumentUpdatesTable } from './00009-create-document-updates-table';
import { createFileStatesTable } from './00010-create-file-states-table';
import { createMutationsTable } from './00011-create-mutations-table';
import { createTombstonesTable } from './00012-create-tombstones-table';
import { createCursorsTable } from './00013-create-cursors-table';
import { createMetadataTable } from './00014-create-metadata-table';
@@ -23,11 +23,11 @@ export const workspaceDatabaseMigrations: Record<string, Migration> = {
'00005-create-node-interactions-table': createNodeInteractionsTable,
'00006-create-node-reactions-table': createNodeReactionsTable,
'00007-create-collaborations-table': createCollaborationsTable,
'00008-create-files-table': createFilesTable,
'00009-create-mutations-table': createMutationsTable,
'00010-create-tombstones-table': createTombstonesTable,
'00011-create-documents-table': createDocumentsTable,
'00012-create-document-updates-table': createDocumentUpdatesTable,
'00008-create-documents-table': createDocumentsTable,
'00009-create-document-updates-table': createDocumentUpdatesTable,
'00010-create-file-states-table': createFileStatesTable,
'00011-create-mutations-table': createMutationsTable,
'00012-create-tombstones-table': createTombstonesTable,
'00013-create-cursors-table': createCursorsTable,
'00014-create-metadata-table': createMetadataTable,
};

106
apps/desktop/src/main/databases/workspace/schema.ts
View File

@@ -1,6 +1,4 @@
import {
FileStatus,
FileType,
MutationType,
NodeType,
WorkspaceRole,
@@ -108,34 +106,61 @@ export type SelectCollaboration = Selectable<CollaborationTable>;
export type CreateCollaboration = Insertable<CollaborationTable>;
export type UpdateCollaboration = Updateable<CollaborationTable>;
interface FileTable {
interface DocumentTable {
id: ColumnType<string, string, never>;
type: ColumnType<FileType, FileType, FileType>;
parent_id: ColumnType<string, string, string>;
root_id: ColumnType<string, string, string>;
revision: ColumnType<bigint, bigint, bigint>;
name: ColumnType<string, string, string>;
original_name: ColumnType<string, string, string>;
mime_type: ColumnType<string, string, string>;
extension: ColumnType<string, string, string>;
size: ColumnType<number, number, number>;
created_by: ColumnType<string, string, never>;
updated_by: ColumnType<string | null, string | null, string | null>;
deleted_at: ColumnType<string | null, never, string | null>;
download_status: ColumnType<DownloadStatus, DownloadStatus, DownloadStatus>;
download_progress: ColumnType<number, number, number>;
download_retries: ColumnType<number, number, number>;
upload_status: ColumnType<UploadStatus, UploadStatus, UploadStatus>;
upload_progress: ColumnType<number, number, number>;
upload_retries: ColumnType<number, number, number>;
state: ColumnType<Uint8Array, Uint8Array, Uint8Array>;
created_at: ColumnType<string, string, never>;
updated_at: ColumnType<string | null, string | null, string>;
status: ColumnType<FileStatus, FileStatus, FileStatus>;
created_by: ColumnType<string, string, never>;
updated_at: ColumnType<string | null, string | null, string | null>;
updated_by: ColumnType<string | null, string | null, string | null>;
}
export type SelectFile = Selectable<FileTable>;
export type CreateFile = Insertable<FileTable>;
export type UpdateFile = Updateable<FileTable>;
export type SelectDocument = Selectable<DocumentTable>;
export type CreateDocument = Insertable<DocumentTable>;
export type UpdateDocument = Updateable<DocumentTable>;
interface DocumentUpdateTable {
id: ColumnType<string, string, never>;
document_id: ColumnType<string, string, never>;
data: ColumnType<Uint8Array, Uint8Array, never>;
created_at: ColumnType<string, string, never>;
}
export type SelectDocumentUpdate = Selectable<DocumentUpdateTable>;
export type CreateDocumentUpdate = Insertable<DocumentUpdateTable>;
export type UpdateDocumentUpdate = Updateable<DocumentUpdateTable>;
interface FileStateTable {
id: ColumnType<string, string, never>;
version: ColumnType<string, string, string>;
download_status: ColumnType<
DownloadStatus | null,
DownloadStatus | null,
DownloadStatus | null
>;
download_progress: ColumnType<number | null, number | null, number | null>;
download_retries: ColumnType<number | null, number | null, number | null>;
download_started_at: ColumnType<string | null, string | null, string | null>;
download_completed_at: ColumnType<
string | null,
string | null,
string | null
>;
upload_status: ColumnType<
UploadStatus | null,
UploadStatus | null,
UploadStatus | null
>;
upload_progress: ColumnType<number | null, number | null, number | null>;
upload_retries: ColumnType<number | null, number | null, number | null>;
upload_started_at: ColumnType<string | null, string | null, string | null>;
upload_completed_at: ColumnType<string | null, string | null, string | null>;
}
export type SelectFileState = Selectable<FileStateTable>;
export type CreateFileState = Insertable<FileStateTable>;
export type UpdateFileState = Updateable<FileStateTable>;
interface MutationTable {
id: ColumnType<string, string, never>;
@@ -181,31 +206,6 @@ export type SelectWorkspaceMetadata = Selectable<MetadataTable>;
export type CreateWorkspaceMetadata = Insertable<MetadataTable>;
export type UpdateWorkspaceMetadata = Updateable<MetadataTable>;
interface DocumentTable {
id: ColumnType<string, string, never>;
revision: ColumnType<bigint, bigint, bigint>;
state: ColumnType<Uint8Array, Uint8Array, Uint8Array>;
created_at: ColumnType<string, string, never>;
created_by: ColumnType<string, string, never>;
updated_at: ColumnType<string | null, string | null, string | null>;
updated_by: ColumnType<string | null, string | null, string | null>;
}
export type SelectDocument = Selectable<DocumentTable>;
export type CreateDocument = Insertable<DocumentTable>;
export type UpdateDocument = Updateable<DocumentTable>;
interface DocumentUpdateTable {
id: ColumnType<string, string, never>;
document_id: ColumnType<string, string, never>;
data: ColumnType<Uint8Array, Uint8Array, never>;
created_at: ColumnType<string, string, never>;
}
export type SelectDocumentUpdate = Selectable<DocumentUpdateTable>;
export type CreateDocumentUpdate = Insertable<DocumentUpdateTable>;
export type UpdateDocumentUpdate = Updateable<DocumentUpdateTable>;
export interface WorkspaceDatabaseSchema {
users: UserTable;
nodes: NodeTable;
@@ -214,11 +214,11 @@ export interface WorkspaceDatabaseSchema {
node_updates: NodeUpdateTable;
node_reactions: NodeReactionTable;
collaborations: CollaborationTable;
files: FileTable;
mutations: MutationTable;
tombstones: TombstoneTable;
documents: DocumentTable;
document_updates: DocumentUpdateTable;
file_states: FileStateTable;
mutations: MutationTable;
tombstones: TombstoneTable;
cursors: CursorTable;
metadata: MetadataTable;
}

25
apps/desktop/src/main/lib/mappers.ts
View File

@@ -9,7 +9,7 @@ import { SelectEmoji } from '@/main/databases/emojis';
import { SelectIcon } from '@/main/databases/icons';
import { SelectWorkspace } from '@/main/databases/account';
import {
SelectFile,
SelectFileState,
SelectMutation,
SelectNode,
SelectUser,
@@ -20,7 +20,7 @@ import {
import { Account } from '@/shared/types/accounts';
import { Server } from '@/shared/types/servers';
import { User } from '@/shared/types/users';
import { File } from '@/shared/types/files';
import { FileState } from '@/shared/types/files';
import {
Workspace,
WorkspaceMetadata,
@@ -136,29 +136,20 @@ export const mapNodeInteraction = (
};
};
export const mapFile = (row: SelectFile): File => {
export const mapFileState = (row: SelectFileState): FileState => {
return {
id: row.id,
type: row.type,
parentId: row.parent_id,
rootId: row.root_id,
revision: row.revision,
name: row.name,
originalName: row.original_name,
extension: row.extension,
mimeType: row.mime_type,
size: row.size,
createdAt: row.created_at,
createdBy: row.created_by,
updatedAt: row.updated_at,
updatedBy: row.updated_by,
status: row.status,
version: row.version,
downloadStatus: row.download_status,
downloadProgress: row.download_progress,
downloadRetries: row.download_retries,
downloadStartedAt: row.download_started_at,
downloadCompletedAt: row.download_completed_at,
uploadStatus: row.upload_status,
uploadProgress: row.upload_progress,
uploadRetries: row.upload_retries,
uploadStartedAt: row.upload_started_at,
uploadCompletedAt: row.upload_completed_at,
};
};

12
apps/desktop/src/main/lib/utils.ts
View File

@@ -1,8 +1,9 @@
import { extractFileType } from '@colanode/core';
import { extractFileSubtype } from '@colanode/core';
import {
DeleteResult,
InsertResult,
Kysely,
sql,
Transaction,
UpdateResult,
} from 'kysely';
@@ -100,7 +101,7 @@ export const getFileMetadata = (filePath: string): FileMetadata | null => {
}
const stats = fs.statSync(filePath);
const type = extractFileType(mimeType);
const type = extractFileSubtype(mimeType);
return {
path: filePath,
@@ -162,8 +163,11 @@ export const fetchUserStorageUsed = async (
userId: string
): Promise<bigint> => {
const storageUsedRow = await database
.selectFrom('files')
.select(({ fn }) => [fn.sum('size').as('storage_used')])
.selectFrom('nodes')
.select(({ fn }) => [
fn.sum(sql`json_extract(attributes, '$.size')`).as('storage_used'),
])
.where('type', '=', 'file')
.where('created_by', '=', userId)
.executeTakeFirst();

14
apps/desktop/src/main/mutations/files/file-create.ts
View File

@@ -1,4 +1,4 @@
import { generateId, IdType, FileAttributes } from '@colanode/core';
import { generateId, IdType } from '@colanode/core';
import { MutationHandler } from '@/main/lib/types';
import {
@@ -15,19 +15,9 @@ export class FileCreateMutationHandler
input: FileCreateMutationInput
): Promise<FileCreateMutationOutput> {
const workspace = this.getWorkspace(input.accountId, input.workspaceId);
const attributes: FileAttributes = {
type: 'file',
parentId: input.parentId,
};
const fileId = generateId(IdType.File);
await workspace.nodes.createNode({
id: fileId,
attributes,
parentId: input.parentId,
});
await workspace.files.createFile(fileId, input.filePath);
await workspace.files.createFile(fileId, input.parentId, input.filePath);
return {
id: fileId,

47
apps/desktop/src/main/mutations/files/file-download.ts
View File

@@ -1,7 +1,7 @@
import { FileStatus } from '@colanode/core';
import { MutationHandler } from '@/main/lib/types';
import { mapFile } from '@/main/lib/mappers';
import { mapFileState, mapNode } from '@/main/lib/mappers';
import { eventBus } from '@/shared/lib/event-bus';
import { MutationError, MutationErrorCode } from '@/shared/mutations';
import {
@@ -10,6 +10,7 @@ import {
} from '@/shared/mutations/files/file-download';
import { DownloadStatus } from '@/shared/types/files';
import { WorkspaceMutationHandlerBase } from '@/main/mutations/workspace-mutation-handler-base';
import { LocalFileNode } from '@/shared/types/nodes';
export class FileDownloadMutationHandler
extends WorkspaceMutationHandlerBase
@@ -20,48 +21,64 @@ export class FileDownloadMutationHandler
): Promise<FileDownloadMutationOutput> {
const workspace = this.getWorkspace(input.accountId, input.workspaceId);
const file = await workspace.database
.selectFrom('files')
const node = await workspace.database
.selectFrom('nodes')
.selectAll()
.where('id', '=', input.fileId)
.executeTakeFirst();
if (!file) {
if (!node) {
throw new MutationError(
MutationErrorCode.FileNotFound,
'The file you are trying to download does not exist.'
);
}
if (file.status !== FileStatus.Ready) {
const file = mapNode(node) as LocalFileNode;
if (file.attributes.status !== FileStatus.Ready) {
throw new MutationError(
MutationErrorCode.FileNotReady,
'The file you are trying to download is not uploaded by the author yet.'
);
}
const fileState = await workspace.database
.selectFrom('file_states')
.selectAll()
.where('id', '=', input.fileId)
.executeTakeFirst();
if (
file.download_status === DownloadStatus.Completed ||
file.download_status === DownloadStatus.Pending
fileState?.download_status === DownloadStatus.Completed ||
fileState?.download_status === DownloadStatus.Pending
) {
return {
success: true,
};
}
const updatedFile = await workspace.database
.updateTable('files')
const updatedFileState = await workspace.database
.insertInto('file_states')
.returningAll()
.set({
.values({
id: input.fileId,
version: file.attributes.version,
download_status: DownloadStatus.Pending,
download_progress: 0,
download_retries: 0,
updated_at: new Date().toISOString(),
download_started_at: new Date().toISOString(),
})
.where('id', '=', input.fileId)
.onConflict((oc) =>
oc.columns(['id']).doUpdateSet({
download_status: DownloadStatus.Pending,
download_progress: 0,
download_retries: 0,
download_started_at: new Date().toISOString(),
})
)
.executeTakeFirst();
if (!updatedFile) {
if (!updatedFileState) {
throw new MutationError(
MutationErrorCode.FileNotFound,
'The file you are trying to download does not exist.'
@@ -71,10 +88,10 @@ export class FileDownloadMutationHandler
workspace.files.triggerDownloads();
eventBus.publish({
type: 'file_updated',
type: 'file_state_updated',
accountId: workspace.accountId,
workspaceId: workspace.id,
file: mapFile(updatedFile),
fileState: mapFileState(updatedFileState),
});
return {

14
apps/desktop/src/main/mutations/messages/message-create.ts
View File

@@ -1,6 +1,5 @@
import {
EditorNodeTypes,
FileAttributes,
generateId,
IdType,
MessageAttributes,
@@ -65,18 +64,7 @@ export class MessageCreateMutationHandler
});
for (const file of filesToCreate) {
const fileAttributes: FileAttributes = {
type: 'file',
parentId: messageId,
};
await workspace.nodes.createNode({
id: file.id,
attributes: fileAttributes,
parentId: messageId,
});
await workspace.files.createFile(file.id, file.path);
await workspace.files.createFile(file.id, messageId, file.path);
}
return {

40
apps/desktop/src/main/queries/files/file-list.ts
View File

@@ -1,22 +1,24 @@
import { ChangeCheckResult, QueryHandler } from '@/main/lib/types';
import { mapFile } from '@/main/lib/mappers';
import { mapNode } from '@/main/lib/mappers';
import { FileListQueryInput } from '@/shared/queries/files/file-list';
import { Event } from '@/shared/types/events';
import { File } from '@/shared/types/files';
import { LocalFileNode } from '@/shared/types/nodes';
import { WorkspaceQueryHandlerBase } from '@/main/queries/workspace-query-handler-base';
export class FileListQueryHandler
extends WorkspaceQueryHandlerBase
implements QueryHandler<FileListQueryInput>
{
public async handleQuery(input: FileListQueryInput): Promise<File[]> {
public async handleQuery(
input: FileListQueryInput
): Promise<LocalFileNode[]> {
return await this.fetchFiles(input);
}
public async checkForChanges(
event: Event,
input: FileListQueryInput,
output: File[]
output: LocalFileNode[]
): Promise<ChangeCheckResult<FileListQueryInput>> {
if (
event.type === 'workspace_deleted' &&
@@ -30,10 +32,10 @@ export class FileListQueryHandler
}
if (
event.type === 'file_created' &&
event.type === 'node_created' &&
event.accountId === input.accountId &&
event.workspaceId === input.workspaceId &&
event.file.parentId === input.parentId
event.node.parentId === input.parentId
) {
const output = await this.handleQuery(input);
return {
@@ -43,16 +45,16 @@ export class FileListQueryHandler
}
if (
event.type === 'file_updated' &&
event.type === 'node_updated' &&
event.accountId === input.accountId &&
event.workspaceId === input.workspaceId &&
event.file.parentId === input.parentId
event.node.parentId === input.parentId
) {
const file = output.find((file) => file.id === event.file.id);
const file = output.find((file) => file.id === event.node.id);
if (file) {
const newResult = output.map((file) => {
if (file.id === event.file.id) {
return event.file;
if (file.id === event.node.id && event.node.type === 'file') {
return event.node;
}
return file;
@@ -66,12 +68,12 @@ export class FileListQueryHandler
}
if (
event.type === 'file_deleted' &&
event.type === 'node_deleted' &&
event.accountId === input.accountId &&
event.workspaceId === input.workspaceId &&
event.file.parentId === input.parentId
event.node.parentId === input.parentId
) {
const file = output.find((file) => file.id === event.file.id);
const file = output.find((file) => file.id === event.node.id);
if (file) {
const output = await this.handleQuery(input);
return {
@@ -86,20 +88,22 @@ export class FileListQueryHandler
};
}
private async fetchFiles(input: FileListQueryInput): Promise<File[]> {
private async fetchFiles(
input: FileListQueryInput
): Promise<LocalFileNode[]> {
const workspace = this.getWorkspace(input.accountId, input.workspaceId);
const offset = (input.page - 1) * input.count;
const files = await workspace.database
.selectFrom('files')
.selectFrom('nodes')
.selectAll()
.where('type', '=', 'file')
.where('parent_id', '=', input.parentId)
.where('deleted_at', 'is', null)
.orderBy('id', 'asc')
.limit(input.count)
.offset(offset)
.execute();
return files.map(mapFile);
return files.map(mapNode) as LocalFileNode[];
}
}

54
apps/desktop/src/main/queries/files/file-get.ts → apps/desktop/src/main/queries/files/file-state-get.ts
View File

@@ -1,23 +1,25 @@
import { ChangeCheckResult, QueryHandler } from '@/main/lib/types';
import { mapFile } from '@/main/lib/mappers';
import { FileGetQueryInput } from '@/shared/queries/files/file-get';
import { mapFileState } from '@/main/lib/mappers';
import { FileStateGetQueryInput } from '@/shared/queries/files/file-state-get';
import { Event } from '@/shared/types/events';
import { File } from '@/shared/types/files';
import { FileState } from '@/shared/types/files';
import { WorkspaceQueryHandlerBase } from '@/main/queries/workspace-query-handler-base';
export class FileGetQueryHandler
export class FileStateGetQueryHandler
extends WorkspaceQueryHandlerBase
implements QueryHandler<FileGetQueryInput>
implements QueryHandler<FileStateGetQueryInput>
{
public async handleQuery(input: FileGetQueryInput): Promise<File | null> {
return await this.fetchFile(input);
public async handleQuery(
input: FileStateGetQueryInput
): Promise<FileState | null> {
return await this.fetchFileState(input);
}
public async checkForChanges(
event: Event,
input: FileGetQueryInput,
_: File | null
): Promise<ChangeCheckResult<FileGetQueryInput>> {
input: FileStateGetQueryInput,
_: FileState | null
): Promise<ChangeCheckResult<FileStateGetQueryInput>> {
if (
event.type === 'workspace_deleted' &&
event.workspace.accountId === input.accountId &&
@@ -30,10 +32,10 @@ export class FileGetQueryHandler
}
if (
event.type === 'file_created' &&
event.type === 'file_state_updated' &&
event.accountId === input.accountId &&
event.workspaceId === input.workspaceId &&
event.file.id === input.id
event.fileState.id === input.id
) {
const output = await this.handleQuery(input);
return {
@@ -43,22 +45,10 @@ export class FileGetQueryHandler
}
if (
event.type === 'file_updated' &&
event.type === 'node_deleted' &&
event.accountId === input.accountId &&
event.workspaceId === input.workspaceId &&
event.file.id === input.id
) {
return {
hasChanges: true,
result: event.file,
};
}
if (
event.type === 'file_deleted' &&
event.accountId === input.accountId &&
event.workspaceId === input.workspaceId &&
event.file.id === input.id
event.node.id === input.id
) {
return {
hasChanges: true,
@@ -71,19 +61,21 @@ export class FileGetQueryHandler
};
}
private async fetchFile(input: FileGetQueryInput): Promise<File | null> {
private async fetchFileState(
input: FileStateGetQueryInput
): Promise<FileState | null> {
const workspace = this.getWorkspace(input.accountId, input.workspaceId);
const file = await workspace.database
.selectFrom('files')
const fileState = await workspace.database
.selectFrom('file_states')
.selectAll()
.where('id', '=', input.id)
.executeTakeFirst();
if (!file || file.deleted_at) {
if (!fileState) {
return null;
}
return mapFile(file);
return mapFileState(fileState);
}
}

4
apps/desktop/src/main/queries/index.ts
View File

@@ -7,7 +7,7 @@ import { EmojiCategoryListQueryHandler } from '@/main/queries/emojis/emoji-categ
import { EmojiSearchQueryHandler } from '@/main/queries/emojis/emoji-search';
import { EmojiGetBySkinIdQueryHandler } from '@/main/queries/emojis/emoji-get-by-skin-id';
import { FileListQueryHandler } from '@/main/queries/files/file-list';
import { FileGetQueryHandler } from '@/main/queries/files/file-get';
import { FileStateGetQueryHandler } from '@/main/queries/files/file-state-get';
import { FileMetadataGetQueryHandler } from '@/main/queries/files/file-metadata-get';
import { IconListQueryHandler } from '@/main/queries/icons/icon-list';
import { IconSearchQueryHandler } from '@/main/queries/icons/icon-search';
@@ -71,7 +71,7 @@ export const queryHandlerMap: QueryHandlerMap = {
database_view_list: new DatabaseViewListQueryHandler(),
record_search: new RecordSearchQueryHandler(),
user_get: new UserGetQueryHandler(),
file_get: new FileGetQueryHandler(),
file_state_get: new FileStateGetQueryHandler(),
chat_list: new ChatListQueryHandler(),
space_list: new SpaceListQueryHandler(),
workspace_metadata_list: new WorkspaceMetadataListQueryHandler(),

5
apps/desktop/src/main/services/workspaces/collaboration-service.ts
View File

@@ -50,11 +50,6 @@ export class CollaborationService {
.where('root_id', '=', collaboration.nodeId)
.execute();
await this.workspace.database
.deleteFrom('files')
.where('root_id', '=', collaboration.nodeId)
.execute();
await this.workspace.database
.deleteFrom('node_interactions')
.where('root_id', '=', collaboration.nodeId)

483
apps/desktop/src/main/services/workspaces/file-service.ts
View File

@@ -1,12 +1,12 @@
import {
CompleteUploadOutput,
CreateDownloadOutput,
CreateFileMutationData,
CreateUploadOutput,
FileAttributes,
FileStatus,
IdType,
SyncFileData,
createDebugger,
extractFileSubtype,
generateId,
} from '@colanode/core';
import axios from 'axios';
@@ -22,14 +22,15 @@ import {
getWorkspaceFilesDirectoryPath,
getWorkspaceTempFilesDirectoryPath,
} from '@/main/lib/utils';
import { mapFile, mapNode } from '@/main/lib/mappers';
import { mapFileState, mapNode } from '@/main/lib/mappers';
import { eventBus } from '@/shared/lib/event-bus';
import { DownloadStatus, UploadStatus } from '@/shared/types/files';
import { WorkspaceService } from '@/main/services/workspaces/workspace-service';
import { EventLoop } from '@/main/lib/event-loop';
import { SelectFile } from '@/main/databases/workspace';
import { SelectFileState, SelectNode } from '@/main/databases/workspace';
import { MutationError, MutationErrorCode } from '@/shared/mutations';
import { formatBytes } from '@/shared/lib/files';
import { LocalFileNode } from '@/shared/types/nodes';
const UPLOAD_RETRIES_LIMIT = 10;
const DOWNLOAD_RETRIES_LIMIT = 10;
@@ -94,7 +95,11 @@ export class FileService {
this.cleanupEventLoop.start();
}
public async createFile(id: string, path: string): Promise<void> {
public async createFile(
id: string,
parentId: string,
path: string
): Promise<void> {
const metadata = getFileMetadata(path);
if (!metadata) {
throw new MutationError(
@@ -128,8 +133,8 @@ export class FileService {
);
}
const node = await fetchNode(this.workspace.database, id);
if (!node || node.type !== 'file') {
const node = await fetchNode(this.workspace.database, parentId);
if (!node) {
throw new MutationError(
MutationErrorCode.NodeNotFound,
'There was an error while creating the file. Please make sure you have access to this node.'
@@ -138,79 +143,70 @@ export class FileService {
this.copyFileToWorkspace(path, id, metadata.extension);
const mutationData: CreateFileMutationData = {
id,
type: metadata.type,
parentId: node.parent_id!,
rootId: node.root_id,
const attributes: FileAttributes = {
type: 'file',
subtype: extractFileSubtype(metadata.mimeType),
parentId: parentId,
name: metadata.name,
originalName: metadata.name,
extension: metadata.extension,
mimeType: metadata.mimeType,
size: metadata.size,
createdAt: new Date().toISOString(),
status: FileStatus.Pending,
version: generateId(IdType.Version),
};
const createdFile = await this.workspace.database
.transaction()
.execute(async (tx) => {
const createdFile = await tx
.insertInto('files')
await this.workspace.nodes.createNode({
id: id,
attributes: attributes,
parentId: parentId,
});
const createdFileState = await this.workspace.database
.insertInto('file_states')
.returningAll()
.values({
id,
type: metadata.type,
parent_id: node.parent_id!,
root_id: node.root_id,
name: metadata.name,
original_name: metadata.name,
mime_type: metadata.mimeType,
size: metadata.size,
extension: metadata.extension,
created_at: new Date().toISOString(),
created_by: this.workspace.userId,
status: FileStatus.Pending,
revision: 0n,
download_status: DownloadStatus.Completed,
id: id,
version: attributes.version,
download_progress: 100,
download_retries: 0,
upload_status: UploadStatus.Pending,
download_status: DownloadStatus.Completed,
download_completed_at: new Date().toISOString(),
upload_progress: 0,
upload_status: UploadStatus.Pending,
upload_retries: 0,
upload_started_at: new Date().toISOString(),
})
.executeTakeFirst();
if (!createdFile) {
throw new Error('Failed to create file.');
if (!createdFileState) {
throw new MutationError(
MutationErrorCode.FileCreateFailed,
'Failed to create file state'
);
}
await tx
.insertInto('mutations')
.values({
id: generateId(IdType.Mutation),
type: 'create_file',
data: JSON.stringify(mutationData),
created_at: new Date().toISOString(),
retries: 0,
})
.execute();
return createdFile;
});
if (createdFile) {
this.workspace.mutations.triggerSync();
eventBus.publish({
type: 'file_created',
type: 'file_state_updated',
accountId: this.workspace.accountId,
workspaceId: this.workspace.id,
file: mapFile(createdFile),
fileState: mapFileState(createdFileState),
});
}
this.triggerUploads();
}
public copyFileToWorkspace(
public async deleteFile(node: SelectNode): Promise<void> {
const file = mapNode(node);
if (file.type !== 'file') {
return;
}
const filePath = this.buildFilePath(file.id, file.attributes.extension);
fs.rmSync(filePath, { force: true });
}
private copyFileToWorkspace(
filePath: string,
fileId: string,
fileExtension: string
@@ -233,13 +229,6 @@ export class FileService {
}
}
public deleteFile(id: string, extension: string): void {
const filePath = this.buildFilePath(id, extension);
this.debug(`Deleting file ${filePath}`);
fs.rmSync(filePath, { force: true });
}
public triggerUploads(): void {
this.uploadsEventLoop.trigger();
}
@@ -262,7 +251,7 @@ export class FileService {
this.debug(`Uploading files for workspace ${this.workspace.id}`);
const uploads = await this.workspace.database
.selectFrom('files')
.selectFrom('file_states')
.selectAll()
.where('upload_status', '=', UploadStatus.Pending)
.execute();
@@ -276,71 +265,77 @@ export class FileService {
}
}
private async uploadFile(file: SelectFile): Promise<void> {
if (file.upload_retries >= UPLOAD_RETRIES_LIMIT) {
private async uploadFile(state: SelectFileState): Promise<void> {
if (state.upload_retries && state.upload_retries >= UPLOAD_RETRIES_LIMIT) {
this.debug(
`File ${file.id} upload retries limit reached, marking as failed`
`File ${state.id} upload retries limit reached, marking as failed`
);
const updatedFile = await this.workspace.database
.updateTable('files')
const updatedFileState = await this.workspace.database
.updateTable('file_states')
.returningAll()
.set({
upload_status: UploadStatus.Failed,
updated_at: new Date().toISOString(),
upload_retries: state.upload_retries + 1,
})
.where('id', '=', file.id)
.where('id', '=', state.id)
.executeTakeFirst();
if (updatedFile) {
if (updatedFileState) {
eventBus.publish({
type: 'file_updated',
type: 'file_state_updated',
accountId: this.workspace.accountId,
workspaceId: this.workspace.id,
file: mapFile(updatedFile),
fileState: mapFileState(updatedFileState),
});
}
return;
}
if (file.revision === BigInt(0)) {
const node = await this.workspace.database
.selectFrom('nodes')
.selectAll()
.where('id', '=', state.id)
.executeTakeFirst();
if (!node) {
return;
}
const file = mapNode(node) as LocalFileNode;
if (node.server_revision === BigInt(0)) {
// file is not synced with the server, we need to wait for the sync to complete
return;
}
if (file.status === FileStatus.Ready) {
const updatedFile = await this.workspace.database
.updateTable('files')
if (file.attributes.status === FileStatus.Ready) {
const updatedFileState = await this.workspace.database
.updateTable('file_states')
.returningAll()
.set({
upload_status: UploadStatus.Completed,
upload_progress: 100,
updated_at: new Date().toISOString(),
upload_completed_at: new Date().toISOString(),
})
.where('id', '=', file.id)
.executeTakeFirst();
if (updatedFile) {
if (updatedFileState) {
eventBus.publish({
type: 'file_updated',
type: 'file_state_updated',
accountId: this.workspace.accountId,
workspaceId: this.workspace.id,
file: mapFile(updatedFile),
fileState: mapFileState(updatedFileState),
});
}
return;
}
const filePath = this.buildFilePath(file.id, file.extension);
const filePath = this.buildFilePath(file.id, file.attributes.extension);
if (!fs.existsSync(filePath)) {
await this.workspace.database
.deleteFrom('files')
.where('id', '=', file.id)
.execute();
this.debug(`File ${file.id} not found, deleting from database`);
return;
}
@@ -360,11 +355,13 @@ export class FileService {
let lastProgress = 0;
await axios.put(presignedUrl, fileStream, {
headers: {
'Content-Type': file.mime_type,
'Content-Length': file.size,
'Content-Type': file.attributes.mimeType,
'Content-Length': file.attributes.size,
},
onUploadProgress: async (progressEvent) => {
const progress = Math.round((progressEvent.loaded / file.size) * 100);
const progress = Math.round(
(progressEvent.loaded / file.attributes.size) * 100
);
if (progress >= lastProgress) {
return;
@@ -372,25 +369,24 @@ export class FileService {
lastProgress = progress;
const updatedFile = await this.workspace.database
.updateTable('files')
const updatedFileState = await this.workspace.database
.updateTable('file_states')
.returningAll()
.set({
upload_progress: progress,
updated_at: new Date().toISOString(),
})
.where('id', '=', file.id)
.executeTakeFirst();
if (!updatedFile) {
if (!updatedFileState) {
return;
}
eventBus.publish({
type: 'file_updated',
type: 'file_state_updated',
accountId: this.workspace.accountId,
workspaceId: this.workspace.id,
file: mapFile(updatedFile),
fileState: mapFileState(updatedFileState),
});
},
});
@@ -400,41 +396,41 @@ export class FileService {
{}
);
const finalFile = await this.workspace.database
.updateTable('files')
const finalFileState = await this.workspace.database
.updateTable('file_states')
.returningAll()
.set({
upload_status: UploadStatus.Completed,
upload_progress: 100,
updated_at: new Date().toISOString(),
upload_completed_at: new Date().toISOString(),
})
.where('id', '=', file.id)
.executeTakeFirst();
if (finalFile) {
if (finalFileState) {
eventBus.publish({
type: 'file_updated',
type: 'file_state_updated',
accountId: this.workspace.accountId,
workspaceId: this.workspace.id,
file: mapFile(finalFile),
fileState: mapFileState(finalFileState),
});
}
this.debug(`File ${file.id} uploaded successfully`);
} catch {
const updatedFile = await this.workspace.database
.updateTable('files')
const updatedFileState = await this.workspace.database
.updateTable('file_states')
.returningAll()
.set((eb) => ({ upload_retries: eb('upload_retries', '+', 1) }))
.where('id', '=', file.id)
.executeTakeFirst();
if (updatedFile) {
if (updatedFileState) {
eventBus.publish({
type: 'file_updated',
type: 'file_state_updated',
accountId: this.workspace.accountId,
workspaceId: this.workspace.id,
file: mapFile(updatedFile),
fileState: mapFileState(updatedFileState),
});
}
}
@@ -448,7 +444,7 @@ export class FileService {
this.debug(`Downloading files for workspace ${this.workspace.id}`);
const downloads = await this.workspace.database
.selectFrom('files')
.selectFrom('file_states')
.selectAll()
.where('download_status', '=', DownloadStatus.Pending)
.execute();
@@ -462,41 +458,73 @@ export class FileService {
}
}
private async downloadFile(file: SelectFile): Promise<void> {
if (file.download_retries >= DOWNLOAD_RETRIES_LIMIT) {
private async downloadFile(fileState: SelectFileState): Promise<void> {
if (
fileState.download_retries &&
fileState.download_retries >= DOWNLOAD_RETRIES_LIMIT
) {
this.debug(
`File ${file.id} download retries limit reached, marking as failed`
`File ${fileState.id} download retries limit reached, marking as failed`
);
await this.workspace.database
.updateTable('files')
const updatedFileState = await this.workspace.database
.updateTable('file_states')
.returningAll()
.set({
download_status: DownloadStatus.Failed,
updated_at: new Date().toISOString(),
download_retries: fileState.download_retries + 1,
})
.where('id', '=', file.id)
.execute();
.where('id', '=', fileState.id)
.executeTakeFirst();
if (updatedFileState) {
eventBus.publish({
type: 'file_state_updated',
accountId: this.workspace.accountId,
workspaceId: this.workspace.id,
fileState: mapFileState(updatedFileState),
});
}
const filePath = this.buildFilePath(file.id, file.extension);
return;
}
const node = await this.workspace.database
.selectFrom('nodes')
.selectAll()
.where('id', '=', fileState.id)
.executeTakeFirst();
if (!node) {
return;
}
const file = mapNode(node) as LocalFileNode;
if (node.server_revision === BigInt(0)) {
// file is not synced with the server, we need to wait for the sync to complete
return;
}
const filePath = this.buildFilePath(file.id, file.attributes.extension);
if (fs.existsSync(filePath)) {
const updatedFile = await this.workspace.database
.updateTable('files')
const updatedFileState = await this.workspace.database
.updateTable('file_states')
.returningAll()
.set({
download_status: DownloadStatus.Completed,
download_progress: 100,
updated_at: new Date().toISOString(),
download_completed_at: new Date().toISOString(),
})
.where('id', '=', file.id)
.where('id', '=', fileState.id)
.executeTakeFirst();
if (updatedFile) {
if (updatedFileState) {
eventBus.publish({
type: 'file_updated',
type: 'file_state_updated',
accountId: this.workspace.accountId,
workspaceId: this.workspace.id,
file: mapFile(updatedFile),
fileState: mapFileState(updatedFileState),
});
}
@@ -519,7 +547,7 @@ export class FileService {
responseType: 'stream',
onDownloadProgress: async (progressEvent) => {
const progress = Math.round(
(progressEvent.loaded / file.size) * 100
(progressEvent.loaded / file.attributes.size) * 100
);
if (progress <= lastProgress) {
@@ -528,22 +556,21 @@ export class FileService {
lastProgress = progress;
const updatedFile = await this.workspace.database
.updateTable('files')
const updatedFileState = await this.workspace.database
.updateTable('file_states')
.returningAll()
.set({
download_progress: progress,
updated_at: new Date().toISOString(),
})
.where('id', '=', file.id)
.where('id', '=', fileState.id)
.executeTakeFirst();
if (updatedFile) {
if (updatedFileState) {
eventBus.publish({
type: 'file_updated',
type: 'file_state_updated',
accountId: this.workspace.accountId,
workspaceId: this.workspace.id,
file: mapFile(updatedFile),
fileState: mapFileState(updatedFileState),
});
}
},
@@ -552,39 +579,39 @@ export class FileService {
response.data.pipe(fileStream);
});
const updatedFile = await this.workspace.database
.updateTable('files')
const updatedFileState = await this.workspace.database
.updateTable('file_states')
.returningAll()
.set({
download_status: DownloadStatus.Completed,
download_progress: 100,
updated_at: new Date().toISOString(),
download_completed_at: new Date().toISOString(),
})
.where('id', '=', file.id)
.where('id', '=', fileState.id)
.executeTakeFirst();
if (updatedFile) {
if (updatedFileState) {
eventBus.publish({
type: 'file_updated',
type: 'file_state_updated',
accountId: this.workspace.accountId,
workspaceId: this.workspace.id,
file: mapFile(updatedFile),
fileState: mapFileState(updatedFileState),
});
}
} catch {
const updatedFile = await this.workspace.database
.updateTable('files')
const updatedFileState = await this.workspace.database
.updateTable('file_states')
.returningAll()
.set((eb) => ({ download_retries: eb('download_retries', '+', 1) }))
.where('id', '=', file.id)
.where('id', '=', fileState.id)
.executeTakeFirst();
if (updatedFile) {
if (updatedFileState) {
eventBus.publish({
type: 'file_updated',
type: 'file_state_updated',
accountId: this.workspace.accountId,
workspaceId: this.workspace.id,
file: mapFile(updatedFile),
fileState: mapFileState(updatedFileState),
});
}
}
@@ -604,15 +631,15 @@ export class FileService {
}
const fileIds = Object.keys(fileIdMap);
const files = await this.workspace.database
.selectFrom('files')
const fileStates = await this.workspace.database
.selectFrom('file_states')
.select(['id'])
.where('id', 'in', fileIds)
.execute();
for (const fileId of fileIds) {
const file = files.find((f) => f.id === fileId);
if (!file) {
const fileState = fileStates.find((f) => f.id === fileId);
if (fileState) {
continue;
}
@@ -647,160 +674,6 @@ export class FileService {
}
}
public async syncServerFile(file: SyncFileData): Promise<void> {
const existingFile = await this.workspace.database
.selectFrom('files')
.selectAll()
.where('id', '=', file.id)
.executeTakeFirst();
const revision = BigInt(file.revision);
if (existingFile) {
if (existingFile.revision === revision) {
this.debug(`Server file ${file.id} is already synced`);
return;
}
const updatedFile = await this.workspace.database
.updateTable('files')
.returningAll()
.set({
name: file.name,
original_name: file.originalName,
mime_type: file.mimeType,
extension: file.extension,
size: file.size,
parent_id: file.parentId,
root_id: file.rootId,
status: file.status,
type: file.type,
updated_at: file.updatedAt ?? undefined,
updated_by: file.updatedBy ?? undefined,
revision,
})
.where('id', '=', file.id)
.executeTakeFirst();
if (!updatedFile) {
return;
}
eventBus.publish({
type: 'file_updated',
accountId: this.workspace.accountId,
workspaceId: this.workspace.id,
file: mapFile(updatedFile),
});
this.triggerUploads();
this.triggerDownloads();
this.debug(`Server file ${file.id} has been synced`);
return;
}
const createdFile = await this.workspace.database
.insertInto('files')
.returningAll()
.values({
id: file.id,
revision,
type: file.type,
parent_id: file.parentId,
root_id: file.rootId,
name: file.name,
original_name: file.originalName,
mime_type: file.mimeType,
extension: file.extension,
size: file.size,
created_at: file.createdAt,
created_by: file.createdBy,
updated_at: file.updatedAt,
updated_by: file.updatedBy,
status: file.status,
download_progress: 0,
download_retries: 0,
upload_progress: 0,
upload_retries: 0,
download_status: DownloadStatus.None,
upload_status: UploadStatus.None,
})
.executeTakeFirst();
if (!createdFile) {
return;
}
eventBus.publish({
type: 'file_created',
accountId: this.workspace.accountId,
workspaceId: this.workspace.id,
file: mapFile(createdFile),
});
this.debug(`Server file ${file.id} has been synced`);
}
public async revertFileCreate(fileId: string) {
const deletedFile = await this.workspace.database
.deleteFrom('files')
.returningAll()
.where('id', '=', fileId)
.executeTakeFirst();
if (!deletedFile) {
return;
}
const deletedNode = await this.workspace.database
.deleteFrom('nodes')
.returningAll()
.where('id', '=', deletedFile.parent_id)
.executeTakeFirst();
if (!deletedNode) {
return;
}
await this.workspace.database
.deleteFrom('node_states')
.where('id', '=', deletedNode.id)
.execute();
await this.workspace.database
.deleteFrom('node_interactions')
.where('node_id', '=', deletedNode.id)
.execute();
await this.workspace.database
.deleteFrom('node_reactions')
.where('node_id', '=', deletedNode.id)
.execute();
const filePath = path.join(
this.filesDir,
`${fileId}${deletedFile.extension}`
);
if (fs.existsSync(filePath)) {
fs.rmSync(filePath, { force: true });
}
eventBus.publish({
type: 'file_deleted',
accountId: this.workspace.accountId,
workspaceId: this.workspace.id,
file: mapFile(deletedFile),
});
eventBus.publish({
type: 'node_deleted',
accountId: this.workspace.accountId,
workspaceId: this.workspace.id,
node: mapNode(deletedNode),
});
}
private buildFilePath(id: string, extension: string): string {
return path.join(this.filesDir, `${id}${extension}`);
}

4
apps/desktop/src/main/services/workspaces/mutation-service.ts
View File

@@ -143,9 +143,7 @@ export class MutationService {
for (const mutationRow of invalidMutations) {
const mutation = mapMutation(mutationRow);
if (mutation.type === 'create_file') {
await this.workspace.files.revertFileCreate(mutation.id);
} else if (mutation.type === 'create_node') {
if (mutation.type === 'create_node') {
await this.workspace.nodes.revertNodeCreate(mutation.data);
} else if (mutation.type === 'update_node') {
await this.workspace.nodes.revertNodeUpdate(mutation.data);

26
apps/desktop/src/main/services/workspaces/node-service.ts
View File

@@ -16,7 +16,7 @@ import {
import { decodeState, encodeState, YDoc } from '@colanode/crdt';
import { fetchNodeTree } from '@/main/lib/utils';
import { mapFile, mapNode } from '@/main/lib/mappers';
import { mapNode } from '@/main/lib/mappers';
import { eventBus } from '@/shared/lib/event-bus';
import { WorkspaceService } from '@/main/services/workspaces/workspace-service';
import { SelectNode } from '@/main/databases/workspace';
@@ -587,7 +587,7 @@ export class NodeService {
`Applying server delete transaction ${tombstone.id} for node ${tombstone.id}`
);
const { deletedNode, deletedFile } = await this.workspace.database
const { deletedNode } = await this.workspace.database
.transaction()
.execute(async (trx) => {
const deletedNode = await trx
@@ -616,12 +616,6 @@ export class NodeService {
.where('node_id', '=', tombstone.id)
.execute();
const deletedFile = await trx
.deleteFrom('files')
.returningAll()
.where('id', '=', tombstone.id)
.executeTakeFirst();
await trx
.deleteFrom('tombstones')
.where('id', '=', tombstone.id)
@@ -637,19 +631,9 @@ export class NodeService {
.where('document_id', '=', tombstone.id)
.execute();
return { deletedNode, deletedFile };
return { deletedNode };
});
if (deletedFile) {
this.workspace.files.deleteFile(deletedFile.id, deletedFile.extension);
eventBus.publish({
type: 'file_deleted',
accountId: this.workspace.accountId,
workspaceId: this.workspace.id,
file: mapFile(deletedFile),
});
}
if (deletedNode) {
eventBus.publish({
type: 'node_deleted',
@@ -702,6 +686,10 @@ export class NodeService {
.execute();
});
if (node.type === 'file') {
await this.workspace.files.deleteFile(node);
}
eventBus.publish({
type: 'node_deleted',
accountId: this.workspace.accountId,

12
apps/desktop/src/main/services/workspaces/sync-service.ts
View File

@@ -5,7 +5,6 @@ import {
SyncNodesInput,
SyncNodeInteractionsInput,
SyncNodeReactionsInput,
SyncFilesInput,
SyncNodeTombstonesInput,
SyncNodeInteractionData,
SyncNodeReactionData,
@@ -13,7 +12,6 @@ import {
SyncNodeData,
SyncUserData,
SyncCollaborationData,
SyncFileData,
SyncDocumentUpdatesInput,
SyncDocumentUpdateData,
} from '@colanode/core';
@@ -28,7 +26,6 @@ interface RootSynchronizers {
nodeInteractions: Synchronizer<SyncNodeInteractionsInput>;
nodeReactions: Synchronizer<SyncNodeReactionsInput>;
nodeTombstones: Synchronizer<SyncNodeTombstonesInput>;
files: Synchronizer<SyncFilesInput>;
documentUpdates: Synchronizer<SyncDocumentUpdatesInput>;
}
@@ -39,7 +36,6 @@ type SyncHandlers = {
nodeInteractions: (data: SyncNodeInteractionData) => Promise<void>;
nodeReactions: (data: SyncNodeReactionData) => Promise<void>;
nodeTombstones: (data: SyncNodeTombstoneData) => Promise<void>;
files: (data: SyncFileData) => Promise<void>;
documentUpdates: (data: SyncDocumentUpdateData) => Promise<void>;
};
@@ -76,7 +72,6 @@ export class SyncService {
nodeTombstones: this.workspace.nodes.syncServerNodeDelete.bind(
this.workspace.nodes
),
files: this.workspace.files.syncServerFile.bind(this.workspace.files),
documentUpdates: this.workspace.documents.syncServerDocumentUpdate.bind(
this.workspace.documents
),
@@ -149,7 +144,6 @@ export class SyncService {
rootSynchronizers.nodeInteractions.destroy();
rootSynchronizers.nodeReactions.destroy();
rootSynchronizers.nodeTombstones.destroy();
rootSynchronizers.files.destroy();
rootSynchronizers.documentUpdates.destroy();
}
@@ -187,12 +181,6 @@ export class SyncService {
`${rootId}_node_tombstones`,
this.syncHandlers.nodeTombstones
),
files: new Synchronizer(
this.workspace,
{ type: 'files', rootId },
`${rootId}_files`,
this.syncHandlers.files
),
documentUpdates: new Synchronizer(
this.workspace,
{ type: 'document_updates', rootId },

9
apps/desktop/src/renderer/components/files/file-block.tsx
View File

@@ -2,6 +2,7 @@ import { FilePreview } from '@/renderer/components/files/file-preview';
import { useWorkspace } from '@/renderer/contexts/workspace';
import { useLayout } from '@/renderer/contexts/layout';
import { useQuery } from '@/renderer/hooks/use-query';
import { LocalFileNode } from '@/shared/types/nodes';
interface FileBlockProps {
id: string;
@@ -12,8 +13,8 @@ export const FileBlock = ({ id }: FileBlockProps) => {
const layout = useLayout();
const { data } = useQuery({
type: 'file_get',
id,
type: 'node_get',
nodeId: id,
accountId: workspace.accountId,
workspaceId: workspace.id,
});
@@ -22,6 +23,8 @@ export const FileBlock = ({ id }: FileBlockProps) => {
return null;
}
const file = data as LocalFileNode;
return (
<div
className="flex h-72 max-h-72 max-w-128 w-full cursor-pointer overflow-hidden rounded-md p-2 hover:bg-gray-100"
@@ -29,7 +32,7 @@ export const FileBlock = ({ id }: FileBlockProps) => {
layout.previewLeft(id, true);
}}
>
<FilePreview file={data} />
<FilePreview file={file} />
</div>
);
};

23
apps/desktop/src/renderer/components/files/file-body.tsx
View File

@@ -4,7 +4,6 @@ import { FilePreview } from '@/renderer/components/files/file-preview';
import { FileSidebar } from '@/renderer/components/files/file-sidebar';
import { Button } from '@/renderer/components/ui/button';
import { useWorkspace } from '@/renderer/contexts/workspace';
import { useQuery } from '@/renderer/hooks/use-query';
import { LocalFileNode } from '@/shared/types/nodes';
interface FileBodyProps {
@@ -13,20 +12,6 @@ interface FileBodyProps {
export const FileBody = ({ file }: FileBodyProps) => {
const workspace = useWorkspace();
const { data, isPending } = useQuery({
type: 'file_get',
id: file.id,
accountId: workspace.accountId,
workspaceId: workspace.id,
});
if (isPending) {
return <p className="text-sm text-muted-foreground">Loading...</p>;
}
if (!data) {
return <p className="text-sm text-muted-foreground">Not found</p>;
}
return (
<div className="flex h-full max-h-full w-full flex-row items-center gap-2">
@@ -40,8 +25,8 @@ export const FileBody = ({ file }: FileBodyProps) => {
type: 'file_open',
accountId: workspace.accountId,
workspaceId: workspace.id,
fileId: data.id,
extension: data.extension,
fileId: file.id,
extension: file.attributes.extension,
})
}
>
@@ -49,11 +34,11 @@ export const FileBody = ({ file }: FileBodyProps) => {
</Button>
</div>
<div className="flex w-full max-w-full flex-grow items-center justify-center overflow-hidden p-10">
<FilePreview file={data} />
<FilePreview file={file} />
</div>
</div>
<div className="h-full w-72 min-w-72 overflow-hidden border-l border-gray-100 p-2 pl-3">
<FileSidebar file={data} />
<FileSidebar file={file} />
</div>
</div>
);

19
apps/desktop/src/renderer/components/files/file-breadcrumb-item.tsx
View File

@@ -1,6 +1,4 @@
import { FileThumbnail } from '@/renderer/components/files/file-thumbnail';
import { useQuery } from '@/renderer/hooks/use-query';
import { useWorkspace } from '@/renderer/contexts/workspace';
import { LocalFileNode } from '@/shared/types/nodes';
interface FileBreadcrumbItemProps {
@@ -8,26 +6,13 @@ interface FileBreadcrumbItemProps {
}
export const FileBreadcrumbItem = ({ file }: FileBreadcrumbItemProps) => {
const workspace = useWorkspace();
const { data } = useQuery({
type: 'file_get',
id: file.id,
accountId: workspace.accountId,
workspaceId: workspace.id,
});
if (!data) {
return null;
}
return (
<div className="flex items-center space-x-2">
<FileThumbnail
file={data}
file={file}
className="size-4 overflow-hidden rounded object-contain"
/>
<span>{data.name}</span>
<span>{file.attributes.name}</span>
</div>
);
};

9
apps/desktop/src/renderer/components/files/file-container-tab.tsx
View File

@@ -1,6 +1,7 @@
import { FileThumbnail } from '@/renderer/components/files/file-thumbnail';
import { useWorkspace } from '@/renderer/contexts/workspace';
import { useQuery } from '@/renderer/hooks/use-query';
import { LocalFileNode } from '@/shared/types/nodes';
interface FileContainerTabProps {
fileId: string;
@@ -10,8 +11,8 @@ export const FileContainerTab = ({ fileId }: FileContainerTabProps) => {
const workspace = useWorkspace();
const { data, isPending } = useQuery({
type: 'file_get',
id: fileId,
type: 'node_get',
nodeId: fileId,
accountId: workspace.accountId,
workspaceId: workspace.id,
});
@@ -20,7 +21,7 @@ export const FileContainerTab = ({ fileId }: FileContainerTabProps) => {
return <p className="text-sm text-muted-foreground">Loading...</p>;
}
const file = data;
const file = data as LocalFileNode;
if (!file) {
return <p className="text-sm text-muted-foreground">Not found</p>;
}
@@ -31,7 +32,7 @@ export const FileContainerTab = ({ fileId }: FileContainerTabProps) => {
file={file}
className="size-4 overflow-hidden rounded object-contain"
/>
<span>{file.name}</span>
<span>{file.attributes.name}</span>
</div>
);
};

17
apps/desktop/src/renderer/components/files/file-download.tsx
View File

@@ -3,24 +3,28 @@ import { Download } from 'lucide-react';
import { Spinner } from '@/renderer/components/ui/spinner';
import { useWorkspace } from '@/renderer/contexts/workspace';
import { toast } from '@/renderer/hooks/use-toast';
import { DownloadStatus, File } from '@/shared/types/files';
import { DownloadStatus, FileState } from '@/shared/types/files';
import { formatBytes } from '@/shared/lib/files';
import { LocalFileNode } from '@/shared/types/nodes';
interface FileDownloadProps {
file: File;
file: LocalFileNode;
state: FileState | null | undefined;
}
export const FileDownload = ({ file }: FileDownloadProps) => {
export const FileDownload = ({ file, state }: FileDownloadProps) => {
const workspace = useWorkspace();
const isDownloading = file.downloadStatus === DownloadStatus.Pending;
const isDownloading = state?.downloadStatus === DownloadStatus.Pending;
return (
<div className="flex h-full w-full items-center justify-center">
{isDownloading ? (
<div className="flex flex-col items-center gap-3 text-muted-foreground">
<Spinner className="size-8" />
<p className="text-sm">Downloading file ({file.downloadProgress}%)</p>
<p className="text-sm">
Downloading file ({state?.downloadProgress}%)
</p>
</div>
) : (
<div
@@ -50,7 +54,8 @@ export const FileDownload = ({ file }: FileDownloadProps) => {
File is not downloaded in your device. Click to download.
</p>
<p className="text-xs text-muted-foreground">
{formatBytes(file.size)} - {file.mimeType}
{formatBytes(file.attributes.size)} -{' '}
{file.attributes.mimeType.split('/')[1]}
</p>
</div>
)}

30
apps/desktop/src/renderer/components/files/file-preview.tsx
View File

@@ -6,28 +6,42 @@ import { FilePreviewOther } from '@/renderer/components/files/previews/file-prev
import { FilePreviewVideo } from '@/renderer/components/files/previews/file-preview-video';
import { useWorkspace } from '@/renderer/contexts/workspace';
import { getFileUrl } from '@/shared/lib/files';
import { File } from '@/shared/types/files';
import { LocalFileNode } from '@/shared/types/nodes';
import { useQuery } from '@/renderer/hooks/use-query';
interface FilePreviewProps {
file: File;
file: LocalFileNode;
}
export const FilePreview = ({ file }: FilePreviewProps) => {
const workspace = useWorkspace();
if (file.downloadProgress !== 100) {
return <FileDownload file={file} />;
const { data, isPending } = useQuery({
type: 'file_state_get',
id: file.id,
accountId: workspace.accountId,
workspaceId: workspace.id,
});
if (isPending) {
return null;
}
if (data?.downloadProgress !== 100) {
return <FileDownload file={file} state={data} />;
}
const url = getFileUrl(
workspace.accountId,
workspace.id,
file.id,
file.extension
file.attributes.extension
);
return match(file.type)
.with('image', () => <FilePreviewImage url={url} name={file.name} />)
return match(file.attributes.subtype)
.with('image', () => (
<FilePreviewImage url={url} name={file.attributes.name} />
))
.with('video', () => <FilePreviewVideo url={url} />)
.otherwise(() => <FilePreviewOther mimeType={file.mimeType} />);
.otherwise(() => <FilePreviewOther mimeType={file.attributes.mimeType} />);
};

14
apps/desktop/src/renderer/components/files/file-sidebar.tsx
View File

@@ -6,10 +6,10 @@ import { FileThumbnail } from '@/renderer/components/files/file-thumbnail';
import { useWorkspace } from '@/renderer/contexts/workspace';
import { useQuery } from '@/renderer/hooks/use-query';
import { formatBytes } from '@/shared/lib/files';
import { File } from '@/shared/types/files';
import { LocalFileNode } from '@/shared/types/nodes';
interface FileSidebarProps {
file: File;
file: LocalFileNode;
}
const FileMeta = ({ title, value }: { title: string; value: string }) => {
@@ -41,16 +41,16 @@ export const FileSidebar = ({ file }: FileSidebarProps) => {
/>
<div
className="line-clamp-3 break-words text-base font-medium"
title={file.name}
title={file.attributes.name}
>
{file.name}
{file.attributes.name}
</div>
</div>
<div className="mt-5 flex flex-col gap-4">
<FileMeta title="Name" value={file.name} />
<FileMeta title="Type" value={file.mimeType} />
<FileMeta title="Size" value={formatBytes(file.size)} />
<FileMeta title="Name" value={file.attributes.name} />
<FileMeta title="Type" value={file.attributes.mimeType} />
<FileMeta title="Size" value={formatBytes(file.attributes.size)} />
<FileMeta title="Created at" value={formatDate(file.createdAt)} />
{user && (

20
apps/desktop/src/renderer/components/files/file-thumbnail.tsx
View File

@@ -1,32 +1,40 @@
import { FileIcon } from '@/renderer/components/files/file-icon';
import { useWorkspace } from '@/renderer/contexts/workspace';
import { useQuery } from '@/renderer/hooks/use-query';
import { getFileUrl } from '@/shared/lib/files';
import { cn } from '@/shared/lib/utils';
import { File } from '@/shared/types/files';
import { LocalFileNode } from '@/shared/types/nodes';
interface FileThumbnailProps {
file: File;
file: LocalFileNode;
className?: string;
}
export const FileThumbnail = ({ file, className }: FileThumbnailProps) => {
const workspace = useWorkspace();
if (file.type === 'image' && file.downloadProgress === 100) {
const { data } = useQuery({
type: 'file_state_get',
id: file.id,
accountId: workspace.accountId,
workspaceId: workspace.id,
});
if (file.attributes.subtype === 'image' && data?.downloadProgress === 100) {
const url = getFileUrl(
workspace.accountId,
workspace.id,
file.id,
file.extension
file.attributes.extension
);
return (
<img
src={url}
alt={file.name}
alt={file.attributes.name}
className={cn('object-contain object-center', className)}
/>
);
}
return <FileIcon mimeType={file.mimeType} className="size-10" />;
return <FileIcon mimeType={file.attributes.mimeType} className="size-10" />;
};

8
apps/desktop/src/renderer/components/folders/grids/grid-file.tsx
View File

@@ -1,10 +1,10 @@
import { File } from '@/shared/types/files';
import { LocalFileNode } from '@/shared/types/nodes';
import { FileContextMenu } from '@/renderer/components/files/file-context-menu';
import { FileThumbnail } from '@/renderer/components/files/file-thumbnail';
import { GridItem } from '@/renderer/components/folders/grids/grid-item';
interface GridFileProps {
file: File;
file: LocalFileNode;
}
export const GridFile = ({ file }: GridFileProps) => {
@@ -16,9 +16,9 @@ export const GridFile = ({ file }: GridFileProps) => {
</div>
<p
className="line-clamp-2 w-full break-words text-center text-xs text-foreground/80"
title={file.name}
title={file.attributes.name}
>
{file.name}
{file.attributes.name}
</p>
</GridItem>
</FileContextMenu>

4
apps/desktop/src/renderer/contexts/folder.ts
View File

@@ -1,11 +1,11 @@
import { createContext, useContext } from 'react';
import { File } from '@/shared/types/files';
import { LocalFileNode } from '@/shared/types/nodes';
interface FolderContext {
id: string;
name: string;
files: File[];
files: LocalFileNode[];
onClick: (event: React.MouseEvent<HTMLElement>, id: string) => void;
onDoubleClick: (event: React.MouseEvent<HTMLElement>, id: string) => void;
onMove: (entryId: string, targetId: string) => void;

17
apps/desktop/src/shared/queries/files/file-get.ts
View File

@@ -1,17 +0,0 @@
import { File } from '@/shared/types/files';
export type FileGetQueryInput = {
type: 'file_get';
id: string;
accountId: string;
workspaceId: string;
};
declare module '@/shared/queries' {
interface QueryMap {
file_get: {
input: FileGetQueryInput;
output: File | null;
};
}
}

4
apps/desktop/src/shared/queries/files/file-list.ts
View File

@@ -1,4 +1,4 @@
import { File } from '@/shared/types/files';
import { LocalFileNode } from '@/shared/types/nodes';
export type FileListQueryInput = {
type: 'file_list';
@@ -13,7 +13,7 @@ declare module '@/shared/queries' {
interface QueryMap {
file_list: {
input: FileListQueryInput;
output: File[];
output: LocalFileNode[];
};
}
}

17
apps/desktop/src/shared/queries/files/file-state-get.ts Normal file
View File

@@ -0,0 +1,17 @@
import { FileState } from '@/shared/types/files';
export type FileStateGetQueryInput = {
type: 'file_state_get';
id: string;
accountId: string;
workspaceId: string;
};
declare module '@/shared/queries' {
interface QueryMap {
file_state_get: {
input: FileStateGetQueryInput;
output: FileState | null;
};
}
}

26
apps/desktop/src/shared/types/events.ts
View File

@@ -5,7 +5,7 @@ import { Account } from '@/shared/types/accounts';
import { Server } from '@/shared/types/servers';
import { Workspace, WorkspaceMetadata } from '@/shared/types/workspaces';
import { User } from '@/shared/types/users';
import { File } from '@/shared/types/files';
import { FileState } from '@/shared/types/files';
import { LocalNode, NodeInteraction, NodeReaction } from '@/shared/types/nodes';
export type UserCreatedEvent = {
@@ -71,25 +71,11 @@ export type NodeReactionDeletedEvent = {
nodeReaction: NodeReaction;
};
export type FileCreatedEvent = {
type: 'file_created';
export type FileStateUpdatedEvent = {
type: 'file_state_updated';
accountId: string;
workspaceId: string;
file: File;
};
export type FileUpdatedEvent = {
type: 'file_updated';
accountId: string;
workspaceId: string;
file: File;
};
export type FileDeletedEvent = {
type: 'file_deleted';
accountId: string;
workspaceId: string;
file: File;
fileState: FileState;
};
export type AccountCreatedEvent = {
@@ -235,9 +221,7 @@ export type Event =
| WorkspaceDeletedEvent
| ServerCreatedEvent
| ServerUpdatedEvent
| FileCreatedEvent
| FileUpdatedEvent
| FileDeletedEvent
| FileStateUpdatedEvent
| QueryResultUpdatedEvent
| RadarDataUpdatedEvent
| ServerAvailabilityChangedEvent

37
apps/desktop/src/shared/types/files.ts
View File

@@ -1,4 +1,4 @@
import { FileStatus, FileType } from '@colanode/core';
import { FileSubtype } from '@colanode/core';
export type FileMetadata = {
path: string;
@@ -6,31 +6,22 @@ export type FileMetadata = {
extension: string;
name: string;
size: number;
type: FileType;
type: FileSubtype;
};
export type File = {
export type FileState = {
id: string;
type: FileType;
parentId: string;
rootId: string;
revision: bigint;
name: string;
originalName: string;
extension: string;
mimeType: string;
size: number;
createdAt: string;
createdBy: string;
updatedAt: string | null;
updatedBy: string | null;
downloadStatus: DownloadStatus;
downloadProgress: number;
downloadRetries: number;
uploadStatus: UploadStatus;
uploadProgress: number;
uploadRetries: number;
status: FileStatus;
version: string;
downloadStatus: DownloadStatus | null;
downloadProgress: number | null;
downloadRetries: number | null;
downloadStartedAt: string | null;
downloadCompletedAt: string | null;
uploadStatus: UploadStatus | null;
uploadProgress: number | null;
uploadRetries: number | null;
uploadStartedAt: string | null;
uploadCompletedAt: string | null;
};
export enum DownloadStatus {

44
apps/server/src/api/client/controllers/workspaces/files/file-download-get.ts
View File

@@ -9,47 +9,51 @@ import {
import { GetObjectCommand } from '@aws-sdk/client-s3';
import { getSignedUrl } from '@aws-sdk/s3-request-presigner';
import { database } from '@/data/database';
import { fetchNode, mapNode } from '@/lib/nodes';
import { fetchNodeTree, mapNode } from '@/lib/nodes';
import { fileS3 } from '@/data/storage';
import { ResponseBuilder } from '@/lib/response-builder';
import { configuration } from '@/lib/configuration';
import { buildFilePath } from '@/lib/files';
export const fileDownloadGetHandler = async (
req: Request,
res: Response
): Promise<void> => {
const workspaceId = req.params.workspaceId as string;
const fileId = req.params.fileId as string;
const file = await database
.selectFrom('files')
.selectAll()
.where('id', '=', fileId)
.executeTakeFirst();
if (!file) {
const tree = await fetchNodeTree(fileId);
if (tree.length === 0) {
return ResponseBuilder.badRequest(res, {
code: ApiErrorCode.FileNotFound,
message: 'File not found.',
});
}
if (file.status !== FileStatus.Ready) {
const nodes = tree.map((node) => mapNode(node));
const file = nodes[nodes.length - 1]!;
if (!file || file.id !== fileId) {
return ResponseBuilder.badRequest(res, {
code: ApiErrorCode.FileNotFound,
message: 'File not found.',
});
}
if (file.type !== 'file') {
return ResponseBuilder.badRequest(res, {
code: ApiErrorCode.FileNotFound,
message: 'This node is not a file.',
});
}
if (file.attributes.status !== FileStatus.Ready) {
return ResponseBuilder.badRequest(res, {
code: ApiErrorCode.FileNotReady,
message: 'File is not ready to be downloaded.',
});
}
const root = await fetchNode(file.root_id);
if (!root) {
return ResponseBuilder.badRequest(res, {
code: ApiErrorCode.RootNotFound,
message: 'Root not found.',
});
}
const role = extractNodeRole(mapNode(root), res.locals.user.id);
const role = extractNodeRole(nodes, res.locals.user.id);
if (role === null || !hasNodeRole(role, 'viewer')) {
return ResponseBuilder.forbidden(res, {
code: ApiErrorCode.FileNoAccess,
@@ -58,7 +62,7 @@ export const fileDownloadGetHandler = async (
}
//generate presigned url for download
const path = `files/${file.workspace_id}/${file.id}${file.extension}`;
const path = buildFilePath(workspaceId, file.id, file.attributes);
const command = new GetObjectCommand({
Bucket: configuration.fileS3.bucketName,
Key: path,

63
apps/server/src/api/client/controllers/workspaces/files/file-upload-complete.ts
View File

@@ -4,9 +4,10 @@ import { FileStatus, ApiErrorCode } from '@colanode/core';
import { database } from '@/data/database';
import { fileS3 } from '@/data/storage';
import { eventBus } from '@/lib/event-bus';
import { ResponseBuilder } from '@/lib/response-builder';
import { configuration } from '@/lib/configuration';
import { mapNode, updateNode } from '@/lib/nodes';
import { buildFilePath } from '@/lib/files';
export const fileUploadCompleteHandler = async (
req: Request,
@@ -15,47 +16,55 @@ export const fileUploadCompleteHandler = async (
const workspaceId = req.params.workspaceId as string;
const fileId = req.params.fileId as string;
const file = await database
.selectFrom('files')
const node = await database
.selectFrom('nodes')
.selectAll()
.where('id', '=', fileId)
.executeTakeFirst();
if (!file) {
if (!node) {
return ResponseBuilder.badRequest(res, {
code: ApiErrorCode.FileNotFound,
message: 'File not found.',
});
}
if (file.created_by !== res.locals.user.id) {
if (node.created_by !== res.locals.user.id) {
return ResponseBuilder.forbidden(res, {
code: ApiErrorCode.FileOwnerMismatch,
message: 'You cannot complete this file upload.',
});
}
if (file.workspace_id !== workspaceId) {
if (node.workspace_id !== workspaceId) {
return ResponseBuilder.badRequest(res, {
code: ApiErrorCode.WorkspaceMismatch,
message: 'File does not belong to this workspace.',
});
}
if (file.status === FileStatus.Ready) {
const file = mapNode(node);
if (file.type !== 'file') {
return ResponseBuilder.badRequest(res, {
code: ApiErrorCode.FileNotFound,
message: 'This node is not a file.',
});
}
if (file.attributes.status === FileStatus.Ready) {
return ResponseBuilder.success(res, {
success: true,
});
}
if (file.status === FileStatus.Error) {
if (file.attributes.status === FileStatus.Error) {
return ResponseBuilder.badRequest(res, {
code: ApiErrorCode.FileError,
message: 'File has failed to upload.',
});
}
const path = `files/${file.workspace_id}/${file.id}${file.extension}`;
const path = buildFilePath(workspaceId, file.id, file.attributes);
// check if the file exists in the bucket
const command = new HeadObjectCommand({
Bucket: configuration.fileS3.bucketName,
@@ -66,7 +75,7 @@ export const fileUploadCompleteHandler = async (
const headObject = await fileS3.send(command);
// Verify file size matches expected size
if (headObject.ContentLength !== file.size) {
if (headObject.ContentLength !== file.attributes.size) {
return ResponseBuilder.badRequest(res, {
code: ApiErrorCode.FileSizeMismatch,
message: 'Uploaded file size does not match expected size',
@@ -74,7 +83,7 @@ export const fileUploadCompleteHandler = async (
}
// Verify mime type matches expected type
if (headObject.ContentType !== file.mime_type) {
if (headObject.ContentType !== file.attributes.mimeType) {
return ResponseBuilder.badRequest(res, {
code: ApiErrorCode.FileMimeTypeMismatch,
message: 'Uploaded file type does not match expected type',
@@ -87,31 +96,27 @@ export const fileUploadCompleteHandler = async (
});
}
const updatedFile = await database
.updateTable('files')
.returningAll()
.set({
status: FileStatus.Ready,
updated_by: res.locals.user.id,
updated_at: new Date(),
})
.where('id', '=', fileId)
.executeTakeFirst();
const result = await updateNode({
nodeId: fileId,
userId: res.locals.user.id,
workspaceId: workspaceId,
updater(attributes) {
if (attributes.type !== 'file') {
throw new Error('Node is not a file');
}
if (!updatedFile) {
attributes.status = FileStatus.Ready;
return attributes;
},
});
if (result === null) {
return ResponseBuilder.internalError(res, {
code: ApiErrorCode.FileUploadCompleteFailed,
message: 'Failed to complete file upload.',
});
}
eventBus.publish({
type: 'file_updated',
fileId: updatedFile.id,
rootId: updatedFile.root_id,
workspaceId: updatedFile.workspace_id,
});
return ResponseBuilder.success(res, {
success: true,
});

24
apps/server/src/api/client/controllers/workspaces/files/file-upload-init.ts
View File

@@ -11,6 +11,8 @@ import { database } from '@/data/database';
import { fileS3 } from '@/data/storage';
import { ResponseBuilder } from '@/lib/response-builder';
import { configuration } from '@/lib/configuration';
import { mapNode } from '@/lib/nodes';
import { buildFilePath } from '@/lib/files';
export const fileUploadInitHandler = async (
req: Request,
@@ -19,20 +21,20 @@ export const fileUploadInitHandler = async (
const workspaceId = req.params.workspaceId as string;
const input = req.body as CreateUploadInput;
const file = await database
.selectFrom('files')
const node = await database
.selectFrom('nodes')
.selectAll()
.where('id', '=', input.fileId)
.executeTakeFirst();
if (!file) {
if (!node) {
return ResponseBuilder.notFound(res, {
code: ApiErrorCode.FileNotFound,
message: 'File not found.',
});
}
if (file.created_by !== res.locals.user.id) {
if (node.created_by !== res.locals.user.id) {
return ResponseBuilder.forbidden(res, {
code: ApiErrorCode.FileOwnerMismatch,
message: 'You do not have access to this file.',
@@ -40,12 +42,20 @@ export const fileUploadInitHandler = async (
}
//generate presigned url for upload
const path = `files/${workspaceId}/${input.fileId}${file.extension}`;
const file = mapNode(node);
if (file.type !== 'file') {
return ResponseBuilder.badRequest(res, {
code: ApiErrorCode.FileNotFound,
message: 'This node is not a file.',
});
}
const path = buildFilePath(workspaceId, input.fileId, file.attributes);
const command = new PutObjectCommand({
Bucket: configuration.fileS3.bucketName,
Key: path,
ContentLength: file.size,
ContentType: file.mime_type,
ContentLength: file.attributes.size,
ContentType: file.attributes.mimeType,
});
const expiresIn = 60 * 60 * 4; // 4 hours

0
apps/server/src/data/migrations/00012-create-documents-table.ts → apps/server/src/data/migrations/00011-create-documents-table.ts
View File

66
apps/server/src/data/migrations/00011-create-files-table.ts
View File

@@ -1,66 +0,0 @@
import { sql, Migration } from 'kysely';
export const createFilesTable: Migration = {
up: async (db) => {
await sql`
CREATE SEQUENCE IF NOT EXISTS files_revision_sequence
START WITH 1000000000
INCREMENT BY 1
NO MINVALUE
NO MAXVALUE
CACHE 1;
`.execute(db);
await db.schema
.createTable('files')
.addColumn('id', 'varchar(30)', (col) => col.notNull().primaryKey())
.addColumn('type', 'varchar(30)', (col) => col.notNull())
.addColumn('parent_id', 'varchar(30)', (col) => col.notNull())
.addColumn('entry_id', 'varchar(30)', (col) => col.notNull())
.addColumn('root_id', 'varchar(30)', (col) => col.notNull())
.addColumn('workspace_id', 'varchar(30)', (col) => col.notNull())
.addColumn('name', 'varchar(256)', (col) => col.notNull())
.addColumn('original_name', 'varchar(256)', (col) => col.notNull())
.addColumn('mime_type', 'varchar(256)', (col) => col.notNull())
.addColumn('extension', 'varchar(256)', (col) => col.notNull())
.addColumn('size', 'integer', (col) => col.notNull())
.addColumn('created_at', 'timestamptz', (col) => col.notNull())
.addColumn('created_by', 'varchar(30)', (col) => col.notNull())
.addColumn('updated_at', 'timestamptz')
.addColumn('updated_by', 'varchar(30)')
.addColumn('status', 'integer', (col) => col.notNull())
.addColumn('revision', 'bigint', (col) =>
col.notNull().defaultTo(sql`nextval('files_revision_sequence')`)
)
.execute();
await db.schema
.createIndex('files_root_id_revision_idx')
.on('files')
.columns(['root_id', 'revision'])
.execute();
await sql`
CREATE OR REPLACE FUNCTION update_file_revision() RETURNS TRIGGER AS $$
BEGIN
NEW.revision = nextval('files_revision_sequence');
RETURN NEW;
END;
$$ LANGUAGE plpgsql;
CREATE TRIGGER trg_update_file_revision
BEFORE UPDATE ON files
FOR EACH ROW
EXECUTE FUNCTION update_file_revision();
`.execute(db);
},
down: async (db) => {
await sql`
DROP TRIGGER IF EXISTS trg_update_file_revision ON files;
DROP FUNCTION IF EXISTS update_file_revision();
`.execute(db);
await db.schema.dropTable('files').execute();
await sql`DROP SEQUENCE IF EXISTS files_revision_sequence`.execute(db);
},
};

0
apps/server/src/data/migrations/00013-create-document-updates-table.ts → apps/server/src/data/migrations/00012-create-document-updates-table.ts
View File

10
apps/server/src/data/migrations/index.ts
View File

@@ -10,9 +10,8 @@ import { createNodeInteractionsTable } from './00007-create-node-interactions-ta
import { createNodeTombstonesTable } from './00008-create-node-tombstones-table';
import { createNodePathsTable } from './00009-create-node-paths-table';
import { createCollaborationsTable } from './00010-create-collaborations-table';
import { createFilesTable } from './00011-create-files-table';
import { createDocumentsTable } from './00012-create-documents-table';
import { createDocumentUpdatesTable } from './00013-create-document-updates-table';
import { createDocumentsTable } from './00011-create-documents-table';
import { createDocumentUpdatesTable } from './00012-create-document-updates-table';
export const databaseMigrations: Record<string, Migration> = {
'00001_create_accounts_table': createAccountsTable,
@@ -25,7 +24,6 @@ export const databaseMigrations: Record<string, Migration> = {
'00008_create_node_tombstones_table': createNodeTombstonesTable,
'00009_create_node_paths_table': createNodePathsTable,
'00010_create_collaborations_table': createCollaborationsTable,
'00011_create_files_table': createFilesTable,
'00012_create_documents_table': createDocumentsTable,
'00013_create_document_updates_table': createDocumentUpdatesTable,
'00011_create_documents_table': createDocumentsTable,
'00012_create_document_updates_table': createDocumentUpdatesTable,
};

26
apps/server/src/data/schema.ts
View File

@@ -1,6 +1,4 @@
import {
FileStatus,
FileType,
NodeAttributes,
NodeRole,
NodeType,
@@ -187,29 +185,6 @@ export type SelectCollaboration = Selectable<CollaborationTable>;
export type CreateCollaboration = Insertable<CollaborationTable>;
export type UpdateCollaboration = Updateable<CollaborationTable>;
interface FileTable {
id: ColumnType<string, string, never>;
type: ColumnType<FileType, FileType, FileType>;
parent_id: ColumnType<string, string, never>;
root_id: ColumnType<string, string, never>;
workspace_id: ColumnType<string, string, never>;
revision: ColumnType<bigint, never, never>;
name: ColumnType<string, string, never>;
original_name: ColumnType<string, string, never>;
mime_type: ColumnType<string, string, never>;
extension: ColumnType<string, string, never>;
size: ColumnType<number, number, never>;
created_at: ColumnType<Date, Date, never>;
created_by: ColumnType<string, string, never>;
updated_at: ColumnType<Date | null, Date | null, Date | null>;
updated_by: ColumnType<string | null, string | null, string | null>;
status: ColumnType<FileStatus, FileStatus, FileStatus>;
}
export type SelectFile = Selectable<FileTable>;
export type CreateFile = Insertable<FileTable>;
export type UpdateFile = Updateable<FileTable>;
interface DocumentTable {
id: ColumnType<string, string, never>;
type: ColumnType<DocumentType, DocumentType, DocumentType>;
@@ -257,7 +232,6 @@ export interface DatabaseSchema {
node_paths: NodePathTable;
node_tombstones: NodeTombstoneTable;
collaborations: CollaborationTable;
files: FileTable;
documents: DocumentTable;
document_updates: DocumentUpdateTable;
}

9
apps/server/src/lib/files.ts Normal file
View File

@@ -0,0 +1,9 @@
import { FileAttributes } from '@colanode/core';
export const buildFilePath = (
workspaceId: string,
fileId: string,
fileAttributes: FileAttributes
) => {
return `files/${workspaceId}/${fileId}_${fileAttributes.version}${fileAttributes.extension}`;
};

7
apps/server/src/services/socket-connection.ts
View File

@@ -27,7 +27,6 @@ import { NodeSynchronizer } from '@/synchronizers/nodes';
import { NodeReactionSynchronizer } from '@/synchronizers/node-reactions';
import { NodeTombstoneSynchronizer } from '@/synchronizers/node-tombstones';
import { NodeInteractionSynchronizer } from '@/synchronizers/node-interactions';
import { FileSynchronizer } from '@/synchronizers/files';
import { DocumentUpdateSynchronizer } from '@/synchronizers/document-updates';
type SocketUser = {
@@ -147,12 +146,6 @@ export class SocketConnection {
message.input,
cursor
);
} else if (message.input.type === 'files') {
if (!user.rootIds.has(message.input.rootId)) {
return null;
}
return new FileSynchronizer(message.id, user.user, message.input, cursor);
} else if (message.input.type === 'nodes') {
if (!user.rootIds.has(message.input.rootId)) {
return null;

104
apps/server/src/synchronizers/files.ts
View File

@@ -1,104 +0,0 @@
import {
SynchronizerOutputMessage,
SyncFilesInput,
SyncFileData,
} from '@colanode/core';
import { BaseSynchronizer } from '@/synchronizers/base';
import { Event } from '@/types/events';
import { database } from '@/data/database';
import { SelectFile } from '@/data/schema';
export class FileSynchronizer extends BaseSynchronizer<SyncFilesInput> {
public async fetchData(): Promise<SynchronizerOutputMessage<SyncFilesInput> | null> {
const files = await this.fetchFiles();
if (files.length === 0) {
return null;
}
return this.buildMessage(files);
}
public async fetchDataFromEvent(
event: Event
): Promise<SynchronizerOutputMessage<SyncFilesInput> | null> {
if (!this.shouldFetch(event)) {
return null;
}
const files = await this.fetchFiles();
if (files.length === 0) {
return null;
}
return this.buildMessage(files);
}
private async fetchFiles() {
if (this.status === 'fetching') {
return [];
}
this.status = 'fetching';
const files = await database
.selectFrom('files')
.selectAll()
.where('root_id', '=', this.input.rootId)
.where('revision', '>', this.cursor)
.orderBy('revision', 'asc')
.limit(20)
.execute();
this.status = 'pending';
return files;
}
private buildMessage(
unsyncedFiles: SelectFile[]
): SynchronizerOutputMessage<SyncFilesInput> {
const items: SyncFileData[] = unsyncedFiles.map((file) => ({
id: file.id,
type: file.type,
parentId: file.parent_id,
rootId: file.root_id,
workspaceId: file.workspace_id,
name: file.name,
originalName: file.original_name,
mimeType: file.mime_type,
size: file.size,
extension: file.extension,
createdAt: file.created_at.toISOString(),
createdBy: file.created_by,
updatedAt: file.updated_at?.toISOString() ?? null,
updatedBy: file.updated_by ?? null,
revision: file.revision.toString(),
status: file.status,
}));
return {
type: 'synchronizer_output',
userId: this.user.userId,
id: this.id,
items: items.map((item) => ({
cursor: item.revision,
data: item,
})),
};
}
private shouldFetch(event: Event) {
if (event.type === 'file_created' && event.rootId === this.input.rootId) {
return true;
}
if (event.type === 'file_updated' && event.rootId === this.input.rootId) {
return true;
}
if (event.type === 'file_deleted' && event.rootId === this.input.rootId) {
return true;
}
return false;
}
}

4
packages/core/src/lib/files.ts
View File

@@ -1,6 +1,6 @@
import { FileType } from '../types/files';
import { FileSubtype } from '../types/files';
export const extractFileType = (mimeType: string): FileType => {
export const extractFileSubtype = (mimeType: string): FileSubtype => {
if (mimeType.startsWith('image/')) {
return 'image';
}

9
packages/core/src/registry/nodes/file.ts
View File

@@ -7,9 +7,16 @@ import { hasNodeRole } from '../../lib/permissions';
export const fileAttributesSchema = z.object({
type: z.literal('file'),
name: z.string().optional(),
subtype: z.string(),
parentId: z.string(),
index: z.string().optional(),
name: z.string(),
originalName: z.string(),
mimeType: z.string(),
extension: z.string(),
size: z.number(),
version: z.string(),
status: z.number(),
});
export type FileAttributes = z.infer<typeof fileAttributesSchema>;

34
packages/core/src/synchronizers/files.ts
View File

@@ -1,34 +0,0 @@
import { FileStatus, FileType } from '../types/files';
export type SyncFilesInput = {
type: 'files';
rootId: string;
};
export type SyncFileData = {
id: string;
type: FileType;
parentId: string;
rootId: string;
workspaceId: string;
revision: string;
name: string;
originalName: string;
mimeType: string;
size: number;
extension: string;
status: FileStatus;
createdAt: string;
createdBy: string;
updatedAt: string | null;
updatedBy: string | null;
};
declare module '@colanode/core' {
interface SynchronizerMap {
files: {
input: SyncFilesInput;
data: SyncFileData;
};
}
}

1
packages/core/src/synchronizers/index.ts
View File

@@ -1,6 +1,5 @@
export * from './nodes';
export * from './users';
export * from './files';
export * from './node-reactions';
export * from './node-interactions';
export * from './node-tombstones';

2
packages/core/src/types/files.ts
View File

@@ -14,7 +14,7 @@ export type CompleteUploadOutput = {
success: boolean;
};
export type FileType = 'image' | 'video' | 'audio' | 'pdf' | 'other';
export type FileSubtype = 'image' | 'video' | 'audio' | 'pdf' | 'other';
export enum FileStatus {
Pending = 0,

4
packages/core/src/types/mutations.ts
View File

@@ -1,4 +1,4 @@
import { FileType } from './files';
import { FileSubtype } from './files';
export type SyncMutationsInput = {
mutations: Mutation[];
@@ -107,7 +107,7 @@ export type MarkNodeOpenedMutation = MutationBase & {
export type CreateFileMutationData = {
id: string;
type: FileType;
type: FileSubtype;
parentId: string;
rootId: string;
name: string;