plane/web/core/local-db/storage.sqlite.ts

import { getActiveSpan, startSpan } from "@sentry/nextjs";
import * as Comlink from "comlink";
import set from "lodash/set";
// plane
import { EIssueGroupBYServerToProperty } from "@plane/constants";
import { TIssue } from "@plane/types";
// lib
import { rootStore } from "@/lib/store-context";
// services
import { IssueService } from "@/services/issue/issue.service";
//
import { ARRAY_FIELDS } from "./utils/constants";
import { getSubIssuesWithDistribution } from "./utils/data.utils";
import createIndexes from "./utils/indexes";
import { addIssuesBulk, syncDeletesToLocal } from "./utils/load-issues";
import { loadWorkSpaceData } from "./utils/load-workspace";
import { issueFilterCountQueryConstructor, issueFilterQueryConstructor } from "./utils/query-constructor";
import { runQuery } from "./utils/query-executor";
import { createTables } from "./utils/tables";
import { clearOPFS, getGroupedIssueResults, getSubGroupedIssueResults, log, logError } from "./utils/utils";

const DB_VERSION = 1;
const PAGE_SIZE = 500;
const BATCH_SIZE = 500;

type TProjectStatus = {
  issues: { status: undefined | "loading" | "ready" | "error" | "syncing"; sync: Promise<void> | undefined };
};

type TDBStatus = "initializing" | "ready" | "error" | undefined;
export class Storage {
  db: any;
  status: TDBStatus = undefined;
  dbName = "plane";
  projectStatus: Record<string, TProjectStatus> = {};
  workspaceSlug: string = "";

  constructor() {
    this.db = null;

    if (typeof window !== "undefined") {
      window.addEventListener("beforeunload", this.closeDBConnection);
    }
  }

  closeDBConnection = async () => {
    if (this.db) {
      await this.db.close();
    }
  };

  reset = () => {
    if (this.db) {
      this.db.close();
    }
    this.db = null;
    this.status = undefined;
    this.projectStatus = {};
    this.workspaceSlug = "";
  };

  clearStorage = async () => {
    try {
      await this.db.close();
      await clearOPFS();
      this.reset();
    } catch (e) {
      console.error("Error clearing sqlite sync storage", e);
    }
  };

  initialize = async (workspaceSlug: string): Promise<boolean> => {
    if (!rootStore.user.localDBEnabled) return false; // return if the window gets hidden

    if (workspaceSlug !== this.workspaceSlug) {
      this.reset();
    }
    try {
      await startSpan({ name: "INIT_DB" }, async () => await this._initialize(workspaceSlug));
      return true;
    } catch (err) {
      logError(err);
      this.status = "error";
      return false;
    }
  };

  _initialize = async (workspaceSlug: string): Promise<boolean> => {
    if (this.status === "initializing") {
      console.warn(`Initialization already in progress for workspace ${workspaceSlug}`);
      return false;
    }
    if (this.status === "ready") {
      console.warn(`Already initialized for workspace ${workspaceSlug}`);
      return true;
    }
    if (this.status === "error") {
      console.warn(`Initialization failed for workspace ${workspaceSlug}`);
      return false;
    }

    try {
      const { DBClass } = await import("./worker/db");
      const worker = new Worker(new URL("./worker/db.ts", import.meta.url));
      const MyWorker = Comlink.wrap<typeof DBClass>(worker);

      // Add cleanup on window unload
      window.addEventListener("unload", () => worker.terminate());

      this.workspaceSlug = workspaceSlug;
      this.dbName = workspaceSlug;
      const instance = await new MyWorker();
      await instance.init(workspaceSlug);

      this.db = {
        exec: instance.exec,
        close: instance.close,
      };

      this.status = "ready";
      // Your SQLite code here.
      await createTables();

      await this.setOption("DB_VERSION", DB_VERSION.toString());
      return true;
    } catch (error) {
      this.status = "error";
      throw new Error(`Failed to initialize database worker: ${error}`);
    }
  };

  syncWorkspace = async () => {
    if (!rootStore.user.localDBEnabled) return;
    const syncInProgress = await this.getIsWriteInProgress("sync_workspace");
    if (syncInProgress) {
      log("Sync in progress, skipping");
      return;
    }
    try {
      await startSpan({ name: "LOAD_WS", attributes: { slug: this.workspaceSlug } }, async () => {
        this.setOption("sync_workspace", new Date().toUTCString());
        await loadWorkSpaceData(this.workspaceSlug);
        this.deleteOption("sync_workspace");
      });
    } catch (e) {
      logError(e);
      this.deleteOption("sync_workspace");
    }
  };

  syncProject = async (projectId: string) => {
    if (
      // document.hidden ||
      !rootStore.user.localDBEnabled
    )
      return false; // return if the window gets hidden

    // Load labels, members, states, modules, cycles
    await this.syncIssues(projectId);

    // // Sync rest of the projects
    // const projects = await getProjectIds();

    // // Exclude the one we just synced
    // const projectsToSync = projects.filter((p: string) => p !== projectId);
    // for (const project of projectsToSync) {
    //   await delay(8000);
    //   await this.syncIssues(project);
    // }
    // this.setOption("workspace_synced_at", new Date().toISOString());
  };

  syncIssues = async (projectId: string) => {
    if (!rootStore.user.localDBEnabled || !this.db) {
      return false;
    }
    try {
      const sync = startSpan({ name: `SYNC_ISSUES` }, () => this._syncIssues(projectId));
      this.setSync(projectId, sync);
      await sync;
    } catch (e) {
      logError(e);
      this.setStatus(projectId, "error");
    }
  };

  _syncIssues = async (projectId: string) => {
    const activeSpan = getActiveSpan();

    log("### Sync started");
    let status = this.getStatus(projectId);
    if (status === "loading" || status === "syncing") {
      log(`Project ${projectId} is already loading or syncing`);
      return;
    }
    const syncPromise = this.getSync(projectId);

    if (syncPromise) {
      // Redundant check?
      return;
    }

    const queryParams: { cursor: string; updated_at__gt?: string; description: boolean } = {
      cursor: `${PAGE_SIZE}:0:0`,
      description: true,
    };

    const syncedAt = await this.getLastSyncTime(projectId);
    const projectSync = await this.getOption(projectId);

    if (syncedAt) {
      queryParams["updated_at__gt"] = syncedAt;
    }

    this.setStatus(projectId, projectSync === "ready" ? "syncing" : "loading");
    status = this.getStatus(projectId);

    log(`### ${projectSync === "ready" ? "Syncing" : "Loading"} issues to local db for project ${projectId}`);

    const start = performance.now();
    const issueService = new IssueService();

    const response = await issueService.getIssuesForSync(this.workspaceSlug, projectId, queryParams);

    await addIssuesBulk(response.results, BATCH_SIZE);
    if (response.total_pages > 1) {
      const promiseArray = [];
      for (let i = 1; i < response.total_pages; i++) {
        queryParams.cursor = `${PAGE_SIZE}:${i}:0`;
        promiseArray.push(issueService.getIssuesForSync(this.workspaceSlug, projectId, queryParams));
      }
      const pages = await Promise.all(promiseArray);
      for (const page of pages) {
        await addIssuesBulk(page.results, BATCH_SIZE);
      }
    }

    if (syncedAt) {
      await syncDeletesToLocal(this.workspaceSlug, projectId, { updated_at__gt: syncedAt });
    }
    log("### Time taken to add issues", performance.now() - start);

    if (status === "loading") {
      await createIndexes();
    }
    this.setOption(projectId, "ready");
    this.setStatus(projectId, "ready");
    this.setSync(projectId, undefined);

    activeSpan?.setAttributes({
      projectId: projectId,
      count: response.total_count,
    });
  };

  getIssueCount = async (projectId: string) => {
    const count = await runQuery(`select count(*) as count from issues where project_id='${projectId}'`);
    return count[0]["count"];
  };

  getLastUpdatedIssue = async (projectId: string) => {
    const lastUpdatedIssue = await runQuery(
      `select id, name, updated_at , sequence_id from issues WHERE project_id='${projectId}' AND is_local_update IS NULL order by datetime(updated_at) desc limit 1 `
    );

    if (lastUpdatedIssue.length) {
      return lastUpdatedIssue[0];
    }
    return;
  };

  getLastSyncTime = async (projectId: string) => {
    const issue = await this.getLastUpdatedIssue(projectId);
    if (!issue) {
      return false;
    }
    return issue.updated_at;
  };

  getIssues = async (workspaceSlug: string, projectId: string, queries: any, config: any) => {
    log("#### Queries", queries);

    const currentProjectStatus = this.getStatus(projectId);
    if (
      !currentProjectStatus ||
      this.status !== "ready" ||
      currentProjectStatus === "loading" ||
      currentProjectStatus === "error" ||
      !rootStore.user.localDBEnabled
    ) {
      if (rootStore.user.localDBEnabled) {
        log(`Project ${projectId} is loading, falling back to server`);
      }
      const issueService = new IssueService();
      return await issueService.getIssuesFromServer(workspaceSlug, projectId, queries, config);
    }

    const { cursor, group_by, sub_group_by } = queries;

    const query = issueFilterQueryConstructor(this.workspaceSlug, projectId, queries);
    const countQuery = issueFilterCountQueryConstructor(this.workspaceSlug, projectId, queries);
    const start = performance.now();
    let issuesRaw: any[] = [];
    let count: any[];
    try {
      [issuesRaw, count] = await startSpan(
        { name: "GET_ISSUES" },
        async () => await Promise.all([runQuery(query), runQuery(countQuery)])
      );
    } catch (e) {
      logError(e);
      const issueService = new IssueService();
      return await issueService.getIssuesFromServer(workspaceSlug, projectId, queries);
    }
    const end = performance.now();

    const { total_count } = count[0];

    const [pageSize, page, offset] = cursor.split(":");

    const groupByProperty: string =
      EIssueGroupBYServerToProperty[group_by as keyof typeof EIssueGroupBYServerToProperty];
    const subGroupByProperty =
      EIssueGroupBYServerToProperty[sub_group_by as keyof typeof EIssueGroupBYServerToProperty];

    const parsingStart = performance.now();
    let issueResults = issuesRaw.map((issue: any) => formatLocalIssue(issue));

    log("#### Issue Results", issueResults.length);

    const parsingEnd = performance.now();

    const grouping = performance.now();
    if (groupByProperty && page === "0") {
      if (subGroupByProperty) {
        issueResults = getSubGroupedIssueResults(issueResults);
      } else {
        issueResults = getGroupedIssueResults(issueResults);
      }
    }
    const groupCount = group_by ? Object.keys(issueResults).length : undefined;
    // const subGroupCount = sub_group_by ? Object.keys(issueResults[Object.keys(issueResults)[0]]).length : undefined;
    const groupingEnd = performance.now();

    const times = {
      IssueQuery: end - start,
      Parsing: parsingEnd - parsingStart,
      Grouping: groupingEnd - grouping,
    };
    if ((window as any).DEBUG) {
      console.table(times);
    }
    const total_pages = Math.ceil(total_count / Number(pageSize));
    const next_page_results = total_pages > parseInt(page) + 1;

    const out = {
      results: issueResults,
      next_cursor: `${pageSize}:${parseInt(page) + 1}:${Number(offset) + Number(pageSize)}`,
      prev_cursor: `${pageSize}:${parseInt(page) - 1}:${Number(offset) - Number(pageSize)}`,
      total_results: total_count,
      total_count,
      next_page_results,
      total_pages,
    };

    const activeSpan = getActiveSpan();
    activeSpan?.setAttributes({
      projectId,
      count: total_count,
      groupBy: group_by,
      subGroupBy: sub_group_by,
      queries: queries,
      local: true,
      groupCount,
      // subGroupCount,
    });
    return out;
  };

  getIssue = async (issueId: string) => {
    try {
      if (!rootStore.user.localDBEnabled) return;

      const issues = await runQuery(`select * from issues where id='${issueId}'`);
      if (issues.length) {
        return formatLocalIssue(issues[0]);
      }
    } catch (err) {
      logError(err);
      console.warn("unable to fetch issue from local db");
    }

    return;
  };

  getSubIssues = async (workspaceSlug: string, projectId: string, issueId: string) => {
    const workspace_synced_at = await this.getOption("workspace_synced_at");
    if (!workspace_synced_at) {
      const issueService = new IssueService();
      return await issueService.subIssues(workspaceSlug, projectId, issueId);
    }
    return await getSubIssuesWithDistribution(issueId);
  };

  getStatus = (projectId: string) => this.projectStatus[projectId]?.issues?.status || undefined;
  setStatus = (projectId: string, status: "loading" | "ready" | "error" | "syncing" | undefined = undefined) => {
    set(this.projectStatus, `${projectId}.issues.status`, status);
  };

  getSync = (projectId: string) => this.projectStatus[projectId]?.issues?.sync;
  setSync = (projectId: string, sync: Promise<void> | undefined) => {
    set(this.projectStatus, `${projectId}.issues.sync`, sync);
  };

  getOption = async (key: string, fallback?: string | boolean | number) => {
    try {
      const options = await runQuery(`select * from options where key='${key}'`);
      if (options.length) {
        return options[0].value;
      }

      return fallback;
    } catch (e) {
      return fallback;
    }
  };
  setOption = async (key: string, value: string) => {
    await runQuery(`insert or replace into options (key, value) values ('${key}', '${value}')`);
  };

  deleteOption = async (key: string) => {
    await runQuery(` DELETE FROM options where key='${key}'`);
  };
  getOptions = async (keys: string[]) => {
    const options = await runQuery(`select * from options where key in ('${keys.join("','")}')`);
    return options.reduce((acc: any, option: any) => {
      acc[option.key] = option.value;
      return acc;
    }, {});
  };

  getIsWriteInProgress = async (op: string) => {
    const writeStartTime = await this.getOption(op, false);
    if (writeStartTime) {
      // Check if it has been more than 5seconds
      const current = new Date();
      const start = new Date(writeStartTime);

      if (current.getTime() - start.getTime() < 5000) {
        return true;
      }
      return false;
    }
    return false;
  };
}

export const persistence = new Storage();

/**
 * format the issue fetched from local db into an issue
 * @param issue
 * @returns
 */
export const formatLocalIssue = (issue: any) => {
  const currIssue = issue;
  ARRAY_FIELDS.forEach((field: string) => {
    currIssue[field] = currIssue[field] ? JSON.parse(currIssue[field]) : [];
  });
  return currIssue as TIssue & { group_id?: string; total_issues: number; sub_group_id?: string };
};