web: move importer into web/desktop app (#7472)

Signed-off-by: 01zulfi <85733202+01zulfi@users.noreply.github.com>

apps/web/package.json

@@ -19,7 +19,7 @@
     "@lingui/react": "5.1.2",
     "@mdi/js": "7.4.47",
     "@mdi/react": "1.6.1",
-    "@notesnook-importer/core": "^2.1.1",
+    "@notesnook-importer/core": "^2.1.2",
     "@notesnook/common": "file:../../packages/common",
     "@notesnook/core": "file:../../packages/core",
     "@notesnook/crypto": "file:../../packages/crypto",

apps/web/src/components/accordion/index.tsx

@@ -45,11 +45,7 @@ export default function Accordion(
     containerSx,
     ...restProps
   } = props;
-  const [isContentHidden, setIsContentHidden] = useState(false);
-
-  useEffect(() => {
-    setIsContentHidden(isClosed);
-  }, [isClosed]);
+  const [isContentHidden, setIsContentHidden] = useState(isClosed);
 
   return (
     <Flex sx={{ flexDirection: "column", ...sx }} {...restProps}>

apps/web/src/components/importer/components/file-provider-handler.tsx

@@ -0,0 +1,314 @@
+/*
+This file is part of the Notesnook project (https://notesnook.com/)
+
+Copyright (C) 2023 Streetwriters (Private) Limited
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+import {
+  IFile,
+  IFileProvider,
+  ProviderSettings,
+  transform
+} from "@notesnook-importer/core";
+import { formatBytes } from "@notesnook/common";
+import { ScrollContainer } from "@notesnook/ui";
+import { Button, Flex, Input, Text } from "@theme-ui/components";
+import { xxhash64 } from "hash-wasm";
+import { useCallback, useEffect, useRef, useState } from "react";
+import { useDropzone } from "react-dropzone";
+import { importNote } from "../../../utils/importer";
+import Accordion from "../../accordion";
+import { TransformResult } from "../types";
+
+type FileProviderHandlerProps = {
+  provider: IFileProvider;
+  onTransformFinished: (result: TransformResult) => void;
+};
+
+type Progress = {
+  total: number;
+  done: number;
+};
+
+export function FileProviderHandler(props: FileProviderHandlerProps) {
+  const { provider, onTransformFinished } = props;
+  const [files, setFiles] = useState<File[]>([]);
+  const [filesProgress, setFilesProgress] = useState<Progress>({
+    done: 0,
+    total: 0
+  });
+  const [totalNoteCount, setTotalNoteCount] = useState(0);
+  const [_, setCounter] = useState<number>(0);
+  const logs = useRef<string[]>([]);
+
+  const onDrop = useCallback((acceptedFiles: File[]) => {
+    setFiles((files) => {
+      const newFiles = [...acceptedFiles, ...files];
+      return newFiles;
+    });
+  }, []);
+
+  const { getRootProps, getInputProps, isDragActive } = useDropzone({
+    onDrop,
+    accept: {
+      file: provider?.supportedExtensions?.concat([".zip"])
+    }
+  });
+
+  useEffect(() => {
+    setFiles([]);
+  }, [provider]);
+
+  async function onStartImport() {
+    let totalNotes = 0;
+    const errors: Error[] = [];
+    const settings: ProviderSettings = {
+      clientType: "browser",
+      hasher: { type: "xxh64", hash: xxhash64 },
+      storage: {
+        clear: async () => undefined,
+        get: async () => [],
+        write: async (data) => {
+          logs.current.push(
+            `[${new Date().toLocaleString()}] Pushing ${
+              data.title
+            } into database`
+          );
+
+          await importNote(data);
+        },
+        iterate: async function* () {
+          return null;
+        }
+      },
+      log: (message) => {
+        logs.current.push(
+          `[${new Date(message.date).toLocaleString()}] ${message.text}`
+        );
+        setCounter((s) => ++s);
+      },
+      reporter: () => {
+        setTotalNoteCount(++totalNotes);
+      }
+    };
+
+    setTotalNoteCount(0);
+    setFilesProgress({
+      total: files.length,
+      done: 0
+    });
+
+    for (const file of files) {
+      setFilesProgress((p) => ({
+        ...p,
+        done: p.done + 1
+      }));
+
+      const providerFile: IFile = {
+        name: file.name,
+        modifiedAt: file.lastModified,
+        size: file.size,
+        data: file
+      };
+      errors.push(...(await transform(provider, [providerFile], settings)));
+    }
+    console.log("DONE", totalNotes);
+    onTransformFinished({
+      totalNotes,
+      errors
+    });
+  }
+
+  if (filesProgress.done) {
+    return (
+      <Flex sx={{ flexDirection: "column", alignItems: "stretch" }}>
+        <Text variant="subtitle">
+          Processing {filesProgress.done} of {filesProgress.total} file(s)
+        </Text>
+        <Text variant="body" sx={{ mt: 4, textAlign: "center" }}>
+          Found {totalNoteCount} notes
+        </Text>
+        {logs.current.length > 0 && (
+          <Accordion
+            title="Logs"
+            isClosed={false}
+            sx={{
+              border: "1px solid var(--border)",
+              mt: 2
+            }}
+          >
+            <ScrollContainer>
+              <Text
+                as="pre"
+                variant="body"
+                sx={{
+                  fontFamily: "monospace",
+                  maxHeight: 250,
+                  p: 2
+                }}
+              >
+                {logs.current.map((c, index) => (
+                  <>
+                    <span key={index.toString()}>{c}</span>
+                    <br />
+                  </>
+                ))}
+              </Text>
+            </ScrollContainer>
+          </Accordion>
+        )}
+      </Flex>
+    );
+  }
+
+  return (
+    <Flex sx={{ flexDirection: "column", alignItems: "stretch" }}>
+      <Text variant="subtitle">Select {provider.name} files</Text>
+      <Text
+        variant="body"
+        as={"div"}
+        sx={{ mt: 1, color: "paragraph", whiteSpace: "pre-wrap" }}
+      >
+        Check out our step-by-step guide on{" "}
+        <a href={provider.helpLink} target="_blank" rel="noreferrer">
+          how to import from {provider?.name}.
+        </a>
+      </Text>
+      <Flex
+        {...getRootProps()}
+        sx={{
+          justifyContent: "center",
+          alignItems: "center",
+          height: 100,
+          border: "2px dashed var(--border)",
+          borderRadius: "default",
+          mt: 2,
+          cursor: "pointer",
+          ":hover": {
+            bg: "background-secondary"
+          }
+        }}
+      >
+        <Input {...getInputProps()} />
+        <Text variant="body" sx={{ textAlign: "center" }}>
+          {isDragActive
+            ? "Drop the files here"
+            : "Drag & drop files here, or click to select files"}
+          <br />
+          <Text variant="subBody">
+            Only {provider?.supportedExtensions.join(", ")} files are supported.{" "}
+            {provider?.supportedExtensions.includes(".zip") ? null : (
+              <>
+                You can also select .zip files containing{" "}
+                {provider?.supportedExtensions.join(", ")} files.
+              </>
+            )}
+            <br />
+            {provider.examples ? (
+              <>For example, {provider.examples.join(", ")}</>
+            ) : null}
+          </Text>
+        </Text>
+      </Flex>
+      {files.length > 0 ? (
+        <Accordion
+          isClosed
+          title={`${files.length} ${
+            files.length > 1 ? "files" : "file"
+          } selected`}
+          sx={{
+            border: "1px solid var(--border)",
+            mt: 2,
+            borderRadius: "default"
+          }}
+        >
+          <Flex
+            sx={{ flexDirection: "column", overflowY: "auto", maxHeight: 400 }}
+          >
+            {files.map((file, index) => (
+              <Flex
+                key={file.name}
+                sx={{
+                  p: 2,
+                  bg: index % 2 ? "transparent" : "background-secondary",
+                  alignItems: "center",
+                  justifyContent: "space-between",
+                  cursor: "pointer",
+                  ":hover": {
+                    bg: "hover"
+                  }
+                }}
+                onClick={() => {
+                  setFiles((files) => {
+                    const _files = files.slice();
+                    _files.splice(index, 1);
+                    return _files;
+                  });
+                }}
+                title="Click to remove"
+              >
+                <Text variant="body">{file.name}</Text>
+                <Text variant="body">{formatBytes(file.size)}</Text>
+              </Flex>
+            ))}
+          </Flex>
+        </Accordion>
+      ) : null}
+
+      {!!files.length && (
+        <>
+          <Text
+            variant="body"
+            sx={{
+              bg: "primary",
+              color: "static",
+              mt: 2,
+              borderRadius: 5,
+              p: 1
+            }}
+          >
+            Please make sure you have at least{" "}
+            {formatBytes(files.reduce((prev, file) => prev + file.size, 0))} of
+            free space before proceeding.
+          </Text>
+          {provider.requiresNetwork ? (
+            <Text
+              variant="body"
+              sx={{
+                bg: "background-error",
+                color: "paragraph-error",
+                mt: 2,
+                borderRadius: 5,
+                p: 1
+              }}
+            >
+              Please make sure you have good Internet access before proceeding.
+              The importer may send network requests in order to download media
+              resources such as images, files, and other attachments.
+            </Text>
+          ) : null}
+          <Button
+            variant="accent"
+            sx={{ alignSelf: "center", mt: 2, px: 4 }}
+            onClick={onStartImport}
+          >
+            Start importing
+          </Button>
+        </>
+      )}
+    </Flex>
+  );
+}

apps/web/src/components/importer/components/import-errors.tsx

@@ -0,0 +1,60 @@
+/*
+This file is part of the Notesnook project (https://notesnook.com/)
+
+Copyright (C) 2023 Streetwriters (Private) Limited
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+import { Button, Flex, Text } from "@theme-ui/components";
+import Accordion from "../../accordion";
+
+type ImportErrorsProps = {
+  errors: Error[];
+};
+
+export function ImportErrors(props: ImportErrorsProps) {
+  return (
+    <Accordion
+      isClosed={false}
+      title={`${props.errors.length} errors occured`}
+      sx={{ bg: "background-error", borderRadius: "default", mt: 2 }}
+      color="paragraph-error"
+    >
+      <Flex sx={{ flexDirection: "column", px: 2, pb: 2, overflowX: "auto" }}>
+        {props.errors.map((error, index) => (
+          <Text
+            variant="body"
+            sx={{ color: "paragraph-error", my: 1, fontFamily: "monospace" }}
+          >
+            {index + 1}. {error.message}
+            <br />
+          </Text>
+        ))}
+        <Button
+          variant="error"
+          sx={{ alignSelf: "start", mt: 2 }}
+          onClick={() =>
+            window.open(
+              "https://github.com/streetwriters/notesnook-importer/issues/new",
+              "_blank"
+            )
+          }
+        >
+          Send us a bug report
+        </Button>
+      </Flex>
+    </Accordion>
+  );
+}

apps/web/src/components/importer/components/import-result.tsx

@@ -0,0 +1,94 @@
+/*
+This file is part of the Notesnook project (https://notesnook.com/)
+
+Copyright (C) 2023 Streetwriters (Private) Limited
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+import { IProvider } from "@notesnook-importer/core";
+import { strings } from "@notesnook/intl";
+import { Button, Flex, Text } from "@theme-ui/components";
+import { CheckCircleOutline } from "../../icons";
+import { TransformResult } from "../types";
+import { ImportErrors } from "./import-errors";
+
+type ImportResultProps = {
+  result: TransformResult;
+  provider: IProvider;
+  onReset: () => void;
+};
+
+export function ImportResult(props: ImportResultProps) {
+  const { result, onReset } = props;
+
+  if (result.totalNotes <= 0) {
+    return (
+      <Flex sx={{ flexDirection: "column", alignItems: "stretch" }}>
+        <Text variant="title">Import unsuccessful</Text>
+        <Text variant="body" sx={{ mt: 2 }}>
+          We failed to import the selected files. Please try again.
+        </Text>
+        {result.errors.length > 0 && <ImportErrors errors={result.errors} />}
+        <Button
+          variant="accent"
+          onClick={onReset}
+          sx={{ alignSelf: "center", mt: 2, px: 4 }}
+        >
+          Start over
+        </Button>
+      </Flex>
+    );
+  }
+
+  return (
+    <>
+      <CheckCircleOutline color="accent" />
+      <Text variant="body" my={2} sx={{ textAlign: "center" }}>
+        {strings.importCompleted()}. {props.result.totalNotes} notes
+        successfully imported.
+        {strings.errorsOccured(result.errors.length)}
+      </Text>
+      <Button
+        variant="secondary"
+        sx={{ alignSelf: "center" }}
+        onClick={async () => {
+          onReset();
+        }}
+      >
+        {strings.startOver()}
+      </Button>
+      {result.errors.length > 0 && (
+        <Flex
+          my={1}
+          bg="var(--background-error)"
+          p={1}
+          sx={{ flexDirection: "column" }}
+        >
+          {result.errors.map((error) => (
+            <Text
+              key={error.message}
+              variant="body"
+              sx={{
+                color: "var(--paragraph-error)"
+              }}
+            >
+              {error.message}
+            </Text>
+          ))}
+        </Flex>
+      )}
+    </>
+  );
+}

apps/web/src/components/importer/components/network-provider-handler.tsx

@@ -0,0 +1,174 @@
+/*
+This file is part of the Notesnook project (https://notesnook.com/)
+
+Copyright (C) 2023 Streetwriters (Private) Limited
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+import {
+  INetworkProvider,
+  OneNote,
+  OneNoteSettings,
+  ProviderSettings,
+  transform
+} from "@notesnook-importer/core";
+import { ScrollContainer } from "@notesnook/ui";
+import { Button, Flex, Text } from "@theme-ui/components";
+import { xxhash64 } from "hash-wasm";
+import { useRef, useState } from "react";
+import { importNote } from "../../../utils/importer";
+import Accordion from "../../accordion";
+import { TransformResult } from "../types";
+
+type NetworkProviderHandlerProps = {
+  provider: INetworkProvider<ProviderSettings>;
+  onTransformFinished: (result: TransformResult) => void;
+};
+
+type Progress = {
+  total: number;
+  done: number;
+};
+
+function getProviderSettings(
+  provider: INetworkProvider<ProviderSettings>,
+  settings: ProviderSettings
+) {
+  if (provider instanceof OneNote) {
+    return {
+      ...settings,
+      cache: false,
+      clientId: "6c32bdbd-c6c6-4cda-bcf0-0c8ec17e5804",
+      redirectUri:
+        process.env.NODE_ENV === "development"
+          ? "http://localhost:3000"
+          : "https://importer.notesnook.com"
+    } as OneNoteSettings;
+  }
+}
+
+export function NetworkProviderHandler(props: NetworkProviderHandlerProps) {
+  const { provider, onTransformFinished } = props;
+  const [totalNoteCount, setTotalNoteCount] = useState(0);
+  const [_, setCounter] = useState<number>(0);
+  const logs = useRef<string[]>([]);
+
+  async function onStartImport() {
+    let totalNotes = 0;
+    const settings = getProviderSettings(provider, {
+      clientType: "browser",
+      hasher: { type: "xxh64", hash: xxhash64 },
+      storage: {
+        clear: async () => undefined,
+        get: async () => [],
+        write: async (data) => {
+          logs.current.push(
+            `[${new Date().toLocaleString()}] Pushing ${
+              data.title
+            } into database`
+          );
+
+          await importNote(data);
+        },
+        iterate: async function* () {
+          return null;
+        }
+      },
+      log: (message) => {
+        logs.current.push(
+          `[${new Date(message.date).toLocaleString()}] ${message.text}`
+        );
+        setCounter((s) => ++s);
+      },
+      reporter: () => {
+        setTotalNoteCount(++totalNotes);
+      }
+    });
+    if (!settings) return;
+
+    setTotalNoteCount(0);
+
+    const errors = await transform(provider, settings);
+    console.log(errors);
+    onTransformFinished({
+      totalNotes,
+      errors
+    });
+  }
+
+  return (
+    <Flex
+      sx={{
+        flexDirection: "column",
+        alignItems: "stretch"
+      }}
+    >
+      {totalNoteCount ? (
+        <>
+          <Text variant="title">Importing your notes from {provider.name}</Text>
+          <Text variant="body" sx={{ mt: 4 }}>
+            Found {totalNoteCount} notes
+          </Text>
+          {logs.current.length > 0 && (
+            <Accordion
+              isClosed={false}
+              title="Logs"
+              sx={{
+                border: "1px solid var(--border)",
+                mt: 2
+              }}
+            >
+              <ScrollContainer>
+                <Text
+                  as="pre"
+                  variant="body"
+                  sx={{
+                    fontFamily: "monospace",
+                    maxHeight: 250,
+                    p: 2
+                  }}
+                >
+                  {logs.current.map((c, index) => (
+                    <>
+                      <span key={index.toString()}>{c}</span>
+                      <br />
+                    </>
+                  ))}
+                </Text>
+              </ScrollContainer>
+            </Accordion>
+          )}
+        </>
+      ) : (
+        <>
+          <Text variant="title">Connect your {provider.name} account</Text>
+          <Text variant="body" sx={{ color: "fontTertiary", mt: [2, 0] }}>
+            Check out our step-by-step guide on{" "}
+            <a href={provider.helpLink} target="_blank" rel="noreferrer">
+              how to import from {provider.name}.
+            </a>
+          </Text>
+          <Button
+            variant="accent"
+            onClick={onStartImport}
+            sx={{ my: 4, alignSelf: "center" }}
+          >
+            Start importing
+          </Button>
+        </>
+      )}
+    </Flex>
+  );
+}

apps/web/src/components/importer/components/provider-selector.tsx

@@ -0,0 +1,79 @@
+/*
+This file is part of the Notesnook project (https://notesnook.com/)
+
+Copyright (C) 2023 Streetwriters (Private) Limited
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+import {
+  IProvider,
+  ProviderFactory,
+  Providers
+} from "@notesnook-importer/core";
+import { Flex, Text } from "@theme-ui/components";
+
+type ProviderSelectorProps = {
+  onProviderChanged: (provider: IProvider) => void;
+};
+
+export function ProviderSelector(props: ProviderSelectorProps) {
+  return (
+    <Flex
+      sx={{
+        flexDirection: "row",
+        justifyContent: "space-between",
+        alignItems: "start",
+        gap: 4
+      }}
+    >
+      <Flex sx={{ flexDirection: "column", flex: 1 }}>
+        <Text variant="subtitle">Select a notes app to import from</Text>
+        <Text
+          variant="body"
+          as="div"
+          sx={{ mt: 1, color: "paragraph", whiteSpace: "pre-wrap" }}
+        >
+          Can&apos;t find your notes app in the list?{" "}
+          <a href="https://github.com/streetwriters/notesnook-importer/issues/new">
+            Send us a request.
+          </a>
+        </Text>
+      </Flex>
+      <select
+        style={{
+          backgroundColor: "var(--background-secondary)",
+          outline: "none",
+          border: "1px solid var(--border-secondary)",
+          borderRadius: "5px",
+          color: "var(--paragraph)",
+          padding: "5px",
+          overflow: "hidden"
+        }}
+        onChange={(e) => {
+          if (e.target.value === "") return;
+          const providerName: Providers = e.target.value as Providers;
+          props.onProviderChanged(ProviderFactory.getProvider(providerName));
+        }}
+      >
+        <option value="">Select notes app</option>
+        {ProviderFactory.getAvailableProviders().map((provider) => (
+          <option key={provider} value={provider}>
+            {ProviderFactory.getProvider(provider as Providers).name}
+          </option>
+        ))}
+      </select>
+    </Flex>
+  );
+}

apps/web/src/components/importer/importer.tsx

@@ -0,0 +1,82 @@
+/*
+This file is part of the Notesnook project (https://notesnook.com/)
+
+Copyright (C) 2023 Streetwriters (Private) Limited
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+import { Flex } from "@theme-ui/components";
+import { useState } from "react";
+import { ProviderSelector } from "./components/provider-selector";
+import { FileProviderHandler } from "./components/file-provider-handler";
+import { ImportResult } from "./components/import-result";
+import { IProvider } from "@notesnook-importer/core";
+import { NetworkProviderHandler } from "./components/network-provider-handler";
+import { TransformResult } from "./types";
+
+export function Importer() {
+  const [selectedProvider, setSelectedProvider] = useState<IProvider>();
+  const [transformResult, setTransformResult] = useState<TransformResult>();
+  const [instanceKey, setInstanceKey] = useState<string>(`${Math.random()}`);
+
+  return (
+    <Flex sx={{ flexDirection: "column" }}>
+      <Flex
+        sx={{
+          flexDirection: "column",
+          alignItems: "stretch",
+          gap: 4
+        }}
+      >
+        <ProviderSelector
+          onProviderChanged={(provider) => {
+            setInstanceKey(`${Math.random()}`);
+            setSelectedProvider(provider);
+            setTransformResult(undefined);
+          }}
+        />
+        {selectedProvider ? (
+          <>
+            {selectedProvider.type === "file" ? (
+              <FileProviderHandler
+                key={instanceKey}
+                provider={selectedProvider}
+                onTransformFinished={setTransformResult}
+              />
+            ) : selectedProvider.type === "network" ? (
+              <NetworkProviderHandler
+                key={instanceKey}
+                provider={selectedProvider}
+                onTransformFinished={setTransformResult}
+              />
+            ) : null}
+          </>
+        ) : null}
+        {transformResult && selectedProvider ? (
+          <>
+            <ImportResult
+              result={transformResult}
+              provider={selectedProvider}
+              onReset={() => {
+                setTransformResult(undefined);
+                setInstanceKey(`${Math.random()}`);
+              }}
+            />
+          </>
+        ) : null}
+      </Flex>
+    </Flex>
+  );
+}

apps/web/src/components/importer/index.ts

@@ -0,0 +1,20 @@
+/*
+This file is part of the Notesnook project (https://notesnook.com/)
+
+Copyright (C) 2023 Streetwriters (Private) Limited
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+export * from "./importer";

apps/web/src/components/importer/types.ts

@@ -0,0 +1,23 @@
+/*
+This file is part of the Notesnook project (https://notesnook.com/)
+
+Copyright (C) 2023 Streetwriters (Private) Limited
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+export type TransformResult = {
+  totalNotes: number;
+  errors: Error[];
+};

apps/web/src/dialogs/settings/components/importer.tsx

@@ -1,318 +0,0 @@
-/*
-This file is part of the Notesnook project (https://notesnook.com/)
-
-Copyright (C) 2023 Streetwriters (Private) Limited
-
-This program is free software: you can redistribute it and/or modify
-it under the terms of the GNU General Public License as published by
-the Free Software Foundation, either version 3 of the License, or
-(at your option) any later version.
-
-This program is distributed in the hope that it will be useful,
-but WITHOUT ANY WARRANTY; without even the implied warranty of
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-GNU General Public License for more details.
-
-You should have received a copy of the GNU General Public License
-along with this program.  If not, see <http://www.gnu.org/licenses/>.
-*/
-
-import { strings } from "@notesnook/intl";
-import { Button, Flex, Input, Link, Text, Box } from "@theme-ui/components";
-import { useCallback, useRef, useState } from "react";
-import { useDropzone } from "react-dropzone";
-import { db } from "../../../common/db";
-import { CheckCircleOutline } from "../../../components/icons";
-import Accordion from "../../../components/accordion";
-import { importFiles } from "../../../utils/importer";
-import { useStore as useAppStore } from "../../../stores/app-store";
-
-type Provider = { title: string; link: string };
-const POPULAR_PROVIDERS: Provider[] = [
-  {
-    title: "Evernote",
-    link: "https://help.notesnook.com/importing-notes/import-notes-from-evernote"
-  },
-  {
-    title: "Simplenote",
-    link: "https://help.notesnook.com/importing-notes/import-notes-from-simplenote"
-  },
-  {
-    title: "Google Keep",
-    link: "https://help.notesnook.com/importing-notes/import-notes-from-googlekeep"
-  },
-  {
-    title: "Obsidian",
-    link: "https://help.notesnook.com/importing-notes/import-notes-from-obsidian"
-  },
-  {
-    title: "Joplin",
-    link: "https://help.notesnook.com/importing-notes/import-notes-from-joplin"
-  },
-  {
-    title: "Markdown files",
-    link: "https://help.notesnook.com/importing-notes/import-notes-from-markdown-files"
-  },
-  {
-    title: "other apps",
-    link: "https://help.notesnook.com/importing-notes/"
-  }
-];
-
-export function Importer() {
-  const [isDone, setIsDone] = useState(false);
-  const [isImporting, setIsImporting] = useState(false);
-  const [files, setFiles] = useState<File[]>([]);
-  const [errors, setErrors] = useState<Error[]>([]);
-  const notesCounter = useRef<HTMLSpanElement>(null);
-  const importProgress = useRef<HTMLDivElement>(null);
-
-  const onDrop = useCallback((acceptedFiles: File[]) => {
-    setFiles((files) => {
-      const newFiles = [...acceptedFiles, ...files];
-      return newFiles;
-    });
-  }, []);
-
-  const { getRootProps, getInputProps, isDragActive } = useDropzone({
-    onDrop,
-    accept: {
-      "application/zip": [".zip"]
-    }
-  });
-
-  return (
-    <Flex
-      sx={{
-        flexDirection: "column",
-        // justifyContent: "center",
-        overflow: "hidden"
-      }}
-    >
-      {isImporting ? (
-        <>
-          <Text variant="title" sx={{ textAlign: "center", mb: 4, mt: 150 }}>
-            <span ref={notesCounter}>0</span> {strings.notesImported()}.
-          </Text>
-
-          <Flex
-            ref={importProgress}
-            sx={{
-              alignSelf: "start",
-              borderRadius: "default",
-              height: "5px",
-              bg: "accent",
-              width: `0%`
-            }}
-          />
-        </>
-      ) : isDone ? (
-        <>
-          <CheckCircleOutline color="accent" sx={{ mt: 150 }} />
-          <Text variant="body" my={2} sx={{ textAlign: "center" }}>
-            {strings.importCompleted()}. {strings.errorsOccured(errors.length)}
-          </Text>
-          <Button
-            variant="secondary"
-            sx={{ alignSelf: "center" }}
-            onClick={async () => {
-              setErrors([]);
-              setFiles([]);
-              setIsDone(false);
-              setIsImporting(false);
-            }}
-          >
-            {strings.startOver()}
-          </Button>
-          {errors.length > 0 && (
-            <Flex
-              my={1}
-              bg="var(--background-error)"
-              p={1}
-              sx={{ flexDirection: "column" }}
-            >
-              {errors.map((error) => (
-                <Text
-                  key={error.message}
-                  variant="body"
-                  sx={{
-                    color: "var(--paragraph-error)"
-                  }}
-                >
-                  {error.message}
-                </Text>
-              ))}
-            </Flex>
-          )}
-        </>
-      ) : (
-        <>
-          <Accordion
-            isClosed={false}
-            title="How to import your notes from other apps?"
-            containerSx={{
-              px: 2,
-              pb: 2,
-              border: "1px solid var(--border)",
-              borderTopWidth: 0,
-              borderRadius: "default",
-              borderTopLeftRadius: 0,
-              borderTopRightRadius: 0
-            }}
-          >
-            <Text variant="subtitle" sx={{ mt: 2 }}>
-              Quick start guide:
-            </Text>
-            <Box as="ol" sx={{ my: 1 }}>
-              <Text as="li" variant="body">
-                Go to{" "}
-                <Link
-                  href="https://importer.notesnook.com/"
-                  target="_blank"
-                  sx={{ color: "accent" }}
-                >
-                  https://importer.notesnook.com/
-                </Link>
-              </Text>
-              <Text as="li" variant="body">
-                Select the app you want to import from.
-              </Text>
-              <Text as="li" variant="body">
-                Drag drop or select the files you exported from the other app.
-              </Text>
-              <Text as="li" variant="body">
-                Start the importer and wait for it to complete processing.
-              </Text>
-              <Text as="li" variant="body">
-                Download the .zip file from the Importer.
-              </Text>
-              <Text as="li" variant="body">
-                Drop the .zip file below to complete your import.
-              </Text>
-            </Box>
-
-            <Text variant={"body"} sx={{ fontWeight: "bold" }}>
-              For detailed steps with screenshots, refer to the help article for
-              each app:
-            </Text>
-            <Box
-              sx={{
-                display: "grid",
-                gridTemplateColumns: "1fr 1fr 1fr",
-                gap: 1,
-                mt: 1
-              }}
-            >
-              {POPULAR_PROVIDERS.map((provider) => (
-                <Button
-                  key={provider.link}
-                  variant="icon"
-                  sx={{
-                    borderRadius: "default",
-                    border: "1px solid var(--border)",
-                    textAlign: "left"
-                  }}
-                  onClick={() => window.open(provider.link, "_blank")}
-                >
-                  Import from {provider.title}
-                </Button>
-              ))}
-            </Box>
-          </Accordion>
-          <Flex
-            {...getRootProps()}
-            data-test-id="import-dialog-select-files"
-            sx={{
-              justifyContent: "center",
-              alignItems: "center",
-              minHeight: 200,
-              flexShrink: 0,
-              width: "full",
-              border: "2px dashed var(--border)",
-              borderRadius: "default",
-              mt: 2,
-              flexDirection: "column"
-            }}
-          >
-            <Input {...getInputProps()} />
-            <Text variant="body" sx={{ textAlign: "center" }}>
-              {isDragActive
-                ? strings.dropFilesHere()
-                : strings.dragAndDropFiles()}
-              <br />
-              <Text variant="subBody">{strings.onlyZipSupported()}</Text>
-            </Text>
-            <Box sx={{ display: "flex", flexWrap: "wrap", mt: 2 }}>
-              {files.map((file, i) => (
-                <Text
-                  key={file.name}
-                  p={1}
-                  sx={{
-                    ":hover": { bg: "hover" },
-                    cursor: "pointer",
-                    borderRadius: "default"
-                  }}
-                  onClick={() => {
-                    setFiles((files) => {
-                      const cloned = files.slice();
-                      cloned.splice(i, 1);
-                      return cloned;
-                    });
-                  }}
-                  variant="body"
-                  title="Click to remove"
-                >
-                  {file.name}
-                </Text>
-              ))}
-            </Box>
-          </Flex>
-          {/* <Flex my={1} sx={{ flexDirection: "column" }}>
-            
-          </Flex> */}
-          <Button
-            variant="accent"
-            sx={{ alignSelf: "end", mt: 1 }}
-            onClick={async () => {
-              setIsDone(false);
-              setIsImporting(true);
-
-              await db.syncer?.acquireLock(async () => {
-                try {
-                  for await (const message of importFiles(files)) {
-                    switch (message.type) {
-                      case "error":
-                        setErrors((errors) => [...errors, message.error]);
-                        break;
-                      case "progress": {
-                        const { count } = message;
-                        if (notesCounter.current)
-                          notesCounter.current.innerText = `${count}`;
-                        break;
-                      }
-                    }
-                  }
-                } catch (e) {
-                  console.error(e);
-                  if (e instanceof Error) {
-                    setErrors((errors) => [...errors, e as Error]);
-                  }
-                }
-              });
-
-              await useAppStore.getState().refresh();
-
-              setIsDone(true);
-              setIsImporting(false);
-            }}
-            disabled={!files.length}
-          >
-            {files.length > 0
-              ? "Start import"
-              : "Select files to start importing"}
-          </Button>
-        </>
-      )}
-    </Flex>
-  );
-}

apps/web/src/dialogs/settings/importer-settings.ts

@@ -18,7 +18,7 @@ along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */
 
 import { SettingsGroup } from "./types";
-import { Importer } from "./components/importer";
+import { Importer } from "../../components/importer";
 
 export const ImporterSettings: SettingsGroup[] = [
   {

apps/web/src/utils/importer.ts

@@ -17,80 +17,22 @@ You should have received a copy of the GNU General Public License
 along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */
 
-import { db } from "../common/db";
 import {
-  Note,
-  Notebook,
   ContentType,
-  LegacyNotebook
+  LegacyNotebook,
+  Note,
+  Notebook
 } from "@notesnook-importer/core/dist/src/models";
-import {
-  ATTACHMENTS_DIRECTORY_NAME,
-  NOTE_DATA_FILENAME
-} from "@notesnook-importer/core/dist/src/utils/note-stream";
-import { path } from "@notesnook-importer/core/dist/src/utils/path";
-import { type ZipEntry } from "./streams/unzip-stream";
-import { hashBuffer, writeEncryptedFile } from "../interfaces/fs";
 import { Notebook as NotebookType } from "@notesnook/core";
+import { SerializedKey } from "@notesnook/crypto";
+import { db } from "../common/db";
+import { writeEncryptedFile } from "../interfaces/fs";
 
-export async function* importFiles(zipFiles: File[]) {
-  const { createUnzipIterator } = await import("./streams/unzip-stream");
-
-  for (const zip of zipFiles) {
-    let count = 0;
-    let filesRead = 0;
-
-    const attachments: Record<string, any> = {};
-
-    for await (const entry of createUnzipIterator(zip)) {
-      ++filesRead;
-
-      const isAttachment = entry.name.includes(
-        `/${ATTACHMENTS_DIRECTORY_NAME}/`
-      );
-      const isNote = !isAttachment && entry.name.endsWith(NOTE_DATA_FILENAME);
-
-      try {
-        if (isAttachment) {
-          await processAttachment(entry, attachments);
-        } else if (isNote) {
-          await processNote(entry, attachments);
-          ++count;
-        }
-      } catch (e) {
-        if (e instanceof Error) yield { type: "error" as const, error: e };
-      }
-
-      yield {
-        type: "progress" as const,
-        count,
-        filesRead
-      };
-    }
-  }
-}
-
-async function processAttachment(
-  entry: ZipEntry,
-  attachments: Record<string, any>
-) {
-  const name = path.basename(entry.name);
-  if (!name || attachments[name] || (await db.attachments?.exists(name)))
-    return;
-
-  const data = await entry.arrayBuffer();
-  const { hash } = await hashBuffer(new Uint8Array(data));
-  if (hash !== name) {
-    throw new Error(`integrity check failed: ${name} !== ${hash}`);
-  }
-
-  const file = new File([data], name, {
-    type: "application/octet-stream"
-  });
-  const key = await db.attachments?.generateKey();
-  const cipherData = await writeEncryptedFile(file, key, name);
-  attachments[name] = { ...cipherData, key };
-}
+type EncryptedAttachmentFields = Awaited<
+  ReturnType<typeof writeEncryptedFile>
+> & {
+  key: SerializedKey;
+};
 
 const colorMap: Record<string, string | undefined> = {
   default: undefined,
@@ -107,18 +49,50 @@ const colorMap: Record<string, string | undefined> = {
   yellow: "#FFC107"
 };
 
-async function processNote(entry: ZipEntry, attachments: Record<string, any>) {
-  const note = await fileToJson<Note>(entry);
-  for (const attachment of note.attachments || []) {
-    const cipherData = attachments[attachment.hash];
-    if (!cipherData || (await db.attachments?.exists(attachment.hash)))
+export async function importNote(note: Note) {
+  const encryptedAttachmentFieldsMap = await processAttachments(
+    note.attachments
+  );
+  await processNote(note, encryptedAttachmentFieldsMap);
+}
+
+async function processAttachments(attachments: Note["attachments"]) {
+  if (!attachments) return {};
+
+  const map: Record<string, EncryptedAttachmentFields | undefined> = {};
+  for (const { hash, filename, data } of attachments) {
+    if (!data || !hash || map[hash]) {
       continue;
+    }
+    const exists = await db.attachments?.exists(hash);
+    if (exists) continue;
+
+    const file = new File([data], filename, {
+      type: "application/octet-stream"
+    });
+    const key = await db.attachments?.generateKey();
+    const cipherData = await writeEncryptedFile(file, key, hash);
+    map[hash] = { ...cipherData, key };
+  }
+  return map;
+}
+
+async function processNote(
+  note: Note,
+  map: Record<string, EncryptedAttachmentFields | undefined>
+) {
+  for (const attachment of note.attachments || []) {
+    const cipherData = map[attachment.hash];
+    if (!cipherData || (await db.attachments?.exists(attachment.hash))) {
+      continue;
+    }
 
     await db.attachments?.add({
       ...cipherData,
       hash: attachment.hash,
       hashType: attachment.hashType,
       filename: attachment.filename,
+      // todo: figure out typescript error
       type: attachment.mime
     });
   }
@@ -198,11 +172,6 @@ async function processNote(entry: ZipEntry, attachments: Record<string, any>) {
   }
 }
 
-async function fileToJson<T>(file: ZipEntry) {
-  const text = await file.text();
-  return JSON.parse(text) as T;
-}
-
 /**
  * @deprecated
  */