Merge branch 'master' into beta

.github/workflows/themes.publish.yml

@@ -0,0 +1,78 @@
+# This workflow uses actions that are not certified by GitHub.
+# They are provided by a third-party and are governed by
+# separate terms of service, privacy policy, and support
+# documentation.
+
+# GitHub recommends pinning actions to a commit SHA.
+# To get a newer version, you will need to update the SHA.
+# You can also reference a tag or branch, but the action may change without warning.
+
+name: Publish @notesnook/themes-server
+
+on: workflow_dispatch
+
+jobs:
+  push_to_registry:
+    name: Push Docker image to Docker Hub
+    runs-on: ubuntu-22.04
+    permissions:
+      packages: write
+      contents: read
+      attestations: write
+      id-token: write
+    steps:
+      - name: Check out the repo
+        uses: actions/checkout@v4
+
+      - name: Collect package metadata
+        id: package_metadata
+        working-directory: ./servers/themes
+        run: |
+          echo ::set-output name=app_version::$(cat package.json | jq -r .version)
+
+      # Setup Buildx
+      - name: Docker Setup Buildx
+        uses: docker/setup-buildx-action@v3
+        with:
+          platforms: linux/amd64,linux/arm64
+
+      - name: Log in to Docker Hub
+        uses: docker/login-action@v3
+        with:
+          username: ${{ secrets.DOCKER_USERNAME }}
+          password: ${{ secrets.DOCKER_PASSWORD }}
+          ecr: auto
+          logout: true
+
+      # Pull previous image from docker hub to use it as cache to improve the image build time.
+      - name: docker pull cache image
+        continue-on-error: true
+        run: docker pull streetwriters/themes-server:latest
+
+      # Setup QEMU
+      # - name: Set up QEMU
+      #   uses: docker/setup-qemu-action@v2
+
+      - name: Extract metadata (tags, labels) for Docker
+        id: meta
+        uses: docker/metadata-action@v5
+        with:
+          images: streetwriters/themes-server
+
+      - name: Build and push Docker image
+        id: push
+        uses: docker/build-push-action@v6
+        with:
+          context: .
+          file: servers/themes/Dockerfile
+          push: true
+          platforms: linux/amd64,linux/arm64
+          tags: streetwriters/themes-server:${{ steps.package_metadata.outputs.app_version }},streetwriters/themes-server:latest
+          cache-from: streetwriters/themes-server:latest
+
+      - name: Generate artifact attestation
+        uses: actions/attest-build-provenance@v1
+        with:
+          subject-name: index.docker.io/streetwriters/themes-server
+          subject-digest: ${{ steps.push.outputs.digest }}
+          push-to-registry: true

apps/mobile/app/screens/settings/restore-backup/index.tsx

@@ -95,6 +95,15 @@ const restoreBackup = async (options: {
   deleteFile?: boolean;
 }) => {
   try {
+    if (
+      !options.uri.endsWith(".nnbackup") &&
+      !options.uri.endsWith(".nnbackupz")
+    ) {
+      throw new Error(
+        `Invalid backup file selected. Only .nnbackup and .nnbackupz files can be restored.`
+      );
+    }
+
     const isLegacyBackup = options.uri.endsWith(".nnbackup");
 
     startProgress({

package.json

@@ -33,6 +33,7 @@
     "start:editor-mobile": "npm run tx editor-mobile:start",
     "start:editor": "npm run tx editor:start",
     "start:server:themes": "npm run tx themes-server:start",
+    "build:server:themes": "npm run tx themes-server:build",
     "build:intl": "npm run tx intl:build",
     "start:intl": "npm run tx intl:start",
     "prettier": "npx prettier . --write",

servers/themes/Dockerfile

@@ -0,0 +1,32 @@
+# Build stage
+FROM node:22.20.0-alpine AS build
+
+# Install dependencies for gyp to work
+RUN --mount=type=cache,target=/var/cache/apk,sharing=locked apk add --update build-base git python3 py3-setuptools
+
+WORKDIR /app
+
+# Copy files
+COPY . .
+
+# Install dependencies
+RUN --mount=type=cache,target=/root/.npm npm ci --ignore-scripts --prefer-offline --no-audit
+
+# Bootstrap the project
+RUN --mount=type=cache,target=/root/.npm npm run bootstrap -- --scope=themes
+
+# Build the server
+RUN npm run build:server:themes
+
+# Production stage
+FROM oven/bun:1.3.5-alpine AS production
+
+# Install git to pull the themes repo
+RUN --mount=type=cache,target=/var/cache/apk,sharing=locked apk add --update git
+
+WORKDIR /app
+
+# Copy server files from previous stage
+COPY --from=build /app/servers/themes/dist ./dist
+
+CMD ["bun", "run", "dist/server.mjs"]

servers/themes/package.json

@@ -1,26 +1,29 @@
 {
   "name": "@notesnook/themes-server",
-  "version": "1.0.0",
+  "version": "1.0.1",
   "description": "A simple rest api for notesnook themes",
   "private": "true",
   "main": "src/index.ts",
   "scripts": {
-    "start": "vite-node -w src/server.ts"
+    "start": "vite-node -w src/server.ts",
+    "build": "esbuild src/server.ts --bundle --platform=node --sourcemap --format=esm --outfile=dist/server.mjs"
   },
   "author": "",
   "license": "ISC",
   "dependencies": {
+    "@notesnook/theme": "file:../../packages/theme",
     "@orama/orama": "^1.0.8",
+    "@sagi.io/workers-kv": "^0.0.15",
     "@trpc/server": "10.45.2",
     "async-mutex": "0.5.0",
     "cors": "^2.8.5",
+    "react": "18.3.1",
     "util": "^0.12.5",
     "zod": "3.23.8"
   },
   "devDependencies": {
-    "@notesnook/theme": "file:../../packages/theme",
     "@types/cors": "^2.8.13",
-    "react": "18.3.1",
+    "esbuild": "0.21.5",
     "vite-node": "2.1.8"
   }
-}
+}

servers/themes/src/constants.ts

@@ -18,12 +18,35 @@ along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */
 
 import path from "path";
-import { Counter } from "./counter";
+import { fileURLToPath } from "url";
+import { FsCounter } from "./counter/fs";
+import { readSecrets } from "./secrets";
+import { KVCounter } from "./counter/kv";
+
+const env = readSecrets([
+  "CLOUDFLARE_ACCOUNT_ID",
+  "CLOUDFLARE_AUTH_TOKEN",
+  "CLOUDFLARE_INSTALLS_KV_NAMESPACE_ID"
+]);
+
+const __dirname = path.dirname(fileURLToPath(import.meta.url));
+
+export const THEMES_REPO_URL =
+  process.env.THEMES_REPO_URL ||
+  "https://github.com/streetwriters/notesnook-themes.git";
 
-export const THEMES_REPO_URL = `https://github.com/streetwriters/notesnook-themes.git`;
 export const THEME_REPO_DIR_NAME = "notesnook-themes";
 export const THEME_METADATA_JSON = path.join(__dirname, "themes-metadata.json");
 export const THEME_REPO_DIR_PATH = path.resolve(
   path.join(__dirname, "..", THEME_REPO_DIR_NAME)
 );
-export const InstallsCounter = new Counter("installs", path.dirname(__dirname));
+export const InstallsCounter =
+  env.CLOUDFLARE_ACCOUNT_ID &&
+  env.CLOUDFLARE_AUTH_TOKEN &&
+  env.CLOUDFLARE_INSTALLS_KV_NAMESPACE_ID
+    ? new KVCounter({
+        cfAccountId: env.CLOUDFLARE_ACCOUNT_ID,
+        cfAuthToken: env.CLOUDFLARE_AUTH_TOKEN,
+        namespaceId: env.CLOUDFLARE_INSTALLS_KV_NAMESPACE_ID
+      })
+    : new FsCounter(path.join(path.dirname(__dirname), "..", "installs.json"));

servers/themes/src/counter/fs.ts

@@ -18,21 +18,20 @@ along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */
 
 import { readFile, writeFile } from "fs/promises";
-import path from "path";
 import { Mutex } from "async-mutex";
 
 type Counts = Record<string, string[]>;
-export class Counter {
+export class FsCounter {
   private path: string;
   private readonly mutex: Mutex;
-  constructor(id: string, baseDirectory: string) {
-    this.path = path.join(baseDirectory, `${id}.json`);
+  constructor(id: string) {
+    this.path = id;
     this.mutex = new Mutex();
   }
 
   async increment(key: string, uid: string) {
     await this.mutex.runExclusive(async () => {
-      const counts = await this.counts();
+      const counts = await this.all();
       counts[key] = counts[key] || [];
       if (counts[key].includes(uid)) return;
       counts[key].push(uid);
@@ -44,7 +43,13 @@ export class Counter {
     await writeFile(this.path, JSON.stringify(counts));
   }
 
-  async counts(): Promise<Counts> {
+  counts(key: string): Promise<number> {
+    return this.all().then((counts) => {
+      return (counts[key] || []).length;
+    });
+  }
+
+  private async all(): Promise<Counts> {
     try {
       return JSON.parse(await readFile(this.path, "utf-8"));
     } catch {

servers/themes/src/counter/kv.ts

@@ -0,0 +1,76 @@
+/*
+This file is part of the Notesnook project (https://notesnook.com/)
+
+Copyright (C) 2023 Streetwriters (Private) Limited
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+import { Mutex } from "async-mutex";
+import WorkersKVREST from "@sagi.io/workers-kv";
+
+export class KVCounter {
+  private readonly kv: WorkersKVREST;
+  private readonly mutex: Mutex;
+  constructor(config: {
+    cfAccountId: string;
+    cfAuthToken: string;
+    namespaceId: string;
+  }) {
+    this.mutex = new Mutex();
+    this.kv = new WorkersKVREST(config);
+  }
+
+  async increment(key: string, uid: string) {
+    await this.mutex.runExclusive(async () => {
+      const existing = await read<string[]>(this.kv, key, []);
+      await write(this.kv, key, Array.from(new Set([...existing, uid])));
+    });
+  }
+
+  async counts(key: string): Promise<number> {
+    const installs = await read<string[]>(this.kv, key, []);
+    return installs.length;
+  }
+}
+
+async function read<T>(
+  kv: WorkersKVREST,
+  key: string,
+  fallback: T
+): Promise<T> {
+  try {
+    const response = await kv.readKey({
+      key
+    });
+    if (typeof response === "object" && !response.success) {
+      // console.error("failed:", response.errors);
+      return fallback;
+    }
+    return (
+      JSON.parse(typeof response === "string" ? response : response.result) ||
+      fallback
+    );
+  } catch (e) {
+    // console.error(e);
+    return fallback;
+  }
+}
+
+async function write<T>(kv: WorkersKVREST, key: string, data: T) {
+  await kv.writeKey({
+    key,
+    value: JSON.stringify(data)
+  });
+}

servers/themes/src/orama.ts

@@ -19,6 +19,7 @@ along with this program.  If not, see <http://www.gnu.org/licenses/>.
 import { Orama, SearchParams, create, search } from "@orama/orama";
 import { CompiledThemeDefinition, ThemeMetadata } from "./sync";
 import { ThemeQuerySchema } from "./schemas";
+import { InstallsCounter } from "./constants";
 
 export let ThemesDatabase: Orama | null = null;
 export async function initializeDatabase(): Promise<Orama> {
@@ -89,13 +90,16 @@ export async function getThemes(query: (typeof ThemeQuerySchema)["_type"]) {
     }
   }
   const results = await search(ThemesDatabase!, searchParams);
-  const themes = results.hits.map((hit) => {
-    return {
-      ...(hit.document as CompiledThemeDefinition),
+  const themes: ThemeMetadata[] = [];
+  for (const hit of results.hits) {
+    const theme = hit.document as CompiledThemeDefinition;
+    themes.push({
+      ...theme,
       scopes: undefined,
-      codeBlockCSS: undefined
-    } as ThemeMetadata;
-  });
+      codeBlockCSS: undefined,
+      totalInstalls: await InstallsCounter.counts(theme.id)
+    } as ThemeMetadata);
+  }
 
   return {
     themes,

servers/themes/src/secrets.ts

@@ -0,0 +1,40 @@
+/*
+This file is part of the Notesnook project (https://notesnook.com/)
+
+Copyright (C) 2023 Streetwriters (Private) Limited
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+import { readFileSync } from "node:fs";
+
+export function readSecrets<T extends string>(
+  names: T[]
+): Record<T, string | undefined> {
+  const result: Record<T, string | undefined> = {} as Record<
+    T,
+    string | undefined
+  >;
+  for (const name of names) result[name] = readSecret(name);
+  return result;
+}
+
+export function readSecret(name: string): string | undefined {
+  const value = process.env[name];
+  if (value) return value;
+  const file = process.env[`${name}_FILE`];
+  if (file) {
+    return readFileSync(file, "utf-8");
+  }
+}

servers/themes/src/server.ts

@@ -27,8 +27,9 @@ const server = createHTTPServer({
   router: ThemesAPI
 });
 const PORT = parseInt(process.env.PORT || "9000");
-server.listen(PORT);
-console.log(`Server started successfully on: http://localhost:${PORT}/`);
+const HOST = process.env.HOST || "localhost";
+server.listen(PORT, HOST);
+console.log(`Server started successfully on: http://${HOST}:${PORT}/`);
 
 syncThemes();
 

servers/themes/src/sync.ts

@@ -17,14 +17,10 @@ You should have received a copy of the GNU General Public License
 along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */
 import { execSync } from "child_process";
-import fs from "fs";
+import fs from "node:fs";
 import { readdir, readFile } from "fs/promises";
 import path from "path";
-import {
-  InstallsCounter,
-  THEMES_REPO_URL,
-  THEME_REPO_DIR_PATH
-} from "./constants";
+import { THEMES_REPO_URL, THEME_REPO_DIR_PATH } from "./constants";
 import { insertMultiple } from "@orama/orama";
 import { initializeDatabase } from "./orama";
 import {
@@ -70,7 +66,6 @@ async function generateThemesMetadata() {
 
   const THEMES_PATH = path.join(THEME_REPO_DIR_PATH, "themes");
   const themes = await readdir(THEMES_PATH);
-  const counts = await InstallsCounter.counts();
 
   for (const themeId of themes) {
     for (const version of THEME_COMPATIBILITY_VERSIONS) {
@@ -96,7 +91,7 @@ async function generateThemesMetadata() {
         ...theme,
         sourceURL: `https://github.com/streetwriters/notesnook-themes/tree/main/themes/${themeId}/v${version}/`,
         codeBlockCSS,
-        totalInstalls: counts[theme.id]?.length || 0,
+        totalInstalls: 0,
         previewColors: getPreviewColors(theme)
       });
     }