core: fix title generation when title format is set to $headline$ (#7739)

* core: generate headline title until user edit && update headline extraction
* when title format is set to $, keep generating headline title until user manually edits the title
* extract headline from first html tag with content rather than first paragraph

Signed-off-by: 01zulfi <85733202+01zulfi@users.noreply.github.com>

* core: extract headline up to 150 chars regardless of html tag
Signed-off-by: 01zulfi <85733202+01zulfi@users.noreply.github.com>

* core: improve note headline title code quality
Signed-off-by: 01zulfi <85733202+01zulfi@users.noreply.github.com>

* core: remove HEADLINE_CHARACTER_LIMIT constant
Signed-off-by: 01zulfi <85733202+01zulfi@users.noreply.github.com>

* web: only update title input if note title changes

---------

Signed-off-by: 01zulfi <85733202+01zulfi@users.noreply.github.com>
Co-authored-by: Abdullah Atta <abdullahatta@streetwriters.co>

apps/web/__e2e__/editor.test.ts

@@ -131,6 +131,35 @@ test("focus should not jump to editor while typing in title input", async ({
   expect(await notes.editor.getContent("text")).toBe("");
 });
 
+test("when title format is set to headline, title should be generated from headline until user edits the title", async ({
+  page
+}) => {
+  const app = new AppModel(page);
+  await app.goto();
+  const settings = await app.goToSettings();
+  await settings.setTitleFormat("$headline$");
+  await settings.close();
+
+  const notes = await app.goToNotes();
+  await notes.createNote({ content: "my precious" });
+
+  expect(await notes.editor.getTitle()).toBe("my precious");
+
+  await notes.editor.setContent(", my precious note");
+  await notes.editor.waitForSaving();
+
+  expect(await notes.editor.getTitle()).toBe("my precious, my precious note");
+
+  await notes.editor.setTitle("not precious");
+
+  expect(await notes.editor.getTitle()).toBe("not precious");
+
+  await notes.editor.setContent(", but...");
+  await notes.editor.waitForSaving();
+
+  expect(await notes.editor.getTitle()).toBe("not precious");
+});
+
 test("select all & backspace should clear all content in editor", async ({
   page
 }) => {

apps/web/__e2e__/models/settings-view.model.ts

@@ -179,4 +179,14 @@ export class SettingsViewModel {
     await fillPasswordDialog(this.page, appLockPassword);
     await this.page.waitForTimeout(100);
   }
+
+  async setTitleFormat(format: string) {
+    const item = await this.navigation.findItem("Editor");
+    await item?.click();
+
+    const titleFormatInput = this.page
+      .locator(getTestId("setting-default-title"))
+      .locator("input");
+    await titleFormatInput.fill(format);
+  }
 }

apps/web/src/components/editor/title-box.tsx

@@ -19,7 +19,7 @@ along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 import React, { useEffect, useLayoutEffect, useMemo, useRef } from "react";
 import { Textarea } from "@theme-ui/components";
-import { useEditorStore } from "../../stores/editor-store";
+import { SaveState, useEditorStore } from "../../stores/editor-store";
 import { debounceWithId } from "@notesnook/common";
 import { useEditorConfig, useEditorManager } from "./manager";
 import { getFontById } from "@notesnook/editor";
@@ -38,8 +38,10 @@ function TitleBox(props: TitleBoxProps) {
   const { readonly, id } = props;
   const inputRef = useRef<HTMLTextAreaElement>(null);
   const pendingChanges = useRef(false);
-  // const id = useStore((store) => store.session.id);
   const sessionType = useEditorStore((store) => store.getSession(id)?.type);
+  const sessionTitle = useEditorStore(
+    (store) => store.getSession(id, ["default"])?.note.title
+  );
   const { editorConfig } = useEditorConfig();
   const dateFormat = useSettingsStore((store) => store.dateFormat);
   const timeFormat = useSettingsStore((store) => store.timeFormat);
@@ -62,7 +64,7 @@ function TitleBox(props: TitleBoxProps) {
       input.value = title || "";
       resizeTextarea(input);
     });
-  }, [sessionType, id]);
+  }, [sessionType, id, sessionTitle]);
 
   useEffect(() => {
     const { unsubscribe } = AppEventManager.subscribe(
@@ -161,12 +163,12 @@ function resizeTextarea(input: HTMLTextAreaElement) {
   });
 }
 
-function onTitleChange(
+async function onTitleChange(
   noteId: string,
   title: string,
   pendingChanges: React.MutableRefObject<boolean>
 ) {
-  useEditorStore.getState().setTitle(noteId, title);
+  await useEditorStore.getState().setTitle(noteId, title);
   pendingChanges.current = false;
 }
 

docs/help/contents/rich-text-editor/personalizing-rich-text-editor.md

@@ -30,15 +30,15 @@ Go to `Settings` > `Editor` > `Title format` to customize the title formatting.
 
 **$time$**: The current time
 
-**$headline$**: Upto first 10 words of of a note headline
-
 **$count$**: Current note count + 1
 
 **$timestamp$**: Full date & time without any spaces or symbols (e.g. 202305261253)
 
 **$timestampz$**: UTC offset added to _timestamp_
 
-You can use a combination of these templates in the note title. For example `$headline$ - $date$` will become `Your note headline - 06-22-2023`.
+You can use a combination of above templates in the note title. For example `Note $count$ - $date$` will become `Note 150 - 06-22-2023`.
+
+**$headline$**: Up to first 10 words of the note's headline. This will keep updating the title as headline of the note changes until you manually edit the title. Shouldn't be used in combination with other templates.
 
 ## Paragraph spacing
 

packages/core/__tests__/notes.test.ts

@@ -134,29 +134,75 @@ test("changing content shouldn't reset the note title ", () =>
     expect(note?.title).toBe("I am a note");
   }));
 
-test("note should get headline from content", () =>
-  noteTest({
-    ...TEST_NOTE,
-    content: {
-      type: TEST_NOTE.content.type,
-      data: "<p>This is a very colorful existence.</p>"
-    }
-  }).then(async ({ db, id }) => {
-    const note = await db.notes.note(id);
-    expect(note?.headline).toBe("This is a very colorful existence.");
+test("note title with headline format should keep generating headline until title is edited", () =>
+  noteTest().then(async ({ db }) => {
+    await db.settings.setTitleFormat("$headline$");
+    const id = await db.notes.add({
+      content: {
+        type: TEST_NOTE.content.type,
+        data: "<p>super delicious note</p>"
+      }
+    });
+
+    let note = await db.notes.note(id);
+    expect(note?.title).toBe("super delicious note");
+
+    await db.notes.add({
+      id,
+      content: {
+        type: TEST_NOTE.content.type,
+        data: "<p>super duper delicious note</p>"
+      }
+    });
+
+    note = await db.notes.note(id);
+    expect(note?.title).toBe("super duper delicious note");
+
+    await db.notes.add({
+      id,
+      title: "not delicious anymore",
+      content: {
+        type: TEST_NOTE.content.type,
+        data: "<p>super duper delicious note</p>"
+      }
+    });
+
+    note = await db.notes.note(id);
+    expect(note?.title).toBe("not delicious anymore");
+
+    await db.notes.add({
+      id,
+      content: {
+        type: TEST_NOTE.content.type,
+        data: "<p>super duper extra delicious note</p>"
+      }
+    });
+
+    note = await db.notes.note(id);
+    expect(note?.title).toBe("not delicious anymore");
   }));
 
-test("note should not get headline if there is no p tag", () =>
-  noteTest({
-    ...TEST_NOTE,
-    content: {
-      type: TEST_NOTE.content.type,
-      data: `<ol style="list-style-type: decimal;" data-mce-style="list-style-type: decimal;"><li>Hello I won't be a headline :(</li><li>Me too.</li><li>Gold.</li></ol>`
-    }
-  }).then(async ({ db, id }) => {
-    const note = await db.notes.note(id);
-    expect(note?.headline).toBe("");
-  }));
+[
+  ["simple p tag", "<p>headline</p>", "headline"],
+  ["across multiple tags", "<h1>title<h1><ol><li>list</li></ol>", "titlelist"],
+  [
+    "content with exceeded HEADLINE_CHARACTER_LIMIT",
+    "<p><strong>head</strong><em>line</em></p><h1>Lorem ipsum dolor sit amet, consectetur adipiscing elit. Nullam rutrum ex ac eros egestas, ut rhoncus felis faucibus. Mauris tempor orci nisl, vitae pulvinar turpis convallis n</h1>",
+    "headlineLorem ipsum dolor sit amet, consectetur adipiscing elit. Nullam rutrum ex ac eros egestas, ut rhoncus felis faucibus. Mauris tempor orci nisl,"
+  ]
+].forEach(([testCase, content, expectedHeadline]) => {
+  test(`note should generate headline up to HEADLINE_CHARACTER_LIMIT characters - ${testCase}`, () =>
+    noteTest({
+      ...TEST_NOTE,
+      content: {
+        type: TEST_NOTE.content.type,
+        data: content
+      }
+    }).then(async ({ db, id }) => {
+      const note = await db.notes.note(id);
+      expect(note?.headline).toBe(expectedHeadline);
+    }));
+});
 
 test("note title should allow trailing space", () =>
   noteTest({ title: "Hello ", content: TEST_NOTE.content }).then(

packages/core/src/collections/notes.ts

@@ -19,7 +19,11 @@ along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 import { getId } from "../utils/id.js";
 import { getContentFromData } from "../content-types/index.js";
-import { NEWLINE_STRIP_REGEX, formatTitle } from "../utils/title-format.js";
+import {
+  HEADLINE_REGEX,
+  NEWLINE_STRIP_REGEX,
+  formatTitle
+} from "../utils/title-format.js";
 import { clone } from "../utils/clone.js";
 import { Tiptap } from "../content-types/tiptap.js";
 import { EMPTY_CONTENT } from "./content.js";
@@ -121,9 +125,36 @@ export class Notes implements ICollection {
             this.db.settings.getTitleFormat(),
             this.db.settings.getDateFormat(),
             this.db.settings.getTimeFormat(),
-            headline?.split(" ").splice(0, 10).join(" ") || "",
+            headline ? headlineToTitle(headline) : "",
             this.totalNotes
           );
+        item.isGeneratedTitle = true;
+      }
+
+      const currentNoteTitleFields = await this.db
+        .sql()
+        .selectFrom("notes")
+        .select("isGeneratedTitle")
+        .select("title")
+        .where("id", "=", id)
+        .executeTakeFirst();
+      if (isUpdating) {
+        const didUserEditTitle = Boolean(item.title);
+        item.isGeneratedTitle =
+          Boolean(currentNoteTitleFields?.isGeneratedTitle) &&
+          !didUserEditTitle;
+        const titleFormat = this.db.settings.getTitleFormat();
+        if (
+          item.isGeneratedTitle &&
+          HEADLINE_REGEX.test(titleFormat) &&
+          headline &&
+          currentNoteTitleFields?.title !== headlineToTitle(headline)
+        ) {
+          item.title = titleFormat.replace(
+            HEADLINE_REGEX,
+            headlineToTitle(headline)
+          );
+        }
       }
 
       if (isUpdating) {
@@ -138,7 +169,9 @@ export class Notes implements ICollection {
           conflicted: item.conflicted,
           readonly: item.readonly,
 
-          dateEdited: item.dateEdited || dateEdited
+          dateEdited: item.dateEdited || dateEdited,
+
+          isGeneratedTitle: item.isGeneratedTitle
         });
       } else {
         await this.collection.upsert({
@@ -156,7 +189,9 @@ export class Notes implements ICollection {
           readonly: item.readonly,
 
           dateCreated: item.dateCreated || Date.now(),
-          dateEdited: item.dateEdited || dateEdited || Date.now()
+          dateEdited: item.dateEdited || dateEdited || Date.now(),
+
+          isGeneratedTitle: item.isGeneratedTitle
         });
         this.totalNotes++;
       }
@@ -457,3 +492,7 @@ export class Notes implements ICollection {
 function getNoteHeadline(content: Tiptap) {
   return content.toHeadline();
 }
+
+function headlineToTitle(headline: string) {
+  return headline.split(" ").splice(0, 10).join(" ");
+}

packages/core/src/content-types/tiptap.ts

@@ -30,7 +30,7 @@ import { parseDocument } from "htmlparser2";
 import dataurl from "../utils/dataurl.js";
 import {
   HTMLParser,
-  extractFirstParagraph,
+  extractHeadline,
   getDummyDocument
 } from "../utils/html-parser.js";
 import { HTMLRewriter } from "../utils/html-rewriter.js";
@@ -89,7 +89,7 @@ export class Tiptap {
   }
 
   toHeadline() {
-    return extractFirstParagraph(this.data);
+    return extractHeadline(this.data, 150);
   }
 
   // isEmpty() {

packages/core/src/database/index.ts

@@ -231,7 +231,8 @@ const BooleanProperties: Set<BooleanFields> = new Set([
   "pinned",
   "readonly",
   "remote",
-  "synced"
+  "synced",
+  "isGeneratedTitle"
 ]);
 
 const DataMappers: Partial<Record<ItemType, (row: any) => void>> = {

packages/core/src/database/migrations.ts

@@ -383,6 +383,14 @@ export class NNMigrationProvider implements MigrationProvider {
           });
           await rebuildSearchIndex(db);
         }
+      },
+      "8": {
+        async up(db) {
+          await db.schema
+            .alterTable("notes")
+            .addColumn("isGeneratedTitle", "boolean")
+            .execute();
+        }
       }
     };
   }

packages/core/src/types.ts

@@ -198,6 +198,8 @@ export interface Note extends BaseItem<"note"> {
   dateDeleted: null;
   itemType: null;
   deletedBy: null;
+
+  isGeneratedTitle?: boolean;
 }
 
 export interface Notebook extends BaseItem<"notebook"> {

packages/core/src/utils/html-parser.ts

@@ -44,23 +44,17 @@ function wrapIntoHTMLDocument(input: string) {
   return `<!doctype html><html lang="en"><head><title>Document Fragment</title></head><body>${input}</body></html>`;
 }
 
-export function extractFirstParagraph(html: string) {
+export function extractHeadline(html: string, headlineCharacterLimit: number) {
   let text = "";
-  let start = false;
   const parser = new Parser(
     {
-      onopentag: (name) => {
-        if (name === "p") start = true;
-      },
-      onclosetag: (name) => {
-        if (name === "p") {
-          start = false;
-          parser.pause();
-          parser.reset();
-        }
-      },
       ontext: (data) => {
-        if (start) text += data;
+        text += data;
+        if (text.length > headlineCharacterLimit) {
+          text = text.slice(0, headlineCharacterLimit);
+          parser.pause();
+          parser.end();
+        }
       }
     },
     {

packages/core/src/utils/title-format.ts

@@ -21,11 +21,11 @@ import { TimeFormat } from "../types.js";
 import { formatDate } from "./date.js";
 
 export const NEWLINE_STRIP_REGEX = /[\r\n\t\v]+/gm;
+export const HEADLINE_REGEX = /\$headline\$/g;
 
 const DATE_REGEX = /\$date\$/g;
 const COUNT_REGEX = /\$count\$/g;
 const TIME_REGEX = /\$time\$/g;
-const HEADLINE_REGEX = /\$headline\$/g;
 const TIMESTAMP_REGEX = /\$timestamp\$/g;
 const TIMESTAMP_Z_REGEX = /\$timestampz\$/g;
 const DATE_TIME_STRIP_REGEX = /[\\\-:./, ]/g;

packages/intl/locale/en.po

@@ -2304,7 +2304,7 @@ msgstr "Enable app lock"
 msgid "Enable editor margins"
 msgstr "Enable editor margins"
 
-#: src/strings.ts:2443
+#: src/strings.ts:2444
 msgid "Enable ligatures for common symbols like →, ←, etc"
 msgstr "Enable ligatures for common symbols like →, ←, etc"
 
@@ -2722,7 +2722,7 @@ msgstr "Follow us on X for updates and news about Notesnook"
 msgid "Font family"
 msgstr "Font family"
 
-#: src/strings.ts:2442
+#: src/strings.ts:2443
 msgid "Font ligatures"
 msgstr "Font ligatures"
 
@@ -6181,7 +6181,7 @@ msgstr "To use app lock, you must enable biometrics such as Fingerprint lock or
 msgid "Toggle dark/light mode"
 msgstr "Toggle dark/light mode"
 
-#: src/strings.ts:2441
+#: src/strings.ts:2442
 msgid "Toggle focus mode"
 msgstr "Toggle focus mode"
 

packages/intl/locale/pseudo-LOCALE.po

@@ -2293,7 +2293,7 @@ msgstr ""
 msgid "Enable editor margins"
 msgstr ""
 
-#: src/strings.ts:2443
+#: src/strings.ts:2444
 msgid "Enable ligatures for common symbols like →, ←, etc"
 msgstr ""
 
@@ -2711,7 +2711,7 @@ msgstr ""
 msgid "Font family"
 msgstr ""
 
-#: src/strings.ts:2442
+#: src/strings.ts:2443
 msgid "Font ligatures"
 msgstr ""
 
@@ -6140,7 +6140,7 @@ msgstr ""
 msgid "Toggle dark/light mode"
 msgstr ""
 
-#: src/strings.ts:2441
+#: src/strings.ts:2442
 msgid "Toggle focus mode"
 msgstr ""