diff --git a/packages/core/content-types/__tests__/__snapshots__/tiptap.test.js.snap b/packages/core/content-types/__tests__/__snapshots__/tiptap.test.js.snap index dff66a4a7..221b10855 100644 --- a/packages/core/content-types/__tests__/__snapshots__/tiptap.test.js.snap +++ b/packages/core/content-types/__tests__/__snapshots__/tiptap.test.js.snap @@ -232,3 +232,240 @@ Nene " `; + +exports[`convert HTML to text with codeblock2: html-to-txt-codeblock2.txt 1`] = ` +"hello +" +`; + +exports[`convert HTML to text with codeblocks: html-to-txt-codeblocks.txt 1`] = ` +"Typescript is one of those languages that appear to be very simple. It's often +described as \\"Javascript with types\\" and it fits that name very well. However, +what many don't realize starting out with Typescript is that Typescript is a +language and like all other languages it has it's own \\"secrets\\", it's own set of +quirks. + +When I started out with Typescript a few years back, I absolutely hated it. It +was unnecessary, a box of clutter, making me write code that would never +actually run. I hated defining interfaces, typing out all my functions, and +thinking in terms I was not used to as a Javascript developer. Before +Javascript, I had coded in C# and I had never really liked C# (I still don't) +mostly for the huge amounts of boilerplate and magic involved. Typescript is +heavily inspired by C# and seeing that contaminate the Javascript ecosystem +irked me no end. + +That was all just fluff. Typescript is a great language solving a considerable +amount of problems for a huge amount of developers. And generics is the hidden +weapon behind it all. + + +UNDERSTANDING TYPESCRIPT GENERICS + +For all points and purposes, generics shouldn't exist. They are one of the main +factors behind abysmal code readability. But in the right hands, generics turn +into a super-weapon. + +The main problem generics try to solve is how to take in multiple types of +parameters. This is a solved problem but requires duplicating your functions or +using conditions to separate out different logic for different types. That is +essentially what generics do as well but hidden from human eyes. + +Think of a container that can take any type of item as long as it is not a +circle. In Javascript, this becomes a problem you have to solve at runtime with +checks & conditions: + +var container = []; +function putIntoContainer(item) { + if (item.type === \\"round\\") throw new Error(\\"Rounded items not supported.\\") + container.push(item); +} + +var square = {type: \\"square\\"} +var circle = {type: \\"round\\"} +putIntoContainer(square) +putIntoContainer(circle) // ERROR! Rounded items not supported! + + +There are many ways to solve this problem and some are even practical. The issue +here isn't of repetition but of doing unnecessary work. Type safe languages +would automatically give an error if there was a wrong type but Javascript knows +nothing about the item. + +In Typescript, this will be solved much more succinctly: + +// first define the types of items we'll handle +// i.e. we don't want to handle any item other +// than square or round. +// This gives us nice auto completion and safety +// against typos. +type ItemTypes = \\"square\\" | \\"round\\"; + +// Define a generic item that can be of any type +// defined in ItemTypes. +// i.e. Item<\\"triangle\\"> will give an error. +type Item = { + type?: TItemType; + width: number; + height: number; +}; + +// This is just syntax sugar to increase readability. +type Square = Item<\\"square\\">; +type Circle = Item<\\"round\\">; + +// Our container is just a simple wrapper around an array +// that accepts items of only a specific type. +type Container = Array>; + +var squareContainer: Container<\\"square\\"> = []; +var roundContainer: Container<\\"round\\"> = []; + +// This wrapper is unnecessary, of course, because array.push +// already does this. Only for demonstration purposes. +function putIntoContainer< + TItemType extends ItemTypes, + TItem extends Item +>(container: Container, item: TItem) { + container.push(item); +} + +var square: Square = { width: 100, height: 200 }; +var circle: Circle = { width: 200, height: 500 }; + +putIntoContainer(squareContainer, square); +putIntoContainer(roundContainer, circle); +putIntoContainer(roundContainer, square); // Error: Argument of type 'Square' is not assignable to parameter of type 'Item<\\"round\\">'. +putIntoContainer(squareContainer, circle); // Error: Argument of type 'Circle' is not assignable to parameter of type 'Item<\\"square\\">'. + + +A lot more code, I know, and if you don't know how generics work that blob of +code is utter nonsense. One of the main reasons I avoided Typescript for a long +time. But look at the benefits: + + 1. You have 100% compile-time type safety. + 2. You can't put a round item in a square container (you will get compiler + error). + 3. You didn't write any extra runtime code. + +Expanding on point #3, after transpilation the above code will turn more-or-less +into: + +var squareContainer = []; +var roundContainer = []; + +function putIntoContainer(container, item) { + container.push(item); +} + +var square = { width: 100, height: 200 }; +var circle = { width: 200, height: 500 }; + +putIntoContainer(squareContainer, square); +putIntoContainer(roundContainer, circle); + + +This is the power of generics. More specifically, this is Typescript generics at +a glance. + +But this post was supposed to be about the \\"Secrets\\" of Typescript Generics, +right? Well, let's get into that. + + +1. TYPE FILTERS USING TERNARY OPERATORS + + +2. DEEPLY RECURSIVE TYPES + + +3. TYPE FUNCTIONS + + +4. TYPE INFERENCE USING INTERFACE PROPERTIES" +`; + +exports[`convert HTML to text with outlinelists: html-to-txt-outlinelists.txt 1`] = ` +"Testing outline list: + + * My outline list + * works + * but sometimes + * It doesn't + * what do I do? + * I need to do something! + * Makes no sense! + * Yes it doesn't!" +`; + +exports[`convert HTML to text with tables: html-to-txt-tables.txt 1`] = ` +"Goal To introduce various features of the app to the user and to + convert a user on trial or basic plan to upgrade. +Frequency 1/week or 2/week +Types Feature intro, upgrade promo, one time emails + + + +EMAILS + + +FEATURE INTRO + +Features: + + 1. Web clipper on mobile + 2. Pin any note to notification + 3. Take notes from notifications + 4. App lock + 5. Importer + 6. Encrypted attachments + 7. Session history & automatic backups + 8. Note publishing + 9. Note exports + 10. Collapsible headers + + +PROMOS + + 1. Trial about to end + 2. Trial ending (with option to request an extension) + 3. Try free for 14 days + + +ONE TIME + + 1. End-of-month progress report + 2. What's coming/roadmap + 3. What we are working on + 4. Join the community" +`; + +exports[`convert HTML to text with tables2: html-to-txt-tables2.txt 1`] = ` +"NOTE 8/6/22, 10:48 AM + + +hell + +what + +SDSDAVAV DASKVJBDSVA VSADJKVSADBVJK +dsvsajkdb dskajvbsadj kjdasvbkj +daskvbkdsa kdsajvbsajkd kjdsavbdsa" +`; + +exports[`convert HTML to text with tasklists: html-to-txt-tasklists.txt 1`] = ` +"Hello + + ✅ Task item 1 + ✅ Task item 2 + ✅ Task item 3 + ☐ Task item 4 + ☐ Sub task item 1 + ☐ Sub task item 2 + ☐ Task Item 5 + +Nene + + * dasvsadv + * adsva\`sd + * vasd + * vsadvdsa" +`; diff --git a/packages/core/content-types/__tests__/tiptap.test.js b/packages/core/content-types/__tests__/tiptap.test.js index 226bef1a7..a11f2e9c6 100644 --- a/packages/core/content-types/__tests__/tiptap.test.js +++ b/packages/core/content-types/__tests__/tiptap.test.js @@ -57,3 +57,10 @@ for (const html in HTMLS) { expect(tiptap.toMD()).toMatchSnapshot(`html-to-md-${html}.md`); }); } + +for (const html in HTMLS) { + test(`convert HTML to text with ${html}`, () => { + const tiptap = new Tiptap(HTMLS[html]); + expect(tiptap.toTXT()).toMatchSnapshot(`html-to-txt-${html}.txt`); + }); +} diff --git a/packages/core/content-types/tiptap.js b/packages/core/content-types/tiptap.js index 1f955e7b7..05fc31711 100644 --- a/packages/core/content-types/tiptap.js +++ b/packages/core/content-types/tiptap.js @@ -29,7 +29,6 @@ const splitter = /\W+/gm; export class Tiptap { constructor(data) { this.data = data; - this.text; this.document = parseHTML(data); } @@ -38,7 +37,34 @@ export class Tiptap { } toTXT() { - return this.document.body.innerText; + return convert(this.data, { + wordwrap: 80, + preserveNewlines: true, + selectors: [ + { selector: "table", format: "dataTable" }, + { selector: "ul.checklist", format: "taskList" } + ], + formatters: { + taskList: (elem, walk, builder, formatOptions) => { + return list(elem, walk, builder, formatOptions, (elem) => { + return elem.attribs.class.includes("checked") ? " ✅ " : " ☐ "; + }); + }, + paragraph: (elem, walk, builder, formatOptions) => { + if (elem.parent && elem.parent.name === "li") { + walk(elem.children, builder); + } else { + builder.openBlock({ + leadingLineBreaks: formatOptions.leadingLineBreaks || 2 + }); + walk(elem.children, builder); + builder.closeBlock({ + trailingLineBreaks: formatOptions.trailingLineBreaks || 2 + }); + } + } + } + }); } toMD() { diff --git a/packages/core/package-lock.json b/packages/core/package-lock.json index 5d94e6eb9..e9ef4fb25 100644 --- a/packages/core/package-lock.json +++ b/packages/core/package-lock.json @@ -17,6 +17,7 @@ "dayjs": "^1.11.3", "entities": "^4.3.1", "fflate": "^0.7.3", + "html-to-text": "github:thecodrr/node-html-to-text", "htmlparser2": "^8.0.1", "linkedom": "^0.14.17", "liqe": "^1.13.0", @@ -2375,6 +2376,32 @@ "node": ">= 10" } }, + "node_modules/@selderee/plugin-htmlparser2": { + "version": "0.9.0", + "resolved": "https://registry.npmjs.org/@selderee/plugin-htmlparser2/-/plugin-htmlparser2-0.9.0.tgz", + "integrity": "sha512-d4zFHnTLNEjLRJxSxjsRSmLUIT5+EEG42rEs/F/LF6EoIE+KLTMqeyfJa+N3ngmehCyZj3cUA53kZda9c8dILw==", + "dependencies": { + "domhandler": "^4.2.2", + "selderee": "^0.9.0" + }, + "funding": { + "url": "https://ko-fi.com/killymxi" + } + }, + "node_modules/@selderee/plugin-htmlparser2/node_modules/domhandler": { + "version": "4.3.1", + "resolved": "https://registry.npmjs.org/domhandler/-/domhandler-4.3.1.tgz", + "integrity": "sha512-GrwoxYN+uWlzO8uhUXRl0P+kHE4GtVPfYzVLcUxPL7KNdHKj66vvlhiweIHqYYXWlw+T8iLMp42Lm67ghw4WMQ==", + "dependencies": { + "domelementtype": "^2.2.0" + }, + "engines": { + "node": ">= 4" + }, + "funding": { + "url": "https://github.com/fb55/domhandler?sponsor=1" + } + }, "node_modules/@sinclair/typebox": { "version": "0.24.46", "dev": true, @@ -4752,7 +4779,6 @@ }, "node_modules/deepmerge": { "version": "4.2.2", - "dev": true, "license": "MIT", "engines": { "node": ">=0.10.0" @@ -5315,8 +5341,28 @@ "dev": true, "license": "MIT" }, + "node_modules/html-to-text": { + "version": "8.2.1", + "resolved": "git+ssh://git@github.com/thecodrr/node-html-to-text.git#8acb9b2fccceacbf5c01664fc15f9bc5f0e82762", + "license": "MIT", + "dependencies": { + "@selderee/plugin-htmlparser2": "^0.9.0", + "deepmerge": "^4.2.2", + "entities": "^4.4.0", + "htmlparser2": "github:thecodrr/htmlparser2", + "minimist": "^1.2.7", + "selderee": "^0.9.0" + }, + "bin": { + "html-to-text": "bin/cli.js" + }, + "engines": { + "node": ">=10.23.2" + } + }, "node_modules/htmlparser2": { - "version": "8.0.1", + "version": "8.1.1", + "resolved": "git+ssh://git@github.com/thecodrr/htmlparser2.git#fed70e52a067bd16b72f5ae71bc7020a34a2d8c9", "funding": [ "https://github.com/fb55/htmlparser2?sponsor=1", { @@ -5327,9 +5373,9 @@ "license": "MIT", "dependencies": { "domelementtype": "^2.3.0", - "domhandler": "^5.0.2", + "domhandler": "^5.0.3", "domutils": "^3.0.1", - "entities": "^4.3.0" + "entities": "^4.4.0" } }, "node_modules/http-proxy-agent": { @@ -7271,6 +7317,14 @@ "node": ">=6" } }, + "node_modules/leac": { + "version": "0.5.1", + "resolved": "https://registry.npmjs.org/leac/-/leac-0.5.1.tgz", + "integrity": "sha512-ItrUZwFdQSJT5sHceYI8qQif98Sc1XV0kLhn0aWpE4f82B+FICsF4egayLMS1iCHxNKWLbvK6DQNaRSTAbjnYA==", + "funding": { + "url": "https://ko-fi.com/killymxi" + } + }, "node_modules/leven": { "version": "3.1.0", "dev": true, @@ -7444,7 +7498,6 @@ }, "node_modules/minimist": { "version": "1.2.7", - "dev": true, "license": "MIT", "funding": { "url": "https://github.com/sponsors/ljharb" @@ -7731,6 +7784,18 @@ "url": "https://github.com/inikulin/parse5?sponsor=1" } }, + "node_modules/parseley": { + "version": "0.10.0", + "resolved": "https://registry.npmjs.org/parseley/-/parseley-0.10.0.tgz", + "integrity": "sha512-8r4sZD2bNjTxACVk9Sqkzc0pXO9G0KznjPnd+UbHs+ZkvFr5irw4P5HwMVoixanPFP5ZlZKsuf1QP+e47SYZAQ==", + "dependencies": { + "leac": "^0.5.1", + "peberminta": "^0.6.0" + }, + "funding": { + "url": "https://ko-fi.com/killymxi" + } + }, "node_modules/path-exists": { "version": "1.0.0", "dev": true, @@ -7760,6 +7825,14 @@ "dev": true, "license": "MIT" }, + "node_modules/peberminta": { + "version": "0.6.0", + "resolved": "https://registry.npmjs.org/peberminta/-/peberminta-0.6.0.tgz", + "integrity": "sha512-qeJGsC0f6dIWtpIN6t9MhpbIviMJ7wNosu+RtzXCyoxlEUlFJWVaMd3KvYO6ciNPi5XF6TvsVpSVq+a8iBtUBw==", + "funding": { + "url": "https://ko-fi.com/killymxi" + } + }, "node_modules/picocolors": { "version": "1.0.0", "dev": true, @@ -8092,6 +8165,17 @@ "node": ">=v12.22.7" } }, + "node_modules/selderee": { + "version": "0.9.0", + "resolved": "https://registry.npmjs.org/selderee/-/selderee-0.9.0.tgz", + "integrity": "sha512-Zsg9YrMit8Z2u4L/f4oGuVjreT3KRml2Ak2aUKr4S+an3vy6ntGv7MLUs4i2hN6bz9b9DcDMl2fS4r1obhDMJA==", + "dependencies": { + "parseley": "^0.10.0" + }, + "funding": { + "url": "https://ko-fi.com/killymxi" + } + }, "node_modules/semver": { "version": "6.3.0", "dev": true, @@ -10220,6 +10304,25 @@ "@msgpack/msgpack": { "version": "2.8.0" }, + "@selderee/plugin-htmlparser2": { + "version": "0.9.0", + "resolved": "https://registry.npmjs.org/@selderee/plugin-htmlparser2/-/plugin-htmlparser2-0.9.0.tgz", + "integrity": "sha512-d4zFHnTLNEjLRJxSxjsRSmLUIT5+EEG42rEs/F/LF6EoIE+KLTMqeyfJa+N3ngmehCyZj3cUA53kZda9c8dILw==", + "requires": { + "domhandler": "^4.2.2", + "selderee": "^0.9.0" + }, + "dependencies": { + "domhandler": { + "version": "4.3.1", + "resolved": "https://registry.npmjs.org/domhandler/-/domhandler-4.3.1.tgz", + "integrity": "sha512-GrwoxYN+uWlzO8uhUXRl0P+kHE4GtVPfYzVLcUxPL7KNdHKj66vvlhiweIHqYYXWlw+T8iLMp42Lm67ghw4WMQ==", + "requires": { + "domelementtype": "^2.2.0" + } + } + } + }, "@sinclair/typebox": { "version": "0.24.46", "dev": true @@ -12195,8 +12298,7 @@ "dev": true }, "deepmerge": { - "version": "4.2.2", - "dev": true + "version": "4.2.2" }, "define-properties": { "version": "1.1.4", @@ -12529,13 +12631,26 @@ "version": "2.0.2", "dev": true }, + "html-to-text": { + "version": "git+ssh://git@github.com/thecodrr/node-html-to-text.git#8acb9b2fccceacbf5c01664fc15f9bc5f0e82762", + "from": "html-to-text@github:thecodrr/node-html-to-text", + "requires": { + "@selderee/plugin-htmlparser2": "^0.9.0", + "deepmerge": "^4.2.2", + "entities": "^4.4.0", + "htmlparser2": "github:thecodrr/htmlparser2", + "minimist": "^1.2.7", + "selderee": "^0.9.0" + } + }, "htmlparser2": { - "version": "8.0.1", + "version": "git+ssh://git@github.com/thecodrr/htmlparser2.git#fed70e52a067bd16b72f5ae71bc7020a34a2d8c9", + "from": "htmlparser2@^8.0.1", "requires": { "domelementtype": "^2.3.0", - "domhandler": "^5.0.2", + "domhandler": "^5.0.3", "domutils": "^3.0.1", - "entities": "^4.3.0" + "entities": "^4.4.0" } }, "http-proxy-agent": { @@ -13202,8 +13317,7 @@ }, "jest-pnp-resolver": { "version": "1.2.2", - "dev": true, - "requires": {} + "dev": true }, "jest-regex-util": { "version": "28.0.2", @@ -13745,8 +13859,7 @@ "dependencies": { "ws": { "version": "8.9.0", - "dev": true, - "requires": {} + "dev": true } } }, @@ -13766,6 +13879,11 @@ "version": "3.0.3", "dev": true }, + "leac": { + "version": "0.5.1", + "resolved": "https://registry.npmjs.org/leac/-/leac-0.5.1.tgz", + "integrity": "sha512-ItrUZwFdQSJT5sHceYI8qQif98Sc1XV0kLhn0aWpE4f82B+FICsF4egayLMS1iCHxNKWLbvK6DQNaRSTAbjnYA==" + }, "leven": { "version": "3.1.0", "dev": true @@ -13882,8 +14000,7 @@ } }, "minimist": { - "version": "1.2.7", - "dev": true + "version": "1.2.7" }, "mkdirp": { "version": "0.5.6", @@ -14059,6 +14176,15 @@ "entities": "^4.4.0" } }, + "parseley": { + "version": "0.10.0", + "resolved": "https://registry.npmjs.org/parseley/-/parseley-0.10.0.tgz", + "integrity": "sha512-8r4sZD2bNjTxACVk9Sqkzc0pXO9G0KznjPnd+UbHs+ZkvFr5irw4P5HwMVoixanPFP5ZlZKsuf1QP+e47SYZAQ==", + "requires": { + "leac": "^0.5.1", + "peberminta": "^0.6.0" + } + }, "path-exists": { "version": "1.0.0", "dev": true @@ -14075,6 +14201,11 @@ "version": "1.0.7", "dev": true }, + "peberminta": { + "version": "0.6.0", + "resolved": "https://registry.npmjs.org/peberminta/-/peberminta-0.6.0.tgz", + "integrity": "sha512-qeJGsC0f6dIWtpIN6t9MhpbIviMJ7wNosu+RtzXCyoxlEUlFJWVaMd3KvYO6ciNPi5XF6TvsVpSVq+a8iBtUBw==" + }, "picocolors": { "version": "1.0.0", "dev": true @@ -14287,6 +14418,14 @@ "xmlchars": "^2.2.0" } }, + "selderee": { + "version": "0.9.0", + "resolved": "https://registry.npmjs.org/selderee/-/selderee-0.9.0.tgz", + "integrity": "sha512-Zsg9YrMit8Z2u4L/f4oGuVjreT3KRml2Ak2aUKr4S+an3vy6ntGv7MLUs4i2hN6bz9b9DcDMl2fS4r1obhDMJA==", + "requires": { + "parseley": "^0.10.0" + } + }, "semver": { "version": "6.3.0", "dev": true @@ -14657,8 +14796,7 @@ } }, "ws": { - "version": "7.5.9", - "requires": {} + "version": "7.5.9" }, "xml-name-validator": { "version": "4.0.0", diff --git a/packages/core/package.json b/packages/core/package.json index 7396ef108..06d4acfd3 100644 --- a/packages/core/package.json +++ b/packages/core/package.json @@ -43,6 +43,7 @@ "entities": "^4.3.1", "fflate": "^0.7.3", "htmlparser2": "^8.0.1", + "html-to-text": "github:thecodrr/node-html-to-text", "linkedom": "^0.14.17", "liqe": "^1.13.0", "qclone": "^1.2.0",