fix: resolve CondeNast#310 added support for more unicode whitespace …

…characters
JJMcConnell · Sep 27, 2020 · 6b7b7cf · 6b7b7cf
1 parent 919123e
commit 6b7b7cf
Show file tree

Hide file tree

Showing 3 changed files with 81 additions and 11 deletions.
diff --git a/packages/@atjson/renderer-commonmark/src/index.ts b/packages/@atjson/renderer-commonmark/src/index.ts
@@ -236,22 +236,67 @@ function escapeHtmlEntities(text: string) {
     .replace(/&([^\s]+);/g, "\\&$1;")
     .replace(/</g, "&lt;")
     .replace(/\u00A0/gu, "&nbsp;")
-    .replace(/\u2003/gu, "&emsp;");
+    .replace(/\u2000/gu, "&#8192;")
+    .replace(/\u2001/gu, "&#8193;")
+    .replace(/\u2002/gu, "&ensp;")
+    .replace(/\u2003/gu, "&emsp;")
+    .replace(/\u2004/gu, "&#8196;")
+    .replace(/\u2005/gu, "&#8197;")
+    .replace(/\u2006/gu, "&#8198;")
+    .replace(/\u2007/gu, "&#8199;")
+    .replace(/\u2008/gu, "&#8200;")
+    .replace(/\u2009/gu, "&#8201;")
+    .replace(/\u200A/gu, "&#8202;")
+    .replace(/\u200B/gu, "&#8203;")
+    .replace(/\u202F/gu, "&#8239;")
+    .replace(/\u205F/gu, "&#8287;")
+    .replace(/\u3000/gu, "&#12288;")
+    .replace(/\uFEFF/gu, "&#65279;");
 }
 
 function escapeEntities(text: string) {
   return text
     .replace(/&([^\s]+);/g, "\\&$1;")
     .replace(/\u00A0/gu, "&nbsp;")
-    .replace(/\u2003/gu, "&emsp;");
+    .replace(/\u2000/gu, "&#8192;")
+    .replace(/\u2001/gu, "&#8193;")
+    .replace(/\u2002/gu, "&ensp;")
+    .replace(/\u2003/gu, "&emsp;")
+    .replace(/\u2004/gu, "&#8196;")
+    .replace(/\u2005/gu, "&#8197;")
+    .replace(/\u2006/gu, "&#8198;")
+    .replace(/\u2007/gu, "&#8199;")
+    .replace(/\u2008/gu, "&#8200;")
+    .replace(/\u2009/gu, "&#8201;")
+    .replace(/\u200A/gu, "&#8202;")
+    .replace(/\u200B/gu, "&#8203;")
+    .replace(/\u202F/gu, "&#8239;")
+    .replace(/\u205F/gu, "&#8287;")
+    .replace(/\u3000/gu, "&#12288;")
+    .replace(/\uFEFF/gu, "&#65279;");
 }
 
 function unescapeEntities(text: string) {
   return text
     .replace(/&amp;/gi, "&")
     .replace(/&lt;/gi, "<")
     .replace(/&nbsp;/gi, "\u00A0")
-    .replace(/&emsp;/gi, "\u2003");
+    .replace(/&emsp;/gi, "\u2003")
+    .replace(/&#8192;/gi, "\u2000")
+    .replace(/&#8193;/gi, "\u2001")
+    .replace(/&ensp;/gi, "\u2002")
+    .replace(/&#8196;/gi, "\u2004")
+    .replace(/&#8197;/gi, "\u2005")
+    .replace(/&#8198;/gi, "\u2006")
+    .replace(/&#8199;/gi, "\u2007")
+    .replace(/&#8200;/gi, "\u2008")
+    .replace(/&#8201;/gi, "\u2009")
+    .replace(/&#8202;/gi, "\u200A")
+    .replace(/&#8203;/gi, "\u200B")
+    .replace(/&#8239;/gi, "\u202F")
+    .replace(/&#8287;/gi, "\u205F")
+    .replace(/&#12288;/gi, "\u3000")
+    .replace(/&#65279;/gi, "\uFEFF");
 }
 
 function escapeAttribute(text: string) {

diff --git a/packages/@atjson/renderer-commonmark/test/commonmark-test.ts b/packages/@atjson/renderer-commonmark/test/commonmark-test.ts
@@ -765,6 +765,21 @@ After all the lists
       test.each([
         ["&emsp;", "\u2003"],
         ["&nbsp;", "\u00a0"],
+        ["&#8192;", "\u2000"],
+        ["&#8193;", "\u2001"],
+        ["&ensp;", "\u2002"],
+        ["&#8196;", "\u2004"],
+        ["&#8197;", "\u2005"],
+        ["&#8198;", "\u2006"],
+        ["&#8199;", "\u2007"],
+        ["&#8200;", "\u2008"],
+        ["&#8201;", "\u2009"],
+        ["&#8202;", "\u200A"],
+        ["&#8203;", "\u200B"],
+        ["&#8239;", "\u202F"],
+        ["&#8287;", "\u205F"],
+        ["&#12288;", "\u3000"],
+        ["&#65279;", "\uFEFF"],
       ])("%s", (entity, unicode) => {
         let doc = new OffsetSource({ content: unicode, annotations: [] });
         expect(CommonmarkRenderer.render(doc)).toBe(entity);
@@ -1379,7 +1394,7 @@ After all the lists
     });
 
     expect(CommonmarkRenderer.render(document)).toBe(
-      "&nbsp;\n\n**text**\n\n\u202F"
+      "&nbsp;\n\n**text**\n\n&#8239;"
     );
   });
 
@@ -1462,22 +1477,23 @@ After all the lists
     );
   });
 
-  test("emspaces are encoded", () => {
+  test("unicode whitespace characters are encoded", () => {
     let document = new OffsetSource({
-      content: "\u2003\u2003\u2003\u2003Hello \n    This is my text",
+      content:
+        "\u00A0\u2000\u2001\u2002\u2003\u2004\u2005\u2006\u2007\u2008\u2009\u200A\u200B\u202F\u205F\u3000\uFEFFHello \n    This is my text",
       annotations: [
         {
           id: "1",
           type: "-offset-paragraph",
           start: 0,
-          end: 11,
+          end: 24,
           attributes: {},
         },
         {
           id: "2",
           type: "-offset-paragraph",
-          start: 11,
-          end: 30,
+          start: 24,
+          end: 43,
           attributes: {},
         },
       ],
@@ -1486,7 +1502,7 @@ After all the lists
     let markdown = CommonmarkRenderer.render(document);
 
     expect(markdown).toBe(
-      "&emsp;&emsp;&emsp;&emsp;Hello\n\nThis is my text\n\n"
+      "&nbsp;&#8192;&#8193;&ensp;&emsp;&#8196;&#8197;&#8198;&#8199;&#8200;&#8201;&#8202;&#8203;&#8239;&#8287;&#12288;&#65279;Hello\n\nThis is my text\n\n"
     );
     // Make sure we're not generating code in the round-trip
     expect(markdown).toEqual(

diff --git a/packages/@atjson/source-commonmark/test/commonmark-test.ts b/packages/@atjson/source-commonmark/test/commonmark-test.ts
@@ -15,7 +15,9 @@ describe("whitespace", () => {
 
   describe("non-breaking spaces", () => {
     test("html entities are converted to unicode characters", () => {
-      let doc = CommonMarkSource.fromRaw("1\n\n&#8239;\n\n&nbsp;&emsp;\n\n2");
+      let doc = CommonMarkSource.fromRaw(
+        "1\n\n&#8239;\n\n&nbsp;&emsp;\n\n&#8192;&#8193;&ensp;&#8196;&#8197;&#8198;&#8199;&#8200;&#8201;&#8202;&#8203;&#8239;&#8287;&#12288;&#65279;\n\n2"
+      );
       let hir = new HIR(doc);
       expect(hir.toJSON()).toMatchObject({
         type: "root",
@@ -36,6 +38,13 @@ describe("whitespace", () => {
             attributes: {},
             children: ["\u00A0\u2003"],
           },
+          {
+            type: "paragraph",
+            attributes: {},
+            children: [
+              "\u2000\u2001\u2002\u2004\u2005\u2006\u2007\u2008\u2009\u200A\u200B\u202F\u205F\u3000\uFEFF",
+            ],
+          },
           {
             type: "paragraph",
             attributes: {},