Support translator comments (#28)

package.json

@@ -1,6 +1,6 @@
 {
   "name": "react-jsx-i18n",
-  "version": "0.8.0",
+  "version": "0.9.0",
   "description": "Provides gettext-enhanced React components, a babel plugin for extracting the strings and a script to compile translated strings to a format usable by the components.",
   "main": "client/index.js",
   "types": "client/index.d.ts",
@@ -31,7 +31,6 @@
   },
   "homepage": "https://github.com/indico/react-jsx-i18n#readme",
   "dependencies": {
-    "babel-plugin-extract-text": "^2.0.0",
     "chalk": "^2.4.2",
     "gettext-parser": "^4.0.3",
     "glob": "^7.1.6",
@@ -42,8 +41,8 @@
   "peerDependencies": {
     "@babel/core": "*",
     "@babel/preset-react": "*",
-    "react": "*",
-    "prop-types": "*"
+    "prop-types": "*",
+    "react": "*"
   },
   "devDependencies": {
     "@babel/cli": "^7.8.4",

src/tools/extract-plugin.js

@@ -1,6 +1,8 @@
 import {relative} from 'path';
 import cleanJSXElementLiteralChild from '@babel/types/lib/utils/react/cleanJSXElementLiteralChild';
 
+const TRANSLATOR_COMMENT_TAG = 'i18n:';
+
 const collapseWhitespace = (string, trim = true) => {
   // for translated strings we never want consecutive or surrounding whitespace
   if (!string) {
@@ -114,16 +116,17 @@ const getContext = path => {
   return contextAttr ? contextAttr.value.value : undefined;
 };
 
-const processTranslate = (cfg, path, state, types) => {
+const processTranslate = (cfg, path, state, comment, types) => {
   const translatableString = processElement(path, types, true);
   return {
     msgid: translatableString,
     msgctxt: getContext(path),
+    extracted: comment,
     reference: getLocation(cfg, path, state),
   };
 };
 
-const processTranslateString = (cfg, path, state, funcName, types) => {
+const processTranslateString = (cfg, path, state, comment, types) => {
   const args = path.node.arguments;
   if (args.length === 0) {
     throw path.buildCodeFrameError('Translate.string() called with no arguments');
@@ -136,10 +139,11 @@ const processTranslateString = (cfg, path, state, funcName, types) => {
     msgid,
     msgctxt,
     reference: getLocation(cfg, path, state),
+    extracted: comment,
   };
 };
 
-const processPluralTranslate = (cfg, path, state, types) => {
+const processPluralTranslate = (cfg, path, state, comment, types) => {
   let singularPath, pluralPath;
   path
     .get('children')
@@ -171,11 +175,12 @@ const processPluralTranslate = (cfg, path, state, types) => {
     msgid: processElement(singularPath, types, true),
     msgid_plural: processElement(pluralPath, types, true),
     msgctxt: getContext(path),
+    extracted: comment,
     reference: getLocation(cfg, path, state),
   };
 };
 
-const processPluralTranslateString = (cfg, path, state, funcName, types) => {
+const processPluralTranslateString = (cfg, path, state, comment, types) => {
   const args = path.node.arguments;
   if (args.length < 2) {
     throw path.buildCodeFrameError('PluralTranslate.string() called with less than 2 arguments');
@@ -191,20 +196,61 @@ const processPluralTranslateString = (cfg, path, state, funcName, types) => {
     msgid_plural,
     msgctxt,
     reference: getLocation(cfg, path, state),
+    extracted: comment,
   };
 };
 
+function getPrecedingComment(line, comments) {
+  if (!comments[line - 1]) {
+    return;
+  }
+  const comment = [];
+  for (let i = line - 1; i >= 0; i--) {
+    if (comments[i]?.length > 0) {
+      comment.push(comments[i].join('\n'));
+    } else {
+      break;
+    }
+  }
+  return comment.reverse().join('\n');
+}
+
 const makeI18nPlugin = cfg => {
   const entries = [];
+  let currFile = '';
+  let translatorComments = {};
   const i18nPlugin = ({types}) => {
     return {
       visitor: {
+        Program(path, state) {
+          if (currFile !== state.file.opts.filename) {
+            // clear translator comments when the file changes
+            currFile = state.file.opts.filename;
+            translatorComments = {};
+          }
+          path.container.comments
+            .filter(comment => comment.value.trim().startsWith(TRANSLATOR_COMMENT_TAG))
+            .forEach(comment => {
+              const endLine = comment.loc.end.line;
+              if (!translatorComments[endLine]) {
+                translatorComments[endLine] = [];
+              }
+              translatorComments[endLine].push(
+                comment.value
+                  .trim()
+                  .slice(TRANSLATOR_COMMENT_TAG.length)
+                  .trimStart()
+              );
+            });
+        },
         JSXElement(path, state) {
           const elementName = path.node.openingElement.name.name;
+          const line = path.node.loc.start.line;
+          const comment = getPrecedingComment(line, translatorComments);
           if (elementName === 'Translate') {
-            entries.push(processTranslate(cfg, path, state, types));
+            entries.push(processTranslate(cfg, path, state, comment, types));
           } else if (elementName === 'PluralTranslate') {
-            entries.push(processPluralTranslate(cfg, path, state, types));
+            entries.push(processPluralTranslate(cfg, path, state, comment, types));
           }
         },
         CallExpression(path, state) {
@@ -226,11 +272,12 @@ const makeI18nPlugin = cfg => {
             return;
           }
           // we got a proper call of one of our translation functions
-          const qualifiedFuncName = `${elementName}.${funcName}`;
+          const line = path.node.loc.start.line;
+          const comment = getPrecedingComment(line, translatorComments);
           if (elementName === 'Translate') {
-            entries.push(processTranslateString(cfg, path, state, qualifiedFuncName, types));
+            entries.push(processTranslateString(cfg, path, state, comment, types));
           } else if (elementName === 'PluralTranslate') {
-            entries.push(processPluralTranslateString(cfg, path, state, qualifiedFuncName, types));
+            entries.push(processPluralTranslateString(cfg, path, state, comment, types));
           }
         },
       },

src/tools/extract.js

@@ -3,7 +3,6 @@
 import * as babel from '@babel/core';
 import gettextParser from 'gettext-parser';
 import moment from 'moment-timezone';
-import {mergeEntries} from 'babel-plugin-extract-text/src/builders';
 import makeI18nPlugin from './extract-plugin';
 
 const extractFromFiles = (files, cfg, headers = undefined, highlightErrors = true) => {
@@ -31,16 +30,77 @@ const extractFromFiles = (files, cfg, headers = undefined, highlightErrors = tru
     return {errors};
   }
 
-  const data = mergeEntries({}, entries);
-  data.headers = headers || {
-    'POT-Creation-Date': moment().format('YYYY-MM-YY HH:mmZZ'),
-    'Content-Type': 'text/plain; charset=utf-8',
-    'Content-Transfer-Encoding': '8bit',
-    'MIME-Version': '1.0',
-    'Generated-By': 'react-jsx-i18n-extract',
-  };
-
+  const data = mergeEntries(entries, headers);
   return {pot: gettextParser.po.compile(data).toString()};
 };
 
 export default extractFromFiles;
+
+function mergeEntries(entries, headers) {
+  const data = {
+    charset: 'UTF-8',
+    headers: headers || {
+      'POT-Creation-Date': moment().format('YYYY-MM-YY HH:mmZZ'),
+      'Content-Type': 'text/plain; charset=utf-8',
+      'Content-Transfer-Encoding': '8bit',
+      'MIME-Version': '1.0',
+      'Generated-By': 'react-jsx-i18n-extract',
+    },
+    translations: {},
+  };
+
+  entries.forEach(entry => {
+    const {msgid, msgid_plural: msgidPlural, msgctxt, extracted, reference} = entry;
+    const context = entry.msgctxt || '';
+
+    if (!data.translations[context]) {
+      data.translations[context] = {};
+    }
+
+    let existingEntry = data.translations[context][msgid];
+    if (!existingEntry) {
+      existingEntry = data.translations[context][msgid] = {};
+    }
+
+    const existingPlural = existingEntry.msgid_plural;
+    const existingTranslation = existingEntry.msgstr;
+    const existingExtracted = existingEntry.comments?.extracted;
+    const existingReference = existingEntry.comments?.reference;
+
+    data.translations[context][msgid] = {
+      msgid,
+      msgctxt,
+      msgstr: mergeTranslation(existingTranslation, msgidPlural ? ['', ''] : ['']),
+      msgid_plural: existingPlural || msgidPlural,
+      comments: {
+        reference: mergeReference(existingReference, reference),
+        extracted: mergeExtracted(existingExtracted, extracted),
+      },
+    };
+  });
+
+  return data;
+}
+
+function mergeReference(existingReference, newReference) {
+  if (existingReference) {
+    if (!newReference || existingReference.includes(newReference)) {
+      return existingReference;
+    }
+
+    return `${existingReference}\n${newReference}`;
+  }
+
+  return newReference;
+}
+
+function mergeExtracted(existingExtracted, extracted) {
+  return mergeReference(existingExtracted, extracted);
+}
+
+function mergeTranslation(existingTranslation, newTranslation) {
+  if (existingTranslation && existingTranslation.length === 2) {
+    return existingTranslation;
+  }
+  return newTranslation;
+}

test-data/comments1.jsx

@@ -0,0 +1,76 @@
+/* eslint-disable babel/quotes, react/prop-types, react/jsx-curly-brace-presence */
+
+import React from 'react';
+import {makeComponents, Singular, Plural, Param} from '../src/client';
+const {Translate, PluralTranslate} = makeComponents();
+
+// i18n: foo comment 1
+Translate.string('foo');
+// i18n: plural foo comment 1
+PluralTranslate.string('foo', 'foos', 42);
+
+// i18n: foo comment 2
+Translate.string('foo');
+// i18n: plural foo comment 2
+PluralTranslate.string('foo', 'foos', 42);
+
+/* i18n: multiline
+   comment
+*/
+Translate.string('multiline');
+
+/* i18n: multiline
+ *  comment with leading asterisks
+ */
+Translate.string('multiline');
+
+// No 'i18n:' prefix, should not be extracted
+Translate.string('foo');
+
+// i18n: Space between the comment and the 'Translate' call, this should not be extracted
+
+Translate.string('foo');
+
+// i18n: Both strings
+// i18n: should be extracted
+Translate.string('bar');
+
+/* i18n: multiple
+  multiline comments
+*/
+/* i18n: are not guaranteed to work
+  because you should not be using this anyway
+*/
+Translate.string('bar');
+
+// i18n: Only the last comment should be extracted
+// some other comment
+// i18n: translator comment
+Translate.string('baz');
+
+export function TestComponent() {
+  return (
+    <div>
+      {/* i18n: Title */}
+      <Translate>Hello & World</Translate>
+      {/* i18n: multiple */} {/* i18n: translator */}
+      {/* i18n: comments */}
+      <Translate>foo bar</Translate>
+      {/* i18n: rat counter */}
+      <PluralTranslate count={42}>
+        <Singular>
+          You have <Param name="count" value={1} /> rat.
+        </Singular>
+        <Plural>
+          You have <Param name="count" value={2} /> rats.
+        </Plural>
+      </PluralTranslate>
+      {/* i18n: xxx comment */}
+      {Translate.string('xxx')}
+      {() => {
+        // i18n: yyy comment
+        return PluralTranslate.string('yyy', 'yyys', 42);
+      }}
+    </div>
+  );
+}

test-data/comments2.jsx

@@ -0,0 +1,10 @@
+/* eslint-disable babel/quotes, react/prop-types, react/jsx-curly-brace-presence */
+
+import {makeComponents} from '../src/client';
+const {Translate, PluralTranslate} = makeComponents();
+
+// Keep this call on the same line as the corresponding call in comments1.jsx
+// This tests that the comments do not get mixed up when babel loads another file
+Translate.string('baz');
+// Same for this call
+PluralTranslate.string('baz', 'bazs', 42);