Revert "Revert "Merge/6.2.0""

.eslintrc.json

@@ -9,12 +9,27 @@
         "structuredClone": true
     },
     "extends": ["airbnb", "prettier"],
-    "plugins": ["chai-friendly", "jsdoc", "prettier", "unicorn"],
+    "plugins": [
+        "chai-friendly",
+        "jsdoc",
+        "prettier",
+        "unicorn",
+        "@typescript-eslint"
+    ],
+    "parser": "@typescript-eslint/parser",
     "parserOptions": {
         "sourceType": "module",
         "ecmaVersion": 2021
     },
     "settings": {
+        "import/parsers": {
+            "@typescript-eslint/parser": [".js", ".ts", ".tsx"]
+        },
+        "import/resolver": {
+            "typescript": {
+                "alwaysTryTypes": true
+            }
+        },
         "jsdoc": {
             "tagNamePreference": {
                 "returns": "return"
@@ -31,7 +46,8 @@
                     "enketo/widgets",
                     "enketo/translator",
                     "enketo/dialog",
-                    "enketo/file-manager"
+                    "enketo/file-manager",
+                    "enketo-transformer/web"
                 ]
             }
         ],

CHANGELOG.md

@@ -3,6 +3,35 @@
 All notable changes to this project will be documented in this file.
 This project adheres to [Semantic Versioning](http://semver.org/).
 
+## 6.2.0 - 2023-07-03
+
+-   Make pdf generation timeout configurable (#560)
+-   Added the Ukrainian translation (#569)
+-   Optimizations: do less redundant work, part 2 (enketo-core#976)
+-   Fix: gracefully handle undefined key (enketo-core#978)
+-   Fix: TOC navigation in pages theme (enketo-core#983)
+-   Fix: Respect existing val of type='time' (enketo-core#980)
+
+## 6.1.0 - 2023-05-10
+
+-   removed: analog-scale widget (#525) _Note: now maintained by OpenClinica (https://github.com/OpenClinica/enketo-express-oc/tree/master/widget/analog-scale). Not considered a breaking change because usage outside of OpenClinica is not known_
+-   fixed: The app UI strings in newly created repeats are not translated (#527)
+-   Fix: validate CSV header names according to XML spec (#530)
+-   Use pm2-runtime on docker/start.sh (#533)
+-   Fix: revert jQuery upgrade (#539)
+-   Use client-side transform for preview-by-URL (#540)
+-   changed: updated Slovak translation (#542)
+-   Use exponential backoff for submission attempts in offline mode (#543)
+-   feat(app): add enketo namespace to all debug statements (#547)
+-   Add support for LineString and Polygon GeoJSON types (#554)
+-   Fix: further simplify geopicker widget selector (#954)
+-   Fix: undesired newlines in print view in select/select1 labels (#957)
+-   Add browser-based transformation (#171)
+-   Reimplement draw/signature/annotate widget to preserve original image size when annotating and improve resize performance (#960)
+-   Fix: focus first active input/widget on load, page navigation, user-added repeat (#969)
+-   Performance improvements: NumberInput widgets, `excludeNonRelevant` (#971)
+-   Fix: localized numeral entry by keyboard in number input widgets (#973)
+
 ## 6.0.0 - 2023-02-08
 
 -   **BREAKING CHANGE**: Removed IE11 support (enketo-core#946)

Gruntfile.js

@@ -6,7 +6,7 @@ module.exports = (grunt) => {
         '!.nyc_output',
         '!**/node_modules/**',
         '!test/client/forms/forms.js',
-        '!public/js/build/*',
+        '!public/js/build/**',
         '!docs/**',
         '!test-coverage/**',
     ];
@@ -105,7 +105,7 @@ module.exports = (grunt) => {
                     'find locales -name "translation-combined.json" -delete && rm -fr locales/??',
             },
             'clean-js': {
-                command: 'rm -f public/js/build/* && rm -f public/js/*.js',
+                command: 'rm -rf public/js/build/* && rm -f public/js/*.js',
             },
             translation: {
                 command:

README.md

@@ -37,7 +37,7 @@ OpenClinica users, in addition to the configuration documentation linked above,
 
 Enketo was initiated in 2009 by Martijn van de Rijdt as a web-based alternative or complement to [ODK Collect](https://docs.getodk.org/collect-intro/). It has become a core component of the ODK ecosystem and been adopted by several organizations beyond that ecosystem.
 
-As of 2022, Enketo is maintained by [Trevor Schmidt](https://github.com/eyelidlessness/) and [the ODK team](https://getodk.org/about/team.html). Martijn continues to provide advice and continuity. The ODK project sets priorities in collaboration with its [Technical Advisory Board](https://getodk.org/about/ecosystem.html).
+As of 2022, Enketo is maintained by the [ODK team](https://getodk.org/about/team.html) (primarily [Trevor Schmidt](https://github.com/eyelidlessness/)). Martijn continues to provide advice and continuity. The ODK project sets priorities in collaboration with its [Technical Advisory Board](https://getodk.org/about/ecosystem.html).
 
 Our current primary goals are:
 
@@ -49,7 +49,7 @@ Feature requests and project discussion are welcome on the [ODK forum](https://f
 
 ### Translation
 
-The user interface was translated by: Karol Kozyra (Swedish), Badisches Rotes Kreuz (German), Serkan Tümbaş (Turkish), Hélène Martin (French), Gurjot Sidhu(Hindi, Panjabi), "Abcmen" (Turkish), Otto Saldadze, Makhare Atchaidze, David Sichinava, Elene Ergeshidze (Georgian), Nancy Shapsough (Arabic), Noel O'Boyle (French), Miguel Moreno (Spanish), Tortue Torche (French), Bekim Kajtazi (Albanian), Marc Kreidler (German), Darío Hereñú (Spanish), Viktor S. (Russian), Alexander Torrado Leon (Spanish), Peter Smith (Portugese, Spanish), Przemysław Gumułka (Polish), Niklas Ljungkvist, Sid Patel (Swedish), Katri Jalava (Finnish), Francesc Garre (Spanish), Sounay Phothisane (Lao), Linxin Guo (Chinese), Emmanuel Jean, Renaud Gaudin (French), Trần Quý Phi (Vietnamese), Reza Doosti, Hossein Azad, Davood Mottalee (Persian), Tomas Skripcak (Slovak, Czech, German), Daniela Baldova (Czech), Robert Michael Lundin (Norwegian), Margaret Ndisha, Charles Mutisya (Swahili), Panzero Mauro (Italian), Gabriel Kreindler (Romanian), Jason Reeder, Omar Nazar, Sara Sameer, David Gessel (Arabic), Tino Kreutzer (German), Wasilis Mandratzis-Walz (German, Greek), Luis Molina (Spanish), Martijn van de Rijdt (Dutch).
+The user interface was translated by: Oleg Zhyliak (Ukrainian), Karol Kozyra (Swedish), Badisches Rotes Kreuz (German), Serkan Tümbaş (Turkish), Hélène Martin (French), Gurjot Sidhu(Hindi, Panjabi), "Abcmen" (Turkish), Otto Saldadze, Makhare Atchaidze, David Sichinava, Elene Ergeshidze (Georgian), Nancy Shapsough (Arabic), Noel O'Boyle (French), Miguel Moreno (Spanish), Tortue Torche (French), Bekim Kajtazi (Albanian), Marc Kreidler (German), Darío Hereñú (Spanish), Viktor S. (Russian), Alexander Torrado Leon (Spanish), Peter Smith (Portugese, Spanish), Przemysław Gumułka (Polish), Niklas Ljungkvist, Sid Patel (Swedish), Katri Jalava (Finnish), Francesc Garre (Spanish), Sounay Phothisane (Lao), Linxin Guo (Chinese), Emmanuel Jean, Renaud Gaudin (French), Trần Quý Phi (Vietnamese), Reza Doosti, Hossein Azad, Davood Mottalee (Persian), Tomas Skripcak (Slovak, Czech, German), Daniela Baldova (Czech), Robert Michael Lundin (Norwegian), Margaret Ndisha, Charles Mutisya (Swahili), Panzero Mauro (Italian), Gabriel Kreindler (Romanian), Jason Reeder, Omar Nazar, Sara Sameer, David Gessel (Arabic), Tino Kreutzer (German), Wasilis Mandratzis-Walz (German, Greek), Luis Molina (Spanish), Martijn van de Rijdt (Dutch).
 
 _Send a message if you'd like to contribute! We use an easy web interface provided by [Transifex](https://www.transifex.com/projects/p/enketo-express/)._
 
@@ -80,13 +80,15 @@ OpenClinica has a few [additional text strings](./locales/src/en/translation-add
 
 ### Funding
 
-The development of this application was funded by [KoBo Toolbox (Harvard Humanitarian Initiative)](http://www.kobotoolbox.org), [iMMAP](http://immap.org), [OpenClinica](https://openclinica.com), [London School of Hygiene and Tropical Medicine](https://opendatakit.lshtm.ac.uk/), [DIAL Open Source Center](https://www.osc.dial.community/) and [Enketo LLC](https://www.linkedin.com/company/enketo-llc). The [Enketo-core](https://github.com/enketo/enketo-core) library (the form engine + themes) used in this application obtained significant funding from [SEL (Columbia University)](http://modi.mech.columbia.edu/), the [Santa Fe Institute](http://www.santafe.edu/), [Ona](https://ona.io) and the [HRP project](http://www.who.int/reproductivehealth/topics/mhealth/en/).
+The development of this application is now led by [ODK](https://getodk.org) and funded by customers of the ODK Cloud hosted service.
+
+Past funders include [KoBo Toolbox (Harvard Humanitarian Initiative)](http://www.kobotoolbox.org), [iMMAP](http://immap.org), [OpenClinica](https://openclinica.com), [London School of Hygiene and Tropical Medicine](https://opendatakit.lshtm.ac.uk/), [DIAL Open Source Center](https://www.osc.dial.community/) and [Enketo LLC](https://www.linkedin.com/company/enketo-llc). Also see [Enketo Core sponsors](https://github.com/enketo/enketo-core#sponsors).
 
 ### License
 
 See [the license document](https://github.com/enketo/enketo-express/blob/master/LICENSE) for this application's license.
 
-Note that some of the libraries used in this app have a different license. In particular note [this one](https://github.com/enketo/enketo-xpathjs).
+Note that some of the libraries used in this app have a different license.
 
 Note the 'Powered by Enketo' footer requirement as explained in [enketo-core](https://github.com/enketo/enketo-core#license). This requirement is applicable to all Enketo apps, including this one, unless an exemption was granted.
 

app/controllers/media-controller.js

@@ -8,7 +8,7 @@ const request = require('request');
 const express = require('express');
 
 const router = express.Router();
-const debug = require('debug')('media-controller');
+const debug = require('debug')('enketo:media-controller');
 const {
     RequestFilteringHttpAgent,
     RequestFilteringHttpsAgent,

app/controllers/transformation-controller.js

@@ -4,7 +4,7 @@
 
 const transformer = require('enketo-transformer');
 const communicator = require('../lib/communicator');
-const { TranslatedError } = require('../lib/custom-error');
+const { ResponseError, TranslatedError } = require('../lib/custom-error');
 const surveyModel = require('../models/survey-model');
 const cacheModel = require('../models/cache-model');
 const account = require('../models/account-model');
@@ -13,7 +13,6 @@ const config = require('../models/config-model').server;
 const utils = require('../lib/utils');
 const routerUtils = require('../lib/router-utils');
 const express = require('express');
-const url = require('url');
 const mediaLib = require('../lib/media');
 
 const router = express.Router();
@@ -101,27 +100,15 @@ async function getSurveyParts(req, res, next) {
     /** @type {string | null} */
     let formId = null;
 
-    /** @type {string | null} */
-    let formFileName = null;
-
     try {
         let survey = await _getSurveyParams(req);
 
-        // A request with "xformUrl" body parameter was used (unlaunched form)
-        if (survey.info != null) {
-            formFileName = survey.info.downloadUrl.replace(
-                /.*\/([^/]+)$/,
-                '$1'
-            );
-            survey = await _getFormDirectly(survey);
-
-            _respond(res, survey);
+        formId = survey.openRosaId;
 
-            return;
+        if (formId == null) {
+            throw new ResponseError(404);
         }
 
-        formId = survey.openRosaId;
-
         const authenticated = await _authenticate(survey);
         const cached = await _getFormFromCache(authenticated);
 
@@ -142,14 +129,10 @@ async function getSurveyParts(req, res, next) {
         });
     } catch (error) {
         if (error.status === 403) {
-            const notFoundError =
-                formId == null
-                    ? new TranslatedError('error.notfounddirectformurl', {
-                          formFileName,
-                      })
-                    : new TranslatedError('error.notfoundinformlist', {
-                          formId,
-                      });
+            const notFoundError = new TranslatedError(
+                'error.notfoundinformlist',
+                { formId }
+            );
 
             notFoundError.status = 404;
 
@@ -183,17 +166,6 @@ function getSurveyHash(req, res, next) {
         .catch(next);
 }
 
-/**
- * @param {module:survey-model~SurveyObject} survey - survey object
- *
- * @return { Promise<module:survey-model~SurveyObject> } a Promise resolving with survey object with form transformation result
- *
- */
-function _getFormDirectly(survey) {
-    survey.openclinica = true;
-    return communicator.getXForm(survey).then(transformer.transform);
-}
-
 /**
  * @param {module:survey-model~SurveyObject} survey - survey object
  *
@@ -233,8 +205,11 @@ function _updateCache(survey) {
                 delete survey.mediaHash;
                 delete survey.mediaUrlHash;
                 delete survey.formHash;
-
-                return _getFormDirectly(survey).then(cacheModel.set);
+                survey.openclinica = true;
+                return communicator
+                    .getXForm(survey)
+                    .then(transformer.transform)
+                    .then(cacheModel.set);
             }
 
             return survey;
@@ -359,7 +334,6 @@ function _setCookieAndCredentials(survey, req) {
  * @return { Promise<module:survey-model~SurveyObject> } a Promise resolving with survey object
  */
 function _getSurveyParams(req) {
-    const params = req.body;
     const customParamName = req.app.get(
         'query parameter to pass to submission'
     );
@@ -376,32 +350,7 @@ function _getSurveyParams(req) {
                 return _setCookieAndCredentials(survey, req);
             });
     }
-    if (params.xformUrl) {
-        const urlObj = url.parse(params.xformUrl);
-        if (!urlObj || !urlObj.protocol || !urlObj.host) {
-            const error = new Error('Bad Request. Form URL is invalid.');
-            error.status = 400;
-            throw error;
-        }
-        const xUrl = `${urlObj.protocol}//${urlObj.host}${urlObj.pathname}`;
-
-        return account
-            .check({
-                openRosaServer: xUrl,
-            })
-            .then(
-                (
-                    survey // no need to check quota
-                ) =>
-                    Promise.resolve({
-                        info: {
-                            downloadUrl: params.xformUrl,
-                        },
-                        account: survey.account,
-                    })
-            )
-            .then((survey) => _setCookieAndCredentials(survey, req));
-    }
+
     const error = new Error('Bad Request. Survey information not complete.');
     error.status = 400;
     throw error;

app/lib/communicator.js

@@ -6,7 +6,7 @@ const request = require('request');
 const { Auth } = require('request/lib/auth');
 const TError = require('./custom-error').TranslatedError;
 const config = require('../models/config-model').server;
-const debug = require('debug')('openrosa-communicator');
+const debug = require('debug')('enketo:openrosa-communicator');
 const Xml2Js = require('xml2js');
 
 const parser = new Xml2Js.Parser();

app/lib/pdf.js

@@ -121,6 +121,7 @@ async function get(
             },
             scale,
             printBackground: true,
+            timeout,
         });
     } catch (e) {
         e.status = e.status || 400;

app/models/cache-model.js

@@ -9,7 +9,7 @@ const utils = require('../lib/utils');
 
 const prefix = 'ca:';
 const expiry = 30 * 24 * 60 * 60;
-const debug = require('debug')('cache-model');
+const debug = require('debug')('enketo:cache-model');
 
 const clientGet = promisify(cacheClient.get).bind(cacheClient);
 const clientSet = promisify(cacheClient.set).bind(cacheClient);

app/models/survey-model.js

@@ -8,7 +8,7 @@ const TError = require('../lib/custom-error').TranslatedError;
 const config = require('./config-model').server;
 
 const pending = {};
-const debug = require('debug')('survey-model');
+const debug = require('debug')('enketo:survey-model');
 
 /**
  * @typedef {import('./account-model').AccountObj} AccountObj

app/views/index.pug

@@ -57,6 +57,8 @@ block content
             li= t("langs.sv")
           when "lo"
             li= t("langs.lo")
+          when "uk"
+            li= t("langs.uk")
           when "sk"
             li= t("langs.sk")
           when "tr"

app/views/surveys/webform.pug

@@ -20,7 +20,7 @@ block style
 
 block script
   - var suffix = (type && type !== 'single') ? '-' + (type === 'full' || type === 'preview' || type === 'fieldsubmission' ? 'oc' : type): ''
-    script#main-script(defer, module, src=`${basePath}${offlinePath || ''}/js/build/enketo-webform${suffix}.js`)
+    script#main-script(defer, type='module', src=`${basePath}${offlinePath || ''}/js/build/enketo-webform${suffix}.js`)
 
   -// load jini stuff asynchronously (OC)
   if jini && !headless

config/build.js

@@ -1,29 +1,28 @@
-const alias = require('esbuild-plugin-alias');
+// @ts-check
+
 const path = require('path');
 const pkg = require('../package.json');
 
 const cwd = process.cwd();
 
 const entryPoints = pkg.entries.map((entry) => path.resolve(cwd, entry));
 
-const isProduction = process.env.NODE_ENV === 'production';
-
-module.exports = {
+module.exports = /** @satisfies {import('esbuild').BuildOptions} */ ({
+    alias: Object.fromEntries(
+        Object.entries(pkg.browser).map(([key, value]) => [
+            key,
+            path.resolve(cwd, `${value}.js`),
+        ])
+    ),
     bundle: true,
+    chunkNames: 'chunks/[name]-[hash]',
     entryPoints,
-    format: 'iife',
-    minify: isProduction,
+    entryNames: '[name]',
+    external: ['crypto', 'libxslt'],
+    format: 'esm',
+    minify: true,
     outdir: path.resolve(cwd, './public/js/build'),
-    plugins: [
-        alias(
-            Object.fromEntries(
-                Object.entries(pkg.browser).map(([key, value]) => [
-                    key,
-                    path.resolve(cwd, `${value}.js`),
-                ])
-            )
-        ),
-    ],
-    sourcemap: isProduction ? false : 'inline',
+    sourcemap: true,
+    splitting: true,
     target: ['chrome89', 'edge89', 'firefox90', 'safari13'],
-};
+});

config/express.js

@@ -14,7 +14,7 @@ const errorHandler = require('../app/controllers/error-handler');
 
 const controllersPath = path.join(__dirname, '../app/controllers');
 const app = express();
-const debug = require('debug')('express');
+const debug = require('debug')('enketo:express');
 const config = require('../app/models/config-model');
 
 // general