Merge branch 'beta'

# Conflicts:
#	docs/backend_api/BAttachment.html
#	docs/backend_api/BNote.html
#	docs/backend_api/BackendScriptApi.html
#	package-lock.json
#	package.json

.gitpod.yml

@@ -2,7 +2,7 @@ image:
   file: .gitpod.dockerfile
 
 tasks:
-    - before: nvm install 18.18.0 && nvm use 18.18.0
+    - before: nvm install 18.18.2 && nvm use 18.18.2
       init: npm install
       command: npm run start-server
 

Dockerfile

@@ -1,5 +1,5 @@
 # !!! Don't try to build this Dockerfile directly, run it through bin/build-docker.sh script !!!
-FROM node:18.18.0-alpine
+FROM node:18.18.2-alpine
 
 # Create app directory
 WORKDIR /usr/src/app

bin/build-server.sh

@@ -1,7 +1,7 @@
 #!/usr/bin/env bash
 
 PKG_DIR=dist/trilium-linux-x64-server
-NODE_VERSION=18.18.0
+NODE_VERSION=18.18.2
 
 if [ "$1" != "DONTCOPY" ]
 then

bin/copy-trilium.sh

@@ -5,7 +5,7 @@ if [[ $# -eq 0 ]] ; then
     exit 1
 fi
 
-n exec 18.18.0 npm run webpack
+n exec 18.18.2 npm run webpack
 
 DIR=$1
 
@@ -27,7 +27,7 @@ cp -r electron.js $DIR/
 cp webpack-* $DIR/
 
 # run in subshell (so we return to original dir)
-(cd $DIR && n exec 18.18.0 npm install --only=prod)
+(cd $DIR && n exec 18.18.2 npm install --only=prod)
 
 # cleanup of useless files in dependencies
 rm -r $DIR/node_modules/image-q/demo

docs/backend_api/services_backend_script_api.js.html

@@ -47,6 +47,7 @@ <h1 class="page-title">Source: services/backend_script_api.js</h1>
 const specialNotesService = require("./special_notes");
 const branchService = require("./branches");
 const exportService = require("./export/zip");
+const syncMutex = require("./sync_mutex.js");
 
 
 /**
@@ -628,6 +629,20 @@ <h1 class="page-title">Source: services/backend_script_api.js</h1>
         }
     };
 
+    /**
+     * Sync process can make data intermittently inconsistent. Scripts which require strong data consistency
+     * can use this function to wait for a possible sync process to finish and prevent new sync process from starting
+     * while it is running.
+     *
+     * Because this is an async process, the inner callback doesn't have automatic transaction handling, so in case
+     * you need to make some DB changes, you need to surround your call with api.transactional(...)
+     *
+     * @method
+     * @param {function} callback - function to be executed while sync process is not running
+     * @returns {Promise} - resolves once the callback is finished (callback is awaited)
+     */
+    this.runOutsideOfSync = syncMutex.doExclusively;
+
     /**
      * This object contains "at your risk" and "no BC guarantees" objects for advanced use cases.
      *

src/public/app/widgets/type_widgets/canvas.js

@@ -217,7 +217,7 @@ export default class ExcalidrawTypeWidget extends TypeWidget {
                 };
             }
 
-            const {elements, files, appState} = content;
+            const {elements, files, appState = {}} = content;
 
             appState.theme = this.themeStyle;
 

src/public/app/widgets/type_widgets/options/advanced/database_anonymization.js

@@ -20,6 +20,10 @@ const TPL = `
     <p>You can decide yourself if you want to provide a fully or lightly anonymized database. Even fully anonymized DB is very useful, however in some cases lightly anonymized database can speed up the process of bug identification and fixing.</p>
     
     <button class="anonymize-light-button btn">Save lightly anonymized database</button>
+    
+    <h5>Existing anonymized databases</h5>
+    
+    <ul class="existing-anonymized-databases"></ul>
 </div>`;
 
 export default class DatabaseAnonymizationOptions extends OptionsWidget {
@@ -38,6 +42,8 @@ export default class DatabaseAnonymizationOptions extends OptionsWidget {
             else {
                 toastService.showMessage(`Created fully anonymized database in ${resp.anonymizedFilePath}`, 10000);
             }
+
+            this.refresh();
         });
 
         this.$anonymizeLightButton.on('click', async () => {
@@ -51,6 +57,24 @@ export default class DatabaseAnonymizationOptions extends OptionsWidget {
             else {
                 toastService.showMessage(`Created lightly anonymized database in ${resp.anonymizedFilePath}`, 10000);
             }
+
+            this.refresh();
+        });
+
+        this.$existingAnonymizedDatabases = this.$widget.find(".existing-anonymized-databases");
+    }
+
+    optionsLoaded(options) {
+        server.get("database/anonymized-databases").then(anonymizedDatabases => {
+            this.$existingAnonymizedDatabases.empty();
+
+            if (!anonymizedDatabases.length) {
+                anonymizedDatabases = [{filePath: "no anonymized database yet"}];
+            }
+
+            for (const {filePath} of anonymizedDatabases) {
+                this.$existingAnonymizedDatabases.append($("<li>").text(filePath));
+            }
         });
     }
 }

src/public/app/widgets/type_widgets/options/backup.js

@@ -37,6 +37,12 @@ const TPL = `
     
     <button class="backup-database-button btn">Backup database now</button>
 </div>
+
+<div class="options-section">
+    <h4>Existing backups</h4>
+    
+    <ul class="existing-backup-list"></ul>
+</div>
 `;
 
 export default class BackupOptions extends OptionsWidget {
@@ -49,6 +55,8 @@ export default class BackupOptions extends OptionsWidget {
             const {backupFile} = await server.post('database/backup-database');
 
             toastService.showMessage(`Database has been backed up to ${backupFile}`, 10000);
+
+            this.refresh();
         });
 
         this.$dailyBackupEnabled = this.$widget.find(".daily-backup-enabled");
@@ -63,11 +71,25 @@ export default class BackupOptions extends OptionsWidget {
 
         this.$monthlyBackupEnabled.on('change', () =>
             this.updateCheckboxOption('monthlyBackupEnabled', this.$monthlyBackupEnabled));
+
+        this.$existingBackupList = this.$widget.find(".existing-backup-list");
     }
 
     optionsLoaded(options) {
         this.setCheckboxState(this.$dailyBackupEnabled, options.dailyBackupEnabled);
         this.setCheckboxState(this.$weeklyBackupEnabled, options.weeklyBackupEnabled);
         this.setCheckboxState(this.$monthlyBackupEnabled, options.monthlyBackupEnabled);
+
+        server.get("database/backups").then(backupFiles => {
+            this.$existingBackupList.empty();
+
+            if (!backupFiles.length) {
+                backupFiles = [{filePath: "no backup yet", ctime: ''}];
+            }
+
+            for (const {filePath, ctime} of backupFiles) {
+                this.$existingBackupList.append($("<li>").text(`${filePath} ${ctime ? ` - ${ctime}` : ''}`));
+            }
+        });
     }
 }

src/routes/api/database.js

@@ -6,8 +6,8 @@ const backupService = require('../../services/backup');
 const anonymizationService = require('../../services/anonymization');
 const consistencyChecksService = require('../../services/consistency_checks');
 
-async function anonymize(req) {
-    return await anonymizationService.createAnonymizedCopy(req.params.type);
+function getExistingBackups() {
+    return backupService.getExistingBackups();
 }
 
 async function backupDatabase() {
@@ -22,6 +22,18 @@ function vacuumDatabase() {
     log.info("Database has been vacuumed.");
 }
 
+function findAndFixConsistencyIssues() {
+    consistencyChecksService.runOnDemandChecks(true);
+}
+
+function getExistingAnonymizedDatabases() {
+    return anonymizationService.getExistingAnonymizedDatabases();
+}
+
+async function anonymize(req) {
+    return await anonymizationService.createAnonymizedCopy(req.params.type);
+}
+
 function checkIntegrity() {
     const results = sql.getRows("PRAGMA integrity_check");
 
@@ -32,14 +44,12 @@ function checkIntegrity() {
     };
 }
 
-function findAndFixConsistencyIssues() {
-    consistencyChecksService.runOnDemandChecks(true);
-}
-
 module.exports = {
+    getExistingBackups,
     backupDatabase,
     vacuumDatabase,
     findAndFixConsistencyIssues,
+    getExistingAnonymizedDatabases,
     anonymize,
     checkIntegrity
 };

src/routes/routes.js

@@ -289,9 +289,11 @@ function register(app) {
     apiRoute(GET, '/api/sql/schema', sqlRoute.getSchema);
     apiRoute(PST, '/api/sql/execute/:noteId', sqlRoute.execute);
     route(PST, '/api/database/anonymize/:type', [auth.checkApiAuthOrElectron, csrfMiddleware], databaseRoute.anonymize, apiResultHandler, false);
+    apiRoute(GET, '/api/database/anonymized-databases', databaseRoute.getExistingAnonymizedDatabases);
 
     // backup requires execution outside of transaction
     route(PST, '/api/database/backup-database', [auth.checkApiAuthOrElectron, csrfMiddleware], databaseRoute.backupDatabase, apiResultHandler, false);
+    apiRoute(GET, '/api/database/backups', databaseRoute.getExistingBackups);
 
     // VACUUM requires execution outside of transaction
     route(PST, '/api/database/vacuum-database', [auth.checkApiAuthOrElectron, csrfMiddleware], databaseRoute.vacuumDatabase, apiResultHandler, false);

src/services/anonymization.js

@@ -4,6 +4,7 @@ const dataDir = require("./data_dir");
 const dateUtils = require("./date_utils");
 const Database = require("better-sqlite3");
 const sql = require("./sql");
+const path = require("path");
 
 function getFullAnonymizationScript() {
     // we want to delete all non-builtin attributes because they can contain sensitive names and values
@@ -70,7 +71,21 @@ async function createAnonymizedCopy(type) {
     };
 }
 
+function getExistingAnonymizedDatabases() {
+    if (!fs.existsSync(dataDir.ANONYMIZED_DB_DIR)) {
+        return [];
+    }
+
+    return fs.readdirSync(dataDir.ANONYMIZED_DB_DIR)
+        .filter(fileName => fileName.includes("anonymized"))
+        .map(fileName => ({
+            fileName: fileName,
+            filePath: path.resolve(dataDir.ANONYMIZED_DB_DIR, fileName)
+        }));
+}
+
 module.exports = {
     getFullAnonymizationScript,
-    createAnonymizedCopy
+    createAnonymizedCopy,
+    getExistingAnonymizedDatabases
 }

src/services/backend_script_api.js

@@ -19,6 +19,7 @@ const SpacedUpdate = require("./spaced_update");
 const specialNotesService = require("./special_notes");
 const branchService = require("./branches");
 const exportService = require("./export/zip");
+const syncMutex = require("./sync_mutex.js");
 
 
 /**
@@ -600,6 +601,20 @@ function BackendScriptApi(currentNote, apiParams) {
         }
     };
 
+    /**
+     * Sync process can make data intermittently inconsistent. Scripts which require strong data consistency
+     * can use this function to wait for a possible sync process to finish and prevent new sync process from starting
+     * while it is running.
+     *
+     * Because this is an async process, the inner callback doesn't have automatic transaction handling, so in case
+     * you need to make some DB changes, you need to surround your call with api.transactional(...)
+     *
+     * @method
+     * @param {function} callback - function to be executed while sync process is not running
+     * @returns {Promise} - resolves once the callback is finished (callback is awaited)
+     */
+    this.runOutsideOfSync = syncMutex.doExclusively;
+
     /**
      * This object contains "at your risk" and "no BC guarantees" objects for advanced use cases.
      *

src/services/backup.js

@@ -8,6 +8,22 @@ const log = require('./log');
 const syncMutexService = require('./sync_mutex');
 const cls = require('./cls');
 const sql = require('./sql');
+const path = require('path');
+
+function getExistingBackups() {
+    if (!fs.existsSync(dataDir.BACKUP_DIR)) {
+        return [];
+    }
+
+    return fs.readdirSync(dataDir.BACKUP_DIR)
+        .filter(fileName => fileName.includes("backup"))
+        .map(fileName => {
+            const filePath = path.resolve(dataDir.BACKUP_DIR, fileName);
+            const stat = fs.statSync(filePath)
+
+            return {fileName, filePath, ctime: stat.ctime};
+        });
+}
 
 function regularBackup() {
     cls.init(() => {
@@ -58,6 +74,7 @@ if (!fs.existsSync(dataDir.BACKUP_DIR)) {
 }
 
 module.exports = {
+    getExistingBackups,
     backupNow,
     regularBackup
 };

src/services/build.js

@@ -1 +1 @@
-module.exports = { buildDate:"2023-09-29T00:54:45+02:00", buildRevision: "e5555beea9a1638fefa218118e0596f4cfc1f4d0" };
+module.exports = { buildDate:"2023-10-07T23:02:47+03:00", buildRevision: "3d15aeae58224ac8716dd58938458e89af9bf7a0" };

src/services/builtin_attributes.js

@@ -68,6 +68,7 @@ module.exports = [
     { type: 'label', name: 'executeDescription'},
     { type: 'label', name: 'newNotesOnTop'},
     { type: 'label', name: 'clipperInbox'},
+    { type: 'label', name: 'webViewSrc', isDangerous: true },
 
     // relation names
     { type: 'relation', name: 'internalLink' },

src/services/cloning.js

@@ -7,7 +7,7 @@ const BBranch = require('../becca/entities/bbranch');
 const becca = require("../becca/becca");
 const log = require("./log");
 
-function cloneNoteToParentNote(noteId, parentNoteId, prefix) {
+function cloneNoteToParentNote(noteId, parentNoteId, prefix = null) {
     if (!(noteId in becca.notes) || !(parentNoteId in becca.notes)) {
         return { success: false, message: 'Note cannot be cloned because either the cloned note or the intended parent is deleted.' };
     }

src/services/import/zip.js

@@ -311,7 +311,7 @@ async function importZip(taskContext, fileBuffer, importRootNote) {
             return /^(?:[a-z]+:)?\/\//i.test(url);
         }
 
-        content = removeTrilumTags(content);
+        content = removeTriliumTags(content);
 
         content = content.replace(/<h1>([^<]*)<\/h1>/gi, (match, text) => {
             if (noteTitle.trim() === text.trim()) {
@@ -393,7 +393,7 @@ async function importZip(taskContext, fileBuffer, importRootNote) {
         return content;
     }
 
-    function removeTrilumTags(content) {
+    function removeTriliumTags(content) {
         const tagsToRemove = [
             '<h1 data-trilium-h1>([^<]*)<\/h1>',
             '<title data-trilium-title>([^<]*)<\/title>'