fix: save large text to fs in syncAdaptor

package.json

@@ -81,6 +81,7 @@
     "zx": "^7.2.3"
   },
   "resolutions": {
-    "react-native-reanimated": "3.3.0"
+    "react-native-reanimated": "$react-native-reanimated",
+    "tw5-typed": "$tw5-typed"
   }
 }

plugins/src/expo-file-system-syncadaptor/file-system-syncadaptor.ts

@@ -6,7 +6,6 @@ import type { AppDataService } from '../../../src/services/AppDataService/index.
 import type { BackgroundSyncService } from '../../../src/services/BackgroundSyncService/index.js';
 import type { NativeService } from '../../../src/services/NativeService/index.js';
 import type { WikiHookService } from '../../../src/services/WikiHookService/index.js';
-import { getFullSaveTiddlers } from '../../../src/services/WikiStorageService/ignoredTiddler.js';
 import type { WikiStorageService } from '../../../src/services/WikiStorageService/index.js';
 
 type ISyncAdaptorGetStatusCallback = (error: Error | null, isLoggedIn?: boolean, username?: string, isReadOnly?: boolean, isAnonymous?: boolean) => void;
@@ -236,11 +235,10 @@ class TidGiMobileFileSystemSyncAdaptor {
       const title = tiddler.fields.title;
       this.logger.log(`saveTiddler ${title}`);
       this.addRecentUpdatedTiddlersFromClient('modifications', title);
-      const saveFullTiddler = getFullSaveTiddlers(title).includes(title);
       const etag = await this.wikiStorageService.saveTiddler(
         title,
-        tiddler.fields.text,
-        saveFullTiddler ? JSON.stringify(tiddler.getFieldStrings()) : JSON.stringify({ ...tiddler.getFieldStrings({ exclude: ['text'] }), _is_skinny: '' }),
+        tiddler.getFieldStrings(),
+        $tw.wiki.isBinaryTiddler(title) ? 'base64' : undefined,
       );
       if (etag === undefined) {
         callback(new Error('Response from server is missing required `etag` header'));

src/services/BackgroundSyncService/index.ts

@@ -196,10 +196,6 @@ export class BackgroundSyncService {
         },
         body: JSON.stringify(request),
       }).then(response => response.json() as Promise<ISyncEndPointResponse>);
-      // DEBUG: console request
-      console.log(`request`, request);
-      // DEBUG: console response
-      console.log(`response`, response);
       if (response === undefined) return false;
       const { deletes, updates } = response;
       await this.#updateTiddlersFromServer(wiki, deletes, updates);
@@ -259,6 +255,7 @@ export class BackgroundSyncService {
             if (text !== undefined && this.checkIsLargeText(text, fieldsToSave.type as string)) {
               // save to fs instead of sqlite. See `WikiStorageService.#loadFromServer` for how we load it later.
               await this.saveToFSFromServer(wiki, title);
+              tiddler.text = null;
             } else {
               tiddler.text = text;
             }

src/services/SQLiteService/orm/index.ts

@@ -7,7 +7,7 @@ export class TiddlerSQLModel {
   title!: string;
 
   @Column('text')
-  text?: string;
+  text?: string | null;
 
   @Column('text')
   fields?: string;

src/services/WikiStorageService/index.ts

@@ -1,7 +1,8 @@
+/* eslint-disable unicorn/no-null */
 /* eslint-disable @typescript-eslint/require-await */
 import * as fs from 'expo-file-system';
 import { Observable } from 'rxjs';
-import type { IChangedTiddlers } from 'tiddlywiki';
+import type { IChangedTiddlers, ITiddlerFieldsParam } from 'tiddlywiki';
 import { getWikiTiddlerPathByTitle } from '../../constants/paths';
 import i18n from '../../i18n';
 import { TiddlersLogOperation } from '../../pages/Importer/createTable';
@@ -11,7 +12,7 @@ import { IWikiWorkspace } from '../../store/workspace';
 import { backgroundSyncService } from '../BackgroundSyncService';
 import { sqliteServiceService } from '../SQLiteService';
 import { TiddlerChangeSQLModel, TiddlerSQLModel } from '../SQLiteService/orm';
-import { getSyncIgnoredTiddlers } from './ignoredTiddler';
+import { getFullSaveTiddlers, getSyncIgnoredTiddlers } from './ignoredTiddler';
 import { IWikiServerStatusObject } from './types';
 
 /**
@@ -52,9 +53,11 @@ export class WikiStorageService {
   /**
    * Return the e-tag
    */
-  async saveTiddler(title: string, text: string, fieldStrings: string): Promise<string> {
+  async saveTiddler(title: string, fields: ITiddlerFieldsParam, encoding?: 'utf8' | 'base64'): Promise<string> {
     try {
       let operation: TiddlersLogOperation = TiddlersLogOperation.INSERT;
+      const saveFullTiddler = getFullSaveTiddlers(title).includes(title);
+      const { text, title: _, ...fieldsToSave } = fields as ITiddlerFieldsParam & { text?: string; title: string };
 
       // Get the database connection for the workspace
       const dataSource = await sqliteServiceService.getDatabase(this.#workspace);
@@ -65,12 +68,37 @@ export class WikiStorageService {
         const tiddlerRepo = transactionalEntityManager.getRepository(TiddlerSQLModel);
         const tiddlerChangeRepo = transactionalEntityManager.getRepository(TiddlerChangeSQLModel);
 
-        // Save or update tiddler
-        await tiddlerRepo.save({
-          title,
-          text,
-          fields: fieldStrings,
-        });
+        /**
+         * Save or update tiddler
+         * See `BackgroundSyncService.#updateTiddlersFromServer` for a similar logic
+         */
+        const tiddler = new TiddlerSQLModel();
+        tiddler.title = title;
+        // for `$:/` tiddlers, if being skinny will throw error like `"Linked List only accepts string values, not " + value;`
+        if (saveFullTiddler) {
+          tiddler.fields = JSON.stringify({
+            text,
+            title,
+            ...fieldsToSave,
+          });
+        } else {
+          // prevent save huge duplicated content to SQLite, if not necessary
+          if (text !== undefined && backgroundSyncService.checkIsLargeText(text, fieldsToSave.type as string)) {
+            // save to fs instead of sqlite. See `WikiStorageService.#loadFromServer` for how we load it later.
+            // `BackgroundSyncService.#updateTiddlersFromServer` will use saveToFSFromServer, but here we already have the text, so we can save it directly
+            await fs.writeAsStringAsync(getWikiTiddlerPathByTitle(this.#workspace, title), text, { encoding });
+            tiddler.text = null;
+          } else {
+            tiddler.text = text;
+          }
+          tiddler.fields = JSON.stringify({
+            _is_skinny: '',
+            title,
+            ...fieldsToSave,
+          });
+        }
+        await tiddlerRepo.save(tiddler);
+
         if (!(getSyncIgnoredTiddlers(title).includes(title))) {
           // Check if a tiddler with the same title already exists
           const existingTiddler = await tiddlerRepo.findOne({ where: { title } });
@@ -142,7 +170,7 @@ export class WikiStorageService {
       const dataSource = await sqliteServiceService.getDatabase(this.#workspace);
       const tiddlerRepo = dataSource.getRepository(TiddlerSQLModel);
       const tiddler = await tiddlerRepo.findOne({ where: { title } });
-      return tiddler?.text;
+      return tiddler?.text ?? undefined;
     } catch (error) {
       console.error(`SQL error when getting ${title} : ${(error as Error).message} ${(error as Error).stack ?? ''}`);
       return undefined;