feat: show download progress

src/constants/paths.ts

@@ -9,7 +9,7 @@ export const getWikiTiddlerFolderPath = (workspace: IWikiWorkspace) => `${worksp
  * Get file path like `file:///data/user/0/host.exp.exponent/files/wikis/wiki_88370/tiddlers/TiddlyWikiIconBlack.png`
  * Will make sure filename don't have invalid characters.
  */
-export const getWikiTiddlerPathByTitle = (workspace: IWikiWorkspace, title: string) => `${getWikiTiddlerFolderPath(workspace)}${title.replaceAll(/["&'*/:<=>?\\{}]/g, '_')}`;
+export const getWikiTiddlerPathByTitle = (workspace: IWikiWorkspace, title: string) => `${getWikiTiddlerFolderPath(workspace)}${title.replaceAll(/["#%&'*/:<=>?\\{}]/g, '_')}`;
 export const WIKI_STORE_NAME = 'tiddlerStore.json';
 /**
  * This JSON is used as-is, so should be a valid JSON, instead of JSON-Line.

src/i18n/localization/locales/en/translation.json

@@ -159,7 +159,9 @@
     "Reset": "Reset",
     "ImportBinaryFiles": "Import Binary Files",
     "ImportBinaryFilesDescription": "Large files such as images are by default loaded from the connected server only when in use, so as not to take up too much mobile storage. If you wish to have them loaded in advance, you can click this button.",
-    "Success": "Import Success"
+    "Success": "Import Success",
+    "DownloadBinaryTimeout": "Timeout download an asset file.",
+    "WarningMessageDescription": "You can ignore the warning message, since after this it can still be loaded normally on demand, when entering the wiki."
   },
   "EditWorkspace": {
     "Path": "Wiki Path",
@@ -557,5 +559,6 @@
     "TiddlersListAndEssential": "Tiddlers List and Essential Tiddlers",
     "TiddlerTexts": "Tiddler Texts",
     "AddToSQLite": "Adding To local SQLite cache DB"
-  }
+  },
+  "WarningMessage": "Warning Message"
 }

src/i18n/localization/locales/zh_CN/translation.json

@@ -172,7 +172,9 @@
     "Reset": "重置",
     "ImportBinaryFiles": "导入二进制文件",
     "ImportBinaryFilesDescription": "图片等大文件默认在使用时才从连接的服务器加载，以免占用过多移动端存储。如果你希望提前加载好，可以点击此按钮。",
-    "Success": "导入成功"
+    "Success": "导入成功",
+    "DownloadBinaryTimeout": "下载一个资源文件时超时",
+    "WarningMessageDescription": "你可以忽略警告信息。因为导入失败的内容，之后可以在进入知识库后正常按需加载。"
   },
   "EditWorkspace": {
     "Path": "Wiki的位置",
@@ -534,5 +536,6 @@
     "TiddlersListAndEssential": "条目列表和关键条目内容",
     "TiddlerTexts": "条目文本内容",
     "AddToSQLite": "添加到本地SQLite快取数据库"
-  }
+  },
+  "WarningMessage": "警告信息"
 }

src/pages/Importer/useImportBinary.ts

@@ -8,6 +8,7 @@ export function useImportBinary(newWorkspace: IWikiWorkspace | undefined) {
   const [importSuccess, setImportSuccess] = useState<boolean>(false);
   const [importBinaryFetchAndWritPercentage, setImportBinaryFetchAndWritPercentage] = useState(0);
   const [importBinaryReadListPercentage, setImportBinaryReadListPercentage] = useState(0);
+  const [importWarning, setImportWarning] = useState<string | undefined>();
   const resetState = useCallback(() => {
     setImportBinaryFetchAndWritPercentage(0);
     setImportBinaryReadListPercentage(0);
@@ -21,6 +22,7 @@ export function useImportBinary(newWorkspace: IWikiWorkspace | undefined) {
       await importService.loadBinaryTiddlersAsFilesFromServer(newWorkspace, {
         setFetchAndWritProgress: setImportBinaryFetchAndWritPercentage,
         setReadListProgress: setImportBinaryReadListPercentage,
+        setWarning: setImportWarning,
       });
       setImportSuccess(true);
     } catch (error) {
@@ -35,6 +37,7 @@ export function useImportBinary(newWorkspace: IWikiWorkspace | undefined) {
     importBinary,
     importingBinary,
     importSuccess,
+    importWarning,
     importPercentage: {
       importBinaryFetchAndWritPercentage,
       importBinaryReadListPercentage,

src/pages/MainMenu/EditItemModel/WikiModelContent.tsx

@@ -15,9 +15,9 @@ import { backgroundSyncService } from '../../../services/BackgroundSyncService';
 import { useServerStore } from '../../../store/server';
 import { IWikiWorkspace, useWorkspaceStore } from '../../../store/workspace';
 import { deleteWikiFile } from '../../Config/Developer/useClearAllWikiData';
+import { ImportBinary } from '../../Importer/ImportBinary';
 import { AddNewServerModelContent } from '../AddNewServerModelContent';
 import { WikiChangesModelContent } from './WikiChangesModelContent';
-import { ImportBinary } from '../../Importer/ImportBinary';
 
 interface WikiEditModalProps {
   id: string | undefined;
@@ -46,7 +46,8 @@ export function WikiEditModalContent({ id, onClose }: WikiEditModalProps): JSX.E
   const [currentOnlineServerToSync, setCurrentOnlineServerToSync] = useState<undefined | Awaited<ReturnType<typeof backgroundSyncService.getOnlineServerForWiki>>>();
   useEffect(() => {
     if (wiki === undefined) return;
-    void backgroundSyncService.getOnlineServerForWiki(wiki, true).then(server => {
+    void backgroundSyncService.updateServerOnlineStatus().then(async () => {
+      const server = await backgroundSyncService.getOnlineServerForWiki(wiki);
       setCurrentOnlineServerToSync(server);
     });
   }, [wiki]);
@@ -95,7 +96,8 @@ export function WikiEditModalContent({ id, onClose }: WikiEditModalProps): JSX.E
         onPress={async () => {
           setInSyncing(true);
           try {
-            const server = await backgroundSyncService.getOnlineServerForWiki(wiki, true);
+            await backgroundSyncService.updateServerOnlineStatus();
+            const server = await backgroundSyncService.getOnlineServerForWiki(wiki);
             if (server !== undefined) {
               await backgroundSyncService.syncWikiWithServer(wiki, server);
               setIsSyncSucceed(true);

src/services/BackgroundSyncService/index.ts

@@ -67,7 +67,7 @@ export class BackgroundSyncService {
     await this.updateServerOnlineStatus();
     for (const wiki of workspaces) {
       if (wiki.type === 'wiki') {
-        const server = await this.getOnlineServerForWiki(wiki);
+        const server = this.getOnlineServerForWiki(wiki);
         if (server !== undefined) {
           haveUpdate ||= await this.syncWikiWithServer(wiki, server);
         }
@@ -103,10 +103,7 @@ export class BackgroundSyncService {
     return response;
   }
 
-  public async getOnlineServerForWiki(wiki: IWikiWorkspace, updated?: boolean): Promise<(IServerInfo & { lastSync: number; syncActive: boolean }) | undefined> {
-    if (updated === true) {
-      await this.updateServerOnlineStatus();
-    }
+  public getOnlineServerForWiki(wiki: IWikiWorkspace): (IServerInfo & { lastSync: number; syncActive: boolean }) | undefined {
     const onlineLastSyncServer = wiki.syncedServers
       .filter(serverInfoInWiki => serverInfoInWiki.syncActive)
       .sort((a, b) => b.lastSync - a.lastSync)
@@ -290,9 +287,8 @@ export class BackgroundSyncService {
     return blob.size > 200 * 1024; // 200KB
   }
 
-  public async saveToFSFromServer(workspace: IWikiWorkspace, title: string) {
+  public async saveToFSFromServer(workspace: IWikiWorkspace, title: string, onlineLastSyncServer = this.getOnlineServerForWiki(workspace)) {
     try {
-      const onlineLastSyncServer = await this.getOnlineServerForWiki(workspace);
       if (onlineLastSyncServer === undefined) {
         console.error(`saveToFSFromServer: Cannot find online server for workspace ${workspace.id}`);
         return;

src/services/ImportService/ExpoReadStream.ts

@@ -56,6 +56,6 @@ class ExpoReadStream extends Readable {
   }
 }
 
-export function createReadStream(fileUri: string, options: { encoding?: fs.EncodingType; end?: number; highWaterMark?: number; start?: number } = {}): ExpoReadStream {
+export function createReadStream(fileUri: string, options: fs.ReadingOptions = {}): ExpoReadStream {
   return new ExpoReadStream(fileUri, options);
 }

src/services/ImportService/index.ts

@@ -166,17 +166,53 @@ export class ImportService {
    */
   public async loadBinaryTiddlersAsFilesFromServer(
     workspace: IWikiWorkspace,
-    setProgress: { setFetchAndWritProgress: (progress: number) => void; setReadListProgress: (progress: number) => void },
+    setProgress: { setFetchAndWritProgress: (progress: number) => void; setReadListProgress: (progress: number) => void; setWarning: (latestWarning: string) => void },
     options?: { chunkSize?: number },
   ): Promise<void> {
     /**
      * We can concurrently load multiple binary files from server.
      * Concurrency is determined by the read stream and chunk number.
      */
-    const MAX_CHUNK_SIZE = options?.chunkSize ?? 20;
+    const MAX_CHUNK_SIZE = options?.chunkSize ?? 50;
+    /** Read 0.5M each time, to slow down the progress of fs read, because the bottom neck is net request. */
+    const JSON_READ_LENGTH = 512 * 1024;
+    // get content length use first pass
+    let dataCount = 0;
+    try {
+      const readStream = createReadStream(getWikiBinaryTiddlersListCachePath(workspace), { length: JSON_READ_LENGTH * 2 });
+      readStream.on('progress', (progress: number) => {
+        setProgress.setReadListProgress(progress);
+      });
+      const countBinaryTiddlerFieldsStream = chain([
+        readStream,
+        StreamArray.withParser(),
+      ]);
+      let lastData: ISkinnyTiddlersJSONBatch[number] | undefined;
+      countBinaryTiddlerFieldsStream.on('data', (data: ISkinnyTiddlersJSONBatch[number]) => {
+        lastData = data;
+      });
+      await new Promise<void>((resolve, reject) => {
+        countBinaryTiddlerFieldsStream.on('end', () => {
+          const lastIndex = lastData?.key;
+          if (lastIndex === undefined) {
+            reject(new Error('loadBinaryTiddlersAsFilesFromServer() Failed to count task, no data'));
+          } else {
+            dataCount = lastIndex + 1;
+            resolve();
+          }
+        });
+        countBinaryTiddlerFieldsStream.on('error', (error) => {
+          reject(error);
+        });
+      });
+    } catch (error) {
+      throw new Error(`loadBinaryTiddlersAsFilesFromServer() Failed to read tiddler text store, ${(error as Error).message}`);
+    }
+    // reset progress, start really processing data.
+    setProgress.setReadListProgress(0);
     let batchedBinaryTiddlerFieldsStream: Chain;
     try {
-      const readStream = createReadStream(getWikiBinaryTiddlersListCachePath(workspace));
+      const readStream = createReadStream(getWikiBinaryTiddlersListCachePath(workspace), { length: JSON_READ_LENGTH });
       readStream.on('progress', (progress: number) => {
         setProgress.setReadListProgress(progress);
       });
@@ -186,26 +222,26 @@ export class ImportService {
         new Batch({ batchSize: MAX_CHUNK_SIZE }),
       ]);
     } catch (error) {
-      throw new Error(`storeFieldsToSQLite() Failed to read tiddler text store, ${(error as Error).message}`);
+      throw new Error(`loadBinaryTiddlersAsFilesFromServer() Failed to read tiddler text store, ${(error as Error).message}`);
     }
+    let completedCount = 0;
+    const onlineLastSyncServer = backgroundSyncService.getOnlineServerForWiki(workspace);
+    setProgress.setFetchAndWritProgress(0);
     const fetchAndWriteStream = new Writable({
       objectMode: true,
-      // FIXME: after first batch, not getting next batch data
       write: async (tiddlerListChunk: ISkinnyTiddlersJSONBatch, encoding, next) => {
-        setProgress.setFetchAndWritProgress(0);
-        let completedCount = 0;
-        const taskLength = tiddlerListChunk.length;
         await Promise.all(
           tiddlerListChunk.map(item => item.value).map(async tiddler => {
             try {
               if (!tiddler.title) return;
-              // TODO: check if file already exists, skip importing.
-              await backgroundSyncService.saveToFSFromServer(workspace, tiddler.title);
+              // TODO: check if file already exists, skip importing. Maybe read the folder, and use that list to compare? Or can skip the compare if folder is empty (first time import)
+              await backgroundSyncService.saveToFSFromServer(workspace, tiddler.title, onlineLastSyncServer);
             } catch (error) {
               console.error(`loadTiddlersAsFileFromServer: Failed to load tiddler ${tiddler.title} from server: ${(error as Error).message} ${(error as Error).stack ?? ''}`);
+              setProgress.setWarning((error as Error).message);
             } finally {
               completedCount += 1;
-              setProgress.setFetchAndWritProgress(completedCount / taskLength);
+              setProgress.setFetchAndWritProgress(completedCount / dataCount);
             }
           }),
         );