Merge pull request #26 from dimitrov-d/cli-1.2.1

Modify file upload, add CLI upload options

packages/cli/README.md

@@ -373,11 +373,14 @@ Upload contents of a local folder to specified bucket.
 
 - `<folder-path>`: Path to the folder containing your files.
 - `-b, --bucket-uuid <string>`: UUID of the bucket to upload files to.
+- `-w, --wrap`: Wrap uploaded files to an IPFS directory
+- `-p, --path <string>`: Path to upload files to (e.g. main/subdir). Required when --wrap is supplied.
+- `--await`: await file CIDs to be resolved
 
 **Example**
 
 ```sh
-apillon storage upload ./my_folder --bucket-uuid "123e4567-e89b-12d3-a456-426655440000"
+apillon storage upload ./my_folder --bucket-uuid "123e4567-e89b-12d3-a456-426655440000" --wrap --path "main/subdir"
 ```
 
 #### `storage get-file`

packages/cli/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@apillon/cli",
   "description": "▶◀ Apillon CLI tools ▶◀",
-  "version": "1.2.0",
+  "version": "1.2.1",
   "author": "Apillon",
   "license": "MIT",
   "main": "./dist/index.js",

packages/cli/src/modules/storage/storage.commands.ts

@@ -65,6 +65,9 @@ export function createStorageCommands(cli: Command) {
     .description('Upload contents of a local folder to bucket')
     .argument('<path>', 'path to local folder')
     .requiredOption('-b, --bucket-uuid <uuid>', 'UUID of destination bucket')
+    .option('-w, --wrap', 'Wrap uploaded files to an IPFS directory')
+    .option('-p, --path <string>', 'Path to upload files to')
+    .option('--await', 'await file CIDs to be resolved')
     .action(async function (path: string) {
       await uploadFromFolder(path, this.optsWithGlobals());
     });

packages/cli/src/modules/storage/storage.service.ts

@@ -44,11 +44,22 @@ export async function uploadFromFolder(
   path: string,
   optsWithGlobals: GlobalOptions,
 ) {
+  if (!!optsWithGlobals.wrap && !optsWithGlobals.path) {
+    console.error(
+      'Error: path option (-p, --path) is required when --wrap option is supplied',
+    );
+    return;
+  }
   await withErrorHandler(async () => {
     console.log(`Uploading files from folder: ${path}`);
-    await new Storage(optsWithGlobals)
+    const files = await new Storage(optsWithGlobals)
       .bucket(optsWithGlobals.bucketUuid)
-      .uploadFromFolder(path);
+      .uploadFromFolder(path, {
+        wrapWithDirectory: !!optsWithGlobals.wrap,
+        directoryPath: optsWithGlobals.path,
+        awaitCid: !!optsWithGlobals.await,
+      });
+    console.log(files);
   });
 }
 

packages/sdk/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@apillon/sdk",
   "description": "▶◀ Apillon SDK for NodeJS ▶◀",
-  "version": "2.0.0",
+  "version": "2.0.1",
   "author": "Apillon",
   "license": "MIT",
   "main": "./dist/index.js",

packages/sdk/src/modules/storage/storage-bucket.ts

@@ -1,6 +1,7 @@
 import { Directory } from './directory';
 import {
   FileMetadata,
+  FileUploadResult,
   IBucketFilesRequest,
   ICreateIpns,
   IFileUploadRequest,
@@ -121,7 +122,7 @@ export class StorageBucket extends ApillonModel {
   public async uploadFromFolder(
     folderPath: string,
     params?: IFileUploadRequest,
-  ): Promise<FileMetadata[]> {
+  ): Promise<FileUploadResult[]> {
     const { files: uploadedFiles, sessionUuid } = await uploadFiles(
       folderPath,
       this.API_PREFIX,
@@ -143,7 +144,7 @@ export class StorageBucket extends ApillonModel {
   public async uploadFiles(
     files: FileMetadata[],
     params?: IFileUploadRequest,
-  ): Promise<FileMetadata[]> {
+  ): Promise<FileUploadResult[]> {
     const { files: uploadedFiles, sessionUuid } = await uploadFiles(
       null,
       this.API_PREFIX,
@@ -179,8 +180,8 @@ export class StorageBucket extends ApillonModel {
   private async resolveFileCIDs(
     sessionUuid: string,
     limit: number,
-  ): Promise<FileMetadata[]> {
-    let resolvedFiles: FileMetadata[] = [];
+  ): Promise<FileUploadResult[]> {
+    let resolvedFiles: FileUploadResult[] = [];
 
     // Resolve CIDs for each file
     let retryTimes = 0;
@@ -192,7 +193,6 @@ export class StorageBucket extends ApillonModel {
           fileUuid: file.uuid,
           CID: file.CID,
           CIDv1: file.CIDv1,
-          content: null,
         }),
       );
 

packages/sdk/src/tests/hosting.test.ts

@@ -51,21 +51,18 @@ describe('Hosting tests', () => {
     );
     try {
       console.time('File upload complete');
-      await hosting.website(websiteUuid).uploadFiles(
-        [
-          {
-            fileName: 'index.html',
-            contentType: 'text/html',
-            content: html,
-          },
-          {
-            fileName: 'style.css',
-            contentType: 'text/css',
-            content: css,
-          },
-        ],
-        { wrapWithDirectory: true, directoryPath: 'main/subdir' },
-      );
+      await hosting.website(websiteUuid).uploadFiles([
+        {
+          fileName: 'index.html',
+          contentType: 'text/html',
+          content: html,
+        },
+        {
+          fileName: 'style.css',
+          contentType: 'text/css',
+          content: css,
+        },
+      ]);
       console.timeEnd('File upload complete');
       // console.log(content);
     } catch (e) {

packages/sdk/src/types/storage.ts

@@ -50,6 +50,8 @@ export interface FileMetadata {
   CID?: string;
 }
 
+export type FileUploadResult = Omit<FileMetadata, 'content'>;
+
 export interface IFileUploadRequest {
   /**
    * Wrap uploaded files to IPFS directory.

packages/sdk/src/util/file-utils.ts

@@ -62,25 +62,12 @@ async function uploadFilesToS3(
       throw new Error(`Can't find file ${link.path}${link.fileName}!`);
     }
     uploadWorkers.push(
-      new Promise<void>(async (resolve, reject) => {
-        // If uploading from local folder read file, otherwise directly upload content
-        if (file.index) {
-          fs.readFile(file.index, async (err, data) => {
-            if (err) {
-              reject(err.message);
-            }
-            // const respS3 =
-            await s3Api.put(link.url, data);
-            // console.log(respS3);
-
-            ApillonLogger.log(`File uploaded: ${file.fileName}`);
-            resolve();
-          });
-        } else if (file.content) {
-          await s3Api.put(link.url, file.content);
-          ApillonLogger.log(`File uploaded: ${file.fileName}`);
-          resolve();
-        }
+      new Promise<void>(async (resolve, _reject) => {
+        // If uploading from local folder then read file, otherwise directly upload content
+        const content = file.index ? fs.readFileSync(file.index) : file.content;
+        await s3Api.put(link.url, content);
+        ApillonLogger.log(`File uploaded: ${file.fileName}`);
+        resolve();
       }),
     );
   }
@@ -114,23 +101,19 @@ export async function uploadFiles(
   ApillonLogger.log(`Total files to upload: ${files.length}`);
 
   // Split files into chunks for parallel uploading
-  const fileChunkSize = 50;
+  const fileChunkSize = 200;
   const sessionUuid = uuidv4();
+  const uploadedFiles = [];
 
-  const uploadedFiles = await Promise.all(
-    chunkify(files, fileChunkSize).map(async (fileGroup) => {
-      const data = await ApillonApi.post<IFileUploadResponse>(
-        `${apiPrefix}/upload`,
-        {
-          files: fileGroup,
-          sessionUuid,
-        },
-      );
-
-      await uploadFilesToS3(data.files, fileGroup);
-      return data.files;
-    }),
-  );
+  for (const fileGroup of chunkify(files, fileChunkSize)) {
+    const { files } = await ApillonApi.post<IFileUploadResponse>(
+      `${apiPrefix}/upload`,
+      { files: fileGroup, sessionUuid },
+    );
+
+    await uploadFilesToS3(files, fileGroup);
+    uploadedFiles.push(files);
+  }
 
   ApillonLogger.logWithTime('File upload complete.');