Merge pull request #27 from dimitrov-d/feature/cli-updates

CLI file upload updates

packages/cli/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@apillon/cli",
   "description": "▶◀ Apillon CLI tools ▶◀",
-  "version": "1.2.1",
+  "version": "1.2.2",
   "author": "Apillon",
   "license": "MIT",
   "main": "./dist/index.js",

packages/cli/src/modules/storage/storage.commands.ts

@@ -68,6 +68,7 @@ export function createStorageCommands(cli: Command) {
     .option('-w, --wrap', 'Wrap uploaded files to an IPFS directory')
     .option('-p, --path <string>', 'Path to upload files to')
     .option('--await', 'await file CIDs to be resolved')
+    .option('--ignore', 'ignore files from .gitignore file')
     .action(async function (path: string) {
       await uploadFromFolder(path, this.optsWithGlobals());
     });

packages/cli/src/modules/storage/storage.service.ts

@@ -58,6 +58,7 @@ export async function uploadFromFolder(
         wrapWithDirectory: !!optsWithGlobals.wrap,
         directoryPath: optsWithGlobals.path,
         awaitCid: !!optsWithGlobals.await,
+        ignoreFiles: !!optsWithGlobals.ignore,
       });
     console.log(files);
   });

packages/sdk/src/modules/storage/storage-bucket.ts

@@ -130,7 +130,7 @@ export class StorageBucket extends ApillonModel {
     );
 
     if (!params?.awaitCid) {
-      return uploadedFiles;
+      return this.getUploadedFiles(sessionUuid, uploadedFiles.length);
     }
 
     return await this.resolveFileCIDs(sessionUuid, uploadedFiles.length);
@@ -153,7 +153,7 @@ export class StorageBucket extends ApillonModel {
     );
 
     if (!params?.awaitCid) {
-      return uploadedFiles;
+      return this.getUploadedFiles(sessionUuid, uploadedFiles.length);
     }
 
     return await this.resolveFileCIDs(sessionUuid, uploadedFiles.length);
@@ -187,14 +187,7 @@ export class StorageBucket extends ApillonModel {
     let retryTimes = 0;
     ApillonLogger.log('Resolving file CIDs...');
     while (resolvedFiles.length === 0 || !resolvedFiles.every((f) => !!f.CID)) {
-      resolvedFiles = (await this.listFiles({ sessionUuid, limit })).items.map(
-        (file) => ({
-          fileName: file.name,
-          fileUuid: file.uuid,
-          CID: file.CID,
-          CIDv1: file.CIDv1,
-        }),
-      );
+      resolvedFiles = await this.getUploadedFiles(sessionUuid, limit);
 
       await new Promise((resolve) => setTimeout(resolve, 1000));
       if (++retryTimes >= 15) {
@@ -205,6 +198,15 @@ export class StorageBucket extends ApillonModel {
     return resolvedFiles;
   }
 
+  private async getUploadedFiles(sessionUuid: string, limit: number) {
+    return (await this.listFiles({ sessionUuid, limit })).items.map((file) => ({
+      fileName: file.name,
+      fileUuid: file.uuid,
+      CID: file.CID,
+      // CIDv1: file.CIDv1,
+    }));
+  }
+
   //#region IPNS methods
 
   /**

packages/sdk/src/tests/helpers/website/.gitignore

@@ -0,0 +1 @@
+index.html

packages/sdk/src/tests/storage.test.ts

@@ -107,6 +107,18 @@ describe('Storage tests', () => {
     expect(files.every((f) => !!f.CID)).toBeTruthy();
   });
 
+  test('upload files from folder with ignoreFiles = false', async () => {
+    const uploadDir = resolve(__dirname, './helpers/website/');
+
+    console.time('File upload complete');
+    const files = await storage
+      .bucket(bucketUuid)
+      .uploadFromFolder(uploadDir, { ignoreFiles: false });
+    expect(files.length).toEqual(3); // .gitignore and index.html are not ignored
+
+    console.timeEnd('File upload complete');
+  });
+
   test('upload files from buffer', async () => {
     const html = fs.readFileSync(
       resolve(__dirname, './helpers/website/index.html'),

packages/sdk/src/types/storage.ts

@@ -74,8 +74,15 @@ export interface IFileUploadRequest {
 
   /**
    * If set to true, the upload action will wait until files receive a CID from IPFS before returning a result
+   * @default false
    */
   awaitCid?: boolean;
+
+  /**
+   * If set to true, will ignore all the files inside the .gitignore file, including .git and .gitignore itself
+   * @default true
+   */
+  ignoreFiles?: boolean;
 }
 
 export interface IFileUploadResponse {

packages/sdk/src/util/file-utils.ts

@@ -12,69 +12,6 @@ import {
 import { LogLevel } from '../types/apillon';
 import { randomBytes } from 'crypto';
 
-function listFilesRecursive(
-  folderPath: string,
-  fileList = [],
-  relativePath = '',
-) {
-  const gitignorePath = path.join(folderPath, '.gitignore');
-  const gitignorePatterns = fs.existsSync(gitignorePath)
-    ? fs.readFileSync(gitignorePath, 'utf-8').split('\n')
-    : [];
-  gitignorePatterns.push('.git'); // Always ignore .git folder.
-
-  const files = fs.readdirSync(folderPath);
-  for (const file of files) {
-    const fullPath = path.join(folderPath, file);
-    const relativeFilePath = path.join(relativePath, file);
-
-    // Skip file if it matches .gitignore patterns
-    if (
-      gitignorePatterns.some((pattern) =>
-        new RegExp(pattern).test(relativeFilePath),
-      )
-    ) {
-      continue;
-    }
-
-    if (fs.statSync(fullPath).isDirectory()) {
-      listFilesRecursive(fullPath, fileList, `${relativeFilePath}/`);
-    } else {
-      fileList.push({ fileName: file, path: relativePath, index: fullPath });
-    }
-  }
-  return fileList.sort((a, b) => a.fileName.localeCompare(b.fileName));
-}
-
-async function uploadFilesToS3(
-  uploadLinks: (FileMetadata & { url?: string })[],
-  files: (FileMetadata & { index?: string })[],
-) {
-  const s3Api = axios.create();
-  const uploadWorkers = [];
-
-  for (const link of uploadLinks) {
-    // console.log(link.url);
-    const file = files.find(
-      (x) => x.fileName === link.fileName && (!x.path || x.path === link.path),
-    );
-    if (!file) {
-      throw new Error(`Can't find file ${link.path}${link.fileName}!`);
-    }
-    uploadWorkers.push(
-      new Promise<void>(async (resolve, _reject) => {
-        // If uploading from local folder then read file, otherwise directly upload content
-        const content = file.index ? fs.readFileSync(file.index) : file.content;
-        await s3Api.put(link.url, content);
-        ApillonLogger.log(`File uploaded: ${file.fileName}`);
-        resolve();
-      }),
-    );
-  }
-
-  await Promise.all(uploadWorkers);
-}
-
 export async function uploadFiles(
   folderPath: string,
   apiPrefix: string,
@@ -88,10 +25,11 @@ export async function uploadFiles(
   } else {
     throw new Error('Invalid upload parameters received');
   }
+
   // If folderPath param passed, read files from local storage
   if (folderPath && !files?.length) {
     try {
-      files = listFilesRecursive(folderPath);
+      files = readFilesFromFolder(folderPath, params?.ignoreFiles);
     } catch (err) {
       ApillonLogger.log(err.message, LogLevel.ERROR);
       throw new Error(`Error reading files in ${folderPath}`);
@@ -124,6 +62,90 @@ export async function uploadFiles(
   return { sessionUuid, files: uploadedFiles.flatMap((f) => f) };
 }
 
+function readFilesFromFolder(
+  folderPath: string,
+  ignoreFiles = true,
+): FileMetadata[] {
+  const gitignorePatterns = [];
+  if (ignoreFiles) {
+    ApillonLogger.log('Ignoring files from .gitignore during upload.');
+
+    const gitignorePath = path.join(folderPath, '.gitignore');
+    if (fs.existsSync(gitignorePath)) {
+      gitignorePatterns.push(
+        ...fs.readFileSync(gitignorePath, 'utf-8').split('\n'),
+      );
+    }
+    // Ignore the following files by default when ignoreFiles = true
+    gitignorePatterns.push(
+      '\\.git/?$',
+      '\\.gitignore$',
+      'node_modules/?',
+      '\\.env$',
+    );
+  }
+
+  const folderFiles = listFilesRecursive(folderPath);
+  return folderFiles.filter(
+    (file) =>
+      // Skip files that match .gitignore patterns
+      !gitignorePatterns.some(
+        (pattern) =>
+          new RegExp(pattern).test(file.fileName) ||
+          new RegExp(pattern).test(file.path),
+      ),
+  );
+}
+
+function listFilesRecursive(
+  folderPath: string,
+  fileList = [],
+  relativePath = '',
+): FileMetadata[] {
+  const files = fs.readdirSync(folderPath);
+
+  for (const file of files) {
+    const fullPath = path.join(folderPath, file);
+    const relativeFilePath = path.join(relativePath, file);
+
+    if (fs.statSync(fullPath).isDirectory()) {
+      listFilesRecursive(fullPath, fileList, `${relativeFilePath}/`);
+    } else {
+      fileList.push({ fileName: file, path: relativePath, index: fullPath });
+    }
+  }
+  return fileList.sort((a, b) => a.fileName.localeCompare(b.fileName));
+}
+
+async function uploadFilesToS3(
+  uploadLinks: (FileMetadata & { url?: string })[],
+  files: (FileMetadata & { index?: string })[],
+) {
+  const s3Api = axios.create();
+  const uploadWorkers = [];
+
+  for (const link of uploadLinks) {
+    // console.log(link.url);
+    const file = files.find(
+      (x) => x.fileName === link.fileName && (!x.path || x.path === link.path),
+    );
+    if (!file) {
+      throw new Error(`Can't find file ${link.path}${link.fileName}!`);
+    }
+    uploadWorkers.push(
+      new Promise<void>(async (resolve, _reject) => {
+        // If uploading from local folder then read file, otherwise directly upload content
+        const content = file.index ? fs.readFileSync(file.index) : file.content;
+        await s3Api.put(link.url, content);
+        ApillonLogger.log(`File uploaded: ${file.fileName}`);
+        resolve();
+      }),
+    );
+  }
+
+  await Promise.all(uploadWorkers);
+}
+
 function chunkify(files: FileMetadata[], chunkSize = 10): FileMetadata[][] {
   // Divide files into chunks for parallel processing and uploading
   const fileChunks: FileMetadata[][] = [];