Merge pull request #175 from forcedotcom/sm/autofetch-bug

Sm/autofetch bug

.circleci/config.yml

@@ -63,7 +63,9 @@ jobs:
           command: |
             yarn
             yarn upgrade @salesforce/source-deploy-retrieve@latest
-            yarn upgrade @salesforce/core@latest
+            yarn upgrade @salesforce/core@v3-beta
+            npx yarn-deduplicate
+            yarn install
       - when:
           condition:
             equal: ['linux', <<parameters.os>>]

package.json

@@ -43,15 +43,15 @@
     "/oclif.manifest.json"
   ],
   "dependencies": {
-    "@salesforce/core": "^3.19.0",
+    "@salesforce/core": "^3.21.2",
     "@salesforce/kit": "^1.5.17",
-    "@salesforce/source-deploy-retrieve": "^6.0.1",
+    "@salesforce/source-deploy-retrieve": "^6.0.4",
     "graceful-fs": "^4.2.9",
     "isomorphic-git": "1.17.0",
     "ts-retry-promise": "^0.6.0"
   },
   "devDependencies": {
-    "@salesforce/cli-plugins-testkit": "^1.3.7",
+    "@salesforce/cli-plugins-testkit": "^2.3.0",
     "@salesforce/dev-config": "^3.0.0",
     "@salesforce/dev-scripts": "^2.0.0",
     "@salesforce/prettier-config": "^0.0.2",

src/shared/localShadowRepo.ts

@@ -233,6 +233,8 @@ export class ShadowRepo {
       const chunks = chunkArray([...new Set(deployedFiles)], this.maxFileAdd);
       for (const chunk of chunks) {
         try {
+          // these need to be done sequentially (it's already batched) because isogit manages file locking
+          // eslint-disable-next-line no-await-in-loop
           await git.add({
             fs,
             dir: this.projectPath,
@@ -253,6 +255,8 @@ export class ShadowRepo {
     }
 
     for (const filepath of [...new Set(deletedFiles)]) {
+      // these need to be done sequentially because isogit manages file locking.  Isogit remove does not support multiple files at once
+      // eslint-disable-next-line no-await-in-loop
       await git.remove({ fs, dir: this.projectPath, gitdir: this.gitDir, filepath });
     }
 

src/shared/metadataKeys.ts

@@ -12,7 +12,7 @@ import { getMetadataKey } from './functions';
 // See UT for examples of the complexity this must handle
 // keys always use forward slashes, even on Windows
 const pathAfterFullName = (fileResponse: RemoteSyncInput): string =>
-  fileResponse && fileResponse.filePath
+  fileResponse?.filePath
     ? join(
         dirname(fileResponse.filePath).substring(dirname(fileResponse.filePath).lastIndexOf(fileResponse.fullName)),
         basename(fileResponse.filePath)

src/shared/remoteSourceTrackingService.ts

@@ -17,6 +17,7 @@ import { env, Duration } from '@salesforce/kit';
 import { ChangeResult, RemoteChangeElement, MemberRevision, SourceMember, RemoteSyncInput } from './types';
 import { getMetadataKeyFromFileResponse, mappingsForSourceMemberTypesToMetadataType } from './metadataKeys';
 import { getMetadataKey } from './functions';
+
 // represents the contents of the config file stored in 'maxRevision.json'
 type Contents = {
   serverMaxRevisionCounter: number;
@@ -327,13 +328,12 @@ export class RemoteSourceTrackingService extends ConfigFile<RemoteSourceTracking
         }
         sourceMember.serverRevisionCounter = change.RevisionCounter;
         sourceMember.isNameObsolete = change.IsNameObsolete;
-      } else {
-        // We are not yet tracking it so we'll insert a new record
-        if (!quiet) {
-          this.logger.debug(
-            `Inserting ${key} with RevisionCounter: ${change.RevisionCounter}${sync ? ' and syncing' : ''}`
-          );
-        }
+      }
+      // We are not yet tracking it so we'll insert a new record
+      else if (!quiet) {
+        this.logger.debug(
+          `Inserting ${key} with RevisionCounter: ${change.RevisionCounter}${sync ? ' and syncing' : ''}`
+        );
       }
 
       // If we are syncing changes then we need to update the lastRetrievedFromServer field to
@@ -516,6 +516,7 @@ export class RemoteSourceTrackingService extends ConfigFile<RemoteSourceTracking
    * Filter out known source tracking issues
    * This prevents the polling from waiting on things that may never return
    */
+  // eslint-disable-next-line class-methods-use-this
   private calculateExpectedSourceMembers(expectedMembers: RemoteSyncInput[]): Map<string, RemoteSyncInput> {
     const outstandingSourceMembers = new Map<string, RemoteSyncInput>();
 
@@ -641,10 +642,8 @@ export class RemoteSourceTrackingService extends ConfigFile<RemoteSourceTracking
     }
 
     try {
-      const results = await this.org.getConnection().tooling.query<SourceMember>(query, {
-        autoFetch: true,
-      });
-      return results.records;
+      return (await this.org.getConnection().tooling.query<SourceMember>(query, { autoFetch: true, maxFetch: 50000 }))
+        .records;
     } catch (error) {
       throw SfError.wrap(error as Error);
     }

src/sourceTracking.ts

@@ -71,6 +71,7 @@ export class SourceTracking extends AsyncCreatable {
     this.hasSfdxTrackingFiles = hasSfdxTrackingFiles(this.org.getOrgId(), this.projectPath);
   }
 
+  // eslint-disable-next-line class-methods-use-this
   public async init(): Promise<void> {
     // reserved for future use
   }

test/nuts/local/tracking-scale.nut.ts

@@ -4,6 +4,7 @@
  * Licensed under the BSD 3-Clause license.
  * For full license text, see LICENSE.txt file in the repo root or https://opensource.org/licenses/BSD-3-Clause
  */
+
 import * as path from 'path';
 import { TestSession } from '@salesforce/cli-plugins-testkit';
 import { expect } from 'chai';
@@ -32,9 +33,12 @@ describe(`verify tracking handles an add of ${classCount.toLocaleString()} class
     const classdir = path.join(session.project.dir, 'force-app', 'main', 'default', 'classes');
     for (let d = 0; d < dirCount; d++) {
       const dirName = path.join(classdir, `dir${d}`);
+      // you might think you could parallelize this, but Error: ENFILE: file table overflow will happen
+      // eslint-disable-next-line no-await-in-loop
       await fs.promises.mkdir(dirName);
       for (let c = 0; c < classesPerDir; c++) {
         const className = `x${d}x${c}`;
+        // eslint-disable-next-line no-await-in-loop
         await Promise.all([
           fs.promises.writeFile(
             path.join(dirName, `${className}.cls`),

test/unit/remoteSourceTracking.test.ts

@@ -56,14 +56,7 @@ describe('remoteSourceTrackingService', () => {
     orgData.username = username;
     orgData.orgId = orgId;
     orgData.tracksSource = true;
-
-    $$.setConfigStubContents('GlobalInfo', {
-      contents: {
-        orgs: {
-          [username]: await orgData.getConfig(),
-        },
-      },
-    });
+    await $$.stubAuths(orgData);
     const org = await Org.create({ aliasOrUsername: username });
     $$.SANDBOX.stub(org.getConnection().tooling, 'query').resolves({ records: [], done: true, totalSize: 0 });
     remoteSourceTrackingService = await RemoteSourceTrackingService.create({
@@ -363,14 +356,7 @@ describe('remoteSourceTrackingService', () => {
       orgData.username = username;
       orgData.orgId = orgId;
       orgData.tracksSource = true;
-
-      $$.setConfigStubContents('GlobalInfo', {
-        contents: {
-          orgs: {
-            [username]: await orgData.getConfig(),
-          },
-        },
-      });
+      await $$.stubAuths(orgData);
       const org = await Org.create({ aliasOrUsername: username });
       $$.SANDBOX.stub(org.getConnection().tooling, 'query').resolves({ records: [], done: true, totalSize: 0 });
       remoteSourceTrackingService = await RemoteSourceTrackingService.create({