Merge pull request #14 from forcedotcom/d/W-14093557

@W-14093557@: Better E2E, integration, and smoke tests

.github/workflows/build-scanner-tarball.yml

@@ -0,0 +1,40 @@
+name: build-scanner-tarball
+on:
+  workflow_call:
+    inputs:
+      target-branch:
+        description: "Which branch of the scanner should be built?"
+        required: false
+        type: string
+        default: "dev"
+
+jobs:
+  build-tarball:
+    runs-on: ubuntu-latest
+    steps:
+      # Install Node and Java.
+      - name: 'Install Node LTS'
+        uses: actions/setup-node@v3
+        with:
+          node-version: 'lts/*' # Always use Node LTS for building the tarball.
+      - name: 'Install Java 1.8'
+        uses: actions/setup-java@v3
+        with:
+          distribution: 'temurin'
+          java-version: '8' # Always use Java v1.8 for building the tarball.
+      - name: 'Check out, build, pack'
+        run: |
+          # Check out the target branch.
+          git clone https://github.com/forcedotcom/sfdx-scanner.git sfdx-scanner
+          cd sfdx-scanner
+          git checkout ${{ inputs.target-branch }}
+          # Install and build dependencies.
+          yarn
+          yarn build
+          # Create the tarball.
+          npm pack
+      # Upload the tarball as an artifact so it's usable elsewhere.
+      - uses: actions/upload-artifact@v3
+        with:
+          name: scanner-tarball
+          path: ./**/salesforce-sfdx-scanner-*.tgz

.github/workflows/daily-smoke-test.yml

@@ -0,0 +1,31 @@
+name: daily-smoke-test
+on:
+  workflow_dispatch: # As per documentation, the colon is needed even though no config is required.
+  schedule:
+    # Cron syntax is "minute[0-59] hour[0-23] date[1-31] month[1-12] day[0-6]". '*' is 'any value', and multiple values
+    # can be specified with comma-separated lists. All times are UTC.
+    # So this expression means "run at 13:30 UTC every day". This time was chosen because it corresponds to
+    # 8:30AM CDT, meaning that any issues will be surfaced towards the start of business.
+    - cron: "30 13 * * *"
+
+jobs:
+  # Step 1: Build the scanner tarball so it can be installed locally.
+  build-scanner-tarball:
+    name: 'Build scanner tarball'
+    uses: ./.github/workflows/build-scanner-tarball.yml
+    with:
+      target-branch: 'dev'
+  # Step 2: Actually run the tests.
+  smoke-test:
+    name: 'Run smoke tests'
+    needs: build-scanner-tarball
+    uses: ./.github/workflows/run-tests.yml
+    with:
+      # For daily builds, we want to make sure we haven't pushed a breaking change
+      # to the scanner's `dev` branch.
+      use-scanner-tarball: true
+  # Step 3: Build a VSIX artifact for use if needed.
+  create-vsix-artifact:
+    name: 'Upload VSIX as artifact'
+    uses: ./.github/workflows/create-vsix-artifact.yml
+

.github/workflows/publishVSCode.yml

@@ -59,8 +59,17 @@ jobs:
         run: echo "SF_CHANGE_CASE_SCHEDULE_BUILD=offcore.tooling.${{ steps.get-package-version.outputs.PACKAGE_VERSION }}" > $GITHUB_OUTPUT
       - run: echo "SF_CHANGE_CASE_SCHEDULE_BUILD is ${{ steps.get-scheduled-build.outputs.SF_CHANGE_CASE_SCHEDULE_BUILD }}"
 
-  publish:
+  run-tests:
+    name: 'Test against production scanner'
     needs: [ 'validate-release-environment' ]
+    uses: ./.github/workflows/run-tests.yml
+    with:
+      # Before publishing, we want to test the extension against whatever
+      # version of the scanner is currently live.
+      use-scanner-tarball: false
+
+  publish:
+    needs: [ 'run-tests' ]
     runs-on: ubuntu-latest
     env:
       VSCE_PERSONAL_ACCESS_TOKEN: ${{ secrets.VSCE_PERSONAL_ACCESS_TOKEN }}

.github/workflows/run-tests.yml

@@ -1,7 +1,12 @@
 name: run-tests
 on:
   workflow_call:
-  workflow_dispatch:
+    inputs:
+      use-scanner-tarball:
+        description: 'If true, install scanner via tarball'
+        required: false
+        type: boolean
+        default: false
 
 jobs:
   build-and-run:
@@ -18,11 +23,41 @@ jobs:
           node-version: 'lts/*' # Node LTS should always be fine.
       - name: 'Install node module dependencies'
         run: yarn install --frozen-lockfile
-      # Install the CLI tool and SFCA, since there are integration tests
-      # that depend on these things being present.
+      # We'll need to install the CLI tool, since some of the tests
+      # are integration tests.
       - name: Install SF CLI
         run: npm install --global @salesforce/cli
-      - name: Install Code Analyzer
+      # We'll need to install Salesforce Code Analyzer, since some
+      # of the tests are integration tests.
+      # NOTE: SFCA can come from a tarball built in a previous step,
+      #       or be installed as the currently-latest version.
+      - name: Download Scanner Tarball
+        if: ${{ inputs.use-scanner-tarball == true }}
+        id: download
+        uses: actions/download-artifact@v3
+        with:
+          name: scanner-tarball
+          # Download the tarball to a subdirectory of HOME, so it's guaranteed
+          # to be somewhere the installation command can see.
+          path: ~/downloads/tarball
+      - name: Install Scanner Tarball
+        if: ${{ inputs.use-scanner-tarball == true }}
+        shell: bash
+        run: |
+          # Determine the tarball's name.
+          TARBALL_NAME=$(ls ~/downloads/tarball/sfdx-scanner | grep salesforce-sfdx-scanner-[0-9]*\\.[0-9]*\\.[0-9]*\\.tgz)
+          echo $TARBALL_NAME
+          # Figure out where the tarball was downloaded to.
+          # To allow compatibility with Windows, replace backslashes with forward slashes
+          # and rip off a leading `C:` if present.
+          DOWNLOAD_PATH=`echo '${{ steps.download.outputs.download-path }}' | tr '\\' '/'`
+          echo $DOWNLOAD_PATH
+          DOWNLOAD_PATH=`[[ $DOWNLOAD_PATH = C* ]] && echo $DOWNLOAD_PATH | cut -d':' -f 2 || echo $DOWNLOAD_PATH`
+          echo $DOWNLOAD_PATH
+          # Pipe in a `y` to simulate agreeing to install an unsigned package. Use a URI of the file's full path.
+          echo y | sfdx plugins:install "file://${DOWNLOAD_PATH}/sfdx-scanner/${TARBALL_NAME}"
+      - name: Install Production scanner
+        if: ${{ inputs.use-scanner-tarball == false }}
         run: sfdx plugins:install @salesforce/sfdx-scanner
       # Run the tests. (Linux and non-Linux need slightly different commands.)
       - name: 'Run Tests (Linux)'

.github/workflows/validate-pr.yml

@@ -4,7 +4,7 @@ on:
     types: [edited, opened, reopened, synchronize]
 
 jobs:
-  # We need to verify that the Pull Request's title matches the desired format.
+  # VALIDATE PR TITLE FORMAT
   verify_pr_title:
     runs-on: ubuntu-latest
     name: Verify that PR title contains well-formed GUS work item tag.
@@ -15,10 +15,23 @@ jobs:
       # TODO: Implement this action
       - name: Verify PR Title
         run: echo 'Nope not yet'
-  # Separately, we also need to run our tests
+  # RUN TESTS
+  # Step 1: Build the scanner tarball so it can be installed locally.
+  build_scanner_tarball:
+    name: 'Build scanner tarball'
+    uses: ./.github/workflows/build-scanner-tarball.yml
+    with:
+      target-branch: 'dev'
+  # Step 2: Actually run the tests.
   run_tests:
     name: 'Run unit tests'
+    needs: build_scanner_tarball
     uses: ./.github/workflows/run-tests.yml
+    with:
+      # We want to validate the extension against whatever version of the scanner we
+      # *plan* to publish, not what's *already* published.
+      use-scanner-tarball: true
+  # BUILD A VSIX ARTIFACT
   # Additionally, build a VSIX that can be downloaded by the user if needed.
   create-vsix-artifact:
     name: 'Upload VSIX as artifact'

code-fixtures/folder-a/MyClassA2.cls

@@ -4,4 +4,8 @@
  * SPDX-License-Identifier: BSD-3-Clause
  * For full license text, see the LICENSE file in the repo root or https://opensource.org/licenses/BSD-3-Clause
  */
-public with sharing class MyClassA2 {}
+public with sharing class MyClassA2 {
+	public static boolean someMethod() {
+		return false;
+	}
+}

src/test/suite/extension.test.ts

@@ -74,10 +74,66 @@ suite('Extension Test Suite', () => {
 			}
 		});
 
-		test('sfca.runOnSelected', async function() {
-			// TODO: Add actual tests for `runOnSelected`.
+		suite('sfca.runOnSelected', () => {
+			test('One file selected', async function() {
+				// ===== SETUP =====
+				// Set the timeout to a frankly absurd value, just to make sure Github Actions
+				// can finish it in time.
+				this.timeout(60000);
+				// Get the URI for a single file.
+				const targetUri: vscode.Uri = vscode.Uri.file(path.join(codeFixturesPath, 'folder-a', 'MyClassA1.cls'));
+
+				// ===== TEST =====
+				// Run the "scan selected files" command.
+				// Pass the URI in as the first parameter, since that's what happens on a single-file selection.
+				await vscode.commands.executeCommand('sfca.runOnSelected', targetUri, []);
+
+				// ===== ASSERTIONS =====
+				// Verify that we added diagnostics.
+				const diagnosticArrays = vscode.languages.getDiagnostics();
+				const [resultsUri, diagnostics] = diagnosticArrays.find(uriDiagPair => uriDiagPair[0].toString() === targetUri.toString());
+				expect(resultsUri, `Expected diagnostics for ${targetUri.toString()}`).to.exist;
+				expect(diagnostics, `Expected non-empty diagnostics for ${targetUri.toString()}`).to.not.be.empty;
+				// At present, we expect only violations for PMD's `ApexDoc` rule.
+				for (const diagnostic of diagnostics) {
+					expect(diagnostic.source).to.equal('pmd via Code Analyzer', 'Wrong source');
+					expect(diagnostic.code).to.have.property('value', 'ApexDoc', 'Wrong rule violated');
+				}
+			});
 
+			test('One folder selected', () => {
+				// TODO: IMPLEMENT THIS TEST
+			});
 
+			test('Multiple files selected', async function() {
+				// ===== SETUP =====
+				// Set the timeout to a frankly absurd value, just to make sure Github Actions
+				// can finish it in time.
+				this.timeout(60000);
+				// Get the URIs for two separate files.
+				const targetUri1: vscode.Uri = vscode.Uri.file(path.join(codeFixturesPath, 'folder-a', 'MyClassA1.cls'));
+				const targetUri2: vscode.Uri = vscode.Uri.file(path.join(codeFixturesPath, 'folder-a', 'MyClassA2.cls'));
+
+				// ===== TEST =====
+				// Run the "scan selected files" command.
+				// Pass the URIs in as the second parameter, since that's what happens on a multi-select pick.
+				await vscode.commands.executeCommand('sfca.runOnSelected', null, [targetUri1, targetUri2]);
+
+				// ===== ASSERTIONS =====
+				// Verify that we added diagnostics.
+				const diagnosticArrays = vscode.languages.getDiagnostics();
+				const [resultsUri1, diagnostics1] = diagnosticArrays.find(uriDiagPair => uriDiagPair[0].toString() === targetUri1.toString());
+				const [resultsUri2, diagnostics2] = diagnosticArrays.find(uriDiagPair => uriDiagPair[0].toString() === targetUri2.toString());
+				expect(resultsUri1, `Expected diagnostics for ${targetUri1.toString()}`).to.exist;
+				expect(resultsUri2, `Expected diagnostics for ${targetUri2.toString()}`).to.exist;
+				expect(diagnostics1, `Expected non-empty diagnostics for ${targetUri1.toString()}`).to.not.be.empty;
+				expect(diagnostics2, `Expected non-empty diagnostics for ${targetUri2.toString()}`).to.not.be.empty;
+				// At present, we expect only violations for PMD's `ApexDoc` rule.
+				for (const diagnostic of [...diagnostics1, ...diagnostics2]) {
+					expect(diagnostic.source).to.equal('pmd via Code Analyzer', 'Wrong source');
+					expect(diagnostic.code).to.have.property('value', 'ApexDoc', 'Wrong rule violated');
+				}
+			});
 		});
 
 		test('sfca.runDfaOnSelected', async () => {