ci: Support for limited tests run from PR description

.github/workflows/pr-automation.yml

@@ -19,6 +19,7 @@ jobs:
         shell: bash
     outputs:
       tags: ${{ steps.parseTags.outputs.tags }}
+      spec: ${{ steps.parseTags.outputs.spec }}
       matrix: ${{ steps.checkAll.outputs.matrix }}
     steps:
 
@@ -42,11 +43,15 @@ jobs:
       - name: Check if @tag.All is present in tags
         id: checkAll
         run: |
-          tags="${{ steps.parseTags.outputs.tags }}"
-          if [[ $tags == "@tag.All" ]]; then
-            echo "matrix=[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59]" >> $GITHUB_OUTPUT
+          if [[ -n "${{ steps.parseTags.outputs.spec }}" ]]; then
+            echo "matrix=[0]" >> $GITHUB_OUTPUT
           else
-            echo "matrix=[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19]" >> $GITHUB_OUTPUT
+            tags="${{ steps.parseTags.outputs.tags }}"
+            if [[ $tags == "@tag.All" ]]; then
+              echo "matrix=[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59]" >> $GITHUB_OUTPUT
+            else
+              echo "matrix=[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19]" >> $GITHUB_OUTPUT
+            fi
           fi
 
       # In case of a runnable command, update the PR with run details
@@ -60,6 +65,7 @@ jobs:
             Commit: ${{ github.event.pull_request.head.sha }}
             Workflow: `${{ github.workflow }}`
             Tags: `${{ steps.parseTags.outputs.tags }}`
+            Spec: `${{ steps.parseTags.outputs.spec }}`
         with:
           script: |
             require("write-cypress-status.js")({core, context, github}, "important", process.env.BODY)
@@ -72,5 +78,6 @@ jobs:
     secrets: inherit
     with:
       tags: ${{ needs.parse-tags.outputs.tags}}
+      spec: ${{ needs.parse-tags.outputs.spec}}
       matrix: ${{ needs.parse-tags.outputs.matrix}}
       is-pg-build: ${{ github.event.pull_request.base.ref == 'pg' }}

.github/workflows/pr-cypress.yml

@@ -194,6 +194,7 @@ jobs:
             Commit: ${{ github.event.pull_request.head.sha }}
             <a href="https://internal.appsmith.com/app/cypress-dashboard/rundetails-65890b3c81d7400d08fa9ee5?branch=master&workflowId=${{ github.run_id }}&attempt=${{ github.run_attempt }}&selectiontype=test&testsstatus=failed&specsstatus=fail" target="_blank">Cypress dashboard</a>.
             Tags: ${{ inputs.tags }}
+            Spec: ${{ inputs.spec }}
             The following are new failures, please fix them before merging the PR: <ol>
             ${{env.new_failed_spec_env}}</ol>
             <a href="https://internal.appsmith.com/app/cypress-dashboard/identified-flaky-tests-65890b3c81d7400d08fa9ee3?branch=master" target="_blank">List of identified flaky tests</a>.
@@ -212,6 +213,7 @@ jobs:
             Commit: ${{ github.event.pull_request.head.sha }}
             <a href="https://internal.appsmith.com/app/cypress-dashboard/rundetails-65890b3c81d7400d08fa9ee5?branch=master&workflowId=${{ github.run_id }}&attempt=${{ github.run_attempt }}" target="_blank">Cypress dashboard</a>.
             Tags: ${{ inputs.tags }}
+            Spec: ${{ inputs.spec }}
             It seems like **no tests ran** 😔. We are not able to recognize it, please check <a href="${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}" target="_blank">workflow here</a>.
         with:
           script: |
@@ -229,6 +231,7 @@ jobs:
             Commit: ${{ github.event.pull_request.head.sha }}
             <a href="https://internal.appsmith.com/app/cypress-dashboard/rundetails-65890b3c81d7400d08fa9ee5?branch=master&workflowId=${{ github.run_id }}&attempt=${{ github.run_attempt }}" target="_blank">Cypress dashboard</a>.
             Tags: `${{ inputs.tags }}`
+            Spec: ${{ inputs.spec }}
         with:
           script: |
             require("write-cypress-status.js")({core, context, github}, "tip", process.env.BODY)

.github/workflows/scripts/test-tag-parser.js

@@ -1,49 +1,60 @@
 module.exports = function ({core, context, github}) {
-  let tags;
+  // Get predictable newlines in the body content. Cause for _so_ much unneeded pain in this world!
+  const body = context.payload.pull_request.body?.replaceAll(/\r(\n)?/g, "\n");
+  if (!body) {
+    core.setFailed("Empty payload body!");
+    return;
+  }
+
+  let parseResult;
   try {
-    tags = parseTags(context.payload.pull_request.body);
+    parseResult = parseTags(body);
   } catch (error) {
     core.setFailed(error.message);
-    core.setOutput("outcome", "failure");
     const body = [
       "Invalid tags. Please use `/ok-to-test tags=\"@tag.All\"` or `/test all` in the PR body to run all tests.",
-      "[Tags documentation](https://www.notion.so/appsmith/7c0fc64d4efb4afebf53348cd6252918)",
-      "[List of valid tags](https://github.com/appsmithorg/appsmith/blob/release/app/client/cypress/tags.js)",
+      "[Tags documentation](https://www.notion.so/appsmith/7c0fc64d4efb4afebf53348cd6252918).",
+      "[List of valid tags](https://github.com/appsmithorg/appsmith/blob/release/app/client/cypress/tags.js).",
     ].join("\n");
-    require("write-cypress-status.js")({core, context, github}, "warning", body);
+    require("./write-cypress-status.js")({core, context, github}, "warning", body);
+    return;
   }
 
-  core.setOutput("tags", tags);
-  core.setOutput("outcome", "success");
-
-  // Shouldn't be needed anymore, but remove in separate PR.
-  return tags;
+  core.setOutput("tags", parseResult.tags ?? "");
+  core.setOutput("spec", parseResult.spec ?? "");
 }
 
 function parseTags(body) {
   const allTags = require(process.env.GITHUB_WORKSPACE + "/app/client/cypress/tags.js").Tag;
 
   // "/ok-to-test" matcher. Takes precedence over the "/test" matcher.
-  const strictMatch = body.match(/\/ok-to-test tags="(.+?)"/)?.[1];
+  const strictMatch = body.match(/^\/ok-to-test tags="(.+?)"/m)?.[1];
   if (strictMatch) {
     if (strictMatch === "@tag.All") {
-      return strictMatch;
+      return { tags: strictMatch };
     }
     const parts = strictMatch.split(/\s*,\s*/);
     for (const part of parts) {
       if (!allTags.includes(part)) {
         throw new Error("Unknown tag: " + part);
       }
     }
-    return strictMatch;
+    return { tags: strictMatch };
+  }
+
+  // "/test" code-fence matcher.
+  const result = matchCodeFence(body);
+  if (result) {
+    console.log("Code fence match:\n" + result.spec);
+    return result;
   }
 
   // "/test" matcher.
   const config = body.match(/^\**\/test\s+(.+?)\**$/m)?.[1] ?? "";
   const concreteTags = [];
 
   if (config.toLowerCase() === "all") {
-    return "@tag.All"
+    return { tags: "@tag.All" };
   }
 
   for (const [rawTag] of config.matchAll(/\w+/g)) {
@@ -78,5 +89,13 @@ function parseTags(body) {
     throw new Error("Tags were not found!")
   }
 
-  return concreteTags.join(", ");
+  return { tags: concreteTags.join(", ") };
+}
+
+function matchCodeFence(body) {
+  const re = /^```\n\/test\n(.+?)^```\n/ms;
+
+  const spec = body.match(re)?.[1];
+
+  return spec ? { spec } : null;
 }