[ci] Upload screenshots, logs, and Xcode test results for drive and i…

…ntegration_test runs (flutter#7430) 1. Native Xcode tests will output a helpful "xcresult" package on failure containing logs, screenshots, and screen recordings. Zip and upload these results when tests fail. 2. Pass `flutter test --debug-logs-dir` flag to upload logs like flutter/flutter#142643. 3. Pass `flutter drive --screenshot` flag to upload screenshots on timeout like flutter/flutter#96973. Example of [failing Xcode analyzer build](https://ci.chromium.org/ui/p/flutter/builders/try/Mac_arm64%20ios_platform_tests_shard_5%20master/17374/overview) has the [zipped xcresult](https://storage.googleapis.com/flutter_logs/flutter/ff98c32e-18ca-4ad4-a910-9db1d7f7e4b0/xcode%20analyze/ff98c32e-18ca-4ad4-a910-9db1d7f7e4b0/xcodebuild-2024-10-25T09:56:46.440913.zip) attached as a log. ![Screenshot 2024-10-25 at 10 10 36â�¯AM](https://github.com/user-attachments/assets/dd7ae9bc-6161-4381-8a4f-f10b8c981801) The unzipped xcresult looks like this in Xcode: ![Screenshot 2024-10-25 at 10 11 55â�¯AM](https://github.com/user-attachments/assets/d4dd8420-f272-459c-9785-ab0c03887a74) A [failing "native test" step build](https://ci.chromium.org/ui/p/flutter/builders/try/Mac_arm64%20macos_platform_tests%20master%20-%20packages/17315/overview): ![Screenshot 2024-10-25 at 10 19 55â�¯AM](https://github.com/user-attachments/assets/76a86a15-2150-482a-8b15-e3e7ac90485e) Fixes flutter/flutter#144795
jesswrd · Nov 6, 2024 · bb5a258 · bb5a258
1 parent d3d563d
commit bb5a258
Show file tree

Hide file tree

Showing 8 changed files with 242 additions and 40 deletions.
diff --git a/script/tool/lib/src/common/core.dart b/script/tool/lib/src/common/core.dart
@@ -3,6 +3,7 @@
 // found in the LICENSE file.
 
 import 'package:file/file.dart';
+import 'package:platform/platform.dart';
 import 'package:pub_semver/pub_semver.dart';
 
 /// The signature for a print handler for commands that allow overriding the
@@ -127,3 +128,13 @@ const int exitCommandFoundErrors = 1;
 
 /// A exit code for [ToolExit] for a failure to run due to invalid arguments.
 const int exitInvalidArguments = 2;
+
+/// The directory to which to write logs and other artifacts, if set in CI.
+Directory? ciLogsDirectory(Platform platform, FileSystem fileSystem) {
+  final String? logsDirectoryPath = platform.environment['FLUTTER_LOGS_DIR'];
+  Directory? logsDirectory;
+  if (logsDirectoryPath != null) {
+    logsDirectory = fileSystem.directory(logsDirectoryPath);
+  }
+  return logsDirectory;
+}
diff --git a/script/tool/lib/src/common/xcode.dart b/script/tool/lib/src/common/xcode.dart
@@ -6,7 +6,9 @@ import 'dart:convert';
 import 'dart:io' as io;
 
 import 'package:file/file.dart';
+import 'package:platform/platform.dart';
 
+import 'core.dart';
 import 'output_utils.dart';
 import 'process_runner.dart';
 
@@ -33,36 +35,81 @@ class Xcode {
   /// Runs an `xcodebuild` in [directory] with the given parameters.
   Future<int> runXcodeBuild(
     Directory exampleDirectory,
-    String platform, {
+    String targetPlatform, {
     List<String> actions = const <String>['build'],
     required String workspace,
     required String scheme,
     String? configuration,
     List<String> extraFlags = const <String>[],
-  }) {
-    File? disabledSandboxEntitlementFile;
-    if (actions.contains('test') && platform.toLowerCase() == 'macos') {
-      disabledSandboxEntitlementFile = _createDisabledSandboxEntitlementFile(
-        exampleDirectory.childDirectory(platform.toLowerCase()),
-        configuration ?? 'Debug',
-      );
-    }
-    final List<String> args = <String>[
-      _xcodeBuildCommand,
-      ...actions,
-      ...<String>['-workspace', workspace],
-      ...<String>['-scheme', scheme],
-      if (configuration != null) ...<String>['-configuration', configuration],
-      ...extraFlags,
-      if (disabledSandboxEntitlementFile != null)
-        'CODE_SIGN_ENTITLEMENTS=${disabledSandboxEntitlementFile.path}',
-    ];
-    final String completeTestCommand = '$_xcRunCommand ${args.join(' ')}';
-    if (log) {
-      print(completeTestCommand);
+    required Platform hostPlatform,
+  }) async {
+    final FileSystem fileSystem = exampleDirectory.fileSystem;
+    String? resultBundlePath;
+    final Directory? logsDirectory = ciLogsDirectory(hostPlatform, fileSystem);
+    Directory? resultBundleTemp;
+    try {
+      if (logsDirectory != null) {
+        resultBundleTemp =
+            fileSystem.systemTempDirectory.createTempSync('flutter_xcresult.');
+        resultBundlePath = resultBundleTemp.childDirectory('result').path;
+      }
+      File? disabledSandboxEntitlementFile;
+      if (actions.contains('test') && targetPlatform.toLowerCase() == 'macos') {
+        disabledSandboxEntitlementFile = _createDisabledSandboxEntitlementFile(
+          exampleDirectory.childDirectory(targetPlatform.toLowerCase()),
+          configuration ?? 'Debug',
+        );
+      }
+      final List<String> args = <String>[
+        _xcodeBuildCommand,
+        ...actions,
+        ...<String>['-workspace', workspace],
+        ...<String>['-scheme', scheme],
+        if (resultBundlePath != null) ...<String>[
+          '-resultBundlePath',
+          resultBundlePath
+        ],
+        if (configuration != null) ...<String>['-configuration', configuration],
+        ...extraFlags,
+        if (disabledSandboxEntitlementFile != null)
+          'CODE_SIGN_ENTITLEMENTS=${disabledSandboxEntitlementFile.path}',
+      ];
+      final String completeTestCommand = '$_xcRunCommand ${args.join(' ')}';
+      if (log) {
+        print(completeTestCommand);
+      }
+      final int resultExit = await processRunner
+          .runAndStream(_xcRunCommand, args, workingDir: exampleDirectory);
+
+      if (resultExit != 0 && resultBundleTemp != null) {
+        final Directory xcresultBundle =
+            resultBundleTemp.childDirectory('result.xcresult');
+        if (logsDirectory != null) {
+          if (xcresultBundle.existsSync()) {
+            // Zip the test results to the artifacts directory for upload.
+            final File zipPath = logsDirectory.childFile(
+                'xcodebuild-${DateTime.now().toLocal().toIso8601String()}.zip');
+            await processRunner.run(
+              'zip',
+              <String>[
+                '-r',
+                '-9',
+                '-q',
+                zipPath.path,
+                xcresultBundle.basename,
+              ],
+              workingDir: resultBundleTemp,
+            );
+          } else {
+            print(
+                'xcresult bundle ${xcresultBundle.path} does not exist, skipping upload');
+          }
+        }
+      }
+      return resultExit;
+    } finally {
+      resultBundleTemp?.deleteSync(recursive: true);
     }
-    return processRunner.runAndStream(_xcRunCommand, args,
-        workingDir: exampleDirectory);
   }
 
   /// Returns true if [project], which should be an .xcodeproj directory,

diff --git a/script/tool/lib/src/drive_examples_command.dart b/script/tool/lib/src/drive_examples_command.dart
@@ -228,8 +228,12 @@ class DriveExamplesCommand extends PackageLoopingCommand {
         }
         for (final File driver in drivers) {
           final List<File> failingTargets = await _driveTests(
-              example, driver, testTargets,
-              deviceFlags: deviceFlags);
+            example,
+            driver,
+            testTargets,
+            deviceFlags: deviceFlags,
+            exampleName: exampleName,
+          );
           for (final File failingTarget in failingTargets) {
             errors.add(
                 getRelativePosixPath(failingTarget, from: package.directory));
@@ -376,10 +380,16 @@ class DriveExamplesCommand extends PackageLoopingCommand {
     File driver,
     List<File> targets, {
     required List<String> deviceFlags,
+    required String exampleName,
   }) async {
     final List<File> failures = <File>[];
 
     final String enableExperiment = getStringArg(kEnableExperiment);
+    final String screenshotBasename =
+        '${exampleName.replaceAll(platform.pathSeparator, '_')}-drive';
+    final Directory? screenshotDirectory =
+        ciLogsDirectory(platform, driver.fileSystem)
+            ?.childDirectory(screenshotBasename);
 
     for (final File target in targets) {
       final int exitCode = await processRunner.runAndStream(
@@ -389,6 +399,8 @@ class DriveExamplesCommand extends PackageLoopingCommand {
             ...deviceFlags,
             if (enableExperiment.isNotEmpty)
               '--enable-experiment=$enableExperiment',
+            if (screenshotDirectory != null)
+              '--screenshot=${screenshotDirectory.path}',
             '--driver',
             getRelativePosixPath(driver, from: example.directory),
             '--target',
@@ -416,6 +428,8 @@ class DriveExamplesCommand extends PackageLoopingCommand {
     required List<File> testFiles,
   }) async {
     final String enableExperiment = getStringArg(kEnableExperiment);
+    final Directory? logsDirectory =
+        ciLogsDirectory(platform, testFiles.first.fileSystem);
 
     // Workaround for https://github.com/flutter/flutter/issues/135673
     // Once that is fixed on stable, this logic can be removed and the command
@@ -438,6 +452,7 @@ class DriveExamplesCommand extends PackageLoopingCommand {
             ...deviceFlags,
             if (enableExperiment.isNotEmpty)
               '--enable-experiment=$enableExperiment',
+            if (logsDirectory != null) '--debug-logs-dir=${logsDirectory.path}',
             target,
           ],
           workingDir: example.directory);

diff --git a/script/tool/lib/src/native_test_command.dart b/script/tool/lib/src/native_test_command.dart
@@ -431,7 +431,7 @@ this command.
   /// usually at "example/{ios,macos}/Runner.xcworkspace".
   Future<_PlatformResult> _runXcodeTests(
     RepositoryPackage plugin,
-    String platform,
+    String targetPlatform,
     _TestMode mode, {
     List<String> extraFlags = const <String>[],
   }) async {
@@ -456,7 +456,7 @@ this command.
       final String? targetToCheck =
           testTarget ?? (mode.unit ? unitTestTarget : null);
       final Directory xcodeProject = example.directory
-          .childDirectory(platform.toLowerCase())
+          .childDirectory(targetPlatform.toLowerCase())
           .childDirectory('Runner.xcodeproj');
       if (targetToCheck != null) {
         final bool? hasTarget =
@@ -473,16 +473,17 @@ this command.
         }
       }
 
-      _printRunningExampleTestsMessage(example, platform);
+      _printRunningExampleTestsMessage(example, targetPlatform);
       final int exitCode = await _xcode.runXcodeBuild(
         example.directory,
-        platform,
+        targetPlatform,
         // Clean before testing to remove cached swiftmodules from previous
         // runs, which can cause conflicts.
         actions: <String>['clean', 'test'],
-        workspace: '${platform.toLowerCase()}/Runner.xcworkspace',
+        workspace: '${targetPlatform.toLowerCase()}/Runner.xcworkspace',
         scheme: 'Runner',
         configuration: 'Debug',
+        hostPlatform: platform,
         extraFlags: <String>[
           if (testTarget != null) '-only-testing:$testTarget',
           ...extraFlags,
@@ -494,9 +495,10 @@ this command.
       const int xcodebuildNoTestExitCode = 66;
       switch (exitCode) {
         case xcodebuildNoTestExitCode:
-          _printNoExampleTestsMessage(example, platform);
+          _printNoExampleTestsMessage(example, targetPlatform);
         case 0:
-          printSuccess('Successfully ran $platform xctest for $exampleName');
+          printSuccess(
+              'Successfully ran $targetPlatform xctest for $exampleName');
           // If this is the first test, assume success until something fails.
           if (overallResult == RunState.skipped) {
             overallResult = RunState.succeeded;

diff --git a/script/tool/lib/src/xcode_analyze_command.dart b/script/tool/lib/src/xcode_analyze_command.dart
@@ -97,36 +97,37 @@ class XcodeAnalyzeCommand extends PackageLoopingCommand {
             multiplePlatformsRequested ? failures : <String>[]);
   }
 
-  /// Analyzes [plugin] for [platform], returning true if it passed analysis.
+  /// Analyzes [plugin] for [targetPlatform], returning true if it passed analysis.
   Future<bool> _analyzePlugin(
     RepositoryPackage plugin,
-    String platform, {
+    String targetPlatform, {
     List<String> extraFlags = const <String>[],
   }) async {
     bool passing = true;
     for (final RepositoryPackage example in plugin.getExamples()) {
       // Running tests and static analyzer.
       final String examplePath = getRelativePosixPath(example.directory,
           from: plugin.directory.parent);
-      print('Running $platform tests and analyzer for $examplePath...');
+      print('Running $targetPlatform tests and analyzer for $examplePath...');
       final int exitCode = await _xcode.runXcodeBuild(
         example.directory,
-        platform,
+        targetPlatform,
         // Clean before analyzing to remove cached swiftmodules from previous
         // runs, which can cause conflicts.
         actions: <String>['clean', 'analyze'],
-        workspace: '${platform.toLowerCase()}/Runner.xcworkspace',
+        workspace: '${targetPlatform.toLowerCase()}/Runner.xcworkspace',
         scheme: 'Runner',
         configuration: 'Debug',
+        hostPlatform: platform,
         extraFlags: <String>[
           ...extraFlags,
           'GCC_TREAT_WARNINGS_AS_ERRORS=YES',
         ],
       );
       if (exitCode == 0) {
-        printSuccess('$examplePath ($platform) passed analysis.');
+        printSuccess('$examplePath ($targetPlatform) passed analysis.');
       } else {
-        printError('$examplePath ($platform) failed analysis.');
+        printError('$examplePath ($targetPlatform) failed analysis.');
         passing = false;
       }
     }

diff --git a/script/tool/test/common/xcode_test.dart b/script/tool/test/common/xcode_test.dart
@@ -165,6 +165,7 @@ void main() {
         'ios',
         workspace: 'A.xcworkspace',
         scheme: 'AScheme',
+        hostPlatform: MockPlatform(),
       );
 
       expect(exitCode, 0);
@@ -193,6 +194,7 @@ void main() {
           workspace: 'A.xcworkspace',
           scheme: 'AScheme',
           configuration: 'Debug',
+          hostPlatform: MockPlatform(),
           extraFlags: <String>['-a', '-b', 'c=d']);
 
       expect(exitCode, 0);
@@ -230,6 +232,7 @@ void main() {
         'ios',
         workspace: 'A.xcworkspace',
         scheme: 'AScheme',
+        hostPlatform: MockPlatform(),
       );
 
       expect(exitCode, 1);
@@ -264,6 +267,7 @@ void main() {
         'macos',
         workspace: 'A.xcworkspace',
         scheme: 'AScheme',
+        hostPlatform: MockPlatform(),
         actions: <String>['test'],
       );