chore: create csv

.github/workflows/analyze_dependency.yaml

@@ -27,10 +27,10 @@ jobs:
         distribution: temurin
         java-version: 17
         cache: maven
-    - name: Set up Maven
-      uses: stCarolas/[email protected]
-      with:
-        maven-version: 3.8.2
+    - name: Install modules
+      shell: bash
+      run: |
+        mvn clean install -V --batch-mode --no-transfer-progress -DskipTests
     - name: Install dependency analyzer
       shell: bash
       run: |
@@ -39,5 +39,5 @@ jobs:
     - name: Check dependency information
       shell: bash
       run: |
-        mvn exec:java -Ddep.system=${{ github.event.inputs.system }} -Ddep.name=${{ github.event.inputs.name }} -Ddep.version=${{ github.event.inputs.version }}
+        mvn exec:java
       working-directory: java-shared-dependencies/dependency-analyzer

java-shared-dependencies/dependency-analyzer/pom.xml

@@ -64,6 +64,16 @@
       <artifactId>guava</artifactId>
       <version>33.3.1-jre</version>
     </dependency>
+    <dependency>
+      <groupId>com.google.cloud.tools</groupId>
+      <artifactId>dependencies</artifactId>
+      <version>1.5.13</version>
+    </dependency>
+    <dependency>
+      <groupId>com.opencsv</groupId>
+      <artifactId>opencsv</artifactId>
+      <version>5.9</version>
+    </dependency>
     <!-- test dependencies   -->
     <dependency>
       <groupId>org.mockito</groupId>

java-shared-dependencies/dependency-analyzer/src/main/java/com/google/cloud/DependencyAnalyzer.java

@@ -1,39 +1,90 @@
 package com.google.cloud;
 
-import static com.google.common.base.Preconditions.checkArgument;
-
 import com.google.cloud.external.DepsDevClient;
+import com.google.cloud.external.GitHubClient;
 import com.google.cloud.model.Advisory;
 import com.google.cloud.model.AdvisoryKey;
 import com.google.cloud.model.AnalysisResult;
 import com.google.cloud.model.License;
-import com.google.cloud.model.ReportResult;
 import com.google.cloud.model.PackageInfo;
+import com.google.cloud.model.ProjectKey;
+import com.google.cloud.model.PullRequestStatistics;
 import com.google.cloud.model.QueryResult;
+import com.google.cloud.model.RelatedProject;
+import com.google.cloud.model.ReportResult;
 import com.google.cloud.model.Result;
 import com.google.cloud.model.Version;
 import com.google.cloud.model.VersionKey;
+import com.google.cloud.tools.opensource.classpath.ClassPathBuilder;
+import com.google.cloud.tools.opensource.classpath.DependencyMediation;
+import com.google.cloud.tools.opensource.dependencies.Bom;
+import com.google.cloud.tools.opensource.dependencies.MavenRepositoryException;
+import com.opencsv.CSVWriter;
+import java.io.File;
+import java.io.FileWriter;
 import java.io.IOException;
 import java.net.URISyntaxException;
 import java.net.http.HttpClient;
+import java.nio.file.Paths;
 import java.util.ArrayDeque;
 import java.util.ArrayList;
 import java.util.HashSet;
 import java.util.List;
+import java.util.Optional;
 import java.util.Queue;
 import java.util.Set;
+import org.eclipse.aether.artifact.Artifact;
+import org.eclipse.aether.version.InvalidVersionSpecificationException;
 
 public class DependencyAnalyzer {
 
   private final DepsDevClient depsDevClient;
+  private final GitHubClient gitHubClient;
 
-  public DependencyAnalyzer(DepsDevClient depsDevClient) {
+  public DependencyAnalyzer(DepsDevClient depsDevClient, GitHubClient gitHubClient) {
     this.depsDevClient = depsDevClient;
+    this.gitHubClient = gitHubClient;
+  }
+
+  public List<AnalysisResult> analyze(String bomPath)
+      throws URISyntaxException, IOException, InterruptedException {
+    List<AnalysisResult> analysisResults = new ArrayList<>();
+    try {
+      Set<VersionKey> roots = getManagedDependenciesFromBom(Bom.readBom(Paths.get(bomPath)));
+      for (VersionKey versionKey : roots) {
+        if (versionKey.isSnapshot()) {
+          continue;
+        }
+        analysisResults.add(AnalysisResult.of(getPackageInfoFrom(versionKey)));
+      }
+
+    } catch (MavenRepositoryException | InvalidVersionSpecificationException ex) {
+      System.out.printf("Caught exception when resolving dependencies from %s.", bomPath);
+      ex.printStackTrace();
+      System.exit(1);
+    }
+
+    return analysisResults;
   }
 
-  public AnalysisResult analyze(String system, String packageName, String packageVersion)
-      throws URISyntaxException, IOException, InterruptedException, IllegalArgumentException {
-    VersionKey root = VersionKey.from(system, packageName, packageVersion);
+  private static Set<VersionKey> getManagedDependenciesFromBom(Bom bom)
+      throws InvalidVersionSpecificationException {
+    Set<VersionKey> res = new HashSet<>();
+    new ClassPathBuilder()
+        .resolve(bom.getManagedDependencies(), false, DependencyMediation.MAVEN)
+        .getClassPath()
+        .forEach(
+            classPath -> {
+              Artifact artifact = classPath.getArtifact();
+              String pkg = String.format("%s:%s", artifact.getGroupId(), artifact.getArtifactId());
+              res.add(VersionKey.from("MAVEN", pkg, artifact.getVersion()));
+            });
+
+    return res;
+  }
+
+  private List<PackageInfo> getPackageInfoFrom(VersionKey root)
+      throws URISyntaxException, IOException, InterruptedException {
     Set<VersionKey> seenPackage = new HashSet<>();
     seenPackage.add(root);
     Queue<VersionKey> queue = new ArrayDeque<>();
@@ -42,19 +93,22 @@ public AnalysisResult analyze(String system, String packageName, String packageV
     while (!queue.isEmpty()) {
       VersionKey versionKey = queue.poll();
       dependencies.add(versionKey);
+      if (versionKey.toString().equals("org.graalvm.sdk:nativeimage:24.1.1")) {
+        continue;
+      }
       List<VersionKey> directDependencies = depsDevClient.getDirectDependencies(versionKey);
       // only add unseen dependencies to the queue.
       directDependencies
           .stream()
           .filter(seenPackage::add)
           .forEach(queue::offer);
     }
-
     List<PackageInfo> result = new ArrayList<>();
     for (VersionKey versionKey : dependencies) {
       QueryResult packageInfo = depsDevClient.getQueryResult(versionKey);
       List<License> licenses = new ArrayList<>();
       List<Advisory> advisories = new ArrayList<>();
+      Optional<PullRequestStatistics> statistics = Optional.empty();
       for (Result res : packageInfo.results()) {
         Version version = res.version();
         for (String license : version.licenses()) {
@@ -63,12 +117,20 @@ public AnalysisResult analyze(String system, String packageName, String packageV
         for (AdvisoryKey advisoryKey : version.advisoryKeys()) {
           advisories.add(depsDevClient.getAdvisory(advisoryKey.id()));
         }
-      }
 
-      result.add(new PackageInfo(versionKey, licenses, advisories));
+        // for (RelatedProject project : version.relatedProjects()) {
+        //   ProjectKey projectKey = project.projectKey();
+        //   if (!projectKey.isGitHubProject()) {
+        //     continue;
+        //   }
+        //   statistics = Optional.of(gitHubClient.listMonthlyPullRequestStatusOf(
+        //       projectKey.organization(), projectKey.repo()));
+        // }
+      }
+      result.add(new PackageInfo(versionKey, licenses, advisories, statistics));
     }
 
-    return AnalysisResult.of(result);
+    return result;
   }
 
   /**
@@ -87,24 +149,13 @@ public AnalysisResult analyze(String system, String packageName, String packageV
    * @throws IllegalArgumentException if the format of package name is incorrect according to the
    * package management system.
    */
-  public static void main(String[] args) throws IllegalArgumentException {
-    checkArgument(args.length == 3,
-        """
-            The length of the inputs should be 3.
-            The 1st input should be the package management system.
-            The 2nd input should be the package name.
-            The 3rd input should be the package version.
-            """
-    );
-
-    String system = args[0];
-    String packageName = args[1];
-    String packageVersion = args[2];
+  public static void main(String[] args) throws IllegalArgumentException, IOException {
     DependencyAnalyzer dependencyAnalyzer = new DependencyAnalyzer(
-        new DepsDevClient(HttpClient.newHttpClient()));
-    AnalysisResult analyzeReport = null;
+        new DepsDevClient(HttpClient.newHttpClient()),
+        new GitHubClient(HttpClient.newHttpClient()));
+    List<AnalysisResult> analysisResults = null;
     try {
-      analyzeReport = dependencyAnalyzer.analyze(system, packageName, packageVersion);
+      analysisResults = dependencyAnalyzer.analyze("../pom.xml");
     } catch (URISyntaxException | IOException | InterruptedException ex) {
       System.out.println(
           "Caught exception when fetching package information from https://deps.dev/");
@@ -113,13 +164,22 @@ public static void main(String[] args) throws IllegalArgumentException {
     }
 
     System.out.println("Please copy and paste the package information below to your ticket.\n");
-    System.out.println(analyzeReport.toString());
-    ReportResult result = analyzeReport.getAnalysisResult();
-    System.out.println(result);
-    if (result.equals(ReportResult.FAIL)) {
-      System.out.println(
-          "Please refer to go/cloud-java-rotations#security-advisories-monitoring for further actions");
-      System.exit(1);
-    }
+    // create CSVWriter object filewriter object as parameter
+    CSVWriter writer = new CSVWriter(new FileWriter("dependency-report.csv"));
+    writer.writeNext(new String[]{"Dependency", "License problem", "Vulnerabilities"});
+    analysisResults.forEach(analysisResult -> {
+      System.out.println(analysisResult.toString());
+      String licenseProblem = "";
+      if (!analysisResult.getNonCompliantLicenses().isEmpty()) {
+        licenseProblem = analysisResult.getNonCompliantLicenses().toString();
+      }
+      String advisories = "";
+      if (!analysisResult.getAdvisories().isEmpty()) {
+        advisories = analysisResult.getAdvisories().toString();
+      }
+      writer.writeNext(
+          new String[]{analysisResult.getRoot().toString(), licenseProblem, advisories});
+    });
+    writer.close();
   }
 }

java-shared-dependencies/dependency-analyzer/src/main/java/com/google/cloud/external/GitHubClient.java

@@ -5,6 +5,7 @@
 import com.google.cloud.model.PullRequestStatistics;
 import com.google.gson.Gson;
 import com.google.gson.GsonBuilder;
+import com.google.gson.JsonSyntaxException;
 import com.google.gson.reflect.TypeToken;
 import java.io.IOException;
 import java.net.URI;
@@ -19,6 +20,7 @@
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Objects;
+import java.util.logging.Logger;
 
 /**
  * GitHubClient is a class that sends HTTP requests to the GitHub RESTful API. It provides methods
@@ -29,6 +31,8 @@
  * requests and {@link com.google.gson.Gson} for handling JSON serialization/deserialization.
  */
 public class GitHubClient {
+
+  private final static Logger LOGGER = Logger.getLogger(GitHubClient.class.getName());
   private final HttpClient client;
   private final Gson gson;
   private static final String PULL_REQUESTS_BASE =
@@ -81,12 +85,21 @@ private List<PullRequest> listPullRequests(String organization, String repo)
     List<PullRequest> pullRequests = new ArrayList<>();
     int page = 1;
     while (pullRequests.size() < MAX_PULL_REQUEST_NUM) {
+      System.out.println(getPullRequestsUrl(organization, repo, page));
       HttpResponse<String> response = getResponse(getPullRequestsUrl(organization, repo, page));
-      pullRequests.addAll(
-          gson.fromJson(response.body(), new TypeToken<List<PullRequest>>() {}.getType()));
+      try {
+        pullRequests.addAll(
+            gson.fromJson(response.body(), new TypeToken<List<PullRequest>>() {
+            }.getType()));
+      } catch (JsonSyntaxException ex) {
+        LOGGER.warning(String.format(
+            "Can't parse response from GitHub API.\nOrganization: %s, repo: %s, page: %s\n",
+            organization, repo, page));
+        break;
+      }
+
       page++;
     }
-
     return pullRequests;
   }
 

java-shared-dependencies/dependency-analyzer/src/main/java/com/google/cloud/model/AnalysisResult.java

@@ -3,6 +3,7 @@
 import static com.google.cloud.external.DepsDevClient.QUERY_URL_BASE;
 
 import java.util.ArrayList;
+import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
@@ -20,6 +21,18 @@ private AnalysisResult(List<PackageInfo> result) {
     this.nonCompliantLicenses = getNonCompliantLicenses(result);
   }
 
+  public Map<VersionKey, List<License>> getNonCompliantLicenses() {
+    return Collections.unmodifiableMap(nonCompliantLicenses);
+  }
+
+  public Map<VersionKey, List<Advisory>> getAdvisories() {
+    return Collections.unmodifiableMap(advisories);
+  }
+
+  public VersionKey getRoot() {
+    return packageInfos.get(0).versionKey();
+  }
+
   public static AnalysisResult of(List<PackageInfo> result) {
     return new AnalysisResult(result);
   }
@@ -85,13 +98,27 @@ private String packageInfoReport() {
     if (packageInfos.size() == 1) {
       builder.append(String.format("%s has no dependency.", root.versionKey()));
     } else {
+      builder.append("==========Non-compliant licenses==========\n");
       for (int i = 1; i < packageInfos.size(); i++) {
         PackageInfo info = packageInfos.get(i);
-        String dependencyInfo = String.format("""
-            ### Package information of %s
-            %s
-            """, info.versionKey(), packageInfoSection(info));
-        builder.append(dependencyInfo);
+        boolean hasNonComplaintLicenses = false;
+        for (License license : info.licenses()) {
+          if (!license.isCompliant()) {
+            hasNonComplaintLicenses = true;
+            break;
+          }
+        }
+        if (hasNonComplaintLicenses) {
+          builder.append(String.format("%s: %s\n", info.versionKey(), info.licenses()));
+        }
+      }
+
+      builder.append("==========Security vulnerabilities==========\n");
+      for (int i = 1; i < packageInfos.size(); i++) {
+        PackageInfo info = packageInfos.get(i);
+        for (Advisory advisory : info.advisories()) {
+          builder.append(String.format("%s: %s\n", info.versionKey(), advisory.url()));
+        }
       }
     }
     builder.append("\n");
@@ -105,11 +132,13 @@ private String packageInfoSection(PackageInfo packageInfo) {
     String packageInfoReport = """
         Licenses: %s
         Vulnerabilities: %s.
+        Pull request freshness: %s.
         Checked in [%s (%s)](%s)
         """;
     return String.format(packageInfoReport,
         packageInfo.licenses(),
         packageInfo.advisories(),
+        packageInfo.pullRequestStatistics().orElse(null),
         versionKey.name(),
         versionKey.version(),
         getQueryUrl(