[Dvaas] Rename time_limit to packet_synthesis_time_limit. Create func…

…tion to access packet_synthesis_timed_out in ValidationResult. Add more comments to PacketSynthesizerTimedOut.Add support for non-entry entities.

dvaas/BUILD.bazel

@@ -43,6 +43,7 @@ cc_library(
         ":test_vector_cc_proto",
         ":user_provided_packet_test_vector",
         ":validation_result",
+        "//gutil:proto",
         "//gutil:status",
         "//gutil:test_artifact_writer",
         "//gutil:version",

dvaas/dataplane_validation.cc

@@ -34,6 +34,7 @@
 #include "dvaas/user_provided_packet_test_vector.h"
 #include "dvaas/validation_result.h"
 #include "glog/logging.h"
+#include "gutil/proto.h"
 #include "gutil/status.h"
 #include "gutil/test_artifact_writer.h"
 #include "gutil/version.h"
@@ -176,10 +177,10 @@ absl::StatusOr<GenerateTestVectorsResult> GenerateTestVectors(
   RETURN_IF_ERROR(writer.AppendToTestArtifact(
       "sut_bmv2_config.txt", p4_spec.bmv2_config.DebugString()));
 
-  // Read P4Info and entries from SUT, sorted for determinism.
+  // Read P4Info and control plane entities from SUT, sorted for determinism.
   ASSIGN_OR_RETURN(pdpi::IrP4Info ir_p4info, pdpi::GetIrP4Info(*sut.p4rt));
-  ASSIGN_OR_RETURN(pdpi::IrTableEntries entries,
-                   pdpi::ReadIrTableEntriesSorted(*sut.p4rt));
+  ASSIGN_OR_RETURN(pdpi::IrEntities entities,
+                   pdpi::ReadIrEntitiesSorted(*sut.p4rt));
 
   // Get enabled Ethernet ports from SUT's GNMI config.
   ASSIGN_OR_RETURN(std::vector<pins_test::P4rtPortId> ports,
@@ -196,12 +197,12 @@ absl::StatusOr<GenerateTestVectorsResult> GenerateTestVectors(
   LOG(INFO) << "Synthesizing test packets";
   ASSIGN_OR_RETURN(generate_test_vectors_result.packet_synthesis_result,
                    backend.SynthesizePackets(
-                       ir_p4info, entries, p4_spec.p4_symbolic_config, ports,
+                       ir_p4info, entities, p4_spec.p4_symbolic_config, ports,
                        [&](absl::string_view stats) {
                          return writer.AppendToTestArtifact(
                              "test_packet_stats.txt", stats);
                        },
-                       params.time_limit));
+                       params.packet_synthesis_time_limit));
 
   RETURN_IF_ERROR(writer.AppendToTestArtifact(
       "synthesized_packets.txt",
@@ -213,7 +214,7 @@ absl::StatusOr<GenerateTestVectorsResult> GenerateTestVectors(
   LOG(INFO) << "Generating test vectors with output prediction";
   ASSIGN_OR_RETURN(generate_test_vectors_result.packet_test_vector_by_id,
                    backend.GeneratePacketTestVectors(
-                       ir_p4info, entries, p4_spec.bmv2_config, ports,
+                       ir_p4info, entities, p4_spec.bmv2_config, ports,
                        generate_test_vectors_result.packet_synthesis_result
                            .synthesized_packets,
                        default_ingress_port));
@@ -234,19 +235,19 @@ absl::StatusOr<ValidationResult> DataplaneValidator::ValidateDataplane(
     ASSIGN_OR_RETURN(pdpi::IrP4Info ir_info,
                      pdpi::GetIrP4Info(*control_switch.p4rt));
 
-    // Clear control switch table entries and install punt entries instead.
-    RETURN_IF_ERROR(pdpi::ClearTableEntries(control_switch.p4rt.get()));
+    // Clear control switch entities and install punt entries instead.
+    RETURN_IF_ERROR(pdpi::ClearEntities(*control_switch.p4rt));
     ASSIGN_OR_RETURN(pdpi::IrEntities punt_entries,
                      backend_->GetEntitiesToPuntAllPackets(ir_info));
     RETURN_IF_ERROR(
         pdpi::InstallIrEntities(*control_switch.p4rt, punt_entries));
   }
 
   // Read and store table entries on SUT as an artifact.
-  ASSIGN_OR_RETURN(pdpi::IrTableEntries entries,
-                   pdpi::ReadIrTableEntriesSorted(*sut.p4rt));
-  RETURN_IF_ERROR(writer->AppendToTestArtifact("sut_ir_table_entries.txt",
-                                               entries.DebugString()));
+  ASSIGN_OR_RETURN(pdpi::IrEntities entities,
+                   pdpi::ReadIrEntitiesSorted(*sut.p4rt));
+  RETURN_IF_ERROR(writer->AppendToTestArtifact(
+      "sut_ir_entities.txtpb", gutil::PrintTextProto(entities)));
 
   // Store port mapping as an artifact (identity if not given a value).
   MirrorTestbedP4rtPortIdMap mirror_testbed_port_map =

dvaas/dataplane_validation.h

@@ -104,11 +104,11 @@ struct DataplaneValidationParams {
   // non-standard testbeds only.
   std::optional<MirrorTestbedP4rtPortIdMap> mirror_testbed_port_map_override;
 
-  // The 'time_limit' sets the maximum allowed time for dataplane validation to
-  // synthesize test packets. If nullopt, packet synthesizer runs to completion
-  // for its coverage goals. Otherwise, if packet synthesis timed out, the
-  // synthesis results cover the coverage goals only partially.
-  std::optional<absl::Duration> time_limit = std::nullopt;
+  // Maximum allowed time for dataplane validation to synthesize test packets.
+  // If nullopt, packet synthesizer runs to completion for its coverage goals.
+  // Otherwise, if packet synthesis timed out, the synthesis results cover the
+  // coverage goals only partially.
+  std::optional<absl::Duration> packet_synthesis_time_limit = std::nullopt;
 };
 
 // Forward declaration. See below for description.
@@ -221,7 +221,7 @@ class DataplaneValidationBackend {
   // even open-source yet), so DVaaS takes the synthesis function as an input
   // parameter.
   virtual absl::StatusOr<PacketSynthesisResult> SynthesizePackets(
-      const pdpi::IrP4Info& ir_p4info, const pdpi::IrTableEntries& ir_entries,
+      const pdpi::IrP4Info& ir_p4info, const pdpi::IrEntities& ir_entities,
       const p4::v1::ForwardingPipelineConfig& p4_symbolic_config,
       absl::Span<const pins_test::P4rtPortId> ports,
       const OutputWriterFunctionType& write_stats,
@@ -243,7 +243,7 @@ class DataplaneValidationBackend {
   //  3. The packet will be padded to minimum size and the computed fields
   //     recomputed.
   virtual absl::StatusOr<PacketTestVectorById> GeneratePacketTestVectors(
-      const pdpi::IrP4Info& ir_p4info, const pdpi::IrTableEntries& ir_entries,
+      const pdpi::IrP4Info& ir_p4info, const pdpi::IrEntities& ir_entities,
       const p4::v1::ForwardingPipelineConfig& bmv2_config,
       absl::Span<const pins_test::P4rtPortId> ports,
       std::vector<p4_symbolic::packet_synthesizer::SynthesizedPacket>&

dvaas/validation_result.cc

@@ -87,4 +87,8 @@ std::vector<std::string> ValidationResult::GetAllFailures() const {
   return failures;
 }
 
+bool ValidationResult::PacketSynthesizerTimedOut() const {
+  return packet_synthesis_result_.packet_synthesis_timed_out;
+}
+
 }  // namespace dvaas

dvaas/validation_result.h

@@ -39,7 +39,7 @@ struct PacketSynthesisResult {
   // True if and only if packet synthesis runs with a time limit and does not
   // finish within that time limit. If true, `synthesized_packets` may not
   // fully cover the target coverage goals.
-  bool packet_synthesis_timed_out;
+  bool packet_synthesis_timed_out = false;
 };
 
 // The result of dataplane validation, as returned to DVaaS users.
@@ -76,6 +76,14 @@ class [[nodiscard]] ValidationResult {
                    const SwitchOutputDiffParams& diff_params,
                    const PacketSynthesisResult& packet_synthesis_result);
 
+  // Returns true if and only if packet synthesis runs with a time limit and
+  // does not finish within that time limit.
+  // NOTE: If true, dataplane validation did not fully cover the target coverage
+  // goals (in the worst case, it may have not tested dataplane at all). This
+  // should be taken into account when interpreting the results of other
+  // functions like `HasSuccessRateOfAtLeast` and `GetAllFailures`.
+  bool PacketSynthesizerTimedOut() const;
+
  private:
   PacketTestOutcomes test_outcomes_;
   TestVectorStats test_vector_stats_;