feat(api): upgrade grpc-gateway to v2 with protoname (#1337)

* upgrade grpc-gateway to v2

* fix common client code for grpc gateway v2

* fix license and test case

* regenerate python client

* fix python http client code

* update healthz class to new python openapi client

* update new python client function class name to new openapi standard;

* overwrite upstream old client function convention

* fix python upload class

* fix python upload class

* update api class

* update upload api swagger codegen manually

* revert old fixes

* fix new sdk payload parameter mapping

* update names to protoname

* revert backend changes to use protoname

* update proto spec

* update tests

* update sdk client to handle new json name

* fix typo

backend/api/Dockerfile

@@ -14,10 +14,10 @@
 
 # Generate client code (go & json) from API protocol buffers
 FROM golang:1.15.10 as generator
-ENV GRPC_GATEWAY_VERSION v1.9.0
-ENV GO_SWAGGER_VERSION v0.18.0
-ENV GOLANG_PROTOBUF_VERSION v1.5.1
-ENV GRPC_VERSION v1.23.0
+ENV GRPC_GATEWAY_VERSION v2.11.3
+ENV GO_SWAGGER_VERSION v0.30.4
+ENV GOLANG_PROTOBUF_VERSION v1.5.2
+ENV GRPC_VERSION v1.48.0
 ENV PROTOC_VERSION 3.17.3
 ENV GOBIN=/go/bin
 
@@ -38,9 +38,10 @@ RUN mkdir grpc && git clone --depth 1 --branch $GRPC_VERSION https://github.com/
 # Install protoc-gen-rpc-gateway && protoc-gen-swagger.
 RUN cd grpc-ecosystem/grpc-gateway && GO111MODULE=on go mod vendor
 RUN go install github.com/grpc-ecosystem/grpc-gateway/protoc-gen-grpc-gateway
-RUN go install github.com/grpc-ecosystem/grpc-gateway/protoc-gen-swagger
+RUN go install github.com/grpc-ecosystem/grpc-gateway/protoc-gen-openapiv2
 
 # Download go-swagger binary.
+# swagger doesn't exist for openapiv2 yet
 RUN curl -LO "https://github.com/go-swagger/go-swagger/releases/download/${GO_SWAGGER_VERSION}/swagger_linux_amd64"
 RUN chmod +x swagger_linux_amd64 && mv swagger_linux_amd64 /usr/bin/swagger
 

backend/api/Makefile

@@ -17,7 +17,9 @@
 IMAGE_TAG=kfp-api-generator
 # Contact one of Bobgy, or zijianjoy if this remote image needs an update.
 REMOTE_IMAGE=gcr.io/ml-pipeline-test/api-generator
-PREBUILT_REMOTE_IMAGE=gcr.io/ml-pipeline-test/api-generator@sha256:10a09669794180b91c8b0fffb803db81eb458372e6ab669051f40d4979cdf43c
+PREBUILT_REMOTE_IMAGE=aipipeline/api-generator:openapiv2.11.3
+# PREBUILT_REMOTE_IMAGE=aipipeline/api-generator:test
+API_VERSION=v1
 
 # Generate clients using a pre-built api-generator image.
 .PHONY: generate

backend/api/build_kfp_server_api_python_package.sh

@@ -34,6 +34,8 @@ if [ -z "$VERSION" ]; then
     echo "ERROR: $REPO_ROOT/VERSION is empty"
     exit 1
 fi
+API_VERSION=v1
+
 
 codegen_file=/tmp/openapi-generator-cli.jar
 # Browse all versions in: https://repo1.maven.org/maven2/org/openapitools/openapi-generator-cli/

backend/api/hack/generator.sh

@@ -35,18 +35,16 @@ mkdir -p backend/api/${API_VERSION}/go_client
 
 # Generate *.pb.go (grpc api client) from *.proto.
 ${PROTOCCOMPILER} -I. -Ibackend/api/${API_VERSION} \
-    -I/go/src/github.com/grpc-ecosystem/grpc-gateway/third_party/googleapis \
     -I/go/src/github.com/grpc-ecosystem/grpc-gateway/ \
-    -I/go/src/github.com/grpc-ecosystem/grpc-gateway/protoc-gen-swagger/options/ \
+    -I/go/src/github.com/grpc-ecosystem/grpc-gateway/protoc-gen-openapiv2/options/ \
     -I/usr/include/ \
     --plugin=protoc-gen-go=/go/bin/protoc-gen-go  \
     --go_out=plugins=grpc:${TMP_OUTPUT} \
     backend/api/${API_VERSION}/*.proto
 # Generate *.pb.gw.go (grpc api rest client) from *.proto.
 ${PROTOCCOMPILER} -I. -Ibackend/api/${API_VERSION} \
-    -I/go/src/github.com/grpc-ecosystem/grpc-gateway/third_party/googleapis \
     -I/go/src/github.com/grpc-ecosystem/grpc-gateway/ \
-    -I/go/src/github.com/grpc-ecosystem/grpc-gateway/protoc-gen-swagger/options/ \
+    -I/go/src/github.com/grpc-ecosystem/grpc-gateway/protoc-gen-openapiv2/options/ \
     -I/usr/include/ \
     --plugin=protoc-gen-grpc-gateway=/go/bin/protoc-gen-grpc-gateway \
     --grpc-gateway_out=logtostderr=true:${TMP_OUTPUT} \
@@ -55,12 +53,11 @@ ${PROTOCCOMPILER} -I. -Ibackend/api/${API_VERSION} \
 cp ${TMP_OUTPUT}/github.com/kubeflow/pipelines/backend/api/${API_VERSION}/go_client/* ./backend/api/${API_VERSION}/go_client
 # Generate *.swagger.json from *.proto into swagger folder.
 ${PROTOCCOMPILER} -I. -Ibackend/api/${API_VERSION} \
-    -I/go/src/github.com/grpc-ecosystem/grpc-gateway/third_party/googleapis \
     -I/go/src/github.com/grpc-ecosystem/grpc-gateway/ \
-    -I/go/src/github.com/grpc-ecosystem/grpc-gateway/protoc-gen-swagger/options/ \
+    -I/go/src/github.com/grpc-ecosystem/grpc-gateway/protoc-gen-openapiv2/options/ \
     -I//usr/include/ \
-    --plugin=protoc-gen-swagger=/go/bin/protoc-gen-swagger \
-    --swagger_out=logtostderr=true:${TMP_OUTPUT} \
+    --plugin=protoc-gen-openapiv2=/go/bin/protoc-gen-openapiv2 \
+    --openapiv2_out=logtostderr=true,json_names_for_fields=false:${TMP_OUTPUT} \
     backend/api/${API_VERSION}/*.proto
 # Move *.swagger.json files into swagger folder.
 cp -a ${TMP_OUTPUT}/backend/api/${API_VERSION}/*.swagger.json ./backend/api/${API_VERSION}/swagger

backend/api/v1/auth.proto

@@ -19,9 +19,9 @@ package v1;
 
 import "google/api/annotations.proto";
 import "google/protobuf/empty.proto";
-import "protoc-gen-swagger/options/annotations.proto";
+import "protoc-gen-openapiv2/options/annotations.proto";
 
-option (grpc.gateway.protoc_gen_swagger.options.openapiv2_swagger) = {
+option (grpc.gateway.protoc_gen_openapiv2.options.openapiv2_swagger) = {
   responses: {
     key: "default";
     value: {

backend/api/v1/error.proto

@@ -20,8 +20,8 @@ package v1;
 import "google/protobuf/any.proto";
 
 message Error {
-  string error_message = 1;
-  string error_details = 2;
+  string error_message = 1 [json_name = "error_message"];
+  string error_details = 2 [json_name = "error_details"];
 }
 
 message Status {

backend/api/v1/experiment.proto

@@ -22,9 +22,9 @@ import "backend/api/v1/resource_reference.proto";
 import "google/api/annotations.proto";
 import "google/protobuf/empty.proto";
 import "google/protobuf/timestamp.proto";
-import "protoc-gen-swagger/options/annotations.proto";
+import "protoc-gen-openapiv2/options/annotations.proto";
 
-option (grpc.gateway.protoc_gen_swagger.options.openapiv2_swagger) = {
+option (grpc.gateway.protoc_gen_openapiv2.options.openapiv2_swagger) = {
   responses: {
     key: "default";
     value: {
@@ -120,16 +120,16 @@ message ListExperimentsRequest {
   // A page token to request the next page of results. The token is acquried
   // from the nextPageToken field of the response from the previous
   // ListExperiment call or can be omitted when fetching the first page.
-  string page_token = 1;
+  string page_token = 1 [json_name = "page_token"];
 
   // The number of experiments to be listed per page. If there are more
   // experiments than this number, the response message will contain a
   // nextPageToken field you can use to fetch the next page.
-  int32 page_size = 2;
+  int32 page_size = 2 [json_name = "page_size"];
 
   // Can be format of "field_name", "field_name asc" or "field_name desc"
   // Ascending by default.
-  string sort_by = 3;
+  string sort_by = 3 [json_name = "sort_by"];
 
   // A url-encoded, JSON-serialized Filter protocol buffer (see
   // [filter.proto](https://github.com/kubeflow/pipelines/blob/master/backend/api/v1/filter.proto)).
@@ -138,18 +138,18 @@ message ListExperimentsRequest {
   // What resource reference to filter on.
   // For Experiment, the only valid resource type is Namespace. An sample query string could be
   // resource_reference_key.type=NAMESPACE&resource_reference_key.id=ns1
-  ResourceKey resource_reference_key = 5;
+  ResourceKey resource_reference_key = 5 [json_name = "resource_reference_key"];
 }
 
 message ListExperimentsResponse {
   // A list of experiments returned.
   repeated Experiment experiments = 1;
 
   // The total number of experiments for the given query.
-  int32 total_size = 3;
+  int32 total_size = 3 [json_name = "total_size"];
 
   // The token to list the next page of experiments.
-  string next_page_token = 2;
+  string next_page_token = 2 [json_name = "next_page_token"];
 }
 
 message DeleteExperimentRequest {
@@ -168,11 +168,11 @@ message Experiment {
   string description = 3;
 
   // Output. The time that the experiment created.
-  google.protobuf.Timestamp created_at = 4;
+  google.protobuf.Timestamp created_at = 4 [json_name = "created_at"];
 
   // Optional input field. Specify which resource this run belongs to.
   // For Experiment, the only valid resource reference is a single Namespace.
-  repeated ResourceReference resource_references = 5;
+  repeated ResourceReference resource_references = 5 [json_name = "resource_references"];
 
   enum StorageState {
     STORAGESTATE_UNSPECIFIED = 0;
@@ -181,7 +181,7 @@ message Experiment {
   }
 
   // Output. Specifies whether this experiment is in archived or available state.
-  StorageState storage_state = 6;
+  StorageState storage_state = 6 [json_name = "storage_state"];
 }
 
 message ArchiveExperimentRequest {

backend/api/v1/filter.proto

@@ -48,18 +48,18 @@ message Predicate {
 
   string key = 2;
   oneof value {
-    int32 int_value = 3;
-    int64 long_value = 4;
-    string string_value = 5;
+    int32 int_value = 3 [json_name = "int_value"];
+    int64 long_value = 4 [json_name = "long_value"];
+    string string_value = 5 [json_name = "string_value"];
 
     // Timestamp values will be converted to Unix time (seconds since the epoch)
     // prior to being used in a filtering operation.
-    google.protobuf.Timestamp timestamp_value = 6;
+    google.protobuf.Timestamp timestamp_value = 6 [json_name = "timestamp_value"];
 
     // Array values below are only meant to be used by the IN operator.
-    IntValues int_values = 7;
-    LongValues long_values = 8;
-    StringValues string_values = 9;
+    IntValues int_values = 7 [json_name = "int_values"];
+    LongValues long_values = 8 [json_name = "long_values"];
+    StringValues string_values = 9 [json_name = "string_values"];
   }
 }