Merge pull request #52 from qaware/10-raw-format-responses

PR: Decode raw responses without type

EXAMPLES.md

@@ -35,6 +35,7 @@ formattedDate: "Thu, 01 Jan 1970 00:00:00 GMT"
 ```
 
 **Explict full message package paths and explicit proto file**
+
 ```bash
 $ docker run -v "$PWD/test/proto:/proto" --network host qaware/protocurl \
   -f happyday.proto -i happyday.HappyDayRequest -o happyday.HappyDayResponse \
@@ -48,6 +49,7 @@ formattedDate: "Thu, 01 Jan 1970 00:00:00 GMT"
 ```
 
 **Using imported and nested messages such as well-known Google Protobuf types**
+
 ```bash
 $ docker run -v "$PWD/test/proto:/proto" --network host qaware/protocurl \
   -i ..HappyDayRequest -o ..HappyDayResponse \
@@ -62,6 +64,27 @@ date: {
 formattedDate: "Wed, 23 Mar 2022 14:15:39 GMT"
 ```
 
+**Omitting -o \<response-type> shows raw format**
+
+```bash
+$ docker run -v "$PWD/test/proto:/proto" --network host qaware/protocurl \
+   -q -f happyday.proto -i happyday.HappyDayRequest \
+   -u http://localhost:8080/happy-day/verify \
+   -d "includeReason: true"
+
+1: 1
+2: "Thursday is a Happy Day! ⭐"
+3: "Thu, 01 Jan 1970 00:00:00 GMT"
+4: ""
+```
+
+When the response type is unknown or one wants to debug and see what is all in the response,
+then this format shows the values for the given field numbers (instead of their field names).
+
+However, since
+[Protobuf is not self-describing](https://developers.google.com/protocol-buffers/docs/techniques#self-description)
+the types cannot be correctly inferred and may be incorrect.
+
 **JSON**
 
 ```bash
@@ -142,6 +165,7 @@ Invoked with following default & parsed arguments:
   "DataText": "date: { seconds: 1648044939}",
   "InTextType": "text",
   "OutTextType": "text",
+  "DecodeRawResponse": false,
   "DisplayBinaryAndHttp": true,
   "RequestHeaders": [
     "Content-Type: application/x-protobuf"
@@ -386,7 +410,7 @@ Total curl args:
 =========================== Response Headers =========================== <<<
 HTTP/1.1 200 OK
 Content-Type: application/x-protobuf
-Date: Tue, 26 Apr 2022 22:44:45 GMT
+Date: Wed, 18 May 2022 20:30:02 GMT
 Connection: keep-alive
 Keep-Alive: timeout=5
 Content-Length: 35

doc/generate-docs.sh

@@ -3,6 +3,11 @@ set -e
 
 WORKING_DIR="$1"
 
+if [[ "$WORKING_DIR" == "" ]]; then
+  echo "Please provide the working directory as a a docker-mount friendly path."
+  exit 1
+fi
+
 source test/suite/setup.sh
 
 ESCAPED="failed to substitute"
@@ -117,6 +122,20 @@ $(docker run -v "$WORKING_DIR/test/proto:/proto" --network host protocurl \
 escapeString "$EXAMPLE_OUTPUT_ONLY"
 EXAMPLE_OUTPUT_ONLY="$ESCAPED"
 
+# EXAMPLE_RAW_FORMAT =============================
+EXAMPLE_RAW_FORMAT="\$ docker run -v \"\$PWD/test/proto:/proto\" --network host qaware/protocurl \\
+   -q -f happyday.proto -i happyday.HappyDayRequest \\
+   -u http://localhost:8080/happy-day/verify \\
+   -d \"includeReason: true\"
+
+$(docker run -v "$WORKING_DIR/test/proto:/proto" --network host protocurl \
+  -q -f happyday.proto -i happyday.HappyDayRequest \
+  -u http://localhost:8080/happy-day/verify \
+  -d "includeReason: true")"
+
+escapeString "$EXAMPLE_RAW_FORMAT"
+EXAMPLE_RAW_FORMAT="$ESCAPED"
+
 # EXAMPLE OUTPUT ONLY WITH ERROR =============================
 EXAMPLE_OUTPUT_ONLY_WITH_ERR_1="\$ docker run -v \"\$PWD/test/proto:/proto\" --network host qaware/protocurl \\
    -q -i ..HappyDayRequest -o ..HappyDayResponse \\
@@ -157,6 +176,7 @@ echo "$EXAMPLES_TEMPLATE" |
   sed "s%___EXAMPLE_1___%$EXAMPLE_1%" |
   sed "s%___EXAMPLE_2___%$EXAMPLE_2%" |
   sed "s%___EXAMPLE_3___%$EXAMPLE_3%" |
+  sed "s%___EXAMPLE_RAW_FORMAT__%$EXAMPLE_RAW_FORMAT%" |
   sed "s%___EXAMPLE_JSON___%$EXAMPLE_JSON%" |
   sed "s%___EXAMPLE_JSON_PRETTY___%$EXAMPLE_JSON_PRETTY%" |
   sed "s%___EXAMPLE_OUTPUT_ONLY___%$EXAMPLE_OUTPUT_ONLY%" |

doc/generated.usage.txt

@@ -11,8 +11,9 @@ The bundle also includes the google protobuf .proto files necessary to create Fi
 If the bundled 'protoc' is used, then these .proto files are included. Otherwise .proto files from the system-wide include are used.
 The Header 'Content-Type: application/x-protobuf' is set as a request header by default.
 When converting between binary and text, the encoding UTF-8 is always used.
+When the correct response type is unknown or being debugged, omitting -o <response-type> will attempt to show the response in raw format.
 
-Bug reports: https://github.com/qaware/protocurl/issues
+Enhancements and bugs: https://github.com/qaware/protocurl/issues
 
 Examples:
   protocurl -I my-protos -i package.path.Req -o package.path.Resp -u http://example.com/api -d "myField: true, otherField: 1337"
@@ -22,6 +23,7 @@ Flags:
   -C, --curl-args string          Additional cURL args which will be passed on to cURL during request invocation for further configuration. Also activates --curl.
       --curl-path string          Uses the given path to invoke curl instead of searching for curl in PATH. Also activates --curl.
   -d, --data-text string          Mandatory: The payload data in Protobuf text format or JSON. It is inferred from the input as JSON if the first token is a '{'. The format can be set explicitly via --in. See https://github.com/qaware/protocurl
+      --decode-raw                Decode the response into textual format without the schema by only showing field numbers and inferred field types. Types may be incorrect. Only output format text is supported. Use -o <response-type> to see correct contents.
   -D, --display-binary-and-http   Displays the binary request and response as well as the non-binary response headers.
   -h, --help                      help for protocurl
       --in string                 Specifies, in which format the input -d should be interpreted in. 'text' (default) uses the Protobuf text format and 'json' uses JSON.
@@ -34,7 +36,7 @@ Flags:
       --protoc-path string        Uses the given path to invoke protoc instead of searching for protoc in PATH. Also activates --protoc.
   -H, --request-header string     Adds the string header to the invocation of cURL. This option is not supported when --no-curl is active. E.g. -H 'MyHeader: FooBar'.
   -i, --request-type string       Mandatory: Message name or full package path of the Protobuf request type. The path can be shortened to '..', if the name of the request message is unique. E.g. mypackage.MyRequest or ..MyRequest
-  -o, --response-type string      Mandatory: The Protobuf response type. See -i <request-type>
+  -o, --response-type string      The Protobuf response type. See -i <request-type>. Overrides --decode-raw. If not set, then --decode-raw is used.
   -q, --show-output-only          Suppresses all output except response Protobuf as text. Overrides and deactivates -v and -D. Errors are still printed to stderr.
   -u, --url string                Mandatory: The url to send the request to
   -v, --verbose                   Prints version and enables verbose output. Also activates -D.

doc/template.EXAMPLES.md

@@ -25,15 +25,30 @@ ___EXAMPLE_1___
 ```
 
 **Explict full message package paths and explicit proto file**
+
 ```bash
 ___EXAMPLE_2___
 ```
 
 **Using imported and nested messages such as well-known Google Protobuf types**
+
 ```bash
 ___EXAMPLE_3___
 ```
 
+**Omitting -o \<response-type> shows raw format**
+
+```bash
+___EXAMPLE_RAW_FORMAT__
+```
+
+When the response type is unknown or one wants to debug and see what is all in the response,
+then this format shows the values for the given field numbers (instead of their field names).
+
+However, since
+[Protobuf is not self-describing](https://developers.google.com/protocol-buffers/docs/techniques#self-description)
+the types cannot be correctly inferred and may be incorrect.
+
 **JSON**
 
 ```bash

src/flags.go

@@ -60,8 +60,10 @@ func intialiseFlags() {
 	AssertSuccess(rootCmd.MarkFlagRequired("request-type"))
 
 	flags.StringVarP(&CurrentConfig.ResponseType, "response-type", "o", "",
-		"Mandatory: The Protobuf response type. See -i <request-type>")
-	AssertSuccess(rootCmd.MarkFlagRequired("response-type"))
+		"The Protobuf response type. See -i <request-type>. Overrides --decode-raw. If not set, then --decode-raw is used.")
+
+	flags.BoolVar(&CurrentConfig.DecodeRawResponse, "decode-raw", false,
+		"Decode the response into textual format without the schema by only showing field numbers and inferred field types. Types may be incorrect. Only output format "+string(OText)+" is supported. Use -o <response-type> to see correct contents.")
 
 	flags.StringVar(&tmpInTextType, "in", "",
 		"Specifies, in which format the input -d should be interpreted in. 'text' (default) uses the Protobuf text format and 'json' uses JSON.")
@@ -124,6 +126,19 @@ func propagateFlags() {
 		CurrentConfig.DisplayBinaryAndHttp = false
 	}
 
+	if CurrentConfig.ResponseType == "" && !CurrentConfig.DecodeRawResponse {
+		CurrentConfig.DecodeRawResponse = true
+		if CurrentConfig.Verbose {
+			fmt.Println("Response type (-o) was not provided, hence --decode-raw will be used.")
+		}
+
+	} else if CurrentConfig.ResponseType != "" && CurrentConfig.DecodeRawResponse {
+		CurrentConfig.DecodeRawResponse = false
+		if CurrentConfig.Verbose {
+			fmt.Println("Response type (-o) was provided, hence --decode-raw will be overidden.")
+		}
+	}
+
 	if strings.HasPrefix(strings.TrimSpace(CurrentConfig.DataText), "{") {
 		tmpDataTextInferredType = IJson
 	} else {
@@ -183,6 +198,10 @@ func propagateFlags() {
 		}
 	}
 
+	if CurrentConfig.DecodeRawResponse && (strings.Contains(string(CurrentConfig.OutTextType), "json")) {
+		PanicWithMessage("Decoding of raw messages is not supported with output format " + string(CurrentConfig.OutTextType) + ". Please use " + string(OText) + " instead.")
+	}
+
 	if CurrentConfig.ForceNoCurl && len(CurrentConfig.RequestHeaders) != 0 {
 		PanicDueToUnsupportedHeadersWhenInternalHttp(CurrentConfig.RequestHeaders)
 	}

src/protoConversions.go

@@ -72,7 +72,7 @@ func protoBinaryToMsgAndText(messageType string, binary []byte, outFormat OutTex
 	var textBytes = []byte{}
 	switch outFormat {
 	case OText:
-		textBytes, err = textFormatOptions.Marshal(msg)
+		textBytes, err = formatUnknownFieldsIfApplicable(textFormatOptions).Marshal(msg)
 	case OJsonDense:
 		textBytes, err = jsonDenseformatOptions.Marshal(msg)
 	case OJsonPretty:
@@ -85,3 +85,9 @@ func protoBinaryToMsgAndText(messageType string, binary []byte, outFormat OutTex
 
 	return text, msg
 }
+
+func formatUnknownFieldsIfApplicable(opts prototext.MarshalOptions) prototext.MarshalOptions {
+	newOpts := opts // shallow copy
+	newOpts.EmitUnknown = true
+	return newOpts
+}

src/protoRegistry.go

@@ -19,10 +19,13 @@ import (
 	"google.golang.org/protobuf/reflect/protoreflect"
 	"google.golang.org/protobuf/reflect/protoregistry"
 	"google.golang.org/protobuf/types/descriptorpb"
+	"google.golang.org/protobuf/types/known/emptypb"
 )
 
 const protoFileExtension = ".proto"
 
+const WellKnownEmptyMessageType = "google.protobuf.Empty"
+
 /*
 Given a directory of .proto files, we use `protoc` to convert these to
 an equivalent FileDescriptorSet payload where imports have been resolved.
@@ -96,9 +99,20 @@ func convertProtoFilesToProtoRegistryFiles() *protoregistry.Files {
 	protoRegistryFiles, err := protodesc.NewFiles(&protoFileDescriptorSet)
 	PanicOnError(err)
 
+	if CurrentConfig.DecodeRawResponse {
+		if CurrentConfig.Verbose {
+			fmt.Printf("Adding %s to proto registry to ensure it can be used for decoding raw Protobuf.\n", WellKnownEmptyMessageType)
+		}
+		_ = protoRegistryFiles.RegisterFile(wellKnownEmptyMessageProtoFileDescriptorForRawFormat())
+	}
+
 	return protoRegistryFiles
 }
 
+func wellKnownEmptyMessageProtoFileDescriptorForRawFormat() protoreflect.FileDescriptor {
+	return emptypb.File_google_protobuf_empty_proto
+}
+
 func collectRelevantProtoFiles() []string {
 	if CurrentConfig.InferProtoFiles {
 		if CurrentConfig.Verbose {

src/protocurl.go

@@ -10,7 +10,7 @@ import (
 )
 
 const GithubRepositoryLink = "https://github.com/qaware/protocurl"
-const BugReportsLink = "https://github.com/qaware/protocurl/issues"
+const EnhancementsAndBugsLink = "https://github.com/qaware/protocurl/issues"
 
 type Config struct {
 	ProtoFilesDir        string
@@ -21,6 +21,7 @@ type Config struct {
 	DataText             string
 	InTextType           InTextType
 	OutTextType          OutTextType
+	DecodeRawResponse    bool
 	DisplayBinaryAndHttp bool
 	RequestHeaders       []string
 	CustomCurlPath       string
@@ -65,8 +66,9 @@ var rootCmd = &cobra.Command{
 		"The bundle also includes the google protobuf .proto files necessary to create FileDescriptorSet payloads via '" + ProtocExecutableName + "'.\n" +
 		"If the bundled '" + ProtocExecutableName + "' is used, then these .proto files are included. Otherwise .proto files from the system-wide include are used.\n" +
 		"The Header 'Content-Type: application/x-protobuf' is set as a request header by default.\n" +
-		"When converting between binary and text, the encoding UTF-8 is always used.\n\n" +
-		"Bug reports: " + BugReportsLink,
+		"When converting between binary and text, the encoding UTF-8 is always used.\n" +
+		"When the correct response type is unknown or being debugged, omitting -o <response-type> will attempt to show the response in raw format.\n\n" +
+		"Enhancements and bugs: " + EnhancementsAndBugsLink,
 	Example:               "  protocurl -I my-protos -i package.path.Req -o package.path.Resp -u http://example.com/api -d \"myField: true, otherField: 1337\"",
 	Args:                  cobra.OnlyValidArgs,
 	DisableFlagsInUseLine: true,
@@ -147,7 +149,9 @@ func decodeResponse(responseBinary []byte, responseHeaders string, registry *pro
 		fmt.Printf("%s Response Binary  %s %s\n%s", VISUAL_SEPARATOR, VISUAL_SEPARATOR, RECV, hex.Dump(responseBinary))
 	}
 
-	responseText, _ := protoBinaryToMsgAndText(CurrentConfig.ResponseType, responseBinary, CurrentConfig.OutTextType, registry)
+	responseMessageType := properResponseTypeIfProvidedOrEmptyType()
+
+	responseText, _ := protoBinaryToMsgAndText(responseMessageType, responseBinary, CurrentConfig.OutTextType, registry)
 
 	if !CurrentConfig.ShowOutputOnly {
 		fmt.Printf("%s Response %s    %s %s\n",
@@ -156,6 +160,17 @@ func decodeResponse(responseBinary []byte, responseHeaders string, registry *pro
 	fmt.Printf("%s\n", responseText)
 }
 
+func properResponseTypeIfProvidedOrEmptyType() string {
+	if CurrentConfig.ResponseType != "" {
+		return CurrentConfig.ResponseType
+	} else {
+		if CurrentConfig.Verbose {
+			fmt.Printf("Decoding response against %s as no response type was provided.\n", WellKnownEmptyMessageType)
+		}
+		return WellKnownEmptyMessageType
+	}
+}
+
 func addDefaultHeaderArgument() {
 	if CurrentConfig.Verbose {
 		fmt.Printf("Adding default header argument to request headers : %s\n", DefaultHeaders)

test/results/additional-curl-args-verbose-expected.txt

@@ -13,6 +13,7 @@ Invoked with following default & parsed arguments:
   "DataText": "includeReason: true, date: { seconds: 1642044939, nanos: 152000000 }",
   "InTextType": "text",
   "OutTextType": "text",
+  "DecodeRawResponse": false,
   "DisplayBinaryAndHttp": true,
   "RequestHeaders": [
     "Content-Type: application/x-protobuf"
@@ -276,7 +277,7 @@ Total curl args:
 * Mark bundle as not supporting multiuse
 < HTTP/1.1 200 OK
 < Content-Type: application/x-protobuf
-< Date: Tue, 26 Apr 2022 22:34:37 GMT
+< Date: Mon, 16 May 2022 22:09:07 GMT
 < Connection: keep-alive
 < Keep-Alive: timeout=5
 < Content-Length: 65
@@ -287,7 +288,7 @@ Total curl args:
 =========================== Response Headers =========================== <<<
 HTTP/1.1 200 OK
 Content-Type: application/x-protobuf
-Date: Tue, 26 Apr 2022 22:34:37 GMT
+Date: Mon, 16 May 2022 22:09:07 GMT
 Connection: keep-alive
 Keep-Alive: timeout=5
 Content-Length: 65

test/results/far-future-json--v-expected.txt

@@ -13,6 +13,7 @@ Invoked with following default & parsed arguments:
   "DataText": "{ \"includeReason\": true, \"date\": \"9999-12-31T23:59:59Z\"}",
   "InTextType": "json",
   "OutTextType": "json",
+  "DecodeRawResponse": false,
   "DisplayBinaryAndHttp": true,
   "RequestHeaders": [
     "Content-Type: application/x-protobuf"
@@ -255,7 +256,7 @@ Total curl args:
 =========================== Response Headers =========================== <<<
 HTTP/1.1 200 OK
 Content-Type: application/x-protobuf
-Date: Tue, 26 Apr 2022 22:34:54 GMT
+Date: Mon, 16 May 2022 22:09:10 GMT
 Connection: keep-alive
 Keep-Alive: timeout=5
 Content-Length: 63

test/results/help-expected.txt

@@ -12,8 +12,9 @@ The bundle also includes the google protobuf .proto files necessary to create Fi
 If the bundled 'protoc' is used, then these .proto files are included. Otherwise .proto files from the system-wide include are used.
 The Header 'Content-Type: application/x-protobuf' is set as a request header by default.
 When converting between binary and text, the encoding UTF-8 is always used.
+When the correct response type is unknown or being debugged, omitting -o <response-type> will attempt to show the response in raw format.
 
-Bug reports: https://github.com/qaware/protocurl/issues
+Enhancements and bugs: https://github.com/qaware/protocurl/issues
 
 Examples:
   protocurl -I my-protos -i package.path.Req -o package.path.Resp -u http://example.com/api -d "myField: true, otherField: 1337"
@@ -23,6 +24,7 @@ Flags:
   -C, --curl-args string          Additional cURL args which will be passed on to cURL during request invocation for further configuration. Also activates --curl.
       --curl-path string          Uses the given path to invoke curl instead of searching for curl in PATH. Also activates --curl.
   -d, --data-text string          Mandatory: The payload data in Protobuf text format or JSON. It is inferred from the input as JSON if the first token is a '{'. The format can be set explicitly via --in. See https://github.com/qaware/protocurl
+      --decode-raw                Decode the response into textual format without the schema by only showing field numbers and inferred field types. Types may be incorrect. Only output format text is supported. Use -o <response-type> to see correct contents.
   -D, --display-binary-and-http   Displays the binary request and response as well as the non-binary response headers.
   -h, --help                      help for protocurl
       --in string                 Specifies, in which format the input -d should be interpreted in. 'text' (default) uses the Protobuf text format and 'json' uses JSON.
@@ -35,7 +37,7 @@ Flags:
       --protoc-path string        Uses the given path to invoke protoc instead of searching for protoc in PATH. Also activates --protoc.
   -H, --request-header string     Adds the string header to the invocation of cURL. This option is not supported when --no-curl is active. E.g. -H 'MyHeader: FooBar'.
   -i, --request-type string       Mandatory: Message name or full package path of the Protobuf request type. The path can be shortened to '..', if the name of the request message is unique. E.g. mypackage.MyRequest or ..MyRequest
-  -o, --response-type string      Mandatory: The Protobuf response type. See -i <request-type>
+  -o, --response-type string      The Protobuf response type. See -i <request-type>. Overrides --decode-raw. If not set, then --decode-raw is used.
   -q, --show-output-only          Suppresses all output except response Protobuf as text. Overrides and deactivates -v and -D. Errors are still printed to stderr.
   -u, --url string                Mandatory: The url to send the request to
   -v, --verbose                   Prints version and enables verbose output. Also activates -D.