Remove buf.yaml and buf.lock as part of API by sending over structure…

…d module and dependency content (#20)

This replaces #17 as #16 was closed.

This removes the sending of `buf.yaml` and `buf.lock` files over the
wire, and consequently these being part of the structure of `Modules` as
we know them, instead sending over the specific information we need on
the backend in a structured manner.

This fully standardizes on the `Node` terminology. A `Node` is a pointer
to some content, either on the request or response side. Within this API
now, we have:

- `FileNode`: A pointer to file content.
- `CreateCommitRequest.ModuleNode` - A set of `FileNodes` and an
associated `ModuleRef` that represents the content of a single
`Module.`.
- `CreateCommitRequest.DepNode` - A set of pointers to dependencies,
which are just the `commit_ids` of the dependencies and their associated
`Digest` (for verification).
- `CreateCommitResponse.CommitNode` - A set of pointers to files and
then actual commit dependencies.

This renames `GetFileNodes` to `GetCommitNodes`.

To be honest, I don't love the `Node` naming, and I think it could use
some work, but that's less important for the purpose of this PR. We can
work on that. Ideas I've had:

- `Info` for new nested types, ie `ModuleInfo, DepInfo, CommitInfo`.
- `Data` for new nested types. In general, I think it's an anti-pattern
to name a Protobuf message `.*Data`, as all Protobuf messages are data,
and it doesn't pluralize well.

Open to other suggestions.

buf/registry/module/v1beta1/commit_service.proto

@@ -17,6 +17,7 @@ syntax = "proto3";
 package buf.registry.module.v1beta1;
 
 import "buf/registry/module/v1beta1/commit.proto";
+import "buf/registry/module/v1beta1/module.proto";
 import "buf/registry/module/v1beta1/resource.proto";
 import "buf/registry/module/v1beta1/vcs_commit.proto";
 import "buf/registry/storage/v1beta1/storage.proto";
@@ -42,12 +43,18 @@ service CommitService {
   rpc CreateCommits(CreateCommitsRequest) returns (CreateCommitsResponse) {
     option idempotency_level = IDEMPOTENT;
   }
-  // Get the FileNodes on a Commit for a given set of Commits, Modules, Branches, Tags, or VCSCommits.
+  // Get the pointers to the content for a given set of Commits, Modules, Branches, Tags, or VCSCommits.
   //
-  // Retrieving content is a two-step process:
-  //   - Get the FileNode messages. FileNodes are a map from path to digest.
-  //   - Get the Blob messages. Blobs are a map from digest to content.
-  rpc GetFileNodes(GetFileNodesRequest) returns (GetFileNodesResponse) {
+  // Nodes consist of:
+  //   - The FileNodes: .proto files, license files, and documentation files.
+  //   - The dependencies.
+  //
+  // Retrieving file content is a multi-step process:
+  //   - Get the FileNode messages via this RPC. FileNodes are a map from path to digest.
+  //   - Get the Blob messages via GetBlobs. Blobs are a map from digest to content.
+  //   - If file content for the dependencies is desired, Get the Commits for the DepNodes,
+  //     and call GetCommitCodes for those dependencies.
+  rpc GetCommitNodes(GetCommitNodesRequest) returns (GetCommitNodesResponse) {
     option idempotency_level = NO_SIDE_EFFECTS;
   }
   // Get the Blobs for files within Commits for a given set of Digests.
@@ -150,52 +157,83 @@ message CreateCommitsRequest {
     string committer_email = 7;
   }
 
-  // All of the Files for the Modules that should have Commits created for them.
+  // The pointers to the content of a single Module that a Commit will be created for.
+  message ModuleNode {
+    // The reference to the Module to create a Commit for.
+    ModuleRef module_ref = 1;
+    // The FileNodes of the Module.
+    //
+    // This consists of the .proto files, license files, and documentation files. This does
+    // not contain any files relating to dependencies of configuration (such as buf.lock or
+    // buf.yaml). A server will respond with an error if an unexpected file is passed here.
+    //
+    // Blobs are shared across Modules within a given CreateCommit call; missing blobs
+    // should be passed via the missing_blobs field. See GetMissingBlobDigests for more details.
+    repeated buf.registry.storage.v1beta1.FileNode file_nodes = 2;
+  }
+
+  // A pointer to a dependency of a Module.
+  message DepNode {
+    // The ID of the the Commit of the dependency.
+    string commit_id = 1 [(buf.validate.field).required = true];
+    // The digest of the dependency.
+    buf.registry.storage.v1beta1.Digest digest = 2 [(buf.validate.field).required = true];
+  }
+
+  // The pointers to the content for the Modules that should have Commits created for them.
+  //
+  // Each ModuleNode must have a unique ModuleRef.
+  // A commit will be created for each ModuleNode.
+  repeated ModuleNode module_nodes = 1 [(buf.validate.field).repeated.min_items = 1];
+  // The dependencies of the Modules that are not contained within module_nodes.
   //
-  // If you have a workspace, all files from the workspace, including buf.yaml and buf.lock, should
-  // be in files. The backend will parse the buf.yaml to discover the ModuleResourceRefs.
-  repeated buf.registry.storage.v1beta1.FileNode file_nodes = 1 [(buf.validate.field).repeated.min_items = 1];
-  // Blobs for the given FileNodes.
+  // No dep should reference a Commit that is within a Module referenced by module_ref on
+  // any of the ModuleNodes in module_nodes.
+  repeated DepNode dep_nodes = 2;
+  // Blobs for the FileNodes referenced by module_nodes that are not present on the server.
   //
   // Only Blobs that were returned as missing from GetMissingBlobDigests need to be sent.
   // Other Blobs already exist on the server, and will be ignored.
-  repeated buf.registry.storage.v1beta1.Blob missing_blobs = 2;
+  //
+  // If a FileNode within module_nodes has a Blob that is not on the server, and is not
+  // within missing_blobs, an error will be returned.
+  repeated buf.registry.storage.v1beta1.Blob missing_blobs = 3;
   // The names of Branches that should be associated with this Commit.
   //
   // If a Branch currently exists on the associated Module with a name, this existing
   // Branch will be used. Otherwise, a new Branch will be created for the corresponding name.
   //
   // If empty, the default branch is assumed as the only branch.
-  repeated string branch_names = 3 [(buf.validate.field).repeated.items = {
+  repeated string branch_names = 4 [(buf.validate.field).repeated.items = {
     string: {max_len: 250}
   }];
   // The names of Tags that should be associated with this Commit.
   //
   // If a Tag currently exists on the assocated Module with a name, the RPC will error, however
   // this will change in the future when we allow Tags to move. If the Tag does not
   // currently exist, a new Tag will be created for each name.
-  repeated string tag_names = 4 [(buf.validate.field).repeated.items = {
+  repeated string tag_names = 5 [(buf.validate.field).repeated.items = {
     string: {max_len: 250}
   }];
   // Associated VCS commit information.
   //
   // If there are already VCSCommits on the associated Module with a given hash, this
   // will result in an error. Otherwise, a new VCSCommit is created.
-  repeated AssociatedVCSCommit associated_vcs_commits = 5;
+  repeated AssociatedVCSCommit associated_vcs_commits = 6;
 }
 
 message CreateCommitsResponse {
-  // The created Commits.
+  // The created Commits, in the order of the nodes given via module_nodes.
   //
   // If the digest was found for an pre-existing Commit, this pre-existing Commit will be returned
   // instead of a new Commit being created.
   repeated Commit commits = 1 [(buf.validate.field).repeated.min_items = 1];
 }
 
-message GetFileNodesRequest {
-  // An individual request for a set of files from a single Commit, Module, Branch, Tag, or VCSCommit.
+message GetCommitNodesRequest {
+  // An individual request for pointers to the content of a single Commit, Module, Branch, Tag, or VCSCommit.
   message Value {
-    // The reference to get files for.
+    // The reference to get nodes for.
     //
     // See the documentation on ResourceRef for resource resolution details.
     //
@@ -229,13 +267,24 @@ message GetFileNodesRequest {
   repeated Value values = 1 [(buf.validate.field).repeated.min_items = 1];
 }
 
-message GetFileNodesResponse {
-  // A single set of FileNodes and their associated commits.
+message GetCommitNodesResponse {
+  // A single set of commits and their associated nodes.
   message Value {
-    repeated buf.registry.storage.v1beta1.FileNode file_nodes = 1 [(buf.validate.field).repeated.min_items = 1];
-    Commit commit = 2 [(buf.validate.field).required = true];
+    // The Commit for the nodes referenced in this message.
+    //
+    // The Digest on the Commit will always match the Digest calculable by the FileNodes and the DepNodes.
+    Commit commit = 1 [(buf.validate.field).required = true];
+    // The FileNodes for the files associated with this Commit.
+    //
+    // This will consist of the .proto files, license files, and documentation files.
+    // Use GetBlobs to get the content of these files.
+    repeated buf.registry.storage.v1beta1.FileNode file_nodes = 2 [(buf.validate.field).repeated.min_items = 1];
+    // The Commits representing the dependencies of the Module at this commit.
+    //
+    // Use GetCommitNodes recursively to get the nodes for dependency commits.
+    repeated Commit deps = 3;
   }
-  // The returned File sets in the same order as requested.
+  // The returned nodes in the same order as requested.
   repeated Value values = 1 [(buf.validate.field).repeated.min_items = 1];
 }