-
Notifications
You must be signed in to change notification settings - Fork 17
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
feat: meta=eof for IPIP-431; ask for and expect (but not require) fro…
…m http fetches Ref: ipfs/specs#431 Ref: ipld/frisbii#15
- Loading branch information
Showing
14 changed files
with
358 additions
and
64 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,19 @@ | ||
| package httputil | ||
|
|
||
| import "fmt" | ||
|
|
||
| const ( | ||
| MimeTypeCar = "application/vnd.ipld.car" // The only accepted MIME type | ||
| MimeTypeCarVersion = "1" // We only accept version 1 of the MIME type | ||
| ResponseAcceptRangesHeader = "none" // We currently don't accept range requests | ||
| ResponseCacheControlHeader = "public, max-age=29030400, immutable" // Magic cache control values | ||
| FilenameExtCar = ".car" // The only valid filename extension | ||
| FormatParameterCar = "car" // The only valid format parameter value | ||
| DefaultIncludeDupes = true // The default value for an unspecified "dups" parameter. See https://github.com/ipfs/specs/pull/412. | ||
| ) | ||
|
|
||
| var ( | ||
| ResponseChunkDelimeter = []byte("0\r\n") // An http/1.1 chunk delimeter, used for specifying an early end to the response | ||
| ResponseContentTypeHeader = fmt.Sprintf("%s; version=%s; order=dfs; dups=y", MimeTypeCar, MimeTypeCarVersion) | ||
| RequestAcceptHeader = fmt.Sprintf("%s; version=%s; order=dfs; dups=y; meta=eof", MimeTypeCar, MimeTypeCarVersion) | ||
| ) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,70 @@ | ||
| package metadata | ||
|
|
||
| import ( | ||
| "fmt" | ||
| "io" | ||
|
|
||
| "github.com/filecoin-project/lassie/pkg/types" | ||
| "github.com/ipfs/go-cid" | ||
| "github.com/ipld/go-ipld-prime/codec/dagjson" | ||
| bindnoderegistry "github.com/ipld/go-ipld-prime/node/bindnode/registry" | ||
| mh "github.com/multiformats/go-multihash" | ||
|
|
||
| _ "embed" | ||
| ) | ||
|
|
||
| //go:embed metadata.ipldsch | ||
| var schema []byte | ||
|
|
||
| var BindnodeRegistry = bindnoderegistry.NewRegistry() | ||
|
|
||
| type CarMetadata struct { | ||
| Metadata *Metadata | ||
| } | ||
|
|
||
| func (cm CarMetadata) Serialize(w io.Writer) error { | ||
| // TODO: do the same checks we do on Deserialize() | ||
| return BindnodeRegistry.TypeToWriter(&cm, w, dagjson.Encode) | ||
| } | ||
|
|
||
| func (cm *CarMetadata) Deserialize(r io.Reader) error { | ||
| cmIface, err := BindnodeRegistry.TypeFromReader(r, &CarMetadata{}, dagjson.Decode) | ||
| if err != nil { | ||
| return fmt.Errorf("invalid CarMetadata: %w", err) | ||
| } | ||
| cmm := cmIface.(*CarMetadata) // safe to assume type | ||
| if cmm.Metadata.Properties == nil && cmm.Metadata.Error == nil { | ||
| return fmt.Errorf("invalid CarMetadata: must contain either properties or error fields") | ||
| } | ||
| if (cmm.Metadata.Properties == nil) == (cmm.Metadata.Error == nil) { | ||
| return fmt.Errorf("invalid CarMetadata: must contain either properties or error fields, not both") | ||
| } | ||
| if cmm.Metadata.Properties != nil { | ||
| if _, err := mh.Decode(cmm.Metadata.Properties.ChecksumMultihash); err != nil { | ||
| return fmt.Errorf("invalid CarMetadata: checksum multihash: %w", err) | ||
| } | ||
| } | ||
| // TODO: parse and check EntityBytes format | ||
| *cm = *cmm | ||
| return nil | ||
| } | ||
|
|
||
| type Metadata struct { | ||
| Request Request | ||
| Properties *types.CarProperties | ||
| Error *string | ||
| } | ||
|
|
||
| type Request struct { | ||
| Root cid.Cid | ||
| Path *string | ||
| Scope types.DagScope | ||
| Duplicates bool | ||
| EntityBytes *string | ||
| } | ||
|
|
||
| func init() { | ||
| if err := BindnodeRegistry.RegisterType((*CarMetadata)(nil), string(schema), "CarMetadata"); err != nil { | ||
| panic(err.Error()) | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,31 @@ | ||
| type CarMetadata union { | ||
| | Metadata "car-metadata/v1" | ||
| } representation keyed | ||
|
|
||
| type Metadata struct { | ||
| request Request | ||
| # must contain either a properties or an error | ||
| properties optional CarProperties | ||
| error optional String | ||
| } | ||
|
|
||
| type Request struct { | ||
| root &Any | ||
| path optional String | ||
| scope DagScope | ||
| duplicates Bool (rename "dups") | ||
| entityBytes optional String (rename "entity-bytes") # Must be a valid entity-bytes param: "from:to" | ||
| } | ||
|
|
||
| type DagScope enum { | ||
| | all | ||
| | entity | ||
| | block | ||
| } | ||
|
|
||
| type CarProperties struct { | ||
| carBytes Int (rename "car_bytes") | ||
| dataBytes Int (rename "data_bytes") | ||
| blockCount Int (rename "block_count") | ||
| checksumMultihash optional Bytes (rename "checksum") # Must be a valid multihash | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,118 @@ | ||
| package metadata_test | ||
|
|
||
| import ( | ||
| "bytes" | ||
| "testing" | ||
|
|
||
| "github.com/filecoin-project/lassie/pkg/httputil/metadata" | ||
| "github.com/filecoin-project/lassie/pkg/types" | ||
| "github.com/ipfs/go-cid" | ||
| "github.com/stretchr/testify/require" | ||
| ) | ||
|
|
||
| var testCid = cid.MustParse("bafybeic56z3yccnla3cutmvqsn5zy3g24muupcsjtoyp3pu5pm5amurjx4") | ||
|
|
||
| func TestCarMetadataRoundtrip(t *testing.T) { | ||
| path := "/birb.mp4" | ||
| orig := metadata.CarMetadata{ | ||
| Metadata: &metadata.Metadata{ | ||
| Request: metadata.Request{ | ||
| Root: testCid, | ||
| Path: &path, | ||
| Scope: types.DagScopeAll, | ||
| Duplicates: true, | ||
| }, | ||
| Properties: &types.CarProperties{ | ||
| CarBytes: 202020, | ||
| DataBytes: 101010, | ||
| BlockCount: 303, | ||
| ChecksumMultihash: testCid.Hash(), | ||
| }, | ||
| }, | ||
| } | ||
| var buf bytes.Buffer | ||
| require.NoError(t, orig.Serialize(&buf)) | ||
|
|
||
| t.Log("metadata dag-json:", buf.String()) | ||
|
|
||
| var roundtrip metadata.CarMetadata | ||
| require.NoError(t, roundtrip.Deserialize(&buf)) | ||
| require.Equal(t, orig, roundtrip) | ||
| require.NotNil(t, roundtrip.Metadata) | ||
| require.Equal(t, testCid, roundtrip.Metadata.Request.Root) | ||
| require.NotNil(t, roundtrip.Metadata.Request.Path) | ||
| require.Equal(t, "/birb.mp4", *roundtrip.Metadata.Request.Path) | ||
| require.Equal(t, types.DagScopeAll, roundtrip.Metadata.Request.Scope) | ||
| require.True(t, roundtrip.Metadata.Request.Duplicates) | ||
| require.NotNil(t, roundtrip.Metadata.Properties) | ||
| require.Nil(t, roundtrip.Metadata.Error) | ||
| require.Equal(t, int64(202020), roundtrip.Metadata.Properties.CarBytes) | ||
| require.Equal(t, int64(101010), roundtrip.Metadata.Properties.DataBytes) | ||
| require.Equal(t, int64(303), roundtrip.Metadata.Properties.BlockCount) | ||
| require.Equal(t, []byte(testCid.Hash()), roundtrip.Metadata.Properties.ChecksumMultihash) | ||
| } | ||
|
|
||
| func TestCarMetadataErrorRoundtrip(t *testing.T) { | ||
| path := "/birb.mp4" | ||
| msg := "something bad happened" | ||
| orig := metadata.CarMetadata{ | ||
| Metadata: &metadata.Metadata{ | ||
| Request: metadata.Request{ | ||
| Root: testCid, | ||
| Path: &path, | ||
| Scope: types.DagScopeAll, | ||
| Duplicates: true, | ||
| }, | ||
| Error: &msg, | ||
| }, | ||
| } | ||
| var buf bytes.Buffer | ||
| require.NoError(t, orig.Serialize(&buf)) | ||
|
|
||
| t.Log("metadata dag-json:", buf.String()) | ||
|
|
||
| var roundtrip metadata.CarMetadata | ||
| require.NoError(t, roundtrip.Deserialize(&buf)) | ||
| require.Equal(t, orig, roundtrip) | ||
| require.NotNil(t, roundtrip.Metadata) | ||
| require.Equal(t, testCid, roundtrip.Metadata.Request.Root) | ||
| require.NotNil(t, roundtrip.Metadata.Request.Path) | ||
| require.Equal(t, "/birb.mp4", *roundtrip.Metadata.Request.Path) | ||
| require.Equal(t, types.DagScopeAll, roundtrip.Metadata.Request.Scope) | ||
| require.True(t, roundtrip.Metadata.Request.Duplicates) | ||
| require.Nil(t, roundtrip.Metadata.Properties) | ||
| require.NotNil(t, roundtrip.Metadata.Error) | ||
| require.Equal(t, "something bad happened", *roundtrip.Metadata.Error) | ||
| } | ||
|
|
||
| func TestBadMetadata(t *testing.T) { | ||
| testCases := []struct { | ||
| name string | ||
| byts string | ||
| err string | ||
| }{ | ||
| {"empty", `{}`, `union structure constraints for CarMetadata caused rejection: a union must have exactly one entry`}, | ||
| {"bad key", `{"not metadata":true}`, `union structure constraints for CarMetadata caused rejection: no member named "not metadata"`}, | ||
| { | ||
| "bad multihash", | ||
| `{"car-metadata/v1":{"properties":{"block_count":303,"car_bytes":202020,"checksum":{"/":{"bytes":"bm90IGEgbXVsdGloYXNo"}},"data_bytes":101010},"request":{"dups":true,"path":"/birb.mp4","root":{"/":"bafybeic56z3yccnla3cutmvqsn5zy3g24muupcsjtoyp3pu5pm5amurjx4"},"scope":"all"}}}`, | ||
| `invalid CarMetadata: checksum multihash:`, | ||
| }, | ||
| { | ||
| "no properties or error", | ||
| `{"car-metadata/v1":{"request":{"dups":true,"path":"/birb.mp4","root":{"/":"bafybeic56z3yccnla3cutmvqsn5zy3g24muupcsjtoyp3pu5pm5amurjx4"},"scope":"all"}}}`, | ||
| `invalid CarMetadata: must contain either properties or error fields`, | ||
| }, | ||
| { | ||
| "both properties and error", | ||
| `{"car-metadata/v1":{"error":"something bad happened","properties":{"block_count":303,"car_bytes":202020,"checksum":{"/":{"bytes":"EiBd9neBCasGxUmysJN7nGza4ylHikmbsP2+nXs6BlIpvw"}},"data_bytes":101010},"request":{"dups":true,"path":"/birb.mp4","root":{"/":"bafybeic56z3yccnla3cutmvqsn5zy3g24muupcsjtoyp3pu5pm5amurjx4"},"scope":"all"}}}`, | ||
| `invalid CarMetadata: must contain either properties or error fields, not both`, | ||
| }, | ||
| } | ||
| for _, tc := range testCases { | ||
| t.Run(tc.name, func(t *testing.T) { | ||
| var roundtrip metadata.CarMetadata | ||
| require.ErrorContains(t, roundtrip.Deserialize(bytes.NewBuffer([]byte(tc.byts))), tc.err) | ||
| }) | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.