manifest: new configuration override, refactoring of manifest reader

cmd/substreams/codegen.go

@@ -54,7 +54,7 @@ func runCodeGen(cmd *cobra.Command, args []string) error {
 		return fmt.Errorf("computing working directory: %w", err)
 	}
 	workingDir := filepath.Dir(manifestAbsPath)
-	manif, err := manifest.LoadManifestFile(manifestAbsPath)
+	manif, err := manifest.LoadManifestFile(manifestAbsPath, workingDir)
 	if err != nil {
 		return fmt.Errorf("loading manifest: %w", err)
 	}

cmd/substreams/pack.go

@@ -2,7 +2,6 @@ package main
 
 import (
 	"fmt"
-	"gopkg.in/yaml.v3"
 	"io/ioutil"
 	"os"
 	"path/filepath"
@@ -38,7 +37,7 @@ func init() {
 		replaced by "-") and "<version>" is "package.version" value. You can use "{version}" which resolves
 		to "package.version".
 	`))
-	packCmd.Flags().StringArrayP("config", "c", []string{}, cli.FlagDescription(`path to a configuration file that contains overrides for the manifest`))
+	//packCmd.Flags().StringArrayP("config", "c", []string{}, cli.FlagDescription(`path to a configuration file that contains overrides for the manifest`))
 }
 
 func runPack(cmd *cobra.Command, args []string) error {
@@ -47,23 +46,22 @@ func runPack(cmd *cobra.Command, args []string) error {
 		manifestPath = args[0]
 	}
 
-	// Get the value of the -c flag
-	overridePaths, _ := cmd.Flags().GetStringArray("config")
-
 	var manifestReaderOptions []manifest.Option
 
-	// If the overridePath is provided, read, decode it and add to manifestReaderOptions
-	if len(overridePaths) > 0 {
-		var overrides []*manifest.ConfigurationOverride
-		for _, overridePath := range overridePaths {
-			overrideConfig, err := readOverrideConfig(overridePath)
-			if err != nil {
-				return fmt.Errorf("reading override config %q: %w", overridePath, err)
-			}
-			overrides = append(overrides, overrideConfig)
-		}
-		manifestReaderOptions = append(manifestReaderOptions, manifest.WithOverrides(overrides...))
-	}
+	//// Get the value of the -c flag
+	//overridePaths, _ := cmd.Flags().GetStringArray("config")
+	//// If the overridePath is provided, read, decode it and add to manifestReaderOptions
+	//if len(overridePaths) > 0 {
+	//	var overrides []*manifest.ConfigurationOverride
+	//	for _, overridePath := range overridePaths {
+	//		overrideConfig, err := readOverrideConfig(overridePath)
+	//		if err != nil {
+	//			return fmt.Errorf("reading override config %q: %w", overridePath, err)
+	//		}
+	//		overrides = append(overrides, overrideConfig)
+	//	}
+	//	manifestReaderOptions = append(manifestReaderOptions, manifest.WithOverrides(overrides...))
+	//}
 
 	// Use the manifestReaderOptions while creating the manifest reader
 	manifestReader, err := manifest.NewReader(manifestPath, manifestReaderOptions...)
@@ -120,19 +118,3 @@ func resolveOutputFile(input string, bindings map[string]string) string {
 
 	return input
 }
-
-// This function reads and decodes the override configuration from a given path
-func readOverrideConfig(path string) (*manifest.ConfigurationOverride, error) {
-	fileBytes, err := os.ReadFile(path)
-	if err != nil {
-		return nil, err
-	}
-
-	overrideConfig := &manifest.ConfigurationOverride{}
-	err = yaml.Unmarshal(fileBytes, overrideConfig)
-	if err != nil {
-		return nil, err
-	}
-
-	return overrideConfig, nil
-}

codegen/init_test.go

@@ -23,7 +23,7 @@ func InitTestGenerator(t *testing.T) *Generator {
 		panic(fmt.Errorf("reading manifest file %s :%w", manifestPath, err))
 	}
 
-	manif, err := manifest.LoadManifestFile(manifestPath)
+	manif, err := manifest.LoadManifestFile(manifestPath, ".")
 	require.NoError(t, err)
 
 	return NewGenerator(pkg, manif, protoDefinitions, "")

docs/release-notes/change-log.md

@@ -29,6 +29,27 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 
 * `substreams info` command now properly displays the content of sink configs, optionally writing the fields that were bundled from files to disk with `--output-sinkconfig-files-path=</some/path>`
 
+### Changed
+
+* The override feature has been overhauled. Users may now override an existing substreams by pointing to an override file in `run` or `gui` command. This override manifest will have a `deriveFrom` field which points to the original substreams which is to be overriden.
+
+Example of an override manifest:
+```
+deriveFrom: path/to/original.spkg #this can also be a remote url
+
+package:
+  name: "overriden_package_name"
+  version: "100.0.0"
+
+network: polygon
+  
+initialBlocks:
+  module_name_1: 17500000
+  module_name_1: 17500000
+params:
+  module_name_1: "override_paramete_value_here"
+```
+
 ## v1.1.14
 
 ### Bug fixes

manifest/manifest.go

@@ -111,7 +111,12 @@ type StreamOutput struct {
 	Type string `yaml:"type"`
 }
 
-func decodeYamlManifestFromFile(yamlFilePath string) (out *Manifest, err error) {
+func decodeYamlManifestFromFile(yamlFilePath, workingDir string) (out *Manifest, err error) {
+	//if yamlFilePath is a relative path, make it absolute
+	if !filepath.IsAbs(yamlFilePath) {
+		yamlFilePath = filepath.Join(workingDir, yamlFilePath)
+	}
+
 	cnt, err := os.ReadFile(yamlFilePath)
 	if err != nil {
 		return nil, fmt.Errorf("reading substreams manifest %q: %w", yamlFilePath, err)

manifest/manifest_test.go

@@ -11,7 +11,7 @@ import (
 )
 
 func TestManifest_YamlUnmarshal(t *testing.T) {
-	manifest, err := decodeYamlManifestFromFile("./test/test_manifest.yaml")
+	manifest, err := decodeYamlManifestFromFile("./test/test_manifest.yaml", ".")
 	assert.NoError(t, err)
 	assert.GreaterOrEqual(t, len(manifest.Modules), 1)
 }

manifest/override.go

@@ -1,47 +1,88 @@
 package manifest
 
 import (
-	"encoding/base64"
 	pbsubstreams "github.com/streamingfast/substreams/pb/sf/substreams/v1"
-	"google.golang.org/protobuf/types/known/anypb"
-	"io"
-	"strings"
 )
 
 type ConfigurationOverride struct {
 	Package       *PackageOverride  `yaml:"package,omitempty"`
-	Network       string            `yaml:"network,omitempty"`
-	InitialBlocks map[string]int64  `yaml:"initialBlocks,omitempty"`
+	Network       *string           `yaml:"network,omitempty"`
+	InitialBlocks map[string]uint64 `yaml:"initialBlocks,omitempty"`
 	Params        map[string]string `yaml:"params,omitempty"`
 
-	SinkConfig *SinkConfigOverride `yaml:"sinkConfig,omitempty"`
-	SinkModule string              `yaml:"sinkModule,omitempty"`
-}
-
-type SinkConfigOverride struct {
-	TypeUrl string `yaml:"typeUrl,omitempty"`
-	Value   string `yaml:"value,omitempty"`
+	DeriveFrom string `yaml:"deriveFrom,omitempty"`
 }
 
 type PackageOverride struct {
-	Name string `yaml:"name,omitempty"`
+	Name    *string `yaml:"name,omitempty"`
+	Version *string `yaml:"version,omitempty"`
 }
 
-func mergeManifests(main *pbsubstreams.Package, override *ConfigurationOverride) error {
-	if override.Package != nil && override.Package.Name != "" {
-		if main.PackageMeta == nil {
-			main.PackageMeta = []*pbsubstreams.PackageMetadata{}
+func mergeOverrides(overrides ...*ConfigurationOverride) *ConfigurationOverride {
+	var merged *ConfigurationOverride
+
+	for _, override := range overrides {
+		if override == nil {
+			continue
+		}
+
+		if merged == nil {
+			merged = &ConfigurationOverride{}
+		}
+
+		if override.Package != nil {
+			if merged.Package == nil {
+				merged.Package = &PackageOverride{}
+			}
+
+			if override.Package.Name != nil {
+				merged.Package.Name = override.Package.Name
+			}
+
+			if override.Package.Version != nil {
+				merged.Package.Version = override.Package.Version
+			}
+		}
+
+		if override.Network != nil {
+			merged.Network = override.Network
+		}
+
+		if override.InitialBlocks != nil {
+			if merged.InitialBlocks == nil {
+				merged.InitialBlocks = make(map[string]uint64)
+			}
+
+			for name, block := range override.InitialBlocks {
+				merged.InitialBlocks[name] = block
+			}
 		}
 
-		if len(main.PackageMeta) == 0 {
-			main.PackageMeta = append(main.PackageMeta, &pbsubstreams.PackageMetadata{Name: override.Package.Name})
-		} else {
-			main.PackageMeta[0].Name = override.Package.Name
+		if override.Params != nil {
+			if merged.Params == nil {
+				merged.Params = make(map[string]string)
+			}
+
+			for name, value := range override.Params {
+				merged.Params[name] = value
+			}
 		}
 	}
 
-	if override.Network != "" {
-		main.Network = override.Network
+	return merged
+}
+
+func applyOverride(main *pbsubstreams.Package, override *ConfigurationOverride) error {
+	if override == nil {
+		return nil
+	}
+
+	if override.Package != nil {
+		mergePackageMeta(main, override)
+	}
+
+	if override.Network != nil {
+		main.Network = *override.Network
 	}
 
 	if override.Params != nil {
@@ -52,26 +93,29 @@ func mergeManifests(main *pbsubstreams.Package, override *ConfigurationOverride)
 		mergeInitialBlocks(main, override)
 	}
 
-	// Overriding SinkModule
-	if override.Package != nil && override.SinkModule != "" {
-		main.SinkModule = override.SinkModule
+	return nil
+}
+
+func mergePackageMeta(main *pbsubstreams.Package, override *ConfigurationOverride) {
+	if override.Package == nil {
+		return
 	}
 
-	// Overriding SinkConfig
-	if override.Package != nil && override.SinkConfig != nil {
-		decoder := base64.NewDecoder(base64.StdEncoding, strings.NewReader(string(override.SinkConfig.Value)))
-		configValueBytes, err := io.ReadAll(decoder)
-		if err != nil {
-			return err
-		}
+	currentPackageMeta := main.GetPackageMeta()
+	if currentPackageMeta == nil {
+		currentPackageMeta = []*pbsubstreams.PackageMetadata{}
+	}
+	if len(currentPackageMeta) == 0 {
+		currentPackageMeta = append(currentPackageMeta, &pbsubstreams.PackageMetadata{})
+	}
 
-		main.SinkConfig = &anypb.Any{
-			TypeUrl: override.SinkConfig.TypeUrl,
-			Value:   configValueBytes,
-		}
+	if override.Package.Name != nil {
+		currentPackageMeta[0].Name = *override.Package.Name
 	}
 
-	return nil
+	if override.Package.Version != nil {
+		currentPackageMeta[0].Version = *override.Package.Version
+	}
 }
 
 func mergeInitialBlocks(main *pbsubstreams.Package, override *ConfigurationOverride) {
@@ -119,53 +163,4 @@ func mergeParams(main *pbsubstreams.Package, override *ConfigurationOverride) {
 			mainMod.Inputs = mainmodInputs
 		}
 	}
-
-}
-
-func mergeOverrides(overrides ...*ConfigurationOverride) *ConfigurationOverride {
-	merged := &ConfigurationOverride{}
-
-	for _, override := range overrides {
-		if override == nil {
-			continue
-		}
-
-		if override.Package != nil {
-			merged.Package = override.Package
-		}
-
-		if override.Network != "" {
-			merged.Network = override.Network
-		}
-
-		if override.InitialBlocks != nil {
-			if merged.InitialBlocks == nil {
-				merged.InitialBlocks = make(map[string]int64)
-			}
-
-			for name, block := range override.InitialBlocks {
-				merged.InitialBlocks[name] = block
-			}
-		}
-
-		if override.Params != nil {
-			if merged.Params == nil {
-				merged.Params = make(map[string]string)
-			}
-
-			for name, value := range override.Params {
-				merged.Params[name] = value
-			}
-		}
-
-		if override.SinkConfig != nil {
-			merged.SinkConfig = override.SinkConfig
-		}
-
-		if override.SinkModule != "" {
-			merged.SinkModule = override.SinkModule
-		}
-	}
-
-	return merged
 }