Merge branch 'master' into ref-dockerfile

.github/workflows/CI.yaml

@@ -15,7 +15,7 @@ jobs:
 
     strategy:
       matrix:
-        go-version: ['>=1.22.0']
+        go-version: ['>=1.22.2']
 
     steps:
       - name: Check out code
@@ -52,6 +52,7 @@ jobs:
       image_tags: ${{ needs.docker_set_env.outputs.tags }}
       dockerfile: ./Dockerfile
       context: .
+      build-args: ""
     secrets:
       dockerhub_registry: ${{ secrets.DOCKERHUB_REGISTRY }}
       dockerhub_username: ${{ secrets.DOCKERHUB_USERNAME }}

CHANGELOG.md

@@ -0,0 +1,28 @@
+# Changelog
+All notable changes to this library will be documented in this file.
+
+The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
+and this library adheres to Rust's notion of
+[Semantic Versioning](https://semver.org/spec/v2.0.0.html).
+
+## [Unreleased]
+
+### Changed
+
+- The `RawTransaction` values returned from a call to `GetMempoolStream`
+  now report a `Height` value of `0`, in order to be consistent with
+  the results of calls to `GetTransaction`. See the documentation of
+  `RawTransaction` in `walletrpc/service.proto` for more details on
+  the semantics of this field.
+
+### Fixed
+
+- Parsing of `getrawtransaction` results is now platform-independent.
+  Previously, values of `-1` returned for the transaction height would
+  be converted to different `RawTransaction.Height` values depending
+  upon whether `lightwalletd` was being run on a 32-bit or 64-bit 
+  platform.
+
+## [Prior Releases]
+
+This changelog was not created until after the release of v0.4.17

Dockerfile

@@ -7,6 +7,7 @@ ARG ZCASHD_CONF_PATH=$APP_HOME/zcash.conf
 # Create layer in case you want to modify local lightwalletd code
 FROM golang:1.22 AS build
 
+
 # Create and change to the app directory.
 WORKDIR /app
 

cmd/root.go

@@ -55,6 +55,7 @@ var rootCmd = &cobra.Command{
 			GenCertVeryInsecure: viper.GetBool("gen-cert-very-insecure"),
 			DataDir:             viper.GetString("data-dir"),
 			Redownload:          viper.GetBool("redownload"),
+			NoCache:             viper.GetBool("nocache"),
 			SyncFromHeight:      viper.GetInt("sync-from-height"),
 			PingEnable:          viper.GetBool("ping-very-insecure"),
 			Darkside:            viper.GetBool("darkside-very-insecure"),
@@ -118,13 +119,13 @@ func startServer(opts *common.Options) error {
 
 	logger.SetLevel(logrus.Level(opts.LogLevel))
 
+	logging.LogToStderr = opts.GRPCLogging
+
 	common.Log.WithFields(logrus.Fields{
 		"gitCommit": common.GitCommit,
 		"buildDate": common.BuildDate,
 		"buildUser": common.BuildUser,
-	}).Infof("Starting gRPC server version %s on %s", common.Version, opts.GRPCBindAddr)
-
-	logging.LogToStderr = opts.GRPCLogging
+	}).Infof("Starting lightwalletd process version %s", common.Version)
 
 	// gRPC initialization
 	var server *grpc.Server
@@ -197,7 +198,7 @@ func startServer(opts *common.Options) error {
 		if err != nil {
 			common.Log.WithFields(logrus.Fields{
 				"error": err,
-			}).Fatal("setting up RPC connection to zcashd")
+			}).Fatal("setting up RPC connection to zebrad or zcashd")
 		}
 		// Indirect function for test mocking (so unit tests can talk to stub functions).
 		common.RawRequest = rpcClient.RawRequest
@@ -209,14 +210,18 @@ func startServer(opts *common.Options) error {
 		if err != nil {
 			common.Log.WithFields(logrus.Fields{
 				"error": err,
-			}).Fatal("getting initial information from zcashd")
+			}).Fatal("getting initial information from zebrad or zcashd")
 		}
 		common.Log.Info("Got sapling height ", getLightdInfo.SaplingActivationHeight,
 			" block height ", getLightdInfo.BlockHeight,
 			" chain ", getLightdInfo.ChainName,
 			" branchID ", getLightdInfo.ConsensusBranchId)
 		saplingHeight = int(getLightdInfo.SaplingActivationHeight)
 		chainName = getLightdInfo.ChainName
+		if strings.Contains(getLightdInfo.ZcashdSubversion, "MagicBean") {
+			// The default is zebrad
+			common.NodeName = "zcashd"
+		}
 	}
 
 	dbPath := filepath.Join(opts.DataDir, "db")
@@ -240,13 +245,22 @@ func startServer(opts *common.Options) error {
 		os.Stderr.WriteString(fmt.Sprintf("\n  ** Can't create db directory: %s\n\n", dbPath))
 		os.Exit(1)
 	}
-	syncFromHeight := opts.SyncFromHeight
-	if opts.Redownload {
-		syncFromHeight = 0
+	var cache *common.BlockCache
+	if opts.NoCache {
+		lengthsName, blocksName := common.DbFileNames(dbPath, chainName)
+		os.Remove(lengthsName)
+		os.Remove(blocksName)
+	} else {
+		syncFromHeight := opts.SyncFromHeight
+		if opts.Redownload {
+			syncFromHeight = 0
+		}
+		cache = common.NewBlockCache(dbPath, chainName, saplingHeight, syncFromHeight)
 	}
-	cache := common.NewBlockCache(dbPath, chainName, saplingHeight, syncFromHeight)
 	if !opts.Darkside {
-		go common.BlockIngestor(cache, 0 /*loop forever*/)
+		if !opts.NoCache {
+			go common.BlockIngestor(cache, 0 /*loop forever*/)
+		}
 	} else {
 		// Darkside wants to control starting the block ingestor.
 		common.DarksideInit(cache, int(opts.DarksideTimeout))
@@ -272,6 +286,8 @@ func startServer(opts *common.Options) error {
 		walletrpc.RegisterDarksideStreamerServer(server, service)
 	}
 
+	common.Log.Infof("Starting gRPC server on %s", opts.GRPCBindAddr)
+
 	// Start listening
 	listener, err := net.Listen("tcp", opts.GRPCBindAddr)
 	if err != nil {
@@ -329,11 +345,12 @@ func init() {
 	rootCmd.Flags().String("rpcport", "", "RPC host port")
 	rootCmd.Flags().Bool("no-tls-very-insecure", false, "run without the required TLS certificate, only for debugging, DO NOT use in production")
 	rootCmd.Flags().Bool("gen-cert-very-insecure", false, "run with self-signed TLS certificate, only for debugging, DO NOT use in production")
-	rootCmd.Flags().Bool("redownload", false, "re-fetch all blocks from zcashd; reinitialize local cache files")
-	rootCmd.Flags().Int("sync-from-height", -1, "re-fetch blocks from zcashd start at this height")
+	rootCmd.Flags().Bool("redownload", false, "re-fetch all blocks from zebrad or zcashd; reinitialize local cache files")
+	rootCmd.Flags().Bool("nocache", false, "don't maintain a compact blocks disk cache (to reduce storage)")
+	rootCmd.Flags().Int("sync-from-height", -1, "re-fetch blocks from zebrad or zcashd, starting at this height")
 	rootCmd.Flags().String("data-dir", "/var/lib/lightwalletd", "data directory (such as db)")
 	rootCmd.Flags().Bool("ping-very-insecure", false, "allow Ping GRPC for testing")
-	rootCmd.Flags().Bool("darkside-very-insecure", false, "run with GRPC-controllable mock zcashd for integration testing (shuts down after 30 minutes)")
+	rootCmd.Flags().Bool("darkside-very-insecure", false, "run with GRPC-controllable mock zebrad for integration testing (shuts down after 30 minutes)")
 	rootCmd.Flags().Int("darkside-timeout", 30, "override 30 minute default darkside timeout")
 
 	viper.BindPFlag("grpc-bind-addr", rootCmd.Flags().Lookup("grpc-bind-addr"))
@@ -362,6 +379,8 @@ func init() {
 	viper.SetDefault("gen-cert-very-insecure", false)
 	viper.BindPFlag("redownload", rootCmd.Flags().Lookup("redownload"))
 	viper.SetDefault("redownload", false)
+	viper.BindPFlag("nocache", rootCmd.Flags().Lookup("nocache"))
+	viper.SetDefault("nocache", false)
 	viper.BindPFlag("sync-from-height", rootCmd.Flags().Lookup("sync-from-height"))
 	viper.SetDefault("sync-from-height", -1)
 	viper.BindPFlag("data-dir", rootCmd.Flags().Lookup("data-dir"))

common/cache.go

@@ -195,7 +195,7 @@ func NewBlockCache(dbPath string, chainName string, startHeight int, syncFromHei
 	c := &BlockCache{}
 	c.firstBlock = startHeight
 	c.nextBlock = startHeight
-	c.lengthsName, c.blocksName = dbFileNames(dbPath, chainName)
+	c.lengthsName, c.blocksName = DbFileNames(dbPath, chainName)
 	var err error
 	if err := os.MkdirAll(filepath.Join(dbPath, chainName), 0755); err != nil {
 		Log.Fatal("mkdir ", dbPath, " failed: ", err)
@@ -243,21 +243,14 @@ func NewBlockCache(dbPath string, chainName string, startHeight int, syncFromHei
 		}
 		offset += int64(length) + 8
 		c.starts = append(c.starts, offset)
-		// Check for corruption.
-		block := c.readBlock(c.nextBlock)
-		if block == nil {
-			Log.Warning("error reading block")
-			c.recoverFromCorruption(c.nextBlock)
-			break
-		}
 		c.nextBlock++
 	}
 	c.setDbFiles(c.nextBlock)
 	Log.Info("Done reading ", c.nextBlock-c.firstBlock, " blocks from disk cache")
 	return c
 }
 
-func dbFileNames(dbPath string, chainName string) (string, string) {
+func DbFileNames(dbPath string, chainName string) (string, string) {
 	return filepath.Join(dbPath, chainName, "lengths"),
 		filepath.Join(dbPath, chainName, "blocks")
 }

common/common.go

@@ -25,6 +25,7 @@ var (
 	Branch    = ""
 	BuildDate = ""
 	BuildUser = ""
+	NodeName  = "zebrad"
 )
 
 type Options struct {
@@ -43,6 +44,7 @@ type Options struct {
 	NoTLSVeryInsecure   bool   `json:"no_tls_very_insecure,omitempty"`
 	GenCertVeryInsecure bool   `json:"gen_cert_very_insecure,omitempty"`
 	Redownload          bool   `json:"redownload"`
+	NoCache             bool   `json:"nocache"`
 	SyncFromHeight      int    `json:"sync_from_height"`
 	DataDir             string `json:"data_dir"`
 	PingEnable          bool   `json:"ping_enable"`
@@ -100,7 +102,7 @@ type (
 	ZcashdRpcRequestGetaddresstxids struct {
 		Addresses []string `json:"addresses"`
 		Start     uint64   `json:"start"`
-		End       uint64   `json:"end"`
+		End       uint64   `json:"end,omitempty"`
 	}
 
 	// zcashd rpc "z_gettreestate"
@@ -126,7 +128,7 @@ type (
 	// many more fields but these are the only ones we current need.
 	ZcashdRpcReplyGetrawtransaction struct {
 		Hex    string
-		Height int
+		Height int64
 	}
 
 	// zcashd rpc "getaddressbalance"
@@ -224,7 +226,7 @@ func FirstRPC() {
 		if retryCount > 10 {
 			Log.WithFields(logrus.Fields{
 				"timeouts": retryCount,
-			}).Fatal("unable to issue getblockchaininfo RPC call to zcashd node")
+			}).Fatal("unable to issue getblockchaininfo RPC call to zebrad or zcashd node")
 		}
 		Log.WithFields(logrus.Fields{
 			"error": err.Error(),
@@ -417,7 +419,7 @@ func BlockIngestor(c *BlockCache, rep int) {
 		if err != nil {
 			Log.WithFields(logrus.Fields{
 				"error": err,
-			}).Fatal("error zcashd getbestblockhash rpc")
+			}).Fatal("error " + NodeName + " getbestblockhash rpc")
 		}
 		var hashHex string
 		err = json.Unmarshal(result, &hashHex)
@@ -461,10 +463,10 @@ func BlockIngestor(c *BlockCache, rep int) {
 		}
 		if height == c.GetFirstHeight() {
 			c.Sync()
-			Log.Info("Waiting for zcashd height to reach Sapling activation height ",
+			Log.Info("Waiting for "+NodeName+" height to reach Sapling activation height ",
 				"(", c.GetFirstHeight(), ")...")
-			Time.Sleep(20 * time.Second)
-			return
+			Time.Sleep(120 * time.Second)
+			continue
 		}
 		Log.Info("REORG: dropping block ", height-1, " ", displayHash(c.GetLatestHash()))
 		c.Reorg(height - 1)
@@ -476,9 +478,12 @@ func BlockIngestor(c *BlockCache, rep int) {
 // nil if no block exists at this height.
 func GetBlock(cache *BlockCache, height int) (*walletrpc.CompactBlock, error) {
 	// First, check the cache to see if we have the block
-	block := cache.Get(height)
-	if block != nil {
-		return block, nil
+	var block *walletrpc.CompactBlock
+	if cache != nil {
+		block := cache.Get(height)
+		if block != nil {
+			return block, nil
+		}
 	}
 
 	// Not in the cache
@@ -518,6 +523,43 @@ func GetBlockRange(cache *BlockCache, blockOut chan<- *walletrpc.CompactBlock, e
 	errOut <- nil
 }
 
+// ParseRawTransaction converts between the JSON result of a `zcashd`
+// `getrawtransaction` call and the `RawTransaction` protobuf type.
+//
+// Due to an error in the original protobuf definition, it is necessary to
+// reinterpret the result of the `getrawtransaction` RPC call. Zcashd will
+// return the int64 value `-1` for the height of transactions that appear in
+// the block index, but which are not mined in the main chain. `service.proto`
+// defines the height field of `RawTransaction` to be a `uint64`, and as such
+// we must map the response from the zcashd RPC API to be representable within
+// this space. Additionally, the `height` field will be absent for transactions
+// in the mempool, resulting in the default value of `0` being set. Therefore,
+// the meanings of the `Height` field of the `RawTransaction` type are as
+// follows:
+//
+// * height 0: the transaction is in the mempool
+// * height 0xffffffffffffffff: the transaction has been mined on a fork that
+//   is not currently the main chain
+// * any other height: the transaction has been mined in the main chain at the
+//   given height
+func ParseRawTransaction(message json.RawMessage) (*walletrpc.RawTransaction, error) {
+		// Many other fields are returned, but we need only these two.
+		var txinfo ZcashdRpcReplyGetrawtransaction
+		err := json.Unmarshal(message, &txinfo)
+		if err != nil {
+			return nil, err
+		}
+		txBytes, err := hex.DecodeString(txinfo.Hex)
+		if err != nil {
+			return nil, err
+		}
+
+		return &walletrpc.RawTransaction{
+			Data:   txBytes,
+			Height: uint64(txinfo.Height),
+		}, nil
+}
+
 func displayHash(hash []byte) string {
 	return hex.EncodeToString(parser.Reverse(hash))
 }