diff --git a/.github/workflows/CI.yaml b/.github/workflows/CI.yaml index 7b16bd76..bedaf829 100644 --- a/.github/workflows/CI.yaml +++ b/.github/workflows/CI.yaml @@ -15,7 +15,7 @@ jobs: strategy: matrix: - go-version: ['>=1.22.0'] + go-version: ['>=1.22.2'] steps: - name: Check out code @@ -52,6 +52,7 @@ jobs: image_tags: ${{ needs.docker_set_env.outputs.tags }} dockerfile: ./Dockerfile context: . + build-args: "" secrets: dockerhub_registry: ${{ secrets.DOCKERHUB_REGISTRY }} dockerhub_username: ${{ secrets.DOCKERHUB_USERNAME }} diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 00000000..be064197 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,28 @@ +# Changelog +All notable changes to this library will be documented in this file. + +The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), +and this library adheres to Rust's notion of +[Semantic Versioning](https://semver.org/spec/v2.0.0.html). + +## [Unreleased] + +### Changed + +- The `RawTransaction` values returned from a call to `GetMempoolStream` + now report a `Height` value of `0`, in order to be consistent with + the results of calls to `GetTransaction`. See the documentation of + `RawTransaction` in `walletrpc/service.proto` for more details on + the semantics of this field. + +### Fixed + +- Parsing of `getrawtransaction` results is now platform-independent. + Previously, values of `-1` returned for the transaction height would + be converted to different `RawTransaction.Height` values depending + upon whether `lightwalletd` was being run on a 32-bit or 64-bit + platform. + +## [Prior Releases] + +This changelog was not created until after the release of v0.4.17 diff --git a/Dockerfile b/Dockerfile index 4596b223..d944b2c6 100644 --- a/Dockerfile +++ b/Dockerfile @@ -7,6 +7,7 @@ ARG ZCASHD_CONF_PATH=$APP_HOME/zcash.conf # Create layer in case you want to modify local lightwalletd code FROM golang:1.22 AS build + # Create and change to the app directory. WORKDIR /app diff --git a/cmd/root.go b/cmd/root.go index b88342e6..7fed4210 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -55,6 +55,7 @@ var rootCmd = &cobra.Command{ GenCertVeryInsecure: viper.GetBool("gen-cert-very-insecure"), DataDir: viper.GetString("data-dir"), Redownload: viper.GetBool("redownload"), + NoCache: viper.GetBool("nocache"), SyncFromHeight: viper.GetInt("sync-from-height"), PingEnable: viper.GetBool("ping-very-insecure"), Darkside: viper.GetBool("darkside-very-insecure"), @@ -118,13 +119,13 @@ func startServer(opts *common.Options) error { logger.SetLevel(logrus.Level(opts.LogLevel)) + logging.LogToStderr = opts.GRPCLogging + common.Log.WithFields(logrus.Fields{ "gitCommit": common.GitCommit, "buildDate": common.BuildDate, "buildUser": common.BuildUser, - }).Infof("Starting gRPC server version %s on %s", common.Version, opts.GRPCBindAddr) - - logging.LogToStderr = opts.GRPCLogging + }).Infof("Starting lightwalletd process version %s", common.Version) // gRPC initialization var server *grpc.Server @@ -197,7 +198,7 @@ func startServer(opts *common.Options) error { if err != nil { common.Log.WithFields(logrus.Fields{ "error": err, - }).Fatal("setting up RPC connection to zcashd") + }).Fatal("setting up RPC connection to zebrad or zcashd") } // Indirect function for test mocking (so unit tests can talk to stub functions). common.RawRequest = rpcClient.RawRequest @@ -209,7 +210,7 @@ func startServer(opts *common.Options) error { if err != nil { common.Log.WithFields(logrus.Fields{ "error": err, - }).Fatal("getting initial information from zcashd") + }).Fatal("getting initial information from zebrad or zcashd") } common.Log.Info("Got sapling height ", getLightdInfo.SaplingActivationHeight, " block height ", getLightdInfo.BlockHeight, @@ -217,6 +218,10 @@ func startServer(opts *common.Options) error { " branchID ", getLightdInfo.ConsensusBranchId) saplingHeight = int(getLightdInfo.SaplingActivationHeight) chainName = getLightdInfo.ChainName + if strings.Contains(getLightdInfo.ZcashdSubversion, "MagicBean") { + // The default is zebrad + common.NodeName = "zcashd" + } } dbPath := filepath.Join(opts.DataDir, "db") @@ -240,13 +245,22 @@ func startServer(opts *common.Options) error { os.Stderr.WriteString(fmt.Sprintf("\n ** Can't create db directory: %s\n\n", dbPath)) os.Exit(1) } - syncFromHeight := opts.SyncFromHeight - if opts.Redownload { - syncFromHeight = 0 + var cache *common.BlockCache + if opts.NoCache { + lengthsName, blocksName := common.DbFileNames(dbPath, chainName) + os.Remove(lengthsName) + os.Remove(blocksName) + } else { + syncFromHeight := opts.SyncFromHeight + if opts.Redownload { + syncFromHeight = 0 + } + cache = common.NewBlockCache(dbPath, chainName, saplingHeight, syncFromHeight) } - cache := common.NewBlockCache(dbPath, chainName, saplingHeight, syncFromHeight) if !opts.Darkside { - go common.BlockIngestor(cache, 0 /*loop forever*/) + if !opts.NoCache { + go common.BlockIngestor(cache, 0 /*loop forever*/) + } } else { // Darkside wants to control starting the block ingestor. common.DarksideInit(cache, int(opts.DarksideTimeout)) @@ -272,6 +286,8 @@ func startServer(opts *common.Options) error { walletrpc.RegisterDarksideStreamerServer(server, service) } + common.Log.Infof("Starting gRPC server on %s", opts.GRPCBindAddr) + // Start listening listener, err := net.Listen("tcp", opts.GRPCBindAddr) if err != nil { @@ -329,11 +345,12 @@ func init() { rootCmd.Flags().String("rpcport", "", "RPC host port") rootCmd.Flags().Bool("no-tls-very-insecure", false, "run without the required TLS certificate, only for debugging, DO NOT use in production") rootCmd.Flags().Bool("gen-cert-very-insecure", false, "run with self-signed TLS certificate, only for debugging, DO NOT use in production") - rootCmd.Flags().Bool("redownload", false, "re-fetch all blocks from zcashd; reinitialize local cache files") - rootCmd.Flags().Int("sync-from-height", -1, "re-fetch blocks from zcashd start at this height") + rootCmd.Flags().Bool("redownload", false, "re-fetch all blocks from zebrad or zcashd; reinitialize local cache files") + rootCmd.Flags().Bool("nocache", false, "don't maintain a compact blocks disk cache (to reduce storage)") + rootCmd.Flags().Int("sync-from-height", -1, "re-fetch blocks from zebrad or zcashd, starting at this height") rootCmd.Flags().String("data-dir", "/var/lib/lightwalletd", "data directory (such as db)") rootCmd.Flags().Bool("ping-very-insecure", false, "allow Ping GRPC for testing") - rootCmd.Flags().Bool("darkside-very-insecure", false, "run with GRPC-controllable mock zcashd for integration testing (shuts down after 30 minutes)") + rootCmd.Flags().Bool("darkside-very-insecure", false, "run with GRPC-controllable mock zebrad for integration testing (shuts down after 30 minutes)") rootCmd.Flags().Int("darkside-timeout", 30, "override 30 minute default darkside timeout") viper.BindPFlag("grpc-bind-addr", rootCmd.Flags().Lookup("grpc-bind-addr")) @@ -362,6 +379,8 @@ func init() { viper.SetDefault("gen-cert-very-insecure", false) viper.BindPFlag("redownload", rootCmd.Flags().Lookup("redownload")) viper.SetDefault("redownload", false) + viper.BindPFlag("nocache", rootCmd.Flags().Lookup("nocache")) + viper.SetDefault("nocache", false) viper.BindPFlag("sync-from-height", rootCmd.Flags().Lookup("sync-from-height")) viper.SetDefault("sync-from-height", -1) viper.BindPFlag("data-dir", rootCmd.Flags().Lookup("data-dir")) diff --git a/common/cache.go b/common/cache.go index 74c56fb9..40aa6c69 100644 --- a/common/cache.go +++ b/common/cache.go @@ -195,7 +195,7 @@ func NewBlockCache(dbPath string, chainName string, startHeight int, syncFromHei c := &BlockCache{} c.firstBlock = startHeight c.nextBlock = startHeight - c.lengthsName, c.blocksName = dbFileNames(dbPath, chainName) + c.lengthsName, c.blocksName = DbFileNames(dbPath, chainName) var err error if err := os.MkdirAll(filepath.Join(dbPath, chainName), 0755); err != nil { Log.Fatal("mkdir ", dbPath, " failed: ", err) @@ -243,13 +243,6 @@ func NewBlockCache(dbPath string, chainName string, startHeight int, syncFromHei } offset += int64(length) + 8 c.starts = append(c.starts, offset) - // Check for corruption. - block := c.readBlock(c.nextBlock) - if block == nil { - Log.Warning("error reading block") - c.recoverFromCorruption(c.nextBlock) - break - } c.nextBlock++ } c.setDbFiles(c.nextBlock) @@ -257,7 +250,7 @@ func NewBlockCache(dbPath string, chainName string, startHeight int, syncFromHei return c } -func dbFileNames(dbPath string, chainName string) (string, string) { +func DbFileNames(dbPath string, chainName string) (string, string) { return filepath.Join(dbPath, chainName, "lengths"), filepath.Join(dbPath, chainName, "blocks") } diff --git a/common/common.go b/common/common.go index 51955e4c..460ce6b6 100644 --- a/common/common.go +++ b/common/common.go @@ -25,6 +25,7 @@ var ( Branch = "" BuildDate = "" BuildUser = "" + NodeName = "zebrad" ) type Options struct { @@ -43,6 +44,7 @@ type Options struct { NoTLSVeryInsecure bool `json:"no_tls_very_insecure,omitempty"` GenCertVeryInsecure bool `json:"gen_cert_very_insecure,omitempty"` Redownload bool `json:"redownload"` + NoCache bool `json:"nocache"` SyncFromHeight int `json:"sync_from_height"` DataDir string `json:"data_dir"` PingEnable bool `json:"ping_enable"` @@ -100,7 +102,7 @@ type ( ZcashdRpcRequestGetaddresstxids struct { Addresses []string `json:"addresses"` Start uint64 `json:"start"` - End uint64 `json:"end"` + End uint64 `json:"end,omitempty"` } // zcashd rpc "z_gettreestate" @@ -126,7 +128,7 @@ type ( // many more fields but these are the only ones we current need. ZcashdRpcReplyGetrawtransaction struct { Hex string - Height int + Height int64 } // zcashd rpc "getaddressbalance" @@ -224,7 +226,7 @@ func FirstRPC() { if retryCount > 10 { Log.WithFields(logrus.Fields{ "timeouts": retryCount, - }).Fatal("unable to issue getblockchaininfo RPC call to zcashd node") + }).Fatal("unable to issue getblockchaininfo RPC call to zebrad or zcashd node") } Log.WithFields(logrus.Fields{ "error": err.Error(), @@ -417,7 +419,7 @@ func BlockIngestor(c *BlockCache, rep int) { if err != nil { Log.WithFields(logrus.Fields{ "error": err, - }).Fatal("error zcashd getbestblockhash rpc") + }).Fatal("error " + NodeName + " getbestblockhash rpc") } var hashHex string err = json.Unmarshal(result, &hashHex) @@ -461,10 +463,10 @@ func BlockIngestor(c *BlockCache, rep int) { } if height == c.GetFirstHeight() { c.Sync() - Log.Info("Waiting for zcashd height to reach Sapling activation height ", + Log.Info("Waiting for "+NodeName+" height to reach Sapling activation height ", "(", c.GetFirstHeight(), ")...") - Time.Sleep(20 * time.Second) - return + Time.Sleep(120 * time.Second) + continue } Log.Info("REORG: dropping block ", height-1, " ", displayHash(c.GetLatestHash())) c.Reorg(height - 1) @@ -476,9 +478,12 @@ func BlockIngestor(c *BlockCache, rep int) { // nil if no block exists at this height. func GetBlock(cache *BlockCache, height int) (*walletrpc.CompactBlock, error) { // First, check the cache to see if we have the block - block := cache.Get(height) - if block != nil { - return block, nil + var block *walletrpc.CompactBlock + if cache != nil { + block := cache.Get(height) + if block != nil { + return block, nil + } } // Not in the cache @@ -518,6 +523,43 @@ func GetBlockRange(cache *BlockCache, blockOut chan<- *walletrpc.CompactBlock, e errOut <- nil } +// ParseRawTransaction converts between the JSON result of a `zcashd` +// `getrawtransaction` call and the `RawTransaction` protobuf type. +// +// Due to an error in the original protobuf definition, it is necessary to +// reinterpret the result of the `getrawtransaction` RPC call. Zcashd will +// return the int64 value `-1` for the height of transactions that appear in +// the block index, but which are not mined in the main chain. `service.proto` +// defines the height field of `RawTransaction` to be a `uint64`, and as such +// we must map the response from the zcashd RPC API to be representable within +// this space. Additionally, the `height` field will be absent for transactions +// in the mempool, resulting in the default value of `0` being set. Therefore, +// the meanings of the `Height` field of the `RawTransaction` type are as +// follows: +// +// * height 0: the transaction is in the mempool +// * height 0xffffffffffffffff: the transaction has been mined on a fork that +// is not currently the main chain +// * any other height: the transaction has been mined in the main chain at the +// given height +func ParseRawTransaction(message json.RawMessage) (*walletrpc.RawTransaction, error) { + // Many other fields are returned, but we need only these two. + var txinfo ZcashdRpcReplyGetrawtransaction + err := json.Unmarshal(message, &txinfo) + if err != nil { + return nil, err + } + txBytes, err := hex.DecodeString(txinfo.Hex) + if err != nil { + return nil, err + } + + return &walletrpc.RawTransaction{ + Data: txBytes, + Height: uint64(txinfo.Height), + }, nil +} + func displayHash(hash []byte) string { return hex.EncodeToString(parser.Reverse(hash)) } diff --git a/common/common_test.go b/common/common_test.go index 9fc19f32..4c4860ab 100644 --- a/common/common_test.go +++ b/common/common_test.go @@ -9,6 +9,7 @@ import ( "encoding/json" "errors" "fmt" + "math" "os" "strings" "testing" @@ -604,7 +605,7 @@ func mempoolStub(method string, params []json.RawMessage) (json.RawMessage, erro if txid != "mempooltxid-1" { testT.Fatal("unexpected txid") } - r, _ := json.Marshal("aabb") + r, _ := json.Marshal(map[string]string{"hex":"aabb"}) return r, nil case 5: // Simulate that still no new block has arrived ... @@ -636,7 +637,7 @@ func mempoolStub(method string, params []json.RawMessage) (json.RawMessage, erro if txid != "mempooltxid-2" { testT.Fatal("unexpected txid") } - r, _ := json.Marshal("ccdd") + r, _ := json.Marshal(map[string]string{"hex":"ccdd"}) return r, nil case 8: // A new block arrives, this will cause these two tx to be returned @@ -668,7 +669,7 @@ func TestMempoolStream(t *testing.T) { return nil }) if err != nil { - t.Fatal("GetMempool failed") + t.Errorf("GetMempool failed: %v", err) } // This should return two transactions. @@ -677,7 +678,7 @@ func TestMempoolStream(t *testing.T) { return nil }) if err != nil { - t.Fatal("GetMempool failed") + t.Errorf("GetMempool failed: %v", err) } if len(replies) != 2 { t.Fatal("unexpected number of tx") @@ -687,13 +688,13 @@ func TestMempoolStream(t *testing.T) { if !bytes.Equal([]byte(replies[0].GetData()), []byte{0xaa, 0xbb}) { t.Fatal("unexpected tx contents") } - if replies[0].GetHeight() != 200 { + if replies[0].GetHeight() != 0 { t.Fatal("unexpected tx height") } if !bytes.Equal([]byte(replies[1].GetData()), []byte{0xcc, 0xdd}) { t.Fatal("unexpected tx contents") } - if replies[1].GetHeight() != 200 { + if replies[1].GetHeight() != 0 { t.Fatal("unexpected tx height") } @@ -703,10 +704,68 @@ func TestMempoolStream(t *testing.T) { t.Fatal("unexpected end time") } if step != 8 { - t.Fatal("unexpected number of zcashd RPCs") + t.Fatal("unexpected number of zebrad RPCs") } step = 0 sleepCount = 0 sleepDuration = 0 } + +func TestZcashdRpcReplyUnmarshalling(t *testing.T) { + var txinfo0 ZcashdRpcReplyGetrawtransaction + err0 := json.Unmarshal([]byte("{\"hex\": \"deadbeef\", \"height\": 123456}"), &txinfo0) + if err0 != nil { + t.Fatal("Failed to unmarshal tx with known height.") + } + if txinfo0.Height != 123456 { + t.Errorf("Unmarshalled incorrect height: got: %d, want: 123456.", txinfo0.Height) + } + + var txinfo1 ZcashdRpcReplyGetrawtransaction + err1 := json.Unmarshal([]byte("{\"hex\": \"deadbeef\", \"height\": -1}"), &txinfo1) + if err1 != nil { + t.Fatal("failed to unmarshal tx not in main chain") + } + if txinfo1.Height != -1 { + t.Errorf("Unmarshalled incorrect height: got: %d, want: -1.", txinfo1.Height) + } + + var txinfo2 ZcashdRpcReplyGetrawtransaction + err2 := json.Unmarshal([]byte("{\"hex\": \"deadbeef\"}"), &txinfo2) + if err2 != nil { + t.Fatal("failed to unmarshal reply lacking height data") + } + if txinfo2.Height != 0 { + t.Errorf("Unmarshalled incorrect height: got: %d, want: 0.", txinfo2.Height) + } +} + +func TestParseRawTransaction(t *testing.T) { + rt0, err0 := ParseRawTransaction([]byte("{\"hex\": \"deadbeef\", \"height\": 123456}")) + if err0 != nil { + t.Fatal("Failed to parse raw transaction response with known height.") + } + if rt0.Height != 123456 { + t.Errorf("Unmarshalled incorrect height: got: %d, expected: 123456.", rt0.Height) + } + + rt1, err1 := ParseRawTransaction([]byte("{\"hex\": \"deadbeef\", \"height\": -1}")) + if err1 != nil { + t.Fatal("Failed to parse raw transaction response for a known tx not in the main chain.") + } + // We expect the int64 value `-1` to have been reinterpreted as a uint64 value in order + // to be representable as a uint64 in `RawTransaction`. The conversion from the twos-complement + // signed representation should map `-1` to `math.MaxUint64`. + if rt1.Height != math.MaxUint64 { + t.Errorf("Unmarshalled incorrect height: got: %d, want: 0x%X.", rt1.Height, uint64(math.MaxUint64)) + } + + rt2, err2 := ParseRawTransaction([]byte("{\"hex\": \"deadbeef\"}")) + if err2 != nil { + t.Fatal("Failed to parse raw transaction response for a tx in the mempool.") + } + if rt2.Height != 0 { + t.Errorf("Unmarshalled incorrect height: got: %d, expected: 0.", rt2.Height) + } +} diff --git a/common/mempool.go b/common/mempool.go index 241e0520..e52082c5 100644 --- a/common/mempool.go +++ b/common/mempool.go @@ -1,7 +1,6 @@ package common import ( - "encoding/hex" "encoding/json" "sync" "time" @@ -105,35 +104,33 @@ func refreshMempoolTxns() error { // We've already fetched this transaction continue } - g_txidSeen[txid(txidstr)] = struct{}{} + // We haven't fetched this transaction already. + g_txidSeen[txid(txidstr)] = struct{}{} txidJSON, err := json.Marshal(txidstr) if err != nil { return err } - // The "0" is because we only need the raw hex, which is returned as - // just a hex string, and not even a json string (with quotes). - params := []json.RawMessage{txidJSON, json.RawMessage("0")} + + params := []json.RawMessage{txidJSON, json.RawMessage("1")} result, rpcErr := RawRequest("getrawtransaction", params) if rpcErr != nil { // Not an error; mempool transactions can disappear continue } - // strip the quotes - var txStr string - err = json.Unmarshal(result, &txStr) - if err != nil { - return err - } - txBytes, err := hex.DecodeString(txStr) + + rawtx, err := ParseRawTransaction(result) if err != nil { return err } - newRtx := &walletrpc.RawTransaction{ - Data: txBytes, - Height: uint64(g_lastBlockChainInfo.Blocks), + + // Skip any transaction that has been mined since the list of txids + // was retrieved. + if (rawtx.Height != 0) { + continue; } - g_txList = append(g_txList, newRtx) + + g_txList = append(g_txList, rawtx) } return nil } diff --git a/docs/rtd/index.html b/docs/rtd/index.html index cfd44c36..35bf4bda 100644 --- a/docs/rtd/index.html +++ b/docs/rtd/index.html @@ -1564,7 +1564,7 @@
RawTransaction contains the complete transaction data. It also optionally includes
the block height in which the transaction was included, or, when returned
by GetMempoolStream(), the latest block height.
+RawTransaction contains the complete transaction data. It also includes the
height for the block in which the transaction was included in the main
chain, if any (as detailed below).
data | bytes | - | exact data returned by Zcash 'getrawtransaction' |
+ The serialized representation of the Zcash transaction. |
height | uint64 | - | height that the transaction was mined (or -1) |
+ The height at which the transaction is mined, or a sentinel value. + +Due to an error in the original protobuf definition, it is necessary to +reinterpret the result of the `getrawtransaction` RPC call. Zcashd will +return the int64 value `-1` for the height of transactions that appear +in the block index, but which are not mined in the main chain. Here, the +height field of `RawTransaction` was erroneously created as a `uint64`, +and as such we must map the response from the zcashd RPC API to be +representable within this space. Additionally, the `height` field will +be absent for transactions in the mempool, resulting in the default +value of `0` being set. Therefore, the meanings of the `height` field of +the `RawTransaction` type are as follows: + +* height 0: the transaction is in the mempool +* height 0xffffffffffffffff: the transaction has been mined on a fork that + is not currently the main chain +* any other height: the transaction has been mined in the main chain at the + given height |