diff --git a/.github/workflows/CI.yaml b/.github/workflows/CI.yaml index 7b16bd76..bedaf829 100644 --- a/.github/workflows/CI.yaml +++ b/.github/workflows/CI.yaml @@ -15,7 +15,7 @@ jobs: strategy: matrix: - go-version: ['>=1.22.0'] + go-version: ['>=1.22.2'] steps: - name: Check out code @@ -52,6 +52,7 @@ jobs: image_tags: ${{ needs.docker_set_env.outputs.tags }} dockerfile: ./Dockerfile context: . + build-args: "" secrets: dockerhub_registry: ${{ secrets.DOCKERHUB_REGISTRY }} dockerhub_username: ${{ secrets.DOCKERHUB_USERNAME }} diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 00000000..be064197 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,28 @@ +# Changelog +All notable changes to this library will be documented in this file. + +The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), +and this library adheres to Rust's notion of +[Semantic Versioning](https://semver.org/spec/v2.0.0.html). + +## [Unreleased] + +### Changed + +- The `RawTransaction` values returned from a call to `GetMempoolStream` + now report a `Height` value of `0`, in order to be consistent with + the results of calls to `GetTransaction`. See the documentation of + `RawTransaction` in `walletrpc/service.proto` for more details on + the semantics of this field. + +### Fixed + +- Parsing of `getrawtransaction` results is now platform-independent. + Previously, values of `-1` returned for the transaction height would + be converted to different `RawTransaction.Height` values depending + upon whether `lightwalletd` was being run on a 32-bit or 64-bit + platform. + +## [Prior Releases] + +This changelog was not created until after the release of v0.4.17 diff --git a/Dockerfile b/Dockerfile index 4596b223..d944b2c6 100644 --- a/Dockerfile +++ b/Dockerfile @@ -7,6 +7,7 @@ ARG ZCASHD_CONF_PATH=$APP_HOME/zcash.conf # Create layer in case you want to modify local lightwalletd code FROM golang:1.22 AS build + # Create and change to the app directory. WORKDIR /app diff --git a/cmd/root.go b/cmd/root.go index b88342e6..7fed4210 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -55,6 +55,7 @@ var rootCmd = &cobra.Command{ GenCertVeryInsecure: viper.GetBool("gen-cert-very-insecure"), DataDir: viper.GetString("data-dir"), Redownload: viper.GetBool("redownload"), + NoCache: viper.GetBool("nocache"), SyncFromHeight: viper.GetInt("sync-from-height"), PingEnable: viper.GetBool("ping-very-insecure"), Darkside: viper.GetBool("darkside-very-insecure"), @@ -118,13 +119,13 @@ func startServer(opts *common.Options) error { logger.SetLevel(logrus.Level(opts.LogLevel)) + logging.LogToStderr = opts.GRPCLogging + common.Log.WithFields(logrus.Fields{ "gitCommit": common.GitCommit, "buildDate": common.BuildDate, "buildUser": common.BuildUser, - }).Infof("Starting gRPC server version %s on %s", common.Version, opts.GRPCBindAddr) - - logging.LogToStderr = opts.GRPCLogging + }).Infof("Starting lightwalletd process version %s", common.Version) // gRPC initialization var server *grpc.Server @@ -197,7 +198,7 @@ func startServer(opts *common.Options) error { if err != nil { common.Log.WithFields(logrus.Fields{ "error": err, - }).Fatal("setting up RPC connection to zcashd") + }).Fatal("setting up RPC connection to zebrad or zcashd") } // Indirect function for test mocking (so unit tests can talk to stub functions). common.RawRequest = rpcClient.RawRequest @@ -209,7 +210,7 @@ func startServer(opts *common.Options) error { if err != nil { common.Log.WithFields(logrus.Fields{ "error": err, - }).Fatal("getting initial information from zcashd") + }).Fatal("getting initial information from zebrad or zcashd") } common.Log.Info("Got sapling height ", getLightdInfo.SaplingActivationHeight, " block height ", getLightdInfo.BlockHeight, @@ -217,6 +218,10 @@ func startServer(opts *common.Options) error { " branchID ", getLightdInfo.ConsensusBranchId) saplingHeight = int(getLightdInfo.SaplingActivationHeight) chainName = getLightdInfo.ChainName + if strings.Contains(getLightdInfo.ZcashdSubversion, "MagicBean") { + // The default is zebrad + common.NodeName = "zcashd" + } } dbPath := filepath.Join(opts.DataDir, "db") @@ -240,13 +245,22 @@ func startServer(opts *common.Options) error { os.Stderr.WriteString(fmt.Sprintf("\n ** Can't create db directory: %s\n\n", dbPath)) os.Exit(1) } - syncFromHeight := opts.SyncFromHeight - if opts.Redownload { - syncFromHeight = 0 + var cache *common.BlockCache + if opts.NoCache { + lengthsName, blocksName := common.DbFileNames(dbPath, chainName) + os.Remove(lengthsName) + os.Remove(blocksName) + } else { + syncFromHeight := opts.SyncFromHeight + if opts.Redownload { + syncFromHeight = 0 + } + cache = common.NewBlockCache(dbPath, chainName, saplingHeight, syncFromHeight) } - cache := common.NewBlockCache(dbPath, chainName, saplingHeight, syncFromHeight) if !opts.Darkside { - go common.BlockIngestor(cache, 0 /*loop forever*/) + if !opts.NoCache { + go common.BlockIngestor(cache, 0 /*loop forever*/) + } } else { // Darkside wants to control starting the block ingestor. common.DarksideInit(cache, int(opts.DarksideTimeout)) @@ -272,6 +286,8 @@ func startServer(opts *common.Options) error { walletrpc.RegisterDarksideStreamerServer(server, service) } + common.Log.Infof("Starting gRPC server on %s", opts.GRPCBindAddr) + // Start listening listener, err := net.Listen("tcp", opts.GRPCBindAddr) if err != nil { @@ -329,11 +345,12 @@ func init() { rootCmd.Flags().String("rpcport", "", "RPC host port") rootCmd.Flags().Bool("no-tls-very-insecure", false, "run without the required TLS certificate, only for debugging, DO NOT use in production") rootCmd.Flags().Bool("gen-cert-very-insecure", false, "run with self-signed TLS certificate, only for debugging, DO NOT use in production") - rootCmd.Flags().Bool("redownload", false, "re-fetch all blocks from zcashd; reinitialize local cache files") - rootCmd.Flags().Int("sync-from-height", -1, "re-fetch blocks from zcashd start at this height") + rootCmd.Flags().Bool("redownload", false, "re-fetch all blocks from zebrad or zcashd; reinitialize local cache files") + rootCmd.Flags().Bool("nocache", false, "don't maintain a compact blocks disk cache (to reduce storage)") + rootCmd.Flags().Int("sync-from-height", -1, "re-fetch blocks from zebrad or zcashd, starting at this height") rootCmd.Flags().String("data-dir", "/var/lib/lightwalletd", "data directory (such as db)") rootCmd.Flags().Bool("ping-very-insecure", false, "allow Ping GRPC for testing") - rootCmd.Flags().Bool("darkside-very-insecure", false, "run with GRPC-controllable mock zcashd for integration testing (shuts down after 30 minutes)") + rootCmd.Flags().Bool("darkside-very-insecure", false, "run with GRPC-controllable mock zebrad for integration testing (shuts down after 30 minutes)") rootCmd.Flags().Int("darkside-timeout", 30, "override 30 minute default darkside timeout") viper.BindPFlag("grpc-bind-addr", rootCmd.Flags().Lookup("grpc-bind-addr")) @@ -362,6 +379,8 @@ func init() { viper.SetDefault("gen-cert-very-insecure", false) viper.BindPFlag("redownload", rootCmd.Flags().Lookup("redownload")) viper.SetDefault("redownload", false) + viper.BindPFlag("nocache", rootCmd.Flags().Lookup("nocache")) + viper.SetDefault("nocache", false) viper.BindPFlag("sync-from-height", rootCmd.Flags().Lookup("sync-from-height")) viper.SetDefault("sync-from-height", -1) viper.BindPFlag("data-dir", rootCmd.Flags().Lookup("data-dir")) diff --git a/common/cache.go b/common/cache.go index 74c56fb9..40aa6c69 100644 --- a/common/cache.go +++ b/common/cache.go @@ -195,7 +195,7 @@ func NewBlockCache(dbPath string, chainName string, startHeight int, syncFromHei c := &BlockCache{} c.firstBlock = startHeight c.nextBlock = startHeight - c.lengthsName, c.blocksName = dbFileNames(dbPath, chainName) + c.lengthsName, c.blocksName = DbFileNames(dbPath, chainName) var err error if err := os.MkdirAll(filepath.Join(dbPath, chainName), 0755); err != nil { Log.Fatal("mkdir ", dbPath, " failed: ", err) @@ -243,13 +243,6 @@ func NewBlockCache(dbPath string, chainName string, startHeight int, syncFromHei } offset += int64(length) + 8 c.starts = append(c.starts, offset) - // Check for corruption. - block := c.readBlock(c.nextBlock) - if block == nil { - Log.Warning("error reading block") - c.recoverFromCorruption(c.nextBlock) - break - } c.nextBlock++ } c.setDbFiles(c.nextBlock) @@ -257,7 +250,7 @@ func NewBlockCache(dbPath string, chainName string, startHeight int, syncFromHei return c } -func dbFileNames(dbPath string, chainName string) (string, string) { +func DbFileNames(dbPath string, chainName string) (string, string) { return filepath.Join(dbPath, chainName, "lengths"), filepath.Join(dbPath, chainName, "blocks") } diff --git a/common/common.go b/common/common.go index 51955e4c..460ce6b6 100644 --- a/common/common.go +++ b/common/common.go @@ -25,6 +25,7 @@ var ( Branch = "" BuildDate = "" BuildUser = "" + NodeName = "zebrad" ) type Options struct { @@ -43,6 +44,7 @@ type Options struct { NoTLSVeryInsecure bool `json:"no_tls_very_insecure,omitempty"` GenCertVeryInsecure bool `json:"gen_cert_very_insecure,omitempty"` Redownload bool `json:"redownload"` + NoCache bool `json:"nocache"` SyncFromHeight int `json:"sync_from_height"` DataDir string `json:"data_dir"` PingEnable bool `json:"ping_enable"` @@ -100,7 +102,7 @@ type ( ZcashdRpcRequestGetaddresstxids struct { Addresses []string `json:"addresses"` Start uint64 `json:"start"` - End uint64 `json:"end"` + End uint64 `json:"end,omitempty"` } // zcashd rpc "z_gettreestate" @@ -126,7 +128,7 @@ type ( // many more fields but these are the only ones we current need. ZcashdRpcReplyGetrawtransaction struct { Hex string - Height int + Height int64 } // zcashd rpc "getaddressbalance" @@ -224,7 +226,7 @@ func FirstRPC() { if retryCount > 10 { Log.WithFields(logrus.Fields{ "timeouts": retryCount, - }).Fatal("unable to issue getblockchaininfo RPC call to zcashd node") + }).Fatal("unable to issue getblockchaininfo RPC call to zebrad or zcashd node") } Log.WithFields(logrus.Fields{ "error": err.Error(), @@ -417,7 +419,7 @@ func BlockIngestor(c *BlockCache, rep int) { if err != nil { Log.WithFields(logrus.Fields{ "error": err, - }).Fatal("error zcashd getbestblockhash rpc") + }).Fatal("error " + NodeName + " getbestblockhash rpc") } var hashHex string err = json.Unmarshal(result, &hashHex) @@ -461,10 +463,10 @@ func BlockIngestor(c *BlockCache, rep int) { } if height == c.GetFirstHeight() { c.Sync() - Log.Info("Waiting for zcashd height to reach Sapling activation height ", + Log.Info("Waiting for "+NodeName+" height to reach Sapling activation height ", "(", c.GetFirstHeight(), ")...") - Time.Sleep(20 * time.Second) - return + Time.Sleep(120 * time.Second) + continue } Log.Info("REORG: dropping block ", height-1, " ", displayHash(c.GetLatestHash())) c.Reorg(height - 1) @@ -476,9 +478,12 @@ func BlockIngestor(c *BlockCache, rep int) { // nil if no block exists at this height. func GetBlock(cache *BlockCache, height int) (*walletrpc.CompactBlock, error) { // First, check the cache to see if we have the block - block := cache.Get(height) - if block != nil { - return block, nil + var block *walletrpc.CompactBlock + if cache != nil { + block := cache.Get(height) + if block != nil { + return block, nil + } } // Not in the cache @@ -518,6 +523,43 @@ func GetBlockRange(cache *BlockCache, blockOut chan<- *walletrpc.CompactBlock, e errOut <- nil } +// ParseRawTransaction converts between the JSON result of a `zcashd` +// `getrawtransaction` call and the `RawTransaction` protobuf type. +// +// Due to an error in the original protobuf definition, it is necessary to +// reinterpret the result of the `getrawtransaction` RPC call. Zcashd will +// return the int64 value `-1` for the height of transactions that appear in +// the block index, but which are not mined in the main chain. `service.proto` +// defines the height field of `RawTransaction` to be a `uint64`, and as such +// we must map the response from the zcashd RPC API to be representable within +// this space. Additionally, the `height` field will be absent for transactions +// in the mempool, resulting in the default value of `0` being set. Therefore, +// the meanings of the `Height` field of the `RawTransaction` type are as +// follows: +// +// * height 0: the transaction is in the mempool +// * height 0xffffffffffffffff: the transaction has been mined on a fork that +// is not currently the main chain +// * any other height: the transaction has been mined in the main chain at the +// given height +func ParseRawTransaction(message json.RawMessage) (*walletrpc.RawTransaction, error) { + // Many other fields are returned, but we need only these two. + var txinfo ZcashdRpcReplyGetrawtransaction + err := json.Unmarshal(message, &txinfo) + if err != nil { + return nil, err + } + txBytes, err := hex.DecodeString(txinfo.Hex) + if err != nil { + return nil, err + } + + return &walletrpc.RawTransaction{ + Data: txBytes, + Height: uint64(txinfo.Height), + }, nil +} + func displayHash(hash []byte) string { return hex.EncodeToString(parser.Reverse(hash)) } diff --git a/common/common_test.go b/common/common_test.go index 9fc19f32..4c4860ab 100644 --- a/common/common_test.go +++ b/common/common_test.go @@ -9,6 +9,7 @@ import ( "encoding/json" "errors" "fmt" + "math" "os" "strings" "testing" @@ -604,7 +605,7 @@ func mempoolStub(method string, params []json.RawMessage) (json.RawMessage, erro if txid != "mempooltxid-1" { testT.Fatal("unexpected txid") } - r, _ := json.Marshal("aabb") + r, _ := json.Marshal(map[string]string{"hex":"aabb"}) return r, nil case 5: // Simulate that still no new block has arrived ... @@ -636,7 +637,7 @@ func mempoolStub(method string, params []json.RawMessage) (json.RawMessage, erro if txid != "mempooltxid-2" { testT.Fatal("unexpected txid") } - r, _ := json.Marshal("ccdd") + r, _ := json.Marshal(map[string]string{"hex":"ccdd"}) return r, nil case 8: // A new block arrives, this will cause these two tx to be returned @@ -668,7 +669,7 @@ func TestMempoolStream(t *testing.T) { return nil }) if err != nil { - t.Fatal("GetMempool failed") + t.Errorf("GetMempool failed: %v", err) } // This should return two transactions. @@ -677,7 +678,7 @@ func TestMempoolStream(t *testing.T) { return nil }) if err != nil { - t.Fatal("GetMempool failed") + t.Errorf("GetMempool failed: %v", err) } if len(replies) != 2 { t.Fatal("unexpected number of tx") @@ -687,13 +688,13 @@ func TestMempoolStream(t *testing.T) { if !bytes.Equal([]byte(replies[0].GetData()), []byte{0xaa, 0xbb}) { t.Fatal("unexpected tx contents") } - if replies[0].GetHeight() != 200 { + if replies[0].GetHeight() != 0 { t.Fatal("unexpected tx height") } if !bytes.Equal([]byte(replies[1].GetData()), []byte{0xcc, 0xdd}) { t.Fatal("unexpected tx contents") } - if replies[1].GetHeight() != 200 { + if replies[1].GetHeight() != 0 { t.Fatal("unexpected tx height") } @@ -703,10 +704,68 @@ func TestMempoolStream(t *testing.T) { t.Fatal("unexpected end time") } if step != 8 { - t.Fatal("unexpected number of zcashd RPCs") + t.Fatal("unexpected number of zebrad RPCs") } step = 0 sleepCount = 0 sleepDuration = 0 } + +func TestZcashdRpcReplyUnmarshalling(t *testing.T) { + var txinfo0 ZcashdRpcReplyGetrawtransaction + err0 := json.Unmarshal([]byte("{\"hex\": \"deadbeef\", \"height\": 123456}"), &txinfo0) + if err0 != nil { + t.Fatal("Failed to unmarshal tx with known height.") + } + if txinfo0.Height != 123456 { + t.Errorf("Unmarshalled incorrect height: got: %d, want: 123456.", txinfo0.Height) + } + + var txinfo1 ZcashdRpcReplyGetrawtransaction + err1 := json.Unmarshal([]byte("{\"hex\": \"deadbeef\", \"height\": -1}"), &txinfo1) + if err1 != nil { + t.Fatal("failed to unmarshal tx not in main chain") + } + if txinfo1.Height != -1 { + t.Errorf("Unmarshalled incorrect height: got: %d, want: -1.", txinfo1.Height) + } + + var txinfo2 ZcashdRpcReplyGetrawtransaction + err2 := json.Unmarshal([]byte("{\"hex\": \"deadbeef\"}"), &txinfo2) + if err2 != nil { + t.Fatal("failed to unmarshal reply lacking height data") + } + if txinfo2.Height != 0 { + t.Errorf("Unmarshalled incorrect height: got: %d, want: 0.", txinfo2.Height) + } +} + +func TestParseRawTransaction(t *testing.T) { + rt0, err0 := ParseRawTransaction([]byte("{\"hex\": \"deadbeef\", \"height\": 123456}")) + if err0 != nil { + t.Fatal("Failed to parse raw transaction response with known height.") + } + if rt0.Height != 123456 { + t.Errorf("Unmarshalled incorrect height: got: %d, expected: 123456.", rt0.Height) + } + + rt1, err1 := ParseRawTransaction([]byte("{\"hex\": \"deadbeef\", \"height\": -1}")) + if err1 != nil { + t.Fatal("Failed to parse raw transaction response for a known tx not in the main chain.") + } + // We expect the int64 value `-1` to have been reinterpreted as a uint64 value in order + // to be representable as a uint64 in `RawTransaction`. The conversion from the twos-complement + // signed representation should map `-1` to `math.MaxUint64`. + if rt1.Height != math.MaxUint64 { + t.Errorf("Unmarshalled incorrect height: got: %d, want: 0x%X.", rt1.Height, uint64(math.MaxUint64)) + } + + rt2, err2 := ParseRawTransaction([]byte("{\"hex\": \"deadbeef\"}")) + if err2 != nil { + t.Fatal("Failed to parse raw transaction response for a tx in the mempool.") + } + if rt2.Height != 0 { + t.Errorf("Unmarshalled incorrect height: got: %d, expected: 0.", rt2.Height) + } +} diff --git a/common/mempool.go b/common/mempool.go index 241e0520..e52082c5 100644 --- a/common/mempool.go +++ b/common/mempool.go @@ -1,7 +1,6 @@ package common import ( - "encoding/hex" "encoding/json" "sync" "time" @@ -105,35 +104,33 @@ func refreshMempoolTxns() error { // We've already fetched this transaction continue } - g_txidSeen[txid(txidstr)] = struct{}{} + // We haven't fetched this transaction already. + g_txidSeen[txid(txidstr)] = struct{}{} txidJSON, err := json.Marshal(txidstr) if err != nil { return err } - // The "0" is because we only need the raw hex, which is returned as - // just a hex string, and not even a json string (with quotes). - params := []json.RawMessage{txidJSON, json.RawMessage("0")} + + params := []json.RawMessage{txidJSON, json.RawMessage("1")} result, rpcErr := RawRequest("getrawtransaction", params) if rpcErr != nil { // Not an error; mempool transactions can disappear continue } - // strip the quotes - var txStr string - err = json.Unmarshal(result, &txStr) - if err != nil { - return err - } - txBytes, err := hex.DecodeString(txStr) + + rawtx, err := ParseRawTransaction(result) if err != nil { return err } - newRtx := &walletrpc.RawTransaction{ - Data: txBytes, - Height: uint64(g_lastBlockChainInfo.Blocks), + + // Skip any transaction that has been mined since the list of txids + // was retrieved. + if (rawtx.Height != 0) { + continue; } - g_txList = append(g_txList, newRtx) + + g_txList = append(g_txList, rawtx) } return nil } diff --git a/docs/rtd/index.html b/docs/rtd/index.html index cfd44c36..35bf4bda 100644 --- a/docs/rtd/index.html +++ b/docs/rtd/index.html @@ -1564,7 +1564,7 @@

PingResponse

RawTransaction

-

RawTransaction contains the complete transaction data. It also optionally includes

the block height in which the transaction was included, or, when returned

by GetMempoolStream(), the latest block height.

+

RawTransaction contains the complete transaction data. It also includes the

height for the block in which the transaction was included in the main

chain, if any (as detailed below).

@@ -1577,14 +1577,31 @@

RawTransaction

- + - + diff --git a/frontend/frontend_test.go b/frontend/frontend_test.go index 3460c6d1..4d1d0593 100644 --- a/frontend/frontend_test.go +++ b/frontend/frontend_test.go @@ -540,52 +540,6 @@ func TestSendTransaction(t *testing.T) { step = 0 } -var sampleconf = ` -testnet = 1 -rpcport = 18232 -rpcbind = 127.0.0.1 -rpcuser = testlightwduser -rpcpassword = testlightwdpassword -` - -func TestNewZRPCFromConf(t *testing.T) { - connCfg, err := connFromConf([]byte(sampleconf)) - if err != nil { - t.Fatal("connFromConf failed") - } - if connCfg.Host != "127.0.0.1:18232" { - t.Fatal("connFromConf returned unexpected Host") - } - if connCfg.User != "testlightwduser" { - t.Fatal("connFromConf returned unexpected User") - } - if connCfg.Pass != "testlightwdpassword" { - t.Fatal("connFromConf returned unexpected User") - } - if !connCfg.HTTPPostMode { - t.Fatal("connFromConf returned unexpected HTTPPostMode") - } - if !connCfg.DisableTLS { - t.Fatal("connFromConf returned unexpected DisableTLS") - } - - // can't pass an integer - _, err = connFromConf(10) - if err == nil { - t.Fatal("connFromConf unexpected success") - } - - // Can't verify returned values, but at least run it - _, err = NewZRPCFromConf([]byte(sampleconf)) - if err != nil { - t.Fatal("NewZRPCFromClient failed") - } - _, err = NewZRPCFromConf(10) - if err == nil { - t.Fatal("NewZRPCFromClient unexpected success") - } -} - func TestMempoolFilter(t *testing.T) { txidlist := []string{ "2e819d0bab5c819dc7d5f92d1bfb4127ce321daf847f6602", diff --git a/frontend/rpc_client.go b/frontend/rpc_client.go index 0026eb25..5aba112c 100644 --- a/frontend/rpc_client.go +++ b/frontend/rpc_client.go @@ -8,14 +8,16 @@ import ( "errors" "fmt" "net" + "path/filepath" + "github.com/BurntSushi/toml" "github.com/btcsuite/btcd/rpcclient" "github.com/zcash/lightwalletd/common" ini "gopkg.in/ini.v1" ) // NewZRPCFromConf reads the zcashd configuration file. -func NewZRPCFromConf(confPath interface{}) (*rpcclient.Client, error) { +func NewZRPCFromConf(confPath string) (*rpcclient.Client, error) { connCfg, err := connFromConf(confPath) if err != nil { return nil, err @@ -36,12 +38,18 @@ func NewZRPCFromFlags(opts *common.Options) (*rpcclient.Client, error) { return rpcclient.New(connCfg, nil) } -// If passed a string, interpret as a path, open and read; if passed -// a byte slice, interpret as the config file content (used in testing). -func connFromConf(confPath interface{}) (*rpcclient.ConnConfig, error) { +func connFromConf(confPath string) (*rpcclient.ConnConfig, error) { + if filepath.Ext(confPath) == ".toml" { + return connFromToml(confPath) + } else { + return connFromIni(confPath) + } +} + +func connFromIni(confPath string) (*rpcclient.ConnConfig, error) { cfg, err := ini.Load(confPath) if err != nil { - return nil, fmt.Errorf("failed to read config file: %w", err) + return nil, fmt.Errorf("failed to read config file in .conf format: %w", err) } rpcaddr := cfg.Section("").Key("rpcbind").String() @@ -76,3 +84,30 @@ func connFromConf(confPath interface{}) (*rpcclient.ConnConfig, error) { // not supported in HTTP POST mode. return connCfg, nil } + +// If passed a string, interpret as a path, open and read; if passed +// a byte slice, interpret as the config file content (used in testing). +func connFromToml(confPath string) (*rpcclient.ConnConfig, error) { + var tomlConf struct { + Rpc struct { + Listen_addr string + RPCUser string + RPCPassword string + } + } + _, err := toml.DecodeFile(confPath, &tomlConf) + if err != nil { + return nil, fmt.Errorf("failed to read config file in .toml format: %w", err) + } + conf := rpcclient.ConnConfig{ + Host: tomlConf.Rpc.Listen_addr, + User: tomlConf.Rpc.RPCUser, + Pass: tomlConf.Rpc.RPCPassword, + HTTPPostMode: true, // Zcash only supports HTTP POST mode + DisableTLS: true, // Zcash does not provide TLS by default + } + + // Notice the notification parameter is nil since notifications are + // not supported in HTTP POST mode. + return &conf, nil +} diff --git a/frontend/service.go b/frontend/service.go index 30df3ed8..00b36857 100644 --- a/frontend/service.go +++ b/frontend/service.go @@ -87,20 +87,21 @@ func (s *lwdStreamer) GetTaddressTxids(addressBlockFilter *walletrpc.Transparent if addressBlockFilter.Range.Start == nil { return errors.New("must specify a start block height") } - if addressBlockFilter.Range.End == nil { - return errors.New("must specify an end block height") - } - params := make([]json.RawMessage, 1) + request := &common.ZcashdRpcRequestGetaddresstxids{ Addresses: []string{addressBlockFilter.Address}, Start: addressBlockFilter.Range.Start.Height, - End: addressBlockFilter.Range.End.Height, } + if addressBlockFilter.Range.End != nil { + request.End = addressBlockFilter.Range.End.Height + } + param, err := json.Marshal(request) if err != nil { return err } - params[0] = param + params := []json.RawMessage{param} + result, rpcErr := common.RawRequest("getaddresstxids", params) // For some reason, the error responses are not JSON @@ -142,6 +143,7 @@ func (s *lwdStreamer) GetBlock(ctx context.Context, id *walletrpc.BlockID) (*wal // Precedence: a hash is more specific than a height. If we have it, use it first. if id.Hash != nil { // TODO: Get block by hash + // see https://github.com/zcash/lightwalletd/pull/309 return nil, errors.New("gRPC GetBlock by Hash is not yet implemented") } cBlock, err := common.GetBlock(s.cache, int(id.Height)) @@ -162,6 +164,7 @@ func (s *lwdStreamer) GetBlockNullifiers(ctx context.Context, id *walletrpc.Bloc // Precedence: a hash is more specific than a height. If we have it, use it first. if id.Hash != nil { // TODO: Get block by hash + // see https://github.com/zcash/lightwalletd/pull/309 return nil, errors.New("gRPC GetBlock by Hash is not yet implemented") } cBlock, err := common.GetBlock(s.cache, int(id.Height)) @@ -286,7 +289,7 @@ func (s *lwdStreamer) GetTreeState(ctx context.Context, id *walletrpc.BlockID) ( params[0] = hashJSON } if gettreestateReply.Sapling.Commitments.FinalState == "" { - return nil, errors.New("zcashd did not return treestate") + return nil, errors.New(common.NodeName + " did not return treestate") } return &walletrpc.TreeState{ Network: s.chainName, @@ -314,34 +317,19 @@ func (s *lwdStreamer) GetTransaction(ctx context.Context, txf *walletrpc.TxFilte if len(txf.Hash) != 32 { return nil, errors.New("transaction ID has invalid length") } - leHashStringJSON, err := json.Marshal(hex.EncodeToString(parser.Reverse(txf.Hash))) + txidJSON, err := json.Marshal(hex.EncodeToString(parser.Reverse(txf.Hash))) if err != nil { return nil, err } - params := []json.RawMessage{ - leHashStringJSON, - json.RawMessage("1"), - } - result, rpcErr := common.RawRequest("getrawtransaction", params) - // For some reason, the error responses are not JSON + params := []json.RawMessage{txidJSON, json.RawMessage("1")} + result, rpcErr := common.RawRequest("getrawtransaction", params) if rpcErr != nil { + // For some reason, the error responses are not JSON return nil, rpcErr } - // Many other fields are returned, but we need only these two. - var txinfo common.ZcashdRpcReplyGetrawtransaction - err = json.Unmarshal(result, &txinfo) - if err != nil { - return nil, err - } - txBytes, err := hex.DecodeString(txinfo.Hex) - if err != nil { - return nil, err - } - return &walletrpc.RawTransaction{ - Data: txBytes, - Height: uint64(txinfo.Height), - }, nil + + return common.ParseRawTransaction(result) } if txf.Block != nil && txf.Block.Hash != nil { diff --git a/go.mod b/go.mod index f373fa57..47d9aa46 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,8 @@ module github.com/zcash/lightwalletd go 1.17 require ( - github.com/btcsuite/btcd v0.24.0 + github.com/BurntSushi/toml v0.3.1 + github.com/btcsuite/btcd v0.24.2 github.com/golang/protobuf v1.5.3 github.com/grpc-ecosystem/go-grpc-middleware v1.4.0 github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 @@ -12,7 +13,7 @@ require ( github.com/spf13/cobra v1.8.0 github.com/spf13/viper v1.18.2 google.golang.org/grpc v1.61.0 - google.golang.org/protobuf v1.32.0 + google.golang.org/protobuf v1.33.0 gopkg.in/ini.v1 v1.67.0 ) @@ -46,10 +47,10 @@ require ( github.com/subosito/gotenv v1.6.0 // indirect go.uber.org/atomic v1.9.0 // indirect go.uber.org/multierr v1.9.0 // indirect - golang.org/x/crypto v0.17.0 // indirect + golang.org/x/crypto v0.21.0 // indirect golang.org/x/exp v0.0.0-20230905200255-921286631fa9 // indirect - golang.org/x/net v0.19.0 // indirect - golang.org/x/sys v0.15.0 // indirect + golang.org/x/net v0.23.0 // indirect + golang.org/x/sys v0.18.0 // indirect golang.org/x/text v0.14.0 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20231120223509-83a465c0220f // indirect gopkg.in/yaml.v3 v3.0.1 // indirect diff --git a/go.sum b/go.sum index ac12a1a5..225ecfe4 100644 --- a/go.sum +++ b/go.sum @@ -1136,6 +1136,7 @@ cloud.google.com/go/workflows v1.12.3/go.mod h1:fmOUeeqEwPzIU81foMjTRQIdwQHADi/v dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= gioui.org v0.0.0-20210308172011-57750fc8a0a6/go.mod h1:RSH6KIUZ0p2xy5zHDxgAM4zumjgTw83q2ge/PI+yyw8= git.sr.ht/~sbinet/gg v0.3.1/go.mod h1:KGYtlADtqsqANL9ueOFkWymvzUvLMQllU5Ixo+8v3pc= +github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/DataDog/datadog-go v3.2.0+incompatible/go.mod h1:LButxg5PwREeZtORoXG3tL4fMGNddJ+vMq1mwgfaqoQ= @@ -1176,8 +1177,8 @@ github.com/boombuler/barcode v1.0.1/go.mod h1:paBWMcWSl3LHKBqUq+rly7CNSldXjb2rDl github.com/btcsuite/btcd v0.20.1-beta/go.mod h1:wVuoA8VJLEcwgqHBwHmzLRazpKxTv13Px/pDuV7OomQ= github.com/btcsuite/btcd v0.22.0-beta.0.20220111032746-97732e52810c/go.mod h1:tjmYdS6MLJ5/s0Fj4DbLgSbDHbEqLJrtnHecBFkdz5M= github.com/btcsuite/btcd v0.23.5-0.20231215221805-96c9fd8078fd/go.mod h1:nm3Bko6zh6bWP60UxwoT5LzdGJsQJaPo6HjduXq9p6A= -github.com/btcsuite/btcd v0.24.0 h1:gL3uHE/IaFj6fcZSu03SvqPMSx7s/dPzfpG/atRwWdo= -github.com/btcsuite/btcd v0.24.0/go.mod h1:K4IDc1593s8jKXIF7yS7yCTSxrknB9z0STzc2j6XgE4= +github.com/btcsuite/btcd v0.24.2 h1:aLmxPguqxza+4ag8R1I2nnJjSu2iFn/kqtHTIImswcY= +github.com/btcsuite/btcd v0.24.2/go.mod h1:5C8ChTkl5ejr3WHj8tkQSCmydiMEPB0ZhQhehpq7Dgg= github.com/btcsuite/btcd/btcec/v2 v2.1.0/go.mod h1:2VzYrv4Gm4apmbVVsSq5bqf1Ec8v56E48Vt0Y/umPgA= github.com/btcsuite/btcd/btcec/v2 v2.1.3 h1:xM/n3yIhHAhHy04z4i43C8p4ehixJZMsnrVJkgl+MTE= github.com/btcsuite/btcd/btcec/v2 v2.1.3/go.mod h1:ctjw4H1kknNJmRN4iP1R7bTQ+v3GJkZBd6mui8ZsAZE= @@ -1754,8 +1755,9 @@ golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliY golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4= golang.org/x/crypto v0.15.0/go.mod h1:4ChreQoLWfG3xLDer1WdlH5NdlQ3+mwnQq1YTKY+72g= golang.org/x/crypto v0.16.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4= -golang.org/x/crypto v0.17.0 h1:r8bRNjWL3GshPW3gkd+RpvzWrZAwPS49OmTGZ/uhM4k= -golang.org/x/crypto v0.17.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4= +golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU= +golang.org/x/crypto v0.21.0 h1:X31++rzVUdKhX5sWmSOFZxx8UW/ldWx55cbf08iNAMA= +golang.org/x/crypto v0.21.0/go.mod h1:0BP7YvVV9gBbVKyeTG0Gyn+gZm94bibOW5BjDEYAOMs= golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20180807140117-3d87b88a115f/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= @@ -1897,8 +1899,10 @@ golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk= golang.org/x/net v0.16.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= golang.org/x/net v0.18.0/go.mod h1:/czyP5RqHAH4odGYxBJ1qz0+CE5WZ+2j1YgoEo8F2jQ= -golang.org/x/net v0.19.0 h1:zTwKpTd2XuCqf8huc7Fo2iSy+4RHPd10s4KzeTnVr1c= golang.org/x/net v0.19.0/go.mod h1:CfAk/cbD4CthTvqiEl8NpboMuiuOYsAr/7NOjZJtv1U= +golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44= +golang.org/x/net v0.23.0 h1:7EYJ93RZ9vYSZAIb2x3lnuvqO5zneoD6IvWjuhfxjTs= +golang.org/x/net v0.23.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -2065,8 +2069,10 @@ golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.14.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/sys v0.15.0 h1:h48lPFYpsTvQJZF4EKyI4aLHaev3CxivZmv7yZig9pc= golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4= +golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= @@ -2084,6 +2090,8 @@ golang.org/x/term v0.12.0/go.mod h1:owVbMEjm3cBLCHdkQu9b1opXd4ETQWc3BhuQGKgXgvU= golang.org/x/term v0.13.0/go.mod h1:LTmsnFJwVN6bCy1rVCoS+qHT1HhALEFxKncY3WNNh4U= golang.org/x/term v0.14.0/go.mod h1:TySc+nGkYR6qt8km8wUhuFRTVSMIX3XPR58y2lC8vww= golang.org/x/term v0.15.0/go.mod h1:BDl952bC7+uMoWR75FIrCDx79TPU9oHkTZ9yRbYOrX0= +golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk= +golang.org/x/term v0.18.0/go.mod h1:ILwASektA3OnRv7amZ1xhE/KTR+u50pbXfZ03+6Nx58= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -2537,8 +2545,8 @@ google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqw google.golang.org/protobuf v1.29.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= google.golang.org/protobuf v1.30.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= -google.golang.org/protobuf v1.32.0 h1:pPC6BG5ex8PDFnkbrGU3EixyhKcQ2aDuBS36lqK/C7I= -google.golang.org/protobuf v1.32.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= +google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI= +google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= diff --git a/walletrpc/service.pb.go b/walletrpc/service.pb.go index 29799339..b3dc73bc 100644 --- a/walletrpc/service.pb.go +++ b/walletrpc/service.pb.go @@ -250,16 +250,35 @@ func (x *TxFilter) GetHash() []byte { return nil } -// RawTransaction contains the complete transaction data. It also optionally includes -// the block height in which the transaction was included, or, when returned -// by GetMempoolStream(), the latest block height. +// RawTransaction contains the complete transaction data. It also includes the +// height for the block in which the transaction was included in the main +// chain, if any (as detailed below). type RawTransaction struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields - Data []byte `protobuf:"bytes,1,opt,name=data,proto3" json:"data,omitempty"` // exact data returned by Zcash 'getrawtransaction' - Height uint64 `protobuf:"varint,2,opt,name=height,proto3" json:"height,omitempty"` // height that the transaction was mined (or -1) + // The serialized representation of the Zcash transaction. + Data []byte `protobuf:"bytes,1,opt,name=data,proto3" json:"data,omitempty"` + // The height at which the transaction is mined, or a sentinel value. + // + // Due to an error in the original protobuf definition, it is necessary to + // reinterpret the result of the `getrawtransaction` RPC call. Zcashd will + // return the int64 value `-1` for the height of transactions that appear + // in the block index, but which are not mined in the main chain. Here, the + // height field of `RawTransaction` was erroneously created as a `uint64`, + // and as such we must map the response from the zcashd RPC API to be + // representable within this space. Additionally, the `height` field will + // be absent for transactions in the mempool, resulting in the default + // value of `0` being set. Therefore, the meanings of the `height` field of + // the `RawTransaction` type are as follows: + // + // - height 0: the transaction is in the mempool + // - height 0xffffffffffffffff: the transaction has been mined on a fork that + // is not currently the main chain + // - any other height: the transaction has been mined in the main chain at the + // given height + Height uint64 `protobuf:"varint,2,opt,name=height,proto3" json:"height,omitempty"` } func (x *RawTransaction) Reset() { diff --git a/walletrpc/service.proto b/walletrpc/service.proto index a94e639f..2f7c9caf 100644 --- a/walletrpc/service.proto +++ b/walletrpc/service.proto @@ -31,12 +31,31 @@ message TxFilter { bytes hash = 3; // transaction ID (hash, txid) } -// RawTransaction contains the complete transaction data. It also optionally includes -// the block height in which the transaction was included, or, when returned -// by GetMempoolStream(), the latest block height. +// RawTransaction contains the complete transaction data. It also includes the +// height for the block in which the transaction was included in the main +// chain, if any (as detailed below). message RawTransaction { - bytes data = 1; // exact data returned by Zcash 'getrawtransaction' - uint64 height = 2; // height that the transaction was mined (or -1) + // The serialized representation of the Zcash transaction. + bytes data = 1; + // The height at which the transaction is mined, or a sentinel value. + // + // Due to an error in the original protobuf definition, it is necessary to + // reinterpret the result of the `getrawtransaction` RPC call. Zcashd will + // return the int64 value `-1` for the height of transactions that appear + // in the block index, but which are not mined in the main chain. Here, the + // height field of `RawTransaction` was erroneously created as a `uint64`, + // and as such we must map the response from the zcashd RPC API to be + // representable within this space. Additionally, the `height` field will + // be absent for transactions in the mempool, resulting in the default + // value of `0` being set. Therefore, the meanings of the `height` field of + // the `RawTransaction` type are as follows: + // + // * height 0: the transaction is in the mempool + // * height 0xffffffffffffffff: the transaction has been mined on a fork that + // is not currently the main chain + // * any other height: the transaction has been mined in the main chain at the + // given height + uint64 height = 2; } // A SendResponse encodes an error code and a string. It is currently used
data bytes

exact data returned by Zcash 'getrawtransaction'

The serialized representation of the Zcash transaction.

height uint64

height that the transaction was mined (or -1)

The height at which the transaction is mined, or a sentinel value. + +Due to an error in the original protobuf definition, it is necessary to +reinterpret the result of the `getrawtransaction` RPC call. Zcashd will +return the int64 value `-1` for the height of transactions that appear +in the block index, but which are not mined in the main chain. Here, the +height field of `RawTransaction` was erroneously created as a `uint64`, +and as such we must map the response from the zcashd RPC API to be +representable within this space. Additionally, the `height` field will +be absent for transactions in the mempool, resulting in the default +value of `0` being set. Therefore, the meanings of the `height` field of +the `RawTransaction` type are as follows: + +* height 0: the transaction is in the mempool +* height 0xffffffffffffffff: the transaction has been mined on a fork that + is not currently the main chain +* any other height: the transaction has been mined in the main chain at the + given height