From 6ff5e98fec3cc35290f14f8e39f41e34e0955ad8 Mon Sep 17 00:00:00 2001 From: hc-github-team-secure-vault-core Date: Wed, 2 Oct 2024 14:24:09 -0600 Subject: [PATCH] backport of commit 53bb78ce5fb1bdbc1e913c4eb41f7f6501a148af (#28571) Co-authored-by: VAL --- changelog/28526.txt | 3 +++ physical/raft/bolt_64bit_test.go | 26 ++++++++++++++++++++++++++ physical/raft/bolt_linux.go | 32 +++++++++++++++++++++++++++----- physical/raft/fsm.go | 8 ++++++++ physical/raft/raft.go | 12 +++++++++++- 5 files changed, 75 insertions(+), 6 deletions(-) create mode 100644 changelog/28526.txt diff --git a/changelog/28526.txt b/changelog/28526.txt new file mode 100644 index 000000000000..b22b093efc9c --- /dev/null +++ b/changelog/28526.txt @@ -0,0 +1,3 @@ +```release-note:improvement +physical/raft: Log when the MAP_POPULATE mmap flag gets disabled before opening the database. +``` \ No newline at end of file diff --git a/physical/raft/bolt_64bit_test.go b/physical/raft/bolt_64bit_test.go index f6f1bbd2dee3..3305d450dd6d 100644 --- a/physical/raft/bolt_64bit_test.go +++ b/physical/raft/bolt_64bit_test.go @@ -43,3 +43,29 @@ func Test_BoltOptions(t *testing.T) { }) } } + +// TestMmapFlags tests the getMmapFlags function, ensuring it returns the appropriate integer representing the desired mmap flag. +func TestMmapFlags(t *testing.T) { + testCases := []struct { + name string + disableMapPopulate bool + }{ + {"MAP_POPULATE is enabled", false}, + {"MAP_POPULATE disabled by env var", true}, + } + + for _, tc := range testCases { + tc := tc + + t.Run(tc.name, func(t *testing.T) { + if tc.disableMapPopulate { + t.Setenv("VAULT_RAFT_DISABLE_MAP_POPULATE", "true") + } + + isEnabled := usingMapPopulate(getMmapFlags("")) + if tc.disableMapPopulate && isEnabled { + t.Error("expected MAP_POPULATE to be disabled but it was enabled") + } + }) + } +} diff --git a/physical/raft/bolt_linux.go b/physical/raft/bolt_linux.go index 811c148a1050..bec6f7e386c7 100644 --- a/physical/raft/bolt_linux.go +++ b/physical/raft/bolt_linux.go @@ -13,27 +13,49 @@ import ( func init() { getMmapFlags = getMmapFlagsLinux + usingMapPopulate = usingMapPopulateLinux } func getMmapFlagsLinux(dbPath string) int { + if setMapPopulateFlag(dbPath) { + return unix.MAP_POPULATE + } + + return 0 +} + +// setMapPopulateFlag determines whether we should set the MAP_POPULATE flag, which +// prepopulates page tables to be mapped in the virtual memory space, +// helping reduce slowness at runtime caused by page faults. +// We only want to set this flag if we've determined there's enough memory on the system available to do so. +func setMapPopulateFlag(dbPath string) bool { if os.Getenv("VAULT_RAFT_DISABLE_MAP_POPULATE") != "" { - return 0 + return false } stat, err := os.Stat(dbPath) if err != nil { - return 0 + return false } size := stat.Size() v, err := mem.VirtualMemoryWithContext(context.Background()) if err != nil { - return 0 + return false } // We won't worry about swap, since we already tell people not to use it. if v.Total > uint64(size) { - return unix.MAP_POPULATE + return true } - return 0 + return false +} + +// the unix.MAP_POPULATE constant only exists on Linux, +// so reference to this constant can only live in a *_linux.go file +func usingMapPopulateLinux(mmapFlag int) bool { + if mmapFlag == unix.MAP_POPULATE { + return true + } + return false } diff --git a/physical/raft/fsm.go b/physical/raft/fsm.go index a0197cdaa6a2..cfbe8374aaff 100644 --- a/physical/raft/fsm.go +++ b/physical/raft/fsm.go @@ -13,6 +13,7 @@ import ( "io" "os" "path/filepath" + "runtime" "strconv" "strings" "sync" @@ -246,9 +247,11 @@ func (f *FSM) openDBFile(dbPath string) error { return errors.New("can not open empty filename") } + vaultDbExists := true st, err := os.Stat(dbPath) switch { case err != nil && os.IsNotExist(err): + vaultDbExists = false case err != nil: return fmt.Errorf("error checking raft FSM db file %q: %v", dbPath, err) default: @@ -260,11 +263,16 @@ func (f *FSM) openDBFile(dbPath string) error { } opts := boltOptions(dbPath) + if runtime.GOOS == "linux" && vaultDbExists && !usingMapPopulate(opts.MmapFlags) { + f.logger.Warn("the MAP_POPULATE mmap flag has not been set before opening the FSM database. This may be due to the database file being larger than the available memory on the system, or due to the VAULT_RAFT_DISABLE_MAP_POPULATE environment variable being set. As a result, Vault may be slower to start up.") + } + start := time.Now() boltDB, err := bolt.Open(dbPath, 0o600, opts) if err != nil { return err } + elapsed := time.Now().Sub(start) f.logger.Debug("time to open database", "elapsed", elapsed, "path", dbPath) metrics.MeasureSince([]string{"raft_storage", "fsm", "open_db_file"}, start) diff --git a/physical/raft/raft.go b/physical/raft/raft.go index a79ee3507dd9..6202b3734f81 100644 --- a/physical/raft/raft.go +++ b/physical/raft/raft.go @@ -14,6 +14,7 @@ import ( "net/url" "os" "path/filepath" + "runtime" "strconv" "sync" "sync/atomic" @@ -74,7 +75,10 @@ const ( defaultMaxBatchSize = 128 * 1024 ) -var getMmapFlags = func(string) int { return 0 } +var ( + getMmapFlags = func(string) int { return 0 } + usingMapPopulate = func(int) bool { return false } +) // Verify RaftBackend satisfies the correct interfaces var ( @@ -447,6 +451,7 @@ func NewRaftBackend(conf map[string]string, logger log.Logger) (physical.Backend } } + // Create the log store. // Build an all in-memory setup for dev mode, otherwise prepare a full // disk-based setup. var logStore raft.LogStore @@ -473,6 +478,7 @@ func NewRaftBackend(conf map[string]string, logger log.Logger) (physical.Backend if err != nil { return nil, fmt.Errorf("failed to check if raft.db already exists: %w", err) } + if backendConfig.RaftWal && raftDbExists { logger.Warn("raft is configured to use raft-wal for storage but existing raft.db detected. raft-wal config will be ignored.") backendConfig.RaftWal = false @@ -504,6 +510,10 @@ func NewRaftBackend(conf map[string]string, logger log.Logger) (physical.Backend MsgpackUseNewTimeFormat: true, } + if runtime.GOOS == "linux" && raftDbExists && !usingMapPopulate(opts.MmapFlags) { + logger.Warn("the MAP_POPULATE mmap flag has not been set before opening the log store database. This may be due to the database file being larger than the available memory on the system, or due to the VAULT_RAFT_DISABLE_MAP_POPULATE environment variable being set. As a result, Vault may be slower to start up.") + } + store, err := raftboltdb.New(raftOptions) if err != nil { return nil, err