Log when MAP_POPULATE gets disabled

changelog/28526.txt

@@ -0,0 +1,3 @@
+```release-note:improvement
+physical/raft: Log when the MAP_POPULATE mmap flag gets disabled before opening the database.
+```

physical/raft/bolt_64bit_test.go

@@ -43,3 +43,30 @@
 		})
 	}
 }
+
+func TestMmapFlags(t *testing.T) {
+	t.Parallel()
+
+	testCases := []struct {
+		name               string
+		disableMapPopulate bool
+	}{
+		{"MAP_POPULATE is enabled", false},
+		{"MAP_POPULATE disabled by env var", true},
+	}
+
+	for _, tc := range testCases {
+		tc := tc
+
+		t.Run(tc.name, func(t *testing.T) {
+			if tc.disableMapPopulate {
+				os.Setenv("VAULT_RAFT_DISABLE_MAP_POPULATE", "true")
+			}
+
+			isEnabled := usingMapPopulate(getMmapFlags(""))
+			if tc.disableMapPopulate && isEnabled {
+				t.Error("expected MAP_POPULATE to be disabled but it was enabled")
+			}
+		})
+	}
+}

physical/raft/bolt_linux.go

@@ -13,27 +13,49 @@ import (
 
 func init() {
 	getMmapFlags = getMmapFlagsLinux
+	usingMapPopulate = usingMapPopulateLinux
 }
 
 func getMmapFlagsLinux(dbPath string) int {
+	if setMapPopulateFlag(dbPath) {
+		return unix.MAP_POPULATE
+	}
+
+	return 0
+}
+
+// setMapPopulateFlag determines whether we should set the MAP_POPULATE flag, which
+// prepopulates page tables to be mapped in the virtual memory space,
+// helping reduce slowness at runtime caused by page faults.
+// We only want to set this flag if we've determined there's enough memory on the system available to do so.
+func setMapPopulateFlag(dbPath string) bool {
 	if os.Getenv("VAULT_RAFT_DISABLE_MAP_POPULATE") != "" {
-		return 0
+		return false
 	}
 	stat, err := os.Stat(dbPath)
 	if err != nil {
-		return 0
+		return false
 	}
 	size := stat.Size()
 
 	v, err := mem.VirtualMemoryWithContext(context.Background())
 	if err != nil {
-		return 0
+		return false
 	}
 
 	// We won't worry about swap, since we already tell people not to use it.
 	if v.Total > uint64(size) {
-		return unix.MAP_POPULATE
+		return true
 	}
 
-	return 0
+	return false
+}
+
+// the unix.MAP_POPULATE constant only exists on Linux,
+// so reference to this constant can only live in a *_linux.go file
+func usingMapPopulateLinux(mmapFlag int) bool {
+	if mmapFlag == unix.MAP_POPULATE {
+		return true
+	}
+	return false
 }

physical/raft/fsm.go

@@ -13,6 +13,7 @@ import (
 	"io"
 	"os"
 	"path/filepath"
+	"runtime"
 	"strconv"
 	"strings"
 	"sync"
@@ -246,9 +247,11 @@ func (f *FSM) openDBFile(dbPath string) error {
 		return errors.New("can not open empty filename")
 	}
 
+	vaultDbExists := true
 	st, err := os.Stat(dbPath)
 	switch {
 	case err != nil && os.IsNotExist(err):
+		vaultDbExists = false
 	case err != nil:
 		return fmt.Errorf("error checking raft FSM db file %q: %v", dbPath, err)
 	default:
@@ -260,11 +263,16 @@ func (f *FSM) openDBFile(dbPath string) error {
 	}
 
 	opts := boltOptions(dbPath)
+	if runtime.GOOS == "linux" && vaultDbExists && !usingMapPopulate(opts.MmapFlags) {
+		f.logger.Warn("the MAP_POPULATE mmap flag has not been set before opening the FSM database. This may be due to the database file being larger than the available memory on the system, or due to the VAULT_RAFT_DISABLE_MAP_POPULATE environment variable being set. As a result, Vault may be slower to start up.")
+	}
+
 	start := time.Now()
 	boltDB, err := bolt.Open(dbPath, 0o600, opts)
 	if err != nil {
 		return err
 	}
+
 	elapsed := time.Now().Sub(start)
 	f.logger.Debug("time to open database", "elapsed", elapsed, "path", dbPath)
 	metrics.MeasureSince([]string{"raft_storage", "fsm", "open_db_file"}, start)

physical/raft/raft.go

@@ -14,6 +14,7 @@ import (
 	"net/url"
 	"os"
 	"path/filepath"
+	"runtime"
 	"strconv"
 	"sync"
 	"sync/atomic"
@@ -74,7 +75,10 @@ const (
 	defaultMaxBatchSize = 128 * 1024
 )
 
-var getMmapFlags = func(string) int { return 0 }
+var (
+	getMmapFlags     = func(string) int { return 0 }
+	usingMapPopulate = func(int) bool { return false }
+)
 
 // Verify RaftBackend satisfies the correct interfaces
 var (
@@ -447,6 +451,7 @@ func NewRaftBackend(conf map[string]string, logger log.Logger) (physical.Backend
 		}
 	}
 
+	// Create the log store.
 	// Build an all in-memory setup for dev mode, otherwise prepare a full
 	// disk-based setup.
 	var logStore raft.LogStore
@@ -473,6 +478,7 @@ func NewRaftBackend(conf map[string]string, logger log.Logger) (physical.Backend
 		if err != nil {
 			return nil, fmt.Errorf("failed to check if raft.db already exists: %w", err)
 		}
+
 		if backendConfig.RaftWal && raftDbExists {
 			logger.Warn("raft is configured to use raft-wal for storage but existing raft.db detected. raft-wal config will be ignored.")
 			backendConfig.RaftWal = false
@@ -504,6 +510,10 @@ func NewRaftBackend(conf map[string]string, logger log.Logger) (physical.Backend
 				MsgpackUseNewTimeFormat: true,
 			}
 
+			if runtime.GOOS == "linux" && raftDbExists && !usingMapPopulate(opts.MmapFlags) {
+				logger.Warn("the MAP_POPULATE mmap flag has not been set before opening the log store database. This may be due to the database file being larger than the available memory on the system, or due to the VAULT_RAFT_DISABLE_MAP_POPULATE environment variable being set. As a result, Vault may be slower to start up.")
+			}
+
 			store, err := raftboltdb.New(raftOptions)
 			if err != nil {
 				return nil, err