Set max conn and and timeout for read and write

fileserver/fileserver.go

@@ -18,6 +18,7 @@ import (
 	"runtime/debug"
 	"strings"
 	"syscall"
+	"time"
 
 	"github.com/go-sql-driver/mysql"
 	"github.com/gorilla/mux"
@@ -28,6 +29,7 @@ import (
 	"github.com/haiwen/seafile-server/fileserver/repomgr"
 	"github.com/haiwen/seafile-server/fileserver/searpc"
 	"github.com/haiwen/seafile-server/fileserver/share"
+	"github.com/haiwen/seafile-server/fileserver/utils"
 	_ "github.com/mattn/go-sqlite3"
 	log "github.com/sirupsen/logrus"
 	"gopkg.in/ini.v1"
@@ -43,7 +45,7 @@ var pidFilePath string
 var logFp *os.File
 
 var dbType string
-var seafileDB, ccnetDB *sql.DB
+var seafileDB, ccnetDB *utils.DB
 var seahubURL, seahubPK string
 
 func init() {
@@ -145,16 +147,15 @@ func loadCcnetDB() {
 		} else {
 			dsn = fmt.Sprintf("%s:%s@unix(%s)/%s", user, password, unixSocket, dbName)
 		}
-		ccnetDB, err = sql.Open("mysql", dsn)
+		db, err := sql.Open("mysql", dsn)
 		if err != nil {
 			log.Fatalf("Failed to open database: %v", err)
 		}
-	} else if strings.EqualFold(dbEngine, "sqlite") {
-		ccnetDBPath := filepath.Join(centralDir, "groupmgr.db")
-		ccnetDB, err = sql.Open("sqlite3", ccnetDBPath)
-		if err != nil {
-			log.Fatalf("Failed to open database %s: %v", ccnetDBPath, err)
-		}
+		db.SetConnMaxLifetime(5 * time.Minute)
+		db.SetMaxOpenConns(8)
+		db.SetMaxIdleConns(8)
+		ccnetDB = new(utils.DB)
+		ccnetDB.DB = db
 	} else {
 		log.Fatalf("Unsupported database %s.", dbEngine)
 	}
@@ -253,16 +254,15 @@ func loadSeafileDB() {
 			dsn = fmt.Sprintf("%s:%s@unix(%s)/%s", user, password, unixSocket, dbName)
 		}
 
-		seafileDB, err = sql.Open("mysql", dsn)
+		db, err := sql.Open("mysql", dsn)
 		if err != nil {
 			log.Fatalf("Failed to open database: %v", err)
 		}
-	} else if strings.EqualFold(dbEngine, "sqlite") {
-		seafileDBPath := filepath.Join(absDataDir, "seafile.db")
-		seafileDB, err = sql.Open("sqlite3", seafileDBPath)
-		if err != nil {
-			log.Fatalf("Failed to open database %s: %v", seafileDBPath, err)
-		}
+		db.SetConnMaxLifetime(5 * time.Minute)
+		db.SetMaxOpenConns(8)
+		db.SetMaxIdleConns(8)
+		seafileDB = new(utils.DB)
+		seafileDB.DB = db
 	} else {
 		log.Fatalf("Unsupported database %s.", dbEngine)
 	}

fileserver/go.mod

@@ -1,6 +1,6 @@
 module github.com/haiwen/seafile-server/fileserver
 
-go 1.17
+go 1.18
 
 require (
 	github.com/dgraph-io/ristretto v0.1.1

fileserver/go.sum

@@ -49,41 +49,21 @@ github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXf
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
 github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk=
 github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
-github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
-golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
-golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
 golang.org/x/net v0.0.0-20180218175443-cbe0f9307d01/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
-golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
-golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
-golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
-golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20221010170243-090e33056c14/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.1.0 h1:kunALQeHf1/185U1i0GOB/fy1IPRDDpuoOOqRReG57U=
 golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
-golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
-golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
-golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
 golang.org/x/text v0.3.8 h1:nAL+RVCQ9uMn3vJZbV+MRnydTJFPf8qqY42YiA6MrqY=
 golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ=
-golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
-golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
-golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
-golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/ini.v1 v1.55.0 h1:E8yzL5unfpW3M6fz/eB7Cb5MQAYSZ7GKo4Qth+N2sgQ=
 gopkg.in/ini.v1 v1.55.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=

fileserver/repomgr/repomgr.go

@@ -2,13 +2,15 @@
 package repomgr
 
 import (
+	"context"
 	"database/sql"
 	"fmt"
 	"time"
 
 	// Change to non-blank imports when use
 	_ "github.com/haiwen/seafile-server/fileserver/blockmgr"
 	"github.com/haiwen/seafile-server/fileserver/commitmgr"
+	"github.com/haiwen/seafile-server/fileserver/utils"
 	log "github.com/sirupsen/logrus"
 )
 
@@ -56,10 +58,10 @@ type VRepoInfo struct {
 	BaseCommitID string
 }
 
-var seafileDB *sql.DB
+var seafileDB *utils.DB
 
 // Init initialize status of repomgr package
-func Init(seafDB *sql.DB) {
+func Init(seafDB *utils.DB) {
 	seafileDB = seafDB
 }
 
@@ -70,14 +72,18 @@ func Get(id string) *Repo {
 		`LEFT JOIN VirtualRepo v ON r.repo_id = v.repo_id ` +
 		`WHERE r.repo_id = ? AND b.name = 'master'`
 
-	stmt, err := seafileDB.Prepare(query)
+	timeout := 60 * time.Second
+	ctx, cancel := context.WithTimeout(context.Background(), timeout)
+	defer cancel()
+
+	stmt, err := seafileDB.PrepareContext(ctx, query)
 	if err != nil {
 		log.Printf("failed to prepare sql : %s ：%v", query, err)
 		return nil
 	}
 	defer stmt.Close()
 
-	rows, err := stmt.Query(id)
+	rows, err := stmt.QueryContext(ctx, id)
 	if err != nil {
 		log.Printf("failed to query sql : %v", err)
 		return nil
@@ -198,14 +204,18 @@ func GetEx(id string) *Repo {
 		`LEFT JOIN VirtualRepo v ON r.repo_id = v.repo_id ` +
 		`WHERE r.repo_id = ? AND b.name = 'master'`
 
-	stmt, err := seafileDB.Prepare(query)
+	timeout := 60 * time.Second
+	ctx, cancel := context.WithTimeout(context.Background(), timeout)
+	defer cancel()
+
+	stmt, err := seafileDB.PrepareContext(ctx, query)
 	if err != nil {
 		log.Printf("failed to prepare sql : %s ：%v", query, err)
 		return nil
 	}
 	defer stmt.Close()
 
-	rows, err := stmt.Query(id)
+	rows, err := stmt.QueryContext(ctx, id)
 	if err != nil {
 		log.Printf("failed to query sql : %v", err)
 		return nil

fileserver/repomgr/repomgr_test.go

@@ -9,6 +9,7 @@ import (
 	_ "github.com/go-sql-driver/mysql"
 	"github.com/haiwen/seafile-server/fileserver/commitmgr"
 	"github.com/haiwen/seafile-server/fileserver/searpc"
+	"github.com/haiwen/seafile-server/fileserver/utils"
 )
 
 const (
@@ -60,10 +61,11 @@ func TestMain(m *testing.M) {
 	client = searpc.Init(pipePath, service)
 	repoID = createRepo()
 	dsn := fmt.Sprintf("%s:%s@tcp(%s:%d)/%s?tls=%t", user, password, host, port, dbName, useTLS)
-	seafDB, err := sql.Open("mysql", dsn)
+	db, err := sql.Open("mysql", dsn)
 	if err != nil {
 		fmt.Printf("Failed to open database: %v", err)
 	}
+	seafDB := &utils.DB{DB: db}
 	Init(seafDB)
 	commitmgr.Init(seafileConfPath, seafileDataDir)
 	code := m.Run()

fileserver/share/share.go

@@ -3,13 +3,16 @@
 package share
 
 import (
+	"context"
 	"database/sql"
 	"fmt"
 	"path/filepath"
 	"strconv"
 	"strings"
+	"time"
 
 	"github.com/haiwen/seafile-server/fileserver/repomgr"
+	"github.com/haiwen/seafile-server/fileserver/utils"
 	log "github.com/sirupsen/logrus"
 )
 
@@ -21,13 +24,13 @@ type group struct {
 	parentGroupID int
 }
 
-var ccnetDB *sql.DB
-var seafileDB *sql.DB
+var ccnetDB *utils.DB
+var seafileDB *utils.DB
 var groupTableName string
 var cloudMode bool
 
 // Init ccnetDB, seafileDB, groupTableName, cloudMode
-func Init(cnDB *sql.DB, seafDB *sql.DB, grpTableName string, clMode bool) {
+func Init(cnDB *utils.DB, seafDB *utils.DB, grpTableName string, clMode bool) {
 	ccnetDB = cnDB
 	seafileDB = seafDB
 	groupTableName = grpTableName
@@ -428,13 +431,17 @@ func GetReposByOwner(email string) ([]*SharedRepo, error) {
 		"v.repo_id IS NULL " +
 		"ORDER BY i.update_time DESC, o.repo_id"
 
-	stmt, err := seafileDB.Prepare(query)
+	timeout := 60 * time.Second
+	ctx, cancel := context.WithTimeout(context.Background(), timeout)
+	defer cancel()
+
+	stmt, err := seafileDB.PrepareContext(ctx, query)
 	if err != nil {
 		return nil, err
 	}
 	defer stmt.Close()
 
-	rows, err := stmt.Query(email)
+	rows, err := stmt.QueryContext(ctx, email)
 
 	if err != nil {
 		return nil, err
@@ -483,13 +490,17 @@ func ListInnerPubRepos() ([]*SharedRepo, error) {
 		"WHERE InnerPubRepo.repo_id=RepoOwner.repo_id AND " +
 		"InnerPubRepo.repo_id = Branch.repo_id AND Branch.name = 'master'"
 
-	stmt, err := seafileDB.Prepare(query)
+	timeout := 60 * time.Second
+	ctx, cancel := context.WithTimeout(context.Background(), timeout)
+	defer cancel()
+
+	stmt, err := seafileDB.PrepareContext(ctx, query)
 	if err != nil {
 		return nil, err
 	}
 	defer stmt.Close()
 
-	rows, err := stmt.Query()
+	rows, err := stmt.QueryContext(ctx)
 	if err != nil {
 		return nil, err
 	}
@@ -552,14 +563,18 @@ func ListShareRepos(email, columnType string) ([]*SharedRepo, error) {
 		return nil, err
 	}
 
-	stmt, err := seafileDB.Prepare(query)
+	timeout := 60 * time.Second
+	ctx, cancel := context.WithTimeout(context.Background(), timeout)
+	defer cancel()
+
+	stmt, err := seafileDB.PrepareContext(ctx, query)
 	if err != nil {
 		return nil, err
 	}
 
 	defer stmt.Close()
 
-	rows, err := stmt.Query(email)
+	rows, err := stmt.QueryContext(ctx, email)
 	if err != nil {
 		return nil, err
 	}

fileserver/utils/db.go

@@ -0,0 +1,37 @@
+package utils
+
+import (
+	"context"
+	"database/sql"
+	"time"
+
+	_ "github.com/go-sql-driver/mysql"
+)
+
+type DB struct {
+	*sql.DB
+}
+
+func (db *DB) Query(query string, args ...any) (*sql.Rows, error) {
+	timeout := 60 * time.Second
+	ctx, cancel := context.WithTimeout(context.Background(), timeout)
+	defer cancel()
+
+	return db.DB.QueryContext(ctx, query, args)
+}
+
+func (db *DB) QueryRow(query string, args ...any) *sql.Row {
+	timeout := 60 * time.Second
+	ctx, cancel := context.WithTimeout(context.Background(), timeout)
+	defer cancel()
+
+	return db.DB.QueryRowContext(ctx, query, args)
+}
+
+func (db *DB) Exec(query string, args ...any) (sql.Result, error) {
+	timeout := 60 * time.Second
+	ctx, cancel := context.WithTimeout(context.Background(), timeout)
+	defer cancel()
+
+	return db.DB.ExecContext(ctx, query, args)
+}

notification-server/event.go

@@ -1,9 +1,11 @@
 package main
 
 import (
+	"context"
 	"encoding/json"
 	"reflect"
 	"runtime/debug"
+	"time"
 
 	log "github.com/sirupsen/logrus"
 )
@@ -113,15 +115,19 @@ func Notify(msg *Message) {
 }
 
 func getGroupMembers(group int) map[string]struct{} {
+	timeout := 60 * time.Second
+	ctx, cancel := context.WithTimeout(context.Background(), timeout)
+	defer cancel()
+
 	query := `SELECT user_name FROM GroupUser WHERE group_id = ?`
-	stmt, err := ccnetDB.Prepare(query)
+	stmt, err := ccnetDB.PrepareContext(ctx, query)
 	if err != nil {
 		log.Printf("failed to prepare sql: %s：%v", query, err)
 		return nil
 	}
 	defer stmt.Close()
 
-	rows, err := stmt.Query(group)
+	rows, err := stmt.QueryContext(ctx, group)
 	if err != nil {
 		log.Printf("failed to query sql: %v", err)
 		return nil

notification-server/server.go

@@ -153,6 +153,9 @@ func loadCcnetDB() {
 	if err := ccnetDB.Ping(); err != nil {
 		log.Fatalf("Failed to connected to mysql: %v", err)
 	}
+	ccnetDB.SetConnMaxLifetime(5 * time.Minute)
+	ccnetDB.SetMaxOpenConns(8)
+	ccnetDB.SetMaxIdleConns(8)
 }
 
 func main() {