From 526964099e89eb3fff061994dbb43fe82c0937a8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=9D=A8=E8=B5=AB=E7=84=B6?= Date: Thu, 4 Jul 2024 16:03:16 +0800 Subject: [PATCH 1/5] Add parse seahub database config --- Makefile.am | 4 +- common/seaf-utils.c | 100 ++++++++++++++++++++++++++++++++++++ common/seaf-utils.h | 3 ++ configure.ac | 1 + fuse/seafile-session.h | 1 + scripts/Makefile.am | 3 ++ scripts/parse_seahub_db.py | 8 +++ server/gc/seafile-session.h | 1 + server/seafile-session.c | 2 + server/seafile-session.h | 1 + 10 files changed, 122 insertions(+), 2 deletions(-) create mode 100644 scripts/Makefile.am create mode 100644 scripts/parse_seahub_db.py diff --git a/Makefile.am b/Makefile.am index 16d03aa3..ae142d7b 100644 --- a/Makefile.am +++ b/Makefile.am @@ -14,9 +14,9 @@ endif MAKE_SERVER = server tools $(MAKE_CONTROLLER) $(MAKE_FUSE) -SUBDIRS = include lib common python $(MAKE_SERVER) doc +SUBDIRS = include lib common python $(MAKE_SERVER) doc scripts -DIST_SUBDIRS = include lib common python server tools controller fuse doc +DIST_SUBDIRS = include lib common python server tools controller fuse doc scripts INTLTOOL = \ intltool-extract.in \ diff --git a/common/seaf-utils.c b/common/seaf-utils.c index f007a963..46029485 100644 --- a/common/seaf-utils.c +++ b/common/seaf-utils.c @@ -379,3 +379,103 @@ load_ccnet_database_config (SeafileSession *session) g_free (engine); return ret; } + +static int +parse_seahub_db_config () +{ + char buf[1024]; + GError *error = NULL; + int retcode = 0; + char *child_stdout = NULL; + char *child_stderr = NULL; + + char *binary_path = g_find_program_in_path ("parse_seahub_db.py"); + + snprintf (buf, + sizeof(buf), + "python3 %s", + binary_path); + g_spawn_command_line_sync (buf, + &child_stdout, + &child_stderr, + &retcode, + &error); + + if (error != NULL) { + seaf_warning ("Failed to run parse_seahub_db.sh : %s\n", error->message); + return -1; + } + if (retcode != 0) { + seaf_warning ("Failed to run parse_seahub_db.sh [%d]: %s\n", retcode, strerror(errno)); + return -1; + } + + return 0; +} + +int +load_seahub_database_config (SeafileSession *session) +{ + int ret = 0; + json_t *object = NULL; + const char *engine = NULL, *name = NULL, *user = NULL, *password = NULL, *host = NULL, *charset = NULL; + int port; + + + char *json_file = g_build_filename ("/tmp", "seahub_db.json", NULL); + + if (parse_seahub_db_config () < 0) { + seaf_warning ("Failed to parse seahub database config.\n"); + ret = -1; + goto out; + } + + object = json_load_file (json_file, 0, NULL); + if (!object) { + seaf_warning ("Failed to load seahub db json file: %s\n", json_file); + ret = -1; + goto out; + } + + engine = json_object_get_string_member (object, "ENGINE"); + name = json_object_get_string_member (object, "NAME"); + user = json_object_get_string_member (object, "USER"); + password = json_object_get_string_member (object, "PASSWORD"); + host = json_object_get_string_member (object, "HOST"); + charset = json_object_get_string_member (object, "CHARSET"); + port = json_object_get_int_member (object, "PORT"); + if (port <= 0) { + port = MYSQL_DEFAULT_PORT; + } + + if (!engine || strstr (engine, "sqlite") != NULL) { + seaf_message ("Use database sqlite\n"); + session->seahub_db = seaf_db_new_sqlite (name, DEFAULT_MAX_CONNECTIONS); + if (!session->seahub_db) { + seaf_warning ("Failed to open seahub database.\n"); + ret = -1; + goto out; + } + } +#ifdef HAVE_MYSQL + else if (strstr (engine, "mysql") != NULL) { + seaf_message("Use database Mysql\n"); + session->seahub_db = seaf_db_new_mysql (host, port, user, password, name, NULL, FALSE, FALSE, NULL, charset, DEFAULT_MAX_CONNECTIONS); + if (!session->seahub_db) { + seaf_warning ("Failed to open seahub database.\n"); + ret = -1; + goto out; + } + } +#endif + else { + seaf_warning ("Unknown database type: %s.\n", engine); + ret = -1; + } + +out: + if (object) + json_decref (object); + g_free (json_file); + return ret; +} diff --git a/common/seaf-utils.h b/common/seaf-utils.h index 2113c441..e5d30e15 100644 --- a/common/seaf-utils.h +++ b/common/seaf-utils.h @@ -16,4 +16,7 @@ load_database_config (struct _SeafileSession *session); int load_ccnet_database_config (struct _SeafileSession *session); +int +load_seahub_database_config (SeafileSession *session); + #endif diff --git a/configure.ac b/configure.ac index d7acd854..23d4de8b 100644 --- a/configure.ac +++ b/configure.ac @@ -317,6 +317,7 @@ AC_CONFIG_FILES( controller/Makefile tools/Makefile doc/Makefile + scripts/Makefile ) AC_OUTPUT diff --git a/fuse/seafile-session.h b/fuse/seafile-session.h index 922e598a..f535f91d 100644 --- a/fuse/seafile-session.h +++ b/fuse/seafile-session.h @@ -26,6 +26,7 @@ struct _SeafileSession { GKeyFile *ccnet_config; SeafDB *db; SeafDB *ccnet_db; + SeafDB *seahub_db; SeafBlockManager *block_mgr; SeafFSManager *fs_mgr; diff --git a/scripts/Makefile.am b/scripts/Makefile.am new file mode 100644 index 00000000..63235534 --- /dev/null +++ b/scripts/Makefile.am @@ -0,0 +1,3 @@ +bin_SCRIPTS = parse_seahub_db.py + +EXTRA_DIST = parse_seahub_db.py diff --git a/scripts/parse_seahub_db.py b/scripts/parse_seahub_db.py new file mode 100644 index 00000000..0ac97fbc --- /dev/null +++ b/scripts/parse_seahub_db.py @@ -0,0 +1,8 @@ +import json +import seahub_settings + +db_infos = seahub_settings.DATABASES['default'] + +with open('/tmp/seahub_db.json', 'w') as f: + json.dump(db_infos, f, indent=4) + diff --git a/server/gc/seafile-session.h b/server/gc/seafile-session.h index 641dc36a..b05f68f2 100644 --- a/server/gc/seafile-session.h +++ b/server/gc/seafile-session.h @@ -24,6 +24,7 @@ struct _SeafileSession { GKeyFile *ccnet_config; SeafDB *db; SeafDB *ccnet_db; + SeafDB *seahub_db; SeafBlockManager *block_mgr; SeafFSManager *fs_mgr; diff --git a/server/seafile-session.c b/server/seafile-session.c index a2409ffd..a697e912 100644 --- a/server/seafile-session.c +++ b/server/seafile-session.c @@ -218,6 +218,8 @@ seafile_session_new(const char *central_config_dir, goto onerror; } + load_seahub_database_config (session); + session->cfg_mgr = seaf_cfg_manager_new (session); if (!session->cfg_mgr) goto onerror; diff --git a/server/seafile-session.h b/server/seafile-session.h index 05a141c8..0dbb1186 100644 --- a/server/seafile-session.h +++ b/server/seafile-session.h @@ -47,6 +47,7 @@ struct _SeafileSession { GKeyFile *ccnet_config; SeafDB *db; CcnetDB *ccnet_db; + SeafDB *seahub_db; SeafBlockManager *block_mgr; SeafFSManager *fs_mgr; From dba36862ebe6176f84e0b9208e11407e74d2826d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=9D=A8=E8=B5=AB=E7=84=B6?= Date: Thu, 4 Jul 2024 16:36:59 +0800 Subject: [PATCH 2/5] Get nickname from seahub database --- common/merge-new.c | 39 +++++++++++++++++++++++++++++++++++++-- common/merge-new.h | 2 ++ common/seaf-utils.c | 4 ++-- server/http-server.c | 6 ++++++ server/repo-op.c | 6 ++++++ server/virtual-repo.c | 12 ++++++++++++ 6 files changed, 65 insertions(+), 4 deletions(-) diff --git a/common/merge-new.c b/common/merge-new.c index 3d5a2e88..890913ec 100644 --- a/common/merge-new.c +++ b/common/merge-new.c @@ -13,6 +13,33 @@ merge_trees_recursive (const char *store_id, int version, const char *basedir, MergeOptions *opt); +static char * +get_nickname_by_modifier (GHashTable *email_to_nickname, const char *modifier) +{ + const char *nickname; + + if (!modifier) { + return NULL; + } + + nickname = g_hash_table_lookup (email_to_nickname, modifier); + if (nickname) { + return g_strdup (nickname); + } + + if (seaf->seahub_db) { + char *sql = "SELECT nickname from profile_profile WHERE user = ?"; + nickname = seaf_db_statement_get_string(seaf->seahub_db, sql, 1, "string", modifier); + } + + if (!nickname) { + nickname = modifier; + } + g_hash_table_insert (email_to_nickname, g_strdup(modifier), g_strdup(nickname)); + + return g_strdup (nickname); +} + static char * merge_conflict_filename (const char *store_id, int version, MergeOptions *opt, @@ -20,6 +47,7 @@ merge_conflict_filename (const char *store_id, int version, const char *filename) { char *path = NULL, *modifier = NULL, *conflict_name = NULL; + char *nickname = NULL; gint64 mtime; SeafCommit *commit; @@ -46,11 +74,14 @@ merge_conflict_filename (const char *store_id, int version, seaf_commit_unref (commit); } - conflict_name = gen_conflict_path (filename, modifier, mtime); + nickname = get_nickname_by_modifier (opt->email_to_nickname, modifier); + + conflict_name = gen_conflict_path (filename, nickname, mtime); out: g_free (path); g_free (modifier); + g_free (nickname); return conflict_name; } @@ -61,6 +92,7 @@ merge_conflict_dirname (const char *store_id, int version, const char *dirname) { char *modifier = NULL, *conflict_name = NULL; + char *nickname = NULL; SeafCommit *commit; commit = seaf_commit_manager_get_commit (seaf->commit_mgr, @@ -74,10 +106,13 @@ merge_conflict_dirname (const char *store_id, int version, modifier = g_strdup(commit->creator_name); seaf_commit_unref (commit); - conflict_name = gen_conflict_path (dirname, modifier, (gint64)time(NULL)); + nickname = get_nickname_by_modifier (opt->email_to_nickname, modifier); + + conflict_name = gen_conflict_path (dirname, nickname, (gint64)time(NULL)); out: g_free (modifier); + g_free (nickname); return conflict_name; } diff --git a/common/merge-new.h b/common/merge-new.h index 0c8d1d49..3e5cfe6f 100644 --- a/common/merge-new.h +++ b/common/merge-new.h @@ -25,6 +25,8 @@ typedef struct MergeOptions { char merged_tree_root[41]; /* merge result */ int visit_dirs; gboolean conflict; + + GHashTable *email_to_nickname; } MergeOptions; int diff --git a/common/seaf-utils.c b/common/seaf-utils.c index 46029485..28c629be 100644 --- a/common/seaf-utils.c +++ b/common/seaf-utils.c @@ -402,11 +402,11 @@ parse_seahub_db_config () &error); if (error != NULL) { - seaf_warning ("Failed to run parse_seahub_db.sh : %s\n", error->message); + seaf_warning ("Failed to run python parse_seahub_db.py : %s\n", error->message); return -1; } if (retcode != 0) { - seaf_warning ("Failed to run parse_seahub_db.sh [%d]: %s\n", retcode, strerror(errno)); + seaf_warning ("Failed to run python parse_seahub_db.py [%d]: %s\n", retcode, strerror(errno)); return -1; } diff --git a/server/http-server.c b/server/http-server.c index 77c137b6..f96fe0bb 100644 --- a/server/http-server.c +++ b/server/http-server.c @@ -945,6 +945,10 @@ fast_forward_or_merge (const char *repo_id, memcpy (opt.remote_repo_id, repo_id, 36); memcpy (opt.remote_head, new_commit->commit_id, 40); opt.do_merge = TRUE; + opt.email_to_nickname = g_hash_table_new_full(g_str_hash, + g_str_equal, + g_free, + g_free); roots[0] = base->root_id; /* base */ roots[1] = current_head->root_id; /* head */ @@ -952,9 +956,11 @@ fast_forward_or_merge (const char *repo_id, if (seaf_merge_trees (repo->store_id, repo->version, 3, roots, &opt) < 0) { seaf_warning ("Failed to merge.\n"); + g_hash_table_destroy (opt.email_to_nickname); ret = -1; goto out; } + g_hash_table_destroy (opt.email_to_nickname); if (!opt.conflict) desc = g_strdup("Auto merge by system"); diff --git a/server/repo-op.c b/server/repo-op.c index 6c89c35d..4d4c0056 100644 --- a/server/repo-op.c +++ b/server/repo-op.c @@ -511,6 +511,10 @@ gen_new_commit (const char *repo_id, memcpy (opt.remote_repo_id, repo_id, 36); memcpy (opt.remote_head, new_commit->commit_id, 40); opt.do_merge = TRUE; + opt.email_to_nickname = g_hash_table_new_full(g_str_hash, + g_str_equal, + g_free, + g_free); roots[0] = base->root_id; /* base */ roots[1] = current_head->root_id; /* head */ @@ -520,9 +524,11 @@ gen_new_commit (const char *repo_id, seaf_warning ("Failed to merge.\n"); g_set_error (error, SEAFILE_DOMAIN, SEAF_ERR_GENERAL, "Internal error"); + g_hash_table_destroy (opt.email_to_nickname); ret = -1; goto out; } + g_hash_table_destroy (opt.email_to_nickname); seaf_debug ("Number of dirs visted in merge %.8s: %d.\n", repo_id, opt.visit_dirs); diff --git a/server/virtual-repo.c b/server/virtual-repo.c index 2698c2de..e2ee3c8e 100644 --- a/server/virtual-repo.c +++ b/server/virtual-repo.c @@ -952,6 +952,10 @@ static void *merge_virtual_repo (void *vtask) memcpy (opt.remote_repo_id, repo_id, 36); memcpy (opt.remote_head, head->commit_id, 40); opt.do_merge = TRUE; + opt.email_to_nickname = g_hash_table_new_full(g_str_hash, + g_str_equal, + g_free, + g_free); roots[0] = base_root; /* base */ roots[1] = orig_root; /* head */ @@ -961,9 +965,11 @@ static void *merge_virtual_repo (void *vtask) if (seaf_merge_trees (orig_repo->store_id, orig_repo->version, 3, roots, &opt) < 0) { seaf_warning ("Failed to merge virtual repo %.10s.\n", repo_id); + g_hash_table_destroy (opt.email_to_nickname); ret = -1; goto out; } + g_hash_table_destroy (opt.email_to_nickname); seaf_debug ("Number of dirs visted in merge: %d.\n", opt.visit_dirs); @@ -1216,6 +1222,10 @@ seaf_repo_manager_repair_virtual_repo (char *repo_id) opt.n_ways = 2; memcpy (opt.remote_repo_id, repo_id, 36); memcpy (opt.remote_head, head->commit_id, 40); + opt.email_to_nickname = g_hash_table_new_full(g_str_hash, + g_str_equal, + g_free, + g_free); roots[0] = orig_root; roots[1] = root; @@ -1224,9 +1234,11 @@ seaf_repo_manager_repair_virtual_repo (char *repo_id) if (seaf_merge_trees (orig_repo->store_id, orig_repo->version, 2, roots, &opt) < 0) { seaf_warning ("Failed to merge virtual repo %.10s.\n", repo_id); + g_hash_table_destroy (opt.email_to_nickname); ret = -1; goto out; } + g_hash_table_destroy (opt.email_to_nickname); seaf_debug ("Number of dirs visted in merge: %d.\n", opt.visit_dirs); From 68067933c7abeb267c51a0a4eafb90cc21212858 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=9D=A8=E8=B5=AB=E7=84=B6?= Date: Thu, 4 Jul 2024 17:33:32 +0800 Subject: [PATCH 3/5] Go get nickname from seahub database --- fileserver/fileop.go | 1 + fileserver/fileserver.go | 50 +++++++++++++++++++++++++++++++++++++- fileserver/merge.go | 36 +++++++++++++++++++++++---- fileserver/merge_test.go | 43 +++++++++++++++++++++----------- fileserver/virtual_repo.go | 1 + 5 files changed, 111 insertions(+), 20 deletions(-) diff --git a/fileserver/fileop.go b/fileserver/fileop.go index 8e5e9623..2162b479 100644 --- a/fileserver/fileop.go +++ b/fileserver/fileop.go @@ -1770,6 +1770,7 @@ func genCommitNeedRetry(repo *repomgr.Repo, base *commitmgr.Commit, commit *comm opt := new(mergeOptions) opt.remoteRepoID = repoID opt.remoteHead = commit.CommitID + opt.emailToNickname = make(map[string]string) err := mergeTrees(repo.StoreID, roots, opt) if err != nil { diff --git a/fileserver/fileserver.go b/fileserver/fileserver.go index 63d8ba8b..c5537ae7 100644 --- a/fileserver/fileserver.go +++ b/fileserver/fileserver.go @@ -5,6 +5,7 @@ import ( "crypto/tls" "crypto/x509" "database/sql" + "encoding/json" "flag" "fmt" "io" @@ -14,6 +15,7 @@ import ( "os/signal" "path/filepath" "runtime/debug" + "strconv" "strings" "syscall" @@ -41,7 +43,7 @@ var pidFilePath string var logFp *os.File var dbType string -var seafileDB, ccnetDB *sql.DB +var seafileDB, ccnetDB, seahubDB *sql.DB // when SQLite is used, user and group db are separated. var userDB, groupDB *sql.DB @@ -58,6 +60,7 @@ func init() { const ( timestampFormat = "[2006-01-02 15:04:05] " + seahubDBPath = "/tmp/seahub_db.json" ) type LogFormatter struct{} @@ -269,6 +272,49 @@ func loadSeafileDB() { dbType = dbEngine } +func loadSeahubDB() { + dbData, err := ioutil.ReadFile(seahubDBPath) + if err != nil { + log.Warnf("Failed to read seahub database json file: %v", err) + return + } + + dbConfig := make(map[string]string) + + err = json.Unmarshal(dbData, &dbConfig) + if err != nil { + log.Warnf("Failed to decode seahub database json file: %v", err) + return + } + + dbEngine := dbConfig["ENGINE"] + dbName := dbConfig["NAME"] + user := dbConfig["USER"] + password := dbConfig["PASSWORD"] + host := dbConfig["HOST"] + portStr := dbConfig["PORT"] + + if strings.Index(dbEngine, "mysql") >= 0 { + port, err := strconv.ParseInt(portStr, 10, 64) + if err != nil || port <= 0 { + port = 3306 + } + dsn := fmt.Sprintf("%s:%s@tcp(%s:%d)/%s?tls=%t", user, password, host, port, dbName, false) + + seahubDB, err = sql.Open("mysql", dsn) + if err != nil { + log.Warnf("Failed to open database: %v", err) + } + } else if strings.Index(dbEngine, "sqlite") >= 0 { + seahubDB, err = sql.Open("sqlite3", dbName) + if err != nil { + log.Warnf("Failed to open database %s: %v", dbName, err) + } + } else { + log.Warnf("Unsupported database %s.", dbEngine) + } +} + func writePidFile(pid_file_path string) error { file, err := os.OpenFile(pid_file_path, os.O_CREATE|os.O_WRONLY, 0664) if err != nil { @@ -367,6 +413,8 @@ func main() { fp.Close() } + loadSeahubDB() + repomgr.Init(seafileDB) fsmgr.Init(centralDir, dataDir, option.FsCacheLimit) diff --git a/fileserver/merge.go b/fileserver/merge.go index f3c7bd6f..2e1d1d31 100644 --- a/fileserver/merge.go +++ b/fileserver/merge.go @@ -12,10 +12,11 @@ import ( ) type mergeOptions struct { - remoteRepoID string - remoteHead string - mergedRoot string - conflict bool + remoteRepoID string + remoteHead string + mergedRoot string + conflict bool + emailToNickname map[string]string } func mergeTrees(storeID string, roots []string, opt *mergeOptions) error { @@ -335,7 +336,9 @@ func mergeConflictFileName(storeID string, opt *mergeOptions, baseDir, fileName mtime = time.Now().Unix() } - conflictName := genConflictPath(fileName, modifier, mtime) + nickname := getNickNameByModifier(opt.emailToNickname, modifier) + + conflictName := genConflictPath(fileName, nickname, mtime) return conflictName, nil } @@ -366,6 +369,29 @@ func genConflictPath(originPath, modifier string, mtime int64) string { return conflictPath } +func getNickNameByModifier(emailToNickname map[string]string, modifier string) string { + if modifier == "" { + return "" + } + nickname, ok := emailToNickname[modifier] + if ok { + return nickname + } + if seahubDB != nil { + sqlStr := "SELECT nickname from profile_profile WHERE user = ?" + row := seahubDB.QueryRow(sqlStr, modifier) + row.Scan(&nickname) + } + + if nickname == "" { + nickname = modifier + } + + emailToNickname[modifier] = nickname + + return nickname +} + func getFileModifierMtime(repoID, storeID, head, filePath string) (string, int64, error) { commit, err := commitmgr.Load(repoID, head) if err != nil { diff --git a/fileserver/merge_test.go b/fileserver/merge_test.go index e34c46bc..d54faf7a 100644 --- a/fileserver/merge_test.go +++ b/fileserver/merge_test.go @@ -30,24 +30,27 @@ var mergeTestTree4CommitID string var mergeTestTree5CommitID string /* - test directory structure: - tree1 - |--bbb - |-- testfile(size:1) +test directory structure: +tree1 +|--bbb - tree2 - |--bbb - |-- testfile(size:10) + |-- testfile(size:1) - tree3 - |--bbb +tree2 +|--bbb - tree4 - |--bbb - |-- testfile(size:100) + |-- testfile(size:10) - tree5 - |-- +tree3 +|--bbb + +tree4 +|--bbb + + |-- testfile(size:100) + +tree5 +|-- */ func mergeTestCreateTestDir() error { modeDir := uint32(syscall.S_IFDIR | 0644) @@ -251,6 +254,7 @@ func testMergeTrees1(t *testing.T) { opt := new(mergeOptions) opt.remoteRepoID = mergeTestRepoID opt.remoteHead = commit.CommitID + opt.emailToNickname = make(map[string]string) err = mergeTrees(mergeTestRepoID, roots, opt) if err != nil { @@ -272,6 +276,7 @@ func testMergeTrees2(t *testing.T) { opt := new(mergeOptions) opt.remoteRepoID = mergeTestRepoID opt.remoteHead = commit.CommitID + opt.emailToNickname = make(map[string]string) err = mergeTrees(mergeTestRepoID, roots, opt) if err != nil { @@ -293,6 +298,7 @@ func testMergeTrees3(t *testing.T) { opt := new(mergeOptions) opt.remoteRepoID = mergeTestRepoID opt.remoteHead = commit.CommitID + opt.emailToNickname = make(map[string]string) err = mergeTrees(mergeTestRepoID, roots, opt) if err != nil { @@ -314,6 +320,7 @@ func testMergeTrees4(t *testing.T) { opt := new(mergeOptions) opt.remoteRepoID = mergeTestRepoID opt.remoteHead = commit.CommitID + opt.emailToNickname = make(map[string]string) err = mergeTrees(mergeTestRepoID, roots, opt) if err != nil { @@ -335,6 +342,7 @@ func testMergeTrees5(t *testing.T) { opt := new(mergeOptions) opt.remoteRepoID = mergeTestRepoID opt.remoteHead = commit.CommitID + opt.emailToNickname = make(map[string]string) err = mergeTrees(mergeTestRepoID, roots, opt) if err != nil { @@ -355,6 +363,7 @@ func testMergeTrees6(t *testing.T) { opt := new(mergeOptions) opt.remoteRepoID = mergeTestRepoID opt.remoteHead = commit.CommitID + opt.emailToNickname = make(map[string]string) err = mergeTrees(mergeTestRepoID, roots, opt) if err != nil { @@ -375,6 +384,7 @@ func testMergeTrees7(t *testing.T) { opt := new(mergeOptions) opt.remoteRepoID = mergeTestRepoID opt.remoteHead = commit.CommitID + opt.emailToNickname = make(map[string]string) err = mergeTrees(mergeTestRepoID, roots, opt) if err != nil { @@ -395,6 +405,7 @@ func testMergeTrees8(t *testing.T) { opt := new(mergeOptions) opt.remoteRepoID = mergeTestRepoID opt.remoteHead = commit.CommitID + opt.emailToNickname = make(map[string]string) err = mergeTrees(mergeTestRepoID, roots, opt) if err != nil { @@ -415,6 +426,7 @@ func testMergeTrees9(t *testing.T) { opt := new(mergeOptions) opt.remoteRepoID = mergeTestRepoID opt.remoteHead = commit.CommitID + opt.emailToNickname = make(map[string]string) err = mergeTrees(mergeTestRepoID, roots, opt) if err != nil { @@ -435,6 +447,7 @@ func testMergeTrees10(t *testing.T) { opt := new(mergeOptions) opt.remoteRepoID = mergeTestRepoID opt.remoteHead = commit.CommitID + opt.emailToNickname = make(map[string]string) err = mergeTrees(mergeTestRepoID, roots, opt) if err != nil { @@ -455,6 +468,7 @@ func testMergeTrees11(t *testing.T) { opt := new(mergeOptions) opt.remoteRepoID = mergeTestRepoID opt.remoteHead = commit.CommitID + opt.emailToNickname = make(map[string]string) err = mergeTrees(mergeTestRepoID, roots, opt) if err != nil { @@ -475,6 +489,7 @@ func testMergeTrees12(t *testing.T) { opt := new(mergeOptions) opt.remoteRepoID = mergeTestRepoID opt.remoteHead = commit.CommitID + opt.emailToNickname = make(map[string]string) err = mergeTrees(mergeTestRepoID, roots, opt) if err != nil { diff --git a/fileserver/virtual_repo.go b/fileserver/virtual_repo.go index fb6be165..2f673d8a 100644 --- a/fileserver/virtual_repo.go +++ b/fileserver/virtual_repo.go @@ -175,6 +175,7 @@ func mergeRepo(repoID string) error { opt := new(mergeOptions) opt.remoteRepoID = repoID opt.remoteHead = head.CommitID + opt.emailToNickname = make(map[string]string) err := mergeTrees(origRepo.StoreID, roots, opt) if err != nil { From 0eb0f936e48e22e53d2ed3f0f2506ed4d098c34e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=9D=A8=E8=B5=AB=E7=84=B6?= Date: Wed, 10 Jul 2024 14:11:59 +0800 Subject: [PATCH 4/5] Save output of script to memory and fix some errors --- common/merge-new.c | 35 ++++++++++++--------- common/seaf-utils.c | 63 +++++++++++++++++++++++++------------- fileserver/fileop.go | 1 - fileserver/fileserver.go | 34 +++++++++++++++----- fileserver/merge.go | 2 ++ fileserver/merge_test.go | 12 -------- fileserver/virtual_repo.go | 1 - scripts/parse_seahub_db.py | 4 +-- server/http-server.c | 6 ---- server/repo-op.c | 6 ---- server/virtual-repo.c | 12 -------- 11 files changed, 93 insertions(+), 83 deletions(-) diff --git a/common/merge-new.c b/common/merge-new.c index 890913ec..35a0bd8c 100644 --- a/common/merge-new.c +++ b/common/merge-new.c @@ -13,10 +13,10 @@ merge_trees_recursive (const char *store_id, int version, const char *basedir, MergeOptions *opt); -static char * +static const char * get_nickname_by_modifier (GHashTable *email_to_nickname, const char *modifier) { - const char *nickname; + const char *nickname = NULL; if (!modifier) { return NULL; @@ -24,20 +24,18 @@ get_nickname_by_modifier (GHashTable *email_to_nickname, const char *modifier) nickname = g_hash_table_lookup (email_to_nickname, modifier); if (nickname) { - return g_strdup (nickname); + return nickname; } - if (seaf->seahub_db) { - char *sql = "SELECT nickname from profile_profile WHERE user = ?"; - nickname = seaf_db_statement_get_string(seaf->seahub_db, sql, 1, "string", modifier); - } + char *sql = "SELECT nickname from profile_profile WHERE user = ?"; + nickname = seaf_db_statement_get_string(seaf->seahub_db, sql, 1, "string", modifier); if (!nickname) { nickname = modifier; } g_hash_table_insert (email_to_nickname, g_strdup(modifier), g_strdup(nickname)); - return g_strdup (nickname); + return nickname; } static char * @@ -47,7 +45,7 @@ merge_conflict_filename (const char *store_id, int version, const char *filename) { char *path = NULL, *modifier = NULL, *conflict_name = NULL; - char *nickname = NULL; + const char *nickname = NULL; gint64 mtime; SeafCommit *commit; @@ -74,14 +72,15 @@ merge_conflict_filename (const char *store_id, int version, seaf_commit_unref (commit); } - nickname = get_nickname_by_modifier (opt->email_to_nickname, modifier); + nickname = modifier; + if (seaf->seahub_db) + nickname = get_nickname_by_modifier (opt->email_to_nickname, modifier); conflict_name = gen_conflict_path (filename, nickname, mtime); out: g_free (path); g_free (modifier); - g_free (nickname); return conflict_name; } @@ -92,7 +91,7 @@ merge_conflict_dirname (const char *store_id, int version, const char *dirname) { char *modifier = NULL, *conflict_name = NULL; - char *nickname = NULL; + const char *nickname = NULL; SeafCommit *commit; commit = seaf_commit_manager_get_commit (seaf->commit_mgr, @@ -106,13 +105,14 @@ merge_conflict_dirname (const char *store_id, int version, modifier = g_strdup(commit->creator_name); seaf_commit_unref (commit); - nickname = get_nickname_by_modifier (opt->email_to_nickname, modifier); + nickname = modifier; + if (seaf->seahub_db) + nickname = get_nickname_by_modifier (opt->email_to_nickname, modifier); conflict_name = gen_conflict_path (dirname, nickname, (gint64)time(NULL)); out: g_free (modifier); - g_free (nickname); return conflict_name; } @@ -722,6 +722,11 @@ seaf_merge_trees (const char *store_id, int version, g_return_val_if_fail (n == 2 || n == 3, -1); + opt->email_to_nickname = g_hash_table_new_full(g_str_hash, + g_str_equal, + g_free, + g_free); + trees = g_new0 (SeafDir *, n); for (i = 0; i < n; ++i) { root = seaf_fs_manager_get_seafdir (seaf->fs_mgr, store_id, version, roots[i]); @@ -739,5 +744,7 @@ seaf_merge_trees (const char *store_id, int version, seaf_dir_free (trees[i]); g_free (trees); + g_hash_table_destroy (opt->email_to_nickname); + return ret; } diff --git a/common/seaf-utils.c b/common/seaf-utils.c index 28c629be..5c7e6e9f 100644 --- a/common/seaf-utils.c +++ b/common/seaf-utils.c @@ -380,7 +380,7 @@ load_ccnet_database_config (SeafileSession *session) return ret; } -static int +static char * parse_seahub_db_config () { char buf[1024]; @@ -402,15 +402,21 @@ parse_seahub_db_config () &error); if (error != NULL) { - seaf_warning ("Failed to run python parse_seahub_db.py : %s\n", error->message); - return -1; + seaf_warning ("Failed to run python parse_seahub_db.py: %s\n", error->message); + g_free (binary_path); + g_clear_error (&error); + return NULL; } - if (retcode != 0) { - seaf_warning ("Failed to run python parse_seahub_db.py [%d]: %s\n", retcode, strerror(errno)); - return -1; + g_spawn_check_exit_status (retcode, &error); + if (error != NULL) { + seaf_warning ("Failed to run python parse_seahub_db.py: %s\n", error->message); + g_free (binary_path); + g_clear_error (&error); + return NULL; } - return 0; + g_free (binary_path); + return g_strdup(child_stdout); } int @@ -418,21 +424,21 @@ load_seahub_database_config (SeafileSession *session) { int ret = 0; json_t *object = NULL; + json_error_t err; const char *engine = NULL, *name = NULL, *user = NULL, *password = NULL, *host = NULL, *charset = NULL; int port; + char *json_str = NULL; - - char *json_file = g_build_filename ("/tmp", "seahub_db.json", NULL); - - if (parse_seahub_db_config () < 0) { + json_str = parse_seahub_db_config (); + if (!json_str){ seaf_warning ("Failed to parse seahub database config.\n"); ret = -1; goto out; } - object = json_load_file (json_file, 0, NULL); + object = json_loadb (json_str, strlen(json_str), 0, &err); if (!object) { - seaf_warning ("Failed to load seahub db json file: %s\n", json_file); + seaf_warning ("Failed to load seahub db json: %s: %s\n", json_str, err.text); ret = -1; goto out; } @@ -449,17 +455,32 @@ load_seahub_database_config (SeafileSession *session) } if (!engine || strstr (engine, "sqlite") != NULL) { - seaf_message ("Use database sqlite\n"); - session->seahub_db = seaf_db_new_sqlite (name, DEFAULT_MAX_CONNECTIONS); - if (!session->seahub_db) { - seaf_warning ("Failed to open seahub database.\n"); - ret = -1; - goto out; - } + goto out; } #ifdef HAVE_MYSQL else if (strstr (engine, "mysql") != NULL) { seaf_message("Use database Mysql\n"); + if (!host) { + seaf_warning ("Seahub DB host not set in config.\n"); + ret = -1; + goto out; + } + if (!user) { + seaf_warning ("Seahub DB user not set in config.\n"); + ret = -1; + goto out; + } + if (!password) { + seaf_warning ("Seahub DB password not set in config.\n"); + ret = -1; + goto out; + } + if (!name) { + seaf_warning ("Seahub DB name not set in config.\n"); + ret = -1; + goto out; + } + session->seahub_db = seaf_db_new_mysql (host, port, user, password, name, NULL, FALSE, FALSE, NULL, charset, DEFAULT_MAX_CONNECTIONS); if (!session->seahub_db) { seaf_warning ("Failed to open seahub database.\n"); @@ -476,6 +497,6 @@ load_seahub_database_config (SeafileSession *session) out: if (object) json_decref (object); - g_free (json_file); + g_free (json_str); return ret; } diff --git a/fileserver/fileop.go b/fileserver/fileop.go index 2162b479..8e5e9623 100644 --- a/fileserver/fileop.go +++ b/fileserver/fileop.go @@ -1770,7 +1770,6 @@ func genCommitNeedRetry(repo *repomgr.Repo, base *commitmgr.Commit, commit *comm opt := new(mergeOptions) opt.remoteRepoID = repoID opt.remoteHead = commit.CommitID - opt.emailToNickname = make(map[string]string) err := mergeTrees(repo.StoreID, roots, opt) if err != nil { diff --git a/fileserver/fileserver.go b/fileserver/fileserver.go index c5537ae7..fa4c169f 100644 --- a/fileserver/fileserver.go +++ b/fileserver/fileserver.go @@ -12,6 +12,7 @@ import ( "io/ioutil" "net/http" "os" + "os/exec" "os/signal" "path/filepath" "runtime/debug" @@ -60,7 +61,6 @@ func init() { const ( timestampFormat = "[2006-01-02 15:04:05] " - seahubDBPath = "/tmp/seahub_db.json" ) type LogFormatter struct{} @@ -273,9 +273,15 @@ func loadSeafileDB() { } func loadSeahubDB() { - dbData, err := ioutil.ReadFile(seahubDBPath) + scriptPath, err := exec.LookPath("parse_seahub_db.py") if err != nil { - log.Warnf("Failed to read seahub database json file: %v", err) + log.Warnf("Failed to find script of parse_seahub_db.py: %v", err) + return + } + cmd := exec.Command("python3", scriptPath) + dbData, err := cmd.CombinedOutput() + if err != nil { + log.Warnf("Failed to run python parse_seahub_db.py: %v", err) return } @@ -299,6 +305,23 @@ func loadSeahubDB() { if err != nil || port <= 0 { port = 3306 } + if dbName == "" { + log.Warnf("Seahub DB name not set in config") + return + } + if user == "" { + log.Warnf("Seahub DB user not set in config") + return + } + if password == "" { + log.Warnf("Seahub DB password not set in config") + return + } + if host == "" { + log.Warnf("Seahub DB host not set in config") + return + } + dsn := fmt.Sprintf("%s:%s@tcp(%s:%d)/%s?tls=%t", user, password, host, port, dbName, false) seahubDB, err = sql.Open("mysql", dsn) @@ -306,10 +329,7 @@ func loadSeahubDB() { log.Warnf("Failed to open database: %v", err) } } else if strings.Index(dbEngine, "sqlite") >= 0 { - seahubDB, err = sql.Open("sqlite3", dbName) - if err != nil { - log.Warnf("Failed to open database %s: %v", dbName, err) - } + return } else { log.Warnf("Unsupported database %s.", dbEngine) } diff --git a/fileserver/merge.go b/fileserver/merge.go index 2e1d1d31..1444eed8 100644 --- a/fileserver/merge.go +++ b/fileserver/merge.go @@ -25,6 +25,8 @@ func mergeTrees(storeID string, roots []string, opt *mergeOptions) error { return err } + opt.emailToNickname = make(map[string]string) + var trees []*fsmgr.SeafDir for i := 0; i < 3; i++ { dir, err := fsmgr.GetSeafdir(storeID, roots[i]) diff --git a/fileserver/merge_test.go b/fileserver/merge_test.go index d54faf7a..07a24bf2 100644 --- a/fileserver/merge_test.go +++ b/fileserver/merge_test.go @@ -254,7 +254,6 @@ func testMergeTrees1(t *testing.T) { opt := new(mergeOptions) opt.remoteRepoID = mergeTestRepoID opt.remoteHead = commit.CommitID - opt.emailToNickname = make(map[string]string) err = mergeTrees(mergeTestRepoID, roots, opt) if err != nil { @@ -276,7 +275,6 @@ func testMergeTrees2(t *testing.T) { opt := new(mergeOptions) opt.remoteRepoID = mergeTestRepoID opt.remoteHead = commit.CommitID - opt.emailToNickname = make(map[string]string) err = mergeTrees(mergeTestRepoID, roots, opt) if err != nil { @@ -298,7 +296,6 @@ func testMergeTrees3(t *testing.T) { opt := new(mergeOptions) opt.remoteRepoID = mergeTestRepoID opt.remoteHead = commit.CommitID - opt.emailToNickname = make(map[string]string) err = mergeTrees(mergeTestRepoID, roots, opt) if err != nil { @@ -320,7 +317,6 @@ func testMergeTrees4(t *testing.T) { opt := new(mergeOptions) opt.remoteRepoID = mergeTestRepoID opt.remoteHead = commit.CommitID - opt.emailToNickname = make(map[string]string) err = mergeTrees(mergeTestRepoID, roots, opt) if err != nil { @@ -342,7 +338,6 @@ func testMergeTrees5(t *testing.T) { opt := new(mergeOptions) opt.remoteRepoID = mergeTestRepoID opt.remoteHead = commit.CommitID - opt.emailToNickname = make(map[string]string) err = mergeTrees(mergeTestRepoID, roots, opt) if err != nil { @@ -363,7 +358,6 @@ func testMergeTrees6(t *testing.T) { opt := new(mergeOptions) opt.remoteRepoID = mergeTestRepoID opt.remoteHead = commit.CommitID - opt.emailToNickname = make(map[string]string) err = mergeTrees(mergeTestRepoID, roots, opt) if err != nil { @@ -384,7 +378,6 @@ func testMergeTrees7(t *testing.T) { opt := new(mergeOptions) opt.remoteRepoID = mergeTestRepoID opt.remoteHead = commit.CommitID - opt.emailToNickname = make(map[string]string) err = mergeTrees(mergeTestRepoID, roots, opt) if err != nil { @@ -405,7 +398,6 @@ func testMergeTrees8(t *testing.T) { opt := new(mergeOptions) opt.remoteRepoID = mergeTestRepoID opt.remoteHead = commit.CommitID - opt.emailToNickname = make(map[string]string) err = mergeTrees(mergeTestRepoID, roots, opt) if err != nil { @@ -426,7 +418,6 @@ func testMergeTrees9(t *testing.T) { opt := new(mergeOptions) opt.remoteRepoID = mergeTestRepoID opt.remoteHead = commit.CommitID - opt.emailToNickname = make(map[string]string) err = mergeTrees(mergeTestRepoID, roots, opt) if err != nil { @@ -447,7 +438,6 @@ func testMergeTrees10(t *testing.T) { opt := new(mergeOptions) opt.remoteRepoID = mergeTestRepoID opt.remoteHead = commit.CommitID - opt.emailToNickname = make(map[string]string) err = mergeTrees(mergeTestRepoID, roots, opt) if err != nil { @@ -468,7 +458,6 @@ func testMergeTrees11(t *testing.T) { opt := new(mergeOptions) opt.remoteRepoID = mergeTestRepoID opt.remoteHead = commit.CommitID - opt.emailToNickname = make(map[string]string) err = mergeTrees(mergeTestRepoID, roots, opt) if err != nil { @@ -489,7 +478,6 @@ func testMergeTrees12(t *testing.T) { opt := new(mergeOptions) opt.remoteRepoID = mergeTestRepoID opt.remoteHead = commit.CommitID - opt.emailToNickname = make(map[string]string) err = mergeTrees(mergeTestRepoID, roots, opt) if err != nil { diff --git a/fileserver/virtual_repo.go b/fileserver/virtual_repo.go index 2f673d8a..fb6be165 100644 --- a/fileserver/virtual_repo.go +++ b/fileserver/virtual_repo.go @@ -175,7 +175,6 @@ func mergeRepo(repoID string) error { opt := new(mergeOptions) opt.remoteRepoID = repoID opt.remoteHead = head.CommitID - opt.emailToNickname = make(map[string]string) err := mergeTrees(origRepo.StoreID, roots, opt) if err != nil { diff --git a/scripts/parse_seahub_db.py b/scripts/parse_seahub_db.py index 0ac97fbc..b20d9648 100644 --- a/scripts/parse_seahub_db.py +++ b/scripts/parse_seahub_db.py @@ -3,6 +3,4 @@ db_infos = seahub_settings.DATABASES['default'] -with open('/tmp/seahub_db.json', 'w') as f: - json.dump(db_infos, f, indent=4) - +print(json.dumps(db_infos)) diff --git a/server/http-server.c b/server/http-server.c index f96fe0bb..77c137b6 100644 --- a/server/http-server.c +++ b/server/http-server.c @@ -945,10 +945,6 @@ fast_forward_or_merge (const char *repo_id, memcpy (opt.remote_repo_id, repo_id, 36); memcpy (opt.remote_head, new_commit->commit_id, 40); opt.do_merge = TRUE; - opt.email_to_nickname = g_hash_table_new_full(g_str_hash, - g_str_equal, - g_free, - g_free); roots[0] = base->root_id; /* base */ roots[1] = current_head->root_id; /* head */ @@ -956,11 +952,9 @@ fast_forward_or_merge (const char *repo_id, if (seaf_merge_trees (repo->store_id, repo->version, 3, roots, &opt) < 0) { seaf_warning ("Failed to merge.\n"); - g_hash_table_destroy (opt.email_to_nickname); ret = -1; goto out; } - g_hash_table_destroy (opt.email_to_nickname); if (!opt.conflict) desc = g_strdup("Auto merge by system"); diff --git a/server/repo-op.c b/server/repo-op.c index 4d4c0056..6c89c35d 100644 --- a/server/repo-op.c +++ b/server/repo-op.c @@ -511,10 +511,6 @@ gen_new_commit (const char *repo_id, memcpy (opt.remote_repo_id, repo_id, 36); memcpy (opt.remote_head, new_commit->commit_id, 40); opt.do_merge = TRUE; - opt.email_to_nickname = g_hash_table_new_full(g_str_hash, - g_str_equal, - g_free, - g_free); roots[0] = base->root_id; /* base */ roots[1] = current_head->root_id; /* head */ @@ -524,11 +520,9 @@ gen_new_commit (const char *repo_id, seaf_warning ("Failed to merge.\n"); g_set_error (error, SEAFILE_DOMAIN, SEAF_ERR_GENERAL, "Internal error"); - g_hash_table_destroy (opt.email_to_nickname); ret = -1; goto out; } - g_hash_table_destroy (opt.email_to_nickname); seaf_debug ("Number of dirs visted in merge %.8s: %d.\n", repo_id, opt.visit_dirs); diff --git a/server/virtual-repo.c b/server/virtual-repo.c index e2ee3c8e..2698c2de 100644 --- a/server/virtual-repo.c +++ b/server/virtual-repo.c @@ -952,10 +952,6 @@ static void *merge_virtual_repo (void *vtask) memcpy (opt.remote_repo_id, repo_id, 36); memcpy (opt.remote_head, head->commit_id, 40); opt.do_merge = TRUE; - opt.email_to_nickname = g_hash_table_new_full(g_str_hash, - g_str_equal, - g_free, - g_free); roots[0] = base_root; /* base */ roots[1] = orig_root; /* head */ @@ -965,11 +961,9 @@ static void *merge_virtual_repo (void *vtask) if (seaf_merge_trees (orig_repo->store_id, orig_repo->version, 3, roots, &opt) < 0) { seaf_warning ("Failed to merge virtual repo %.10s.\n", repo_id); - g_hash_table_destroy (opt.email_to_nickname); ret = -1; goto out; } - g_hash_table_destroy (opt.email_to_nickname); seaf_debug ("Number of dirs visted in merge: %d.\n", opt.visit_dirs); @@ -1222,10 +1216,6 @@ seaf_repo_manager_repair_virtual_repo (char *repo_id) opt.n_ways = 2; memcpy (opt.remote_repo_id, repo_id, 36); memcpy (opt.remote_head, head->commit_id, 40); - opt.email_to_nickname = g_hash_table_new_full(g_str_hash, - g_str_equal, - g_free, - g_free); roots[0] = orig_root; roots[1] = root; @@ -1234,11 +1224,9 @@ seaf_repo_manager_repair_virtual_repo (char *repo_id) if (seaf_merge_trees (orig_repo->store_id, orig_repo->version, 2, roots, &opt) < 0) { seaf_warning ("Failed to merge virtual repo %.10s.\n", repo_id); - g_hash_table_destroy (opt.email_to_nickname); ret = -1; goto out; } - g_hash_table_destroy (opt.email_to_nickname); seaf_debug ("Number of dirs visted in merge: %d.\n", opt.visit_dirs); From 80ada6cbabf6b8e067a69fb2c009069eb50c3ba1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=9D=A8=E8=B5=AB=E7=84=B6?= Date: Wed, 10 Jul 2024 16:22:09 +0800 Subject: [PATCH 5/5] Add exec permission and free child_output --- common/seaf-utils.c | 7 ++++++- scripts/parse_seahub_db.py | 0 2 files changed, 6 insertions(+), 1 deletion(-) mode change 100644 => 100755 scripts/parse_seahub_db.py diff --git a/common/seaf-utils.c b/common/seaf-utils.c index 5c7e6e9f..fd2f9814 100644 --- a/common/seaf-utils.c +++ b/common/seaf-utils.c @@ -404,6 +404,8 @@ parse_seahub_db_config () if (error != NULL) { seaf_warning ("Failed to run python parse_seahub_db.py: %s\n", error->message); g_free (binary_path); + g_free (child_stdout); + g_free (child_stderr); g_clear_error (&error); return NULL; } @@ -411,12 +413,15 @@ parse_seahub_db_config () if (error != NULL) { seaf_warning ("Failed to run python parse_seahub_db.py: %s\n", error->message); g_free (binary_path); + g_free (child_stdout); + g_free (child_stderr); g_clear_error (&error); return NULL; } g_free (binary_path); - return g_strdup(child_stdout); + g_free (child_stderr); + return child_stdout; } int diff --git a/scripts/parse_seahub_db.py b/scripts/parse_seahub_db.py old mode 100644 new mode 100755