From f59232c336bc3db35481ade0a08f0e7760d85591 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=9D=A8=E8=B5=AB=E7=84=B6?= Date: Sat, 7 Sep 2024 10:28:01 +0800 Subject: [PATCH 1/5] Add batch del files RPC --- common/rpc-service.c | 40 ++++++ include/seafile-rpc.h | 6 + python/seafile/rpcclient.py | 5 + python/seaserv/api.py | 3 + server/repo-mgr.h | 7 + server/repo-op.c | 120 +++++++++++++++++- server/seaf-server.c | 5 + .../test_file_operation.py | 36 ++++-- 8 files changed, 213 insertions(+), 9 deletions(-) diff --git a/common/rpc-service.c b/common/rpc-service.c index cf7e9542..1354685d 100644 --- a/common/rpc-service.c +++ b/common/rpc-service.c @@ -2812,6 +2812,46 @@ seafile_del_file (const char *repo_id, const char *parent_dir, return ret; } +int +seafile_batch_del_files (const char *repo_id, + const char *filepaths, + const char *user, + GError **error) +{ + char *norm_file_list = NULL, *rpath = NULL; + int ret = 0; + + if (!repo_id || !filepaths || !user) { + g_set_error (error, 0, SEAF_ERR_BAD_ARGS, "Argument should not be null"); + return -1; + } + + if (!is_uuid_valid (repo_id)) { + g_set_error (error, SEAFILE_DOMAIN, SEAF_ERR_BAD_ARGS, "Invalid repo id"); + return -1; + } + + + norm_file_list = normalize_utf8_path (filepaths); + if (!norm_file_list) { + g_set_error (error, SEAFILE_DOMAIN, SEAF_ERR_BAD_ARGS, + "Path is in valid UTF8 encoding"); + ret = -1; + goto out; + } + + if (seaf_repo_manager_batch_del_files (seaf->repo_mgr, repo_id, + norm_file_list, + user, error) < 0) { + ret = -1; + } + +out: + g_free (norm_file_list); + + return ret; +} + GObject * seafile_copy_file (const char *src_repo_id, const char *src_dir, diff --git a/include/seafile-rpc.h b/include/seafile-rpc.h index b03d5415..acd55e82 100644 --- a/include/seafile-rpc.h +++ b/include/seafile-rpc.h @@ -754,6 +754,12 @@ seafile_del_file (const char *repo_id, const char *user, GError **error); +int +seafile_batch_del_files (const char *repo_id, + const char *file_list, + const char *user, + GError **error); + /** * copy a file/directory from a repo to another on server. */ diff --git a/python/seafile/rpcclient.py b/python/seafile/rpcclient.py index 531578cf..fc3cd452 100644 --- a/python/seafile/rpcclient.py +++ b/python/seafile/rpcclient.py @@ -126,6 +126,11 @@ def seafile_del_file(repo_id, parent_dir, filename, user): pass del_file = seafile_del_file + @searpc_func("int", ["string", "string", "string"]) + def seafile_batch_del_files(repo_id, filepaths, user): + pass + batch_del_files = seafile_batch_del_files + @searpc_func("object", ["string", "string", "string", "string", "string", "string", "string", "int", "int"]) def seafile_copy_file(src_repo, src_dir, src_filename, dst_repo, dst_dir, dst_filename, user, need_progress, synchronous): pass diff --git a/python/seaserv/api.py b/python/seaserv/api.py index d4b7270f..b8fb770c 100644 --- a/python/seaserv/api.py +++ b/python/seaserv/api.py @@ -300,6 +300,9 @@ def put_file(self, repo_id, tmp_file_path, parent_dir, filename, def del_file(self, repo_id, parent_dir, filename, username): return seafserv_threaded_rpc.del_file(repo_id, parent_dir, filename, username) + def batch_del_files(self, repo_id, filepaths, username): + return seafserv_threaded_rpc.batch_del_files(repo_id, filepaths, username) + ''' If you want to move or copy multiple files in a batch, @src_filename and @dst_filename should be json array, make sure the number of files diff --git a/server/repo-mgr.h b/server/repo-mgr.h index 70b8577d..3c2ae0ad 100644 --- a/server/repo-mgr.h +++ b/server/repo-mgr.h @@ -432,6 +432,13 @@ seaf_repo_manager_del_file (SeafRepoManager *mgr, const char *user, GError **error); +int +seaf_repo_manager_batch_del_files (SeafRepoManager *mgr, + const char *repo_id, + const char *file_list, + const char *user, + GError **error); + SeafileCopyResult * seaf_repo_manager_copy_file (SeafRepoManager *mgr, const char *src_repo_id, diff --git a/server/repo-op.c b/server/repo-op.c index 20c134bf..4f30471d 100644 --- a/server/repo-op.c +++ b/server/repo-op.c @@ -1735,7 +1735,7 @@ del_file_recursive(SeafRepo *repo, out: if (p_deleted_num) - *p_deleted_num = deleted_num; + *p_deleted_num += deleted_num; g_free (to_path_dup); g_free (id); @@ -1844,6 +1844,124 @@ seaf_repo_manager_del_file (SeafRepoManager *mgr, return ret; } +static char * +do_batch_del_files (SeafRepo *repo, + const char *root_id, + const char *file_list, + int *mode, int *deleted_num, char **desc_file) +{ + char *ret = NULL; + GList *filenames = NULL, *ptr; + char *name; + const char *next_root_id = root_id; + + filenames = json_to_file_list (file_list); + + for (ptr = filenames; ptr; ptr = ptr->next) { + name = ptr->data; + char *base_name = g_path_get_basename (name); + char *parent_dir = g_path_get_dirname (name); + char *canon_path = get_canonical_path (parent_dir); + char *tmp_file_list = g_strdup_printf ("[\"%s\"]", base_name); + + char *new_root_id = do_del_file (repo, next_root_id, canon_path, tmp_file_list, mode, deleted_num, desc_file); + if (new_root_id) { + g_free (ret); + ret = g_strdup (new_root_id); + g_free (new_root_id); + next_root_id = ret; + } + g_free (base_name); + g_free (parent_dir); + g_free (canon_path); + g_free (tmp_file_list); + } + + string_list_free (filenames); + + return ret; +} + +int +seaf_repo_manager_batch_del_files (SeafRepoManager *mgr, + const char *repo_id, + const char *file_list, + const char *user, + GError **error) +{ + SeafRepo *repo = NULL; + SeafCommit *head_commit = NULL; + SeafDir *dir = NULL; + char buf[SEAF_PATH_MAX]; + char *root_id = NULL; + char *desc_file = NULL; + int mode = 0; + int ret = 0; + int deleted_num = 0; + + GET_REPO_OR_FAIL(repo, repo_id); + GET_COMMIT_OR_FAIL(head_commit, repo->id, repo->version, repo->head->commit_id); + + dir = seaf_fs_manager_get_seafdir (seaf->fs_mgr, + repo->store_id, repo->version, + head_commit->root_id); + if (!dir) { + seaf_warning ("root dir doesn't exist in repo %s.\n", + repo->store_id); + ret = -1; + goto out; + } + + root_id = do_batch_del_files (repo, + head_commit->root_id, file_list, &mode, + &deleted_num, &desc_file); + if (!root_id) { + seaf_warning ("[batch del files] Failed to del files in repo %s.\n", + repo->id); + g_set_error (error, SEAFILE_DOMAIN, SEAF_ERR_GENERAL, + "Failed to batch del files"); + ret = -1; + goto out; + } + if (deleted_num == 0) { + goto out; + } + + /* Commit. */ + if (deleted_num > 1) { + snprintf(buf, SEAF_PATH_MAX, "Deleted \"%s\" and %d more files", + desc_file, deleted_num - 1); + } else if (S_ISDIR(mode)) { + snprintf(buf, SEAF_PATH_MAX, "Removed directory \"%s\"", desc_file); + } else { + snprintf(buf, SEAF_PATH_MAX, "Deleted \"%s\"", desc_file); + } + + if (gen_new_commit (repo_id, head_commit, root_id, + user, buf, NULL, TRUE, error) < 0) { + ret = -1; + goto out; + } + + seaf_repo_manager_merge_virtual_repo (mgr, repo_id, NULL); + +out: + if (repo) + seaf_repo_unref (repo); + if (head_commit) + seaf_commit_unref(head_commit); + if (dir) + seaf_dir_free (dir); + g_free (root_id); + g_free (desc_file); + + if (ret == 0) { + update_repo_size (repo_id); + } + + return ret; +} + static SeafDirent * get_dirent_by_path (SeafRepo *repo, const char *root_id, diff --git a/server/seaf-server.c b/server/seaf-server.c index d250b68e..31610f92 100644 --- a/server/seaf-server.c +++ b/server/seaf-server.c @@ -200,6 +200,11 @@ static void start_rpc_service (const char *seafile_dir, "seafile_del_file", searpc_signature_int__string_string_string_string()); + searpc_server_register_function ("seafserv-threaded-rpcserver", + seafile_batch_del_files, + "seafile_batch_del_files", + searpc_signature_int__string_string_string()); + searpc_server_register_function ("seafserv-threaded-rpcserver", seafile_copy_file, "seafile_copy_file", diff --git a/tests/test_file_operation/test_file_operation.py b/tests/test_file_operation/test_file_operation.py index 994abcb4..fd5c6136 100644 --- a/tests/test_file_operation/test_file_operation.py +++ b/tests/test_file_operation/test_file_operation.py @@ -1,6 +1,7 @@ import pytest import os import time +import json from tests.config import USER from seaserv import seafile_api as api @@ -17,7 +18,9 @@ def create_the_file (): with open(file_path, 'w') as fp: fp.write(file_content) -def test_file_operation(): +@pytest.mark.parametrize('in_batch', + [True, False]) +def test_file_operation(in_batch): t_repo_version = 1 t_repo_id1 = api.create_repo('test_file_operation1', '', USER, passwd = None) @@ -130,7 +133,10 @@ def test_file_operation(): assert t_commit_list[0].creator_name == USER # test del_file - assert api.del_file(t_repo_id2, '/', '[\"'+file_name+'\"]', USER) == 0 + if in_batch: + assert api.batch_del_files(t_repo_id2, '[\"'+'/'+file_name+'\"]', USER) == 0 + else: + assert api.del_file(t_repo_id2, '/', '[\"'+file_name+'\"]', USER) == 0 # test get_deleted t_deleted_file_list = api.get_deleted(t_repo_id2, 1) @@ -140,12 +146,26 @@ def test_file_operation(): assert t_deleted_file_list[0].basedir == '/' # test del a non-exist file. should return 0. - assert api.del_file(t_repo_id2, '/', '[\"'+file_name+'\"]', USER) == 0 - - assert api.del_file(t_repo_id1, '/' + dir_name, '[\"'+new_empty_file_name+'\"]', USER) == 0 - assert api.del_file(t_repo_id1, '/' + dir_name, '[\"'+new_file_name+'\"]', USER) == 0 - assert api.del_file(t_repo_id2, '/', '[\"'+new_file_name+'\"]', USER) == 0 - assert api.del_file(t_repo_id1, '/', '[\"'+new_file_name_2+'\"]', USER) == 0 + if in_batch: + file_list = ["/"+file_name, "/"+new_file_name] + assert api.batch_del_files(t_repo_id2, json.dumps(file_list), USER) == 0 + t_deleted_file_list = api.get_deleted(t_repo_id2, 1) + assert t_deleted_file_list + assert len(t_deleted_file_list) == 3 + + file_list = ["/"+dir_name+"/"+new_empty_file_name, "/"+dir_name+"/"+new_file_name, "/"+new_file_name_2] + assert api.batch_del_files(t_repo_id1, json.dumps(file_list), USER) == 0 + t_deleted_file_list = api.get_deleted(t_repo_id1, 1) + assert t_deleted_file_list + assert len(t_deleted_file_list) == 4 + else: + assert api.del_file(t_repo_id2, '/', '[\"'+file_name+'\"]', USER) == 0 + + assert api.del_file(t_repo_id1, '/' + dir_name, '[\"'+new_empty_file_name+'\"]', USER) == 0 + assert api.del_file(t_repo_id1, '/' + dir_name, '[\"'+new_file_name+'\"]', USER) == 0 + assert api.del_file(t_repo_id2, '/', '[\"'+new_file_name+'\"]', USER) == 0 + assert api.del_file(t_repo_id1, '/', '[\"'+new_file_name_2+'\"]', USER) == 0 time.sleep(1) api.remove_repo(t_repo_id1) + api.remove_repo(t_repo_id2) From dfd6ca7d79110b5002fff2a8d7a4d352d7f1b44d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=9D=A8=E8=B5=AB=E7=84=B6?= Date: Sat, 7 Sep 2024 14:49:34 +0800 Subject: [PATCH 2/5] Add pack multi-level files --- server/zip-download-mgr.c | 46 ++++++++++++++----- .../test_file_operation/test_zip_download.py | 31 +++++++++++++ 2 files changed, 66 insertions(+), 11 deletions(-) diff --git a/server/zip-download-mgr.c b/server/zip-download-mgr.c index ba821f47..f89846f1 100644 --- a/server/zip-download-mgr.c +++ b/server/zip-download-mgr.c @@ -330,7 +330,7 @@ parse_download_multi_data (DownloadObj *obj, const char *data) for (i = 0; i < len; i++) { file_name = json_string_value (json_array_get (name_array, i)); - if (strcmp (file_name, "") == 0 || strchr (file_name, '/') != NULL) { + if (strcmp (file_name, "") == 0) { seaf_warning ("Invalid download file name: %s.\n", file_name); if (dirent_list) { g_list_free_full (dirent_list, (GDestroyNotify)seaf_dirent_free); @@ -339,18 +339,42 @@ parse_download_multi_data (DownloadObj *obj, const char *data) break; } - dirent = g_hash_table_lookup (dirent_hash, file_name); - if (!dirent) { - seaf_warning ("Failed to get dirent for %s in dir %s in repo %.8s.\n", - file_name, parent_dir, repo->store_id); - if (dirent_list) { - g_list_free_full (dirent_list, (GDestroyNotify)seaf_dirent_free); - dirent_list = NULL; + // Packing files in multi-level directories. + if (strchr (file_name, '/') != NULL) { + char *fullpath = g_build_path ("/", parent_dir, file_name, NULL); + dirent = seaf_fs_manager_get_dirent_by_path (seaf->fs_mgr, repo->store_id, repo->version, repo->root_id, fullpath, &error); + if (!dirent) { + if (error) { + seaf_warning ("Failed to get path %s repo %.8s: %s.\n", + fullpath, repo->store_id, error->message); + g_clear_error(&error); + } else { + seaf_warning ("Path %s doesn't exist in repo %.8s.\n", + parent_dir, repo->store_id); + } + if (dirent_list) { + g_list_free_full (dirent_list, (GDestroyNotify)seaf_dirent_free); + dirent_list = NULL; + } + g_free (fullpath); + break; + } + g_free (fullpath); + dirent_list = g_list_prepend (dirent_list, dirent); + } else { + dirent = g_hash_table_lookup (dirent_hash, file_name); + if (!dirent) { + seaf_warning ("Failed to get dirent for %s in dir %s in repo %.8s.\n", + file_name, parent_dir, repo->store_id); + if (dirent_list) { + g_list_free_full (dirent_list, (GDestroyNotify)seaf_dirent_free); + dirent_list = NULL; + } + break; } - break; - } - dirent_list = g_list_prepend (dirent_list, seaf_dirent_dup(dirent)); + dirent_list = g_list_prepend (dirent_list, seaf_dirent_dup(dirent)); + } } g_hash_table_unref(dirent_hash); diff --git a/tests/test_file_operation/test_zip_download.py b/tests/test_file_operation/test_zip_download.py index 3082379a..90d130b1 100644 --- a/tests/test_file_operation/test_zip_download.py +++ b/tests/test_file_operation/test_zip_download.py @@ -131,6 +131,37 @@ def test_zip_download(): os.remove(download_dir_path + '/file2.txt') os.remove(download_dir_path + '/multi_files.zip') + #test zip download mutliple files in multi-level + api.post_file(t_repo_id, file2_path, '/dir', file2_name, USER) + obj_id = {'parent_dir': '/', 'file_list': [file1_name, 'dir/'+file2_name], 'is_windows' : 0} + obj_id_json_str = json.dumps(obj_id) + token = api.get_fileserver_access_token(t_repo_id, obj_id_json_str, + 'download-multi', USER) + + time.sleep(1) + download_url = base_url + 'zip/' + token + response = requests.get(download_url) + assert response.status_code == 200 + + download_zipfile_path = download_dir_path + '/multi_files.zip' + with open(download_zipfile_path, 'wb') as fp: + fp.write(response.content) + zipFile = zipfile.ZipFile(download_zipfile_path) + for name in zipFile.namelist(): + zipFile.extract(name, download_dir_path) + zipFile.close() + assert os.path.exists(download_dir_path + '/file1.txt') + assert os.path.exists(download_dir_path + '/file2.txt') + with open(download_dir_path + '/file1.txt', 'r') as fp1: + line = fp1.read() + assert line == file1_content + with open(download_dir_path + '/file2.txt', 'r') as fp2: + line = fp2.read() + assert line == file2_content + os.remove(download_dir_path + '/file1.txt') + os.remove(download_dir_path + '/file2.txt') + os.remove(download_dir_path + '/multi_files.zip') + remove_test_files() api.remove_repo(t_repo_id) From ee80867b887491cef985e36be3dceeabcbe21934 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=9D=A8=E8=B5=AB=E7=84=B6?= Date: Sat, 7 Sep 2024 15:42:01 +0800 Subject: [PATCH 3/5] Go add pack multi-level files --- fileserver/fileop.go | 24 +++++++++++++++++++----- fileserver/fsmgr/fsmgr.go | 28 ++++++++++++++++++++++++++++ 2 files changed, 47 insertions(+), 5 deletions(-) diff --git a/fileserver/fileop.go b/fileserver/fileop.go index a30ef171..be83f60b 100644 --- a/fileserver/fileop.go +++ b/fileserver/fileop.go @@ -802,14 +802,28 @@ func parseDirFilelist(repo *repomgr.Repo, obj map[string]interface{}) ([]fsmgr.S err := fmt.Errorf("invalid download multi data") return nil, err } - - v, ok := direntHash[name] - if !ok { - err := fmt.Errorf("invalid download multi data") + if name == "" { + err := fmt.Errorf("invalid download file name") return nil, err } - direntList = append(direntList, v) + if strings.Contains(name, "/") { + rpath := filepath.Join(parentDir, name) + dent, err := fsmgr.GetDirentByPath(repo.StoreID, repo.RootID, rpath) + if err != nil { + err := fmt.Errorf("failed to get path %s for repo %s: %v", rpath, repo.StoreID, err) + return nil, err + } + direntList = append(direntList, *dent) + } else { + v, ok := direntHash[name] + if !ok { + err := fmt.Errorf("invalid download multi data") + return nil, err + } + + direntList = append(direntList, v) + } } return direntList, nil diff --git a/fileserver/fsmgr/fsmgr.go b/fileserver/fsmgr/fsmgr.go index 180c087f..dd348c7b 100644 --- a/fileserver/fsmgr/fsmgr.go +++ b/fileserver/fsmgr/fsmgr.go @@ -908,3 +908,31 @@ func getFileCountInfo(repoID, dirID string) (*FileCountInfo, error) { return info, nil } + +func GetDirentByPath(repoID, rootID, rpath string) (*SeafDirent, error) { + parentDir := filepath.Dir(rpath) + fileName := filepath.Base(rpath) + + var dir *SeafDir + var err error + + if parentDir == "." { + dir, err = GetSeafdir(repoID, rootID) + if err != nil { + return nil, err + } + } else { + dir, err = GetSeafdirByPath(repoID, rootID, parentDir) + if err != nil { + return nil, err + } + } + + for _, de := range dir.Entries { + if de.Name == fileName { + return de, nil + } + } + + return nil, fmt.Errorf("failed to get dirent %s", rpath) +} From 482c075951b14e789c28ccf1fe27506d0b97b5b7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=9D=A8=E8=B5=AB=E7=84=B6?= Date: Wed, 11 Sep 2024 15:24:50 +0800 Subject: [PATCH 4/5] Use change set to batch del files --- fileserver/fsmgr/fsmgr.go | 2 +- server/Makefile.am | 4 +- server/change-set.c | 381 ++++++++++++++++++++++++++++++++++++++ server/change-set.h | 37 ++++ server/repo-op.c | 71 ++++--- 5 files changed, 463 insertions(+), 32 deletions(-) create mode 100644 server/change-set.c create mode 100644 server/change-set.h diff --git a/fileserver/fsmgr/fsmgr.go b/fileserver/fsmgr/fsmgr.go index dd348c7b..eac9818d 100644 --- a/fileserver/fsmgr/fsmgr.go +++ b/fileserver/fsmgr/fsmgr.go @@ -934,5 +934,5 @@ func GetDirentByPath(repoID, rootID, rpath string) (*SeafDirent, error) { } } - return nil, fmt.Errorf("failed to get dirent %s", rpath) + return nil, ErrPathNoExist } diff --git a/server/Makefile.am b/server/Makefile.am index de1f15fe..51cd66db 100644 --- a/server/Makefile.am +++ b/server/Makefile.am @@ -36,7 +36,8 @@ noinst_HEADERS = web-accesstoken-mgr.h seafile-session.h \ ../common/org-mgr.h \ index-blocks-mgr.h \ http-tx-mgr.h \ - notif-mgr.h + notif-mgr.h \ + change-set.h seaf_server_SOURCES = \ seaf-server.c \ @@ -58,6 +59,7 @@ seaf_server_SOURCES = \ fileserver-config.c \ http-tx-mgr.c \ notif-mgr.c \ + change-set.c \ ../common/seaf-db.c \ ../common/branch-mgr.c ../common/fs-mgr.c \ ../common/config-mgr.c \ diff --git a/server/change-set.c b/server/change-set.c new file mode 100644 index 00000000..b053ca10 --- /dev/null +++ b/server/change-set.c @@ -0,0 +1,381 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +#include "common.h" + +#include "seafile-session.h" + +#include "utils.h" +#include "log.h" + +#include "change-set.h" + +struct _ChangeSetDir { + int version; + char dir_id[41]; + /* A hash table of dirents for fast lookup and insertion. */ + GHashTable *dents; + +}; +typedef struct _ChangeSetDir ChangeSetDir; + +struct _ChangeSetDirent { + guint32 mode; + char id[41]; + char *name; + gint64 mtime; + char *modifier; + gint64 size; + /* Only used for directory. Most of time this is NULL + * unless we change the subdir too. + */ + ChangeSetDir *subdir; +}; +typedef struct _ChangeSetDirent ChangeSetDirent; + +/* Change set dirent. */ + +static ChangeSetDirent * +changeset_dirent_new (const char *id, guint32 mode, const char *name, + gint64 mtime, const char *modifier, gint64 size) +{ + ChangeSetDirent *dent = g_new0 (ChangeSetDirent, 1); + + dent->mode = mode; + memcpy (dent->id, id, 40); + dent->name = g_strdup(name); + dent->mtime = mtime; + dent->modifier = g_strdup(modifier); + dent->size = size; + + return dent; +} + +static ChangeSetDirent * +seaf_dirent_to_changeset_dirent (SeafDirent *seaf_dent) +{ + return changeset_dirent_new (seaf_dent->id, seaf_dent->mode, seaf_dent->name, + seaf_dent->mtime, seaf_dent->modifier, seaf_dent->size); +} + +static SeafDirent * +changeset_dirent_to_seaf_dirent (int version, ChangeSetDirent *dent) +{ + return seaf_dirent_new (version, dent->id, dent->mode, dent->name, + dent->mtime, dent->modifier, dent->size); +} + +static void +changeset_dir_free (ChangeSetDir *dir); + +static void +changeset_dirent_free (ChangeSetDirent *dent) +{ + if (!dent) + return; + + g_free (dent->name); + g_free (dent->modifier); + /* Recursively free subdir. */ + if (dent->subdir) + changeset_dir_free (dent->subdir); + g_free (dent); +} + +/* Change set dir. */ + +static void +add_dent_to_dir (ChangeSetDir *dir, ChangeSetDirent *dent) +{ + g_hash_table_insert (dir->dents, + g_strdup(dent->name), + dent); +} + +static void +remove_dent_from_dir (ChangeSetDir *dir, const char *dname) +{ + char *key; + + if (g_hash_table_lookup_extended (dir->dents, dname, + (gpointer*)&key, NULL)) { + g_hash_table_steal (dir->dents, dname); + g_free (key); + } +} + +static ChangeSetDir * +changeset_dir_new (int version, const char *id, GList *dirents) +{ + ChangeSetDir *dir = g_new0 (ChangeSetDir, 1); + GList *ptr; + SeafDirent *dent; + ChangeSetDirent *changeset_dent; + + dir->version = version; + if (id) + memcpy (dir->dir_id, id, 40); + dir->dents = g_hash_table_new_full (g_str_hash, g_str_equal, + g_free, (GDestroyNotify)changeset_dirent_free); + for (ptr = dirents; ptr; ptr = ptr->next) { + dent = ptr->data; + changeset_dent = seaf_dirent_to_changeset_dirent(dent); + add_dent_to_dir (dir, changeset_dent); + } + + return dir; +} + +static void +changeset_dir_free (ChangeSetDir *dir) +{ + if (!dir) + return; + g_hash_table_destroy (dir->dents); + g_free (dir); +} + +static ChangeSetDir * +seaf_dir_to_changeset_dir (SeafDir *seaf_dir) +{ + return changeset_dir_new (seaf_dir->version, seaf_dir->dir_id, seaf_dir->entries); +} + +static gint +compare_dents (gconstpointer a, gconstpointer b) +{ + const SeafDirent *denta = a, *dentb = b; + + return strcmp(dentb->name, denta->name); +} + +static SeafDir * +changeset_dir_to_seaf_dir (ChangeSetDir *dir) +{ + GList *dents = NULL, *seaf_dents = NULL; + GList *ptr; + ChangeSetDirent *dent; + SeafDirent *seaf_dent; + SeafDir *seaf_dir; + + dents = g_hash_table_get_values (dir->dents); + for (ptr = dents; ptr; ptr = ptr->next) { + dent = ptr->data; + seaf_dent = changeset_dirent_to_seaf_dirent (dir->version, dent); + seaf_dents = g_list_prepend (seaf_dents, seaf_dent); + } + /* Sort it in descending order. */ + seaf_dents = g_list_sort (seaf_dents, compare_dents); + + /* seaf_dir_new() computes the dir id. */ + seaf_dir = seaf_dir_new (NULL, seaf_dents, dir->version); + + g_list_free (dents); + return seaf_dir; +} + +/* Change set. */ + +ChangeSet * +changeset_new (const char *repo_id, SeafDir *dir) +{ + ChangeSetDir *changeset_dir = NULL; + ChangeSet *changeset = NULL; + + changeset_dir = seaf_dir_to_changeset_dir (dir); + if (!changeset_dir) + goto out; + + changeset = g_new0 (ChangeSet, 1); + memcpy (changeset->repo_id, repo_id, 36); + changeset->tree_root = changeset_dir; + +out: + return changeset; +} + +void +changeset_free (ChangeSet *changeset) +{ + if (!changeset) + return; + + changeset_dir_free (changeset->tree_root); + g_free (changeset); +} + +static ChangeSetDirent * +delete_from_tree (ChangeSet *changeset, + const char *path, + gboolean *parent_empty) +{ + char *repo_id = changeset->repo_id; + ChangeSetDir *root = changeset->tree_root; + char **parts, *dname; + int n, i; + ChangeSetDir *dir; + ChangeSetDirent *dent, *ret = NULL; + ChangeSetDirent *parent_dent = NULL; + SeafDir *seaf_dir; + + *parent_empty = FALSE; + + parts = g_strsplit (path, "/", 0); + n = g_strv_length(parts); + dir = root; + for (i = 0; i < n; i++) { + dname = parts[i]; + + dent = g_hash_table_lookup (dir->dents, dname); + if (!dent) + break; + + if (S_ISDIR(dent->mode)) { + if (i == (n-1)) { + /* Remove from hash table without freeing dent. */ + remove_dent_from_dir (dir, dname); + if (g_hash_table_size (dir->dents) == 0) + *parent_empty = TRUE; + ret = dent; + // update parent dir mtime when delete dirs locally. + if (parent_dent) { + parent_dent->mtime = time (NULL); + } + break; + } + + if (!dent->subdir) { + seaf_dir = seaf_fs_manager_get_seafdir(seaf->fs_mgr, + repo_id, + root->version, + dent->id); + if (!seaf_dir) { + seaf_warning ("Failed to load seafdir %s:%s\n", + repo_id, dent->id); + break; + } + dent->subdir = seaf_dir_to_changeset_dir (seaf_dir); + seaf_dir_free (seaf_dir); + } + dir = dent->subdir; + parent_dent = dent; + } else if (S_ISREG(dent->mode)) { + if (i == (n-1)) { + /* Remove from hash table without freeing dent. */ + remove_dent_from_dir (dir, dname); + if (g_hash_table_size (dir->dents) == 0) + *parent_empty = TRUE; + ret = dent; + // update parent dir mtime when delete files locally. + if (parent_dent) { + parent_dent->mtime = time (NULL); + } + break; + } + } + } + + g_strfreev (parts); + return ret; +} + +static void +remove_from_changeset_recursive (ChangeSet *changeset, + const char *path, + gboolean remove_parent, + const char *top_dir, + int *mode) +{ + ChangeSetDirent *dent; + gboolean parent_empty = FALSE; + + dent = delete_from_tree (changeset, path, &parent_empty); + if (mode && dent) + *mode = dent->mode; + changeset_dirent_free (dent); + + if (remove_parent && parent_empty) { + char *parent = g_strdup(path); + char *slash = strrchr (parent, '/'); + if (slash) { + *slash = '\0'; + if (strlen(parent) >= strlen(top_dir)) { + /* Recursively remove parent dirs. */ + remove_from_changeset_recursive (changeset, + parent, + remove_parent, + top_dir, + mode); + } + } + g_free (parent); + } +} + +void +remove_from_changeset (ChangeSet *changeset, + const char *path, + gboolean remove_parent, + const char *top_dir, + int *mode) +{ + remove_from_changeset_recursive (changeset, path, remove_parent, top_dir, mode); +} + +static char * +commit_tree_recursive (const char *repo_id, ChangeSetDir *dir) +{ + ChangeSetDirent *dent; + GHashTableIter iter; + gpointer key, value; + char *new_id; + SeafDir *seaf_dir; + char *ret = NULL; + + g_hash_table_iter_init (&iter, dir->dents); + while (g_hash_table_iter_next (&iter, &key, &value)) { + dent = value; + if (dent->subdir) { + new_id = commit_tree_recursive (repo_id, dent->subdir); + if (!new_id) + return NULL; + + memcpy (dent->id, new_id, 40); + g_free (new_id); + } + } + + seaf_dir = changeset_dir_to_seaf_dir (dir); + + memcpy (dir->dir_id, seaf_dir->dir_id, 40); + + if (!seaf_fs_manager_object_exists (seaf->fs_mgr, + repo_id, dir->version, + seaf_dir->dir_id)) { + if (seaf_dir_save (seaf->fs_mgr, repo_id, dir->version, seaf_dir) < 0) { + seaf_warning ("Failed to save dir object %s to repo %s.\n", + seaf_dir->dir_id, repo_id); + goto out; + } + } + + ret = g_strdup(seaf_dir->dir_id); + +out: + seaf_dir_free (seaf_dir); + return ret; +} + +/* + * This function does two things: + * - calculate dir id from bottom up; + * - create and save seaf dir objects. + * It returns root dir id of the new commit. + */ +char * +commit_tree_from_changeset (ChangeSet *changeset) +{ + char *root_id = commit_tree_recursive (changeset->repo_id, + changeset->tree_root); + + return root_id; +} diff --git a/server/change-set.h b/server/change-set.h new file mode 100644 index 00000000..a1433b5e --- /dev/null +++ b/server/change-set.h @@ -0,0 +1,37 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +#ifndef SEAF_CHANGE_SET_H +#define SEAF_CHANGE_SET_H + +#include +#include "utils.h" + +struct _ChangeSetDir; + +struct _ChangeSet { + char repo_id[37]; + /* A partial tree for all changed directories. */ + struct _ChangeSetDir *tree_root; +}; +typedef struct _ChangeSet ChangeSet; + +ChangeSet * +changeset_new (const char *repo_id, SeafDir *dir); + +void +changeset_free (ChangeSet *changeset); + +/* + @remove_parent: remove the parent dir when it becomes empty. +*/ +void +remove_from_changeset (ChangeSet *changeset, + const char *path, + gboolean remove_parent, + const char *top_dir, + int *mode); + +char * +commit_tree_from_changeset (ChangeSet *changeset); + +#endif diff --git a/server/repo-op.c b/server/repo-op.c index 4f30471d..4d36aa85 100644 --- a/server/repo-op.c +++ b/server/repo-op.c @@ -21,6 +21,7 @@ #include "seafile-crypt.h" #include "diff-simple.h" #include "merge-new.h" +#include "change-set.h" #include "seaf-db.h" @@ -1844,42 +1845,41 @@ seaf_repo_manager_del_file (SeafRepoManager *mgr, return ret; } -static char * -do_batch_del_files (SeafRepo *repo, - const char *root_id, +void +do_batch_del_files (ChangeSet *changeset, const char *file_list, int *mode, int *deleted_num, char **desc_file) { - char *ret = NULL; GList *filenames = NULL, *ptr; char *name; - const char *next_root_id = root_id; filenames = json_to_file_list (file_list); for (ptr = filenames; ptr; ptr = ptr->next) { name = ptr->data; - char *base_name = g_path_get_basename (name); - char *parent_dir = g_path_get_dirname (name); - char *canon_path = get_canonical_path (parent_dir); - char *tmp_file_list = g_strdup_printf ("[\"%s\"]", base_name); - - char *new_root_id = do_del_file (repo, next_root_id, canon_path, tmp_file_list, mode, deleted_num, desc_file); - if (new_root_id) { - g_free (ret); - ret = g_strdup (new_root_id); - g_free (new_root_id); - next_root_id = ret; + if (!name || g_strcmp0 (name, "") == 0) { + continue; } - g_free (base_name); - g_free (parent_dir); + char *canon_path = get_canonical_path (name); + char *parent_dir = g_path_get_dirname (canon_path); + char *base_name= g_path_get_basename (canon_path); + char *del_path = canon_path; + if (canon_path[0] == '/') { + del_path = canon_path + 1; + } + + remove_from_changeset (changeset, del_path, FALSE, parent_dir, mode); + + (*deleted_num)++; + if (desc_file && *desc_file == NULL) + *desc_file = g_strdup (base_name); + g_free (canon_path); - g_free (tmp_file_list); + g_free (parent_dir); + g_free (base_name); } string_list_free (filenames); - - return ret; } int @@ -1895,6 +1895,7 @@ seaf_repo_manager_batch_del_files (SeafRepoManager *mgr, char buf[SEAF_PATH_MAX]; char *root_id = NULL; char *desc_file = NULL; + ChangeSet *changeset = NULL; int mode = 0; int ret = 0; int deleted_num = 0; @@ -1902,9 +1903,9 @@ seaf_repo_manager_batch_del_files (SeafRepoManager *mgr, GET_REPO_OR_FAIL(repo, repo_id); GET_COMMIT_OR_FAIL(head_commit, repo->id, repo->version, repo->head->commit_id); - dir = seaf_fs_manager_get_seafdir (seaf->fs_mgr, - repo->store_id, repo->version, - head_commit->root_id); + dir = seaf_fs_manager_get_seafdir_sorted (seaf->fs_mgr, + repo->store_id, repo->version, + head_commit->root_id); if (!dir) { seaf_warning ("root dir doesn't exist in repo %s.\n", repo->store_id); @@ -1912,21 +1913,28 @@ seaf_repo_manager_batch_del_files (SeafRepoManager *mgr, goto out; } - root_id = do_batch_del_files (repo, - head_commit->root_id, file_list, &mode, - &deleted_num, &desc_file); - if (!root_id) { - seaf_warning ("[batch del files] Failed to del files in repo %s.\n", - repo->id); + changeset = changeset_new (repo_id, dir); + if (!changeset) { g_set_error (error, SEAFILE_DOMAIN, SEAF_ERR_GENERAL, "Failed to batch del files"); ret = -1; goto out; } + + do_batch_del_files (changeset, file_list, &mode, &deleted_num, &desc_file); + if (deleted_num == 0) { goto out; } + root_id = commit_tree_from_changeset (changeset); + if (!root_id) { + g_set_error (error, SEAFILE_DOMAIN, SEAF_ERR_GENERAL, + "Failed to batch del files"); + ret = -1; + goto out; + } + /* Commit. */ if (deleted_num > 1) { snprintf(buf, SEAF_PATH_MAX, "Deleted \"%s\" and %d more files", @@ -1946,6 +1954,7 @@ seaf_repo_manager_batch_del_files (SeafRepoManager *mgr, seaf_repo_manager_merge_virtual_repo (mgr, repo_id, NULL); out: + changeset_free (changeset); if (repo) seaf_repo_unref (repo); if (head_commit) @@ -6753,6 +6762,7 @@ seaf_repo_manager_get_deleted_entries (SeafRepoManager *mgr, g_hash_table_destroy (entries); seaf_repo_unref (repo); g_free (data.path); + g_free (next_scan_stat); return NULL; } @@ -6774,6 +6784,7 @@ seaf_repo_manager_get_deleted_entries (SeafRepoManager *mgr, seaf_repo_unref (repo); g_free (data.path); + g_free (next_scan_stat); return ret; } From 8f855b39fe865c04c926ede01e173a39ccaec5e9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=9D=A8=E8=B5=AB=E7=84=B6?= Date: Thu, 12 Sep 2024 14:40:29 +0800 Subject: [PATCH 5/5] Improve args --- server/repo-op.c | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/server/repo-op.c b/server/repo-op.c index 4d36aa85..637fba95 100644 --- a/server/repo-op.c +++ b/server/repo-op.c @@ -1736,7 +1736,7 @@ del_file_recursive(SeafRepo *repo, out: if (p_deleted_num) - *p_deleted_num += deleted_num; + *p_deleted_num = deleted_num; g_free (to_path_dup); g_free (id); @@ -1850,36 +1850,34 @@ do_batch_del_files (ChangeSet *changeset, const char *file_list, int *mode, int *deleted_num, char **desc_file) { - GList *filenames = NULL, *ptr; + GList *filepaths = NULL, *ptr; char *name; - filenames = json_to_file_list (file_list); + filepaths = json_to_file_list (file_list); - for (ptr = filenames; ptr; ptr = ptr->next) { + for (ptr = filepaths; ptr; ptr = ptr->next) { name = ptr->data; if (!name || g_strcmp0 (name, "") == 0) { continue; } char *canon_path = get_canonical_path (name); - char *parent_dir = g_path_get_dirname (canon_path); char *base_name= g_path_get_basename (canon_path); char *del_path = canon_path; if (canon_path[0] == '/') { del_path = canon_path + 1; } - remove_from_changeset (changeset, del_path, FALSE, parent_dir, mode); + remove_from_changeset (changeset, del_path, FALSE, NULL, mode); (*deleted_num)++; if (desc_file && *desc_file == NULL) *desc_file = g_strdup (base_name); g_free (canon_path); - g_free (parent_dir); g_free (base_name); } - string_list_free (filenames); + string_list_free (filepaths); } int @@ -1929,6 +1927,7 @@ seaf_repo_manager_batch_del_files (SeafRepoManager *mgr, root_id = commit_tree_from_changeset (changeset); if (!root_id) { + seaf_warning ("Failed to commit changeset for repo %s.\n", repo_id); g_set_error (error, SEAFILE_DOMAIN, SEAF_ERR_GENERAL, "Failed to batch del files"); ret = -1;