From 5ead1217a64100bdb02806dd222af97b004fdd9e Mon Sep 17 00:00:00 2001 From: liyaxing Date: Fri, 12 May 2023 17:40:02 +0800 Subject: [PATCH 1/4] Add filter support for key data backup --- src/Cargo.lock | 1 + .../src/backup/uni_backup_task.rs | 3 + src/component/cyfs-backup/Cargo.toml | 1 + .../cyfs-backup/src/backup/uni_backup_task.rs | 4 +- .../cyfs-backup/src/key_data/backup.rs | 29 +++++----- .../cyfs-backup/src/key_data/key_data.rs | 55 +++++++++++++++++-- .../cyfs-backup/src/key_data/stat.rs | 23 +++++--- .../cyfs-backup/src/key_data/zip_helper.rs | 9 +++ 8 files changed, 96 insertions(+), 29 deletions(-) diff --git a/src/Cargo.lock b/src/Cargo.lock index 1f6d6b78..0dc1dea8 100644 --- a/src/Cargo.lock +++ b/src/Cargo.lock @@ -1568,6 +1568,7 @@ dependencies = [ "cyfs-util", "file-rotate", "futures", + "globset", "hmac 0.12.1", "http-types", "log 0.4.17", diff --git a/src/component/cyfs-backup-lib/src/backup/uni_backup_task.rs b/src/component/cyfs-backup-lib/src/backup/uni_backup_task.rs index f44d0934..b2c32765 100644 --- a/src/component/cyfs-backup-lib/src/backup/uni_backup_task.rs +++ b/src/component/cyfs-backup-lib/src/backup/uni_backup_task.rs @@ -35,4 +35,7 @@ pub struct UniBackupParams { pub password: Option, pub target_file: LocalFileBackupParam, + + // Key data filters in glob format + pub key_data_filters: Vec, } \ No newline at end of file diff --git a/src/component/cyfs-backup/Cargo.toml b/src/component/cyfs-backup/Cargo.toml index c8bd83e6..8a3d7402 100644 --- a/src/component/cyfs-backup/Cargo.toml +++ b/src/component/cyfs-backup/Cargo.toml @@ -47,6 +47,7 @@ tide = "0.16" http-types = "2.12" surf = { version = '2.3', default-features = false, features = ['h1-client-rustls'] } futures = "0.3" +globset = '0.4' [dev-dependencies] rand = "0.8" \ No newline at end of file diff --git a/src/component/cyfs-backup/src/backup/uni_backup_task.rs b/src/component/cyfs-backup/src/backup/uni_backup_task.rs index b9d6c49f..75b4c1c3 100644 --- a/src/component/cyfs-backup/src/backup/uni_backup_task.rs +++ b/src/component/cyfs-backup/src/backup/uni_backup_task.rs @@ -111,7 +111,7 @@ impl UniBackupTask { let uni_stat = UniBackupStat::new(self.noc.clone(), self.ndc.clone()); let uni_stat = uni_stat.stat().await?; - let keydata = KeyDataManager::new_uni(¶ms.isolate); + let keydata = KeyDataManager::new_uni(¶ms.isolate, ¶ms.key_data_filters)?; let keydata_stat = KeyDataBackupStat::new(keydata); let keydata_stat = keydata_stat.stat(); @@ -225,7 +225,7 @@ impl UniBackupTask { } let keydata_meta = { - let keydata = KeyDataManager::new_uni(¶ms.isolate); + let keydata = KeyDataManager::new_uni(¶ms.isolate, ¶ms.key_data_filters)?; let keydata_backup = KeyDataBackupManager::new(keydata, data_writer); keydata_backup.run().await.map_err(|e| { diff --git a/src/component/cyfs-backup/src/key_data/backup.rs b/src/component/cyfs-backup/src/key_data/backup.rs index e5b3ffa3..fd0bc2c4 100644 --- a/src/component/cyfs-backup/src/key_data/backup.rs +++ b/src/component/cyfs-backup/src/key_data/backup.rs @@ -5,27 +5,23 @@ use crate::meta::{KeyDataMeta, KeyDataType}; use cyfs_base::*; use cyfs_util::AsyncReadWithSeek; -use std::path::PathBuf; - pub struct KeyDataBackupManager { - cyfs_root: PathBuf, - list: Vec, + key_data_manager: KeyDataManager, data_writer: BackupDataWriterRef, } impl KeyDataBackupManager { - pub fn new(keydata: KeyDataManager, data_writer: BackupDataWriterRef) -> Self { + pub fn new(key_data_manager: KeyDataManager, data_writer: BackupDataWriterRef) -> Self { Self { - cyfs_root: keydata.cyfs_root, - list: keydata.list, + key_data_manager, data_writer, } } pub async fn run(&self) -> BuckyResult> { - let mut list = Vec::with_capacity(self.list.len()); + let mut list = Vec::with_capacity(self.key_data_manager.list().len()); - for item in &self.list { + for item in self.key_data_manager.list() { let chunk_id = self.backup_data(item).await?; if chunk_id.is_none() { continue; @@ -50,12 +46,17 @@ impl KeyDataBackupManager { } async fn backup_data(&self, data: &KeyData) -> BuckyResult> { - let file = self.cyfs_root.join(&data.local_path); + let file = self.key_data_manager.cyfs_root().join(&data.local_path); if !file.exists() { warn!("target key data not exists! {}", file.display()); return Ok(None); } + if !self.key_data_manager.check_filter(&file) { + warn!("key data will be ignored by filter: {}", file.display()); + return Ok(None); + } + let data = match data.data_type { KeyDataType::File => async_std::fs::read(&file).await.map_err(|e| { let msg = format!( @@ -66,9 +67,11 @@ impl KeyDataBackupManager { error!("{}", msg); BuckyError::new(BuckyErrorCode::IoError, msg) })?, - KeyDataType::Dir => { - ZipHelper::zip_dir_to_buffer(&file, zip::CompressionMethod::Stored)? - } + KeyDataType::Dir => ZipHelper::zip_dir_to_buffer( + &file, + zip::CompressionMethod::Stored, + &self.key_data_manager, + )?, }; let chunk_id = ChunkId::calculate_sync(&data).unwrap(); diff --git a/src/component/cyfs-backup/src/key_data/key_data.rs b/src/component/cyfs-backup/src/key_data/key_data.rs index 0b8057db..9671cbfc 100644 --- a/src/component/cyfs-backup/src/key_data/key_data.rs +++ b/src/component/cyfs-backup/src/key_data/key_data.rs @@ -1,7 +1,8 @@ use crate::meta::KeyDataType; +use cyfs_base::*; use std::borrow::Cow; -use std::path::PathBuf; +use std::path::{PathBuf, Path}; #[derive(Clone, Debug)] pub struct KeyData { @@ -32,19 +33,38 @@ impl KeyData { } pub struct KeyDataManager { - pub cyfs_root: PathBuf, - pub list: Vec, + cyfs_root: PathBuf, + list: Vec, + filter_list: Vec, } impl KeyDataManager { - pub fn new_uni(isolate: &str) -> Self { + pub fn new_uni(isolate: &str, filters: &Vec) -> BuckyResult { + let mut filter_list = vec![]; + for filter in filters { + let glob = globset::GlobBuilder::new(filter) + .case_insensitive(true) + .literal_separator(true) + .build() + .map_err(|e| { + let msg = format!( + "parse key data filter as glob error! token={}, {}", + filter, e + ); + error!("{}", msg); + BuckyError::new(BuckyErrorCode::InvalidFormat, msg) + })?; + + filter_list.push(glob.compile_matcher()); + } + let mut list = vec![]; let data = if isolate.is_empty() { KeyData::new_dir("etc") } else { KeyData::new_dir(format!("etc/{}", isolate)) }; - + list.push(data); let data_dir = if isolate.is_empty() { @@ -78,6 +98,29 @@ impl KeyDataManager { list.push(data); let cyfs_root = cyfs_util::get_cyfs_root_path(); - Self { cyfs_root, list } + let ret = Self { + cyfs_root, + list, + filter_list, + }; + + Ok(ret) + } + + pub fn cyfs_root(&self) -> &Path { + &self.cyfs_root + } + + pub fn list(&self) -> &Vec { + &self.list + } + pub fn check_filter(&self, path: &Path) -> bool { + for filter in &self.filter_list { + if filter.is_match(path) { + return false; + } + } + + true } } diff --git a/src/component/cyfs-backup/src/key_data/stat.rs b/src/component/cyfs-backup/src/key_data/stat.rs index 7f8ba421..ea1535ca 100644 --- a/src/component/cyfs-backup/src/key_data/stat.rs +++ b/src/component/cyfs-backup/src/key_data/stat.rs @@ -1,25 +1,22 @@ use crate::meta::*; use super::key_data::*; -use std::path::PathBuf; pub struct KeyDataBackupStat { - cyfs_root: PathBuf, - list: Vec, + key_data_manager: KeyDataManager } impl KeyDataBackupStat { - pub fn new(keydata: KeyDataManager) -> Self { + pub fn new(key_data_manager: KeyDataManager) -> Self { Self { - cyfs_root: keydata.cyfs_root, - list: keydata.list, + key_data_manager, } } pub fn stat(&self) -> ObjectArchiveDataMeta { let mut result = ObjectArchiveDataMeta::default(); - for item in &self.list { + for item in self.key_data_manager.list() { self.stat_data(&mut result, item); } @@ -27,12 +24,17 @@ impl KeyDataBackupStat { } fn stat_data(&self, result: &mut ObjectArchiveDataMeta, data: &KeyData) { - let file = self.cyfs_root.join(&data.local_path); + let file = self.key_data_manager.cyfs_root().join(&data.local_path); if !file.exists() { warn!("target key data not exists! {}", file.display()); return; } + if !self.key_data_manager.check_filter(&file) { + warn!("key data will be ignored by filter: {}", file.display()); + return; + } + match data.data_type { KeyDataType::File => { result.count += 1; @@ -40,6 +42,11 @@ impl KeyDataBackupStat { KeyDataType::Dir => { let walkdir = walkdir::WalkDir::new(file); for item in walkdir.into_iter().filter_map(|e| e.ok()) { + if !self.key_data_manager.check_filter(&item.path()) { + warn!("key data will be ignored by filter: {}", item.path().display()); + return; + } + if item.path().is_file() { result.count += 1; } diff --git a/src/component/cyfs-backup/src/key_data/zip_helper.rs b/src/component/cyfs-backup/src/key_data/zip_helper.rs index 58ca3850..efb00d7b 100644 --- a/src/component/cyfs-backup/src/key_data/zip_helper.rs +++ b/src/component/cyfs-backup/src/key_data/zip_helper.rs @@ -1,4 +1,5 @@ use cyfs_base::*; +use super::KeyDataManager; use std::fs::File; use std::io::prelude::*; @@ -7,12 +8,14 @@ use std::iter::Iterator; use std::path::Path; use walkdir::{DirEntry, WalkDir}; + pub struct ZipHelper {} impl ZipHelper { pub fn zip_dir_to_buffer( src_dir: &Path, method: zip::CompressionMethod, + key_data_manager: &KeyDataManager, ) -> BuckyResult> { let walkdir = WalkDir::new(src_dir); let it = walkdir.into_iter(); @@ -26,6 +29,7 @@ impl ZipHelper { src_dir, &mut cursor, method, + key_data_manager, )?; Ok(buf) @@ -136,6 +140,7 @@ impl ZipHelper { prefix: &Path, writer: T, method: zip::CompressionMethod, + key_data_manager: &KeyDataManager, ) -> BuckyResult<()> where T: Write + Seek, @@ -148,6 +153,10 @@ impl ZipHelper { let mut buffer = Vec::new(); for entry in it { let path = entry.path(); + if !key_data_manager.check_filter(path) { + warn!("key data will be ignored by filter: {}", path.display()); + } + let name = path.strip_prefix(prefix).unwrap(); // Write file or directory explicitly From 06ec3bcc277336f52b15ffbd579f280b0555e2e0 Mon Sep 17 00:00:00 2001 From: liyaxing Date: Fri, 12 May 2023 17:51:20 +0800 Subject: [PATCH 2/4] Improve key data test case with filter --- .../cyfs-backup/src/key_data/key_data.rs | 1 + src/component/cyfs-backup/src/key_data/test.rs | 15 +++++++++++++-- .../cyfs-backup/src/key_data/zip_helper.rs | 1 + src/tests/cyfs-stack-test/src/case/backup.rs | 1 + 4 files changed, 16 insertions(+), 2 deletions(-) diff --git a/src/component/cyfs-backup/src/key_data/key_data.rs b/src/component/cyfs-backup/src/key_data/key_data.rs index 9671cbfc..a5cc702c 100644 --- a/src/component/cyfs-backup/src/key_data/key_data.rs +++ b/src/component/cyfs-backup/src/key_data/key_data.rs @@ -42,6 +42,7 @@ impl KeyDataManager { pub fn new_uni(isolate: &str, filters: &Vec) -> BuckyResult { let mut filter_list = vec![]; for filter in filters { + info!("new key data filter: {}", filter); let glob = globset::GlobBuilder::new(filter) .case_insensitive(true) .literal_separator(true) diff --git a/src/component/cyfs-backup/src/key_data/test.rs b/src/component/cyfs-backup/src/key_data/test.rs index c5bedad1..34c417be 100644 --- a/src/component/cyfs-backup/src/key_data/test.rs +++ b/src/component/cyfs-backup/src/key_data/test.rs @@ -1,4 +1,4 @@ -use super::zip_helper::*; +use super::{zip_helper::*, KeyDataManager}; use cyfs_util::get_cyfs_root_path; #[test] @@ -11,8 +11,19 @@ fn test() { #[test] fn test_zip() { + cyfs_base::init_simple_log("test-key-data-backup", None); + let root = get_cyfs_root_path().join("etc"); - let buf = ZipHelper::zip_dir_to_buffer(&root, zip::CompressionMethod::Stored).unwrap(); + + let filter_dir = get_cyfs_root_path().join("etc").join("gateway\\**"); + let filters = vec![ + filter_dir.as_os_str().to_string_lossy().to_string(), + ]; + + info!("filters: {:?}", filters); + + let key_data_manager = KeyDataManager::new_uni("", &filters).unwrap(); + let buf = ZipHelper::zip_dir_to_buffer(&root, zip::CompressionMethod::Stored, &key_data_manager).unwrap(); let data = std::io::Cursor::new(buf); let target = get_cyfs_root_path().join("tmp/etc"); diff --git a/src/component/cyfs-backup/src/key_data/zip_helper.rs b/src/component/cyfs-backup/src/key_data/zip_helper.rs index efb00d7b..9f3f6c44 100644 --- a/src/component/cyfs-backup/src/key_data/zip_helper.rs +++ b/src/component/cyfs-backup/src/key_data/zip_helper.rs @@ -155,6 +155,7 @@ impl ZipHelper { let path = entry.path(); if !key_data_manager.check_filter(path) { warn!("key data will be ignored by filter: {}", path.display()); + continue; } let name = path.strip_prefix(prefix).unwrap(); diff --git a/src/tests/cyfs-stack-test/src/case/backup.rs b/src/tests/cyfs-stack-test/src/case/backup.rs index b3d77238..b3cfebfc 100644 --- a/src/tests/cyfs-stack-test/src/case/backup.rs +++ b/src/tests/cyfs-stack-test/src/case/backup.rs @@ -24,6 +24,7 @@ pub async fn test() { isolate: isolate.clone(), target_file: LocalFileBackupParam::default(), password: Some(ProtectedPassword::new("123456")), + key_data_filters: vec![], }; let target_dir = UniBackupTask::backup_dir(¶ms).to_path_buf(); From 5bd5777718152e74e08227f9b1ec545ed84ebc71 Mon Sep 17 00:00:00 2001 From: liyaxing Date: Fri, 12 May 2023 20:35:23 +0800 Subject: [PATCH 3/4] Add key-data-filter params for cyfs-backup-tool --- src/tools/cyfs-backup-tool/src/main.rs | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/src/tools/cyfs-backup-tool/src/main.rs b/src/tools/cyfs-backup-tool/src/main.rs index 63accd5f..2b587b52 100644 --- a/src/tools/cyfs-backup-tool/src/main.rs +++ b/src/tools/cyfs-backup-tool/src/main.rs @@ -91,6 +91,12 @@ async fn main_run() { .long("exit-on-done") .takes_value(false) .help("After the Backup & restore task is completed, the process will exits. default is true, and if with --iqf option, default is false"), + ).arg( + Arg::with_name("key-data-filter") + .long("key-data-filter") + .multiple(true) + .takes_value(true) + .help("The key data that meets the filter condition will be ignored, and the filter supports glob pattern") ) .get_matches(); @@ -194,11 +200,17 @@ async fn main_run() { .unwrap(); } + let mut key_data_filters = vec![]; + if let Some(filters) = matches.values_of("key-data-filter") { + key_data_filters = filters.map(|v| v.to_owned()).collect(); + } + let params = UniBackupParams { id: id.to_owned(), isolate: isolate.to_owned(), target_file, password, + key_data_filters, }; let backup_manager = backup::BackupService::new(¶ms.isolate) From 1e17b0e0abf801e2c4b06e3c481f57fd6f94999e Mon Sep 17 00:00:00 2001 From: liyaxing Date: Fri, 12 May 2023 20:36:04 +0800 Subject: [PATCH 4/4] Improve error and missing log for backup service --- src/component/cyfs-backup/src/data/log.rs | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/src/component/cyfs-backup/src/data/log.rs b/src/component/cyfs-backup/src/data/log.rs index 264f028a..7e4d8c7b 100644 --- a/src/component/cyfs-backup/src/data/log.rs +++ b/src/component/cyfs-backup/src/data/log.rs @@ -15,7 +15,8 @@ impl BackupLogFile { AppendCount::new(1024), ContentLimit::BytesSurpassed(1024 * 1024 * 10), Compression::None, - #[cfg(unix)] None, + #[cfg(unix)] + None, ); Self { writer } @@ -58,17 +59,19 @@ impl BackupLogManager { id: &ObjectId, e: BuckyError, ) { + let t = if id.is_chunk_id() { "chunk" } else { "object" }; + let msg = match isolate_id { Some(isolate_id) => { let dec_id = dec_id.unwrap(); if self.state_default_isolate == Some(*isolate_id) { - format!("[{}] [{}] {}\n", dec_id, id, e) + format!("[{}] [{}] [{}] {}\n", t, dec_id, id, e) } else { - format!("[{}] [{}] [{}] {}\n", isolate_id, dec_id, id, e) + format!("[{}] [{}] [{}] [{}] {}\n", t, isolate_id, dec_id, id, e) } } None => { - format!("[{}] {}\n", id, e) + format!("[{}] [{}] {}\n", t, id, e) } }; @@ -81,17 +84,19 @@ impl BackupLogManager { dec_id: Option<&ObjectId>, id: &ObjectId, ) { + let t = if id.is_chunk_id() { "chunk" } else { "object" }; + let msg = match isolate_id { Some(isolate_id) => { let dec_id = dec_id.unwrap(); if self.state_default_isolate == Some(*isolate_id) { - format!("[{}] [{}]\n", dec_id, id,) + format!("[{}] [{}] [{}]\n", t, dec_id, id,) } else { - format!("[{}] [{}] [{}]\n", isolate_id, dec_id, id) + format!("[{}] [{}] [{}] [{}]\n", t, isolate_id, dec_id, id) } } None => { - format!("[{}]\n", id) + format!("[{}] [{}]\n", t, id) } };