From 0ca44adf75119db99348555dcfe2fc0642749e1c Mon Sep 17 00:00:00 2001 From: Yifan Yuan Date: Fri, 15 Sep 2023 21:29:10 +0800 Subject: [PATCH] [e2e] add E2E test add E2E test for reading a gzip file by gzstream and save its metadata(gz index & tarmeta) into Overlaybd-turboOCI Signed-off-by: Yifan Yuan --- src/overlaybd/gzindex/gzfile.cpp | 1 - src/overlaybd/gzip/gz.cpp | 8 -- src/overlaybd/gzip/gz.h | 2 +- .../stream_convertor/stream_conv.cpp | 27 +--- src/overlaybd/tar/test/test.cpp | 128 +++++++++++------- src/tools/sha256file.h | 2 +- 6 files changed, 80 insertions(+), 88 deletions(-) diff --git a/src/overlaybd/gzindex/gzfile.cpp b/src/overlaybd/gzindex/gzfile.cpp index 15c174fd..14a1f893 100644 --- a/src/overlaybd/gzindex/gzfile.cpp +++ b/src/overlaybd/gzindex/gzfile.cpp @@ -19,7 +19,6 @@ #include #include #include -#include #include #include #include "gzfile_index.h" diff --git a/src/overlaybd/gzip/gz.cpp b/src/overlaybd/gzip/gz.cpp index 52bff7c1..18bf15a5 100644 --- a/src/overlaybd/gzip/gz.cpp +++ b/src/overlaybd/gzip/gz.cpp @@ -15,23 +15,15 @@ */ #include "gz.h" -#include -#include -#include -#include #include -#include -#include #include #include #include - #include #include #include #include #include -#include #include "../../tools/sha256file.h" #include "../gzindex/gzfile_index.h" class GzAdaptorFile : public photon::fs::VirtualReadOnlyFile { diff --git a/src/overlaybd/gzip/gz.h b/src/overlaybd/gzip/gz.h index 9d8d70e4..5c8bcc1d 100644 --- a/src/overlaybd/gzip/gz.h +++ b/src/overlaybd/gzip/gz.h @@ -23,7 +23,7 @@ class IGzFile : public photon::fs::VirtualReadOnlyFile { public : - // return filename + // return full filename of gzip index virtual std::string save_index() = 0; virtual std::string sha256_checksum() = 0; }; diff --git a/src/overlaybd/stream_convertor/stream_conv.cpp b/src/overlaybd/stream_convertor/stream_conv.cpp index b17bfaef..8a87e759 100644 --- a/src/overlaybd/stream_convertor/stream_conv.cpp +++ b/src/overlaybd/stream_convertor/stream_conv.cpp @@ -111,30 +111,5 @@ int main(int argc, char *argv[]){ server = new StreamConvertor; DEFER(delete server); server->start_uds_server(); - + return 0; } - -// TEST_F(GzIndexTest, stream) { -// set_log_output_level(0); -// LOG_INFO("start streamFile test"); -// std::string fn_test_tgz = "/tmp/go1.17.6.linux-amd64.tar.gz"; -// ASSERT_EQ( -// 0, download("https://dadi-shared.oss-cn-beijing.aliyuncs.com/go1.17.6.linux-amd64.tar.gz", -// fn_test_tgz.c_str())); - -// auto jh1 = photon::thread_enable_join(photon::thread_create11(uds_server)); - -// auto file = photon::fs::open_localfile_adaptor(fn_test_tgz.c_str(), O_RDONLY); -// // auto dst = photon::fs::open_localfile_adaptor("/tmp/dest", O_TRUNC | O_CREAT |O_RDWR, 0644); -// ASSERT_NE(file, nullptr); -// // inf(file, dst); -// uds_client(file); - -// photon::thread_join(jh1); -// remove(uds_path); -// file->lseek(0, SEEK_SET); -// auto fn_test_tgz_idx = fn_test_tgz + ".index"; -// if (::access(fn_test_tgz_idx.c_str(), 0) != 0){ -// ASSERT_EQ(create_gz_index(file, fn_test_tgz_idx.c_str()), 0); -// } -// } \ No newline at end of file diff --git a/src/overlaybd/tar/test/test.cpp b/src/overlaybd/tar/test/test.cpp index a2308dcc..a9d85388 100644 --- a/src/overlaybd/tar/test/test.cpp +++ b/src/overlaybd/tar/test/test.cpp @@ -14,13 +14,13 @@ limitations under the License. */ -#include #include #include #include #include #include #include +#include #include #include "../../gzindex/gzfile.h" #include "../../extfs/extfs.h" @@ -55,7 +55,7 @@ class TarTest : public ::testing::Test { delete fs; } - int download(const std::string &url, std::string out) { + int download(const std::string &url, std::string out = "") { if (out == "") { out = workdir + "/" + std::string(basename(url.c_str())); } @@ -132,8 +132,9 @@ class TarTest : public ::testing::Test { ssize_t LEN = 1UL<<20; char vbuf[1UL<<20], tbuf[1UL<<20]; + // set_log_output_level(0); for (off_t i = 0; i < count; i+=LEN) { - // LOG_INFO("`", i); + LOG_DEBUG("`", i); auto ret_v = verify->pread(vbuf, LEN, i); auto ret_t = test->pread(tbuf, LEN, i); if (ret_v == -1 || ret_t == -1) { @@ -255,54 +256,79 @@ TEST_F(TarTest, stream) { } } -// TEST_F(TarTest, stream_tar_meta) { -// // set_log_output_level(0); -// std::vector filelist { -// "https://dadi-shared.oss-cn-beijing.aliyuncs.com/go1.17.6.linux-amd64.tar.gz" -// }; -// for (auto file : filelist){ -// ASSERT_EQ(0, download(, "")); - - -// auto src_file = fs->open("latest.tar", O_RDONLY, 0666); -// ASSERT_NE(nullptr, src_file); -// DEFER(delete src_file); -// auto verify_dev = createDevice("verify", src_file); -// make_extfs(verify_dev); -// auto verify_ext4fs = new_extfs(verify_dev, false); -// auto verifyfs = new_subfs(verify_ext4fs, "/", true); -// auto turboOCI_verify = new UnTar(src_file, verifyfs, 0, 4096, verify_dev, true); -// ASSERT_EQ(0, turboOCI_verify->extract_all()); -// verify_ext4fs->sync(); -// delete turboOCI_verify; -// delete verifyfs; - -// src_file->lseek(0, 0); - -// auto tar_idx = fs->open("latest.tar.meta", O_TRUNC | O_CREAT | O_RDWR, 0644); -// auto imgfile = createDevice("mock", src_file); -// DEFER(delete imgfile;); -// auto tar = new UnTar(src_file, nullptr, 0, 4096, nullptr, true); -// auto obj_count = tar->dump_tar_headers(tar_idx); -// EXPECT_NE(-1, obj_count); -// LOG_INFO("objects count: `", obj_count); -// tar_idx->lseek(0,0); - -// make_extfs(imgfile); -// auto extfs = new_extfs(imgfile, false); -// auto target = new_subfs(extfs, "/", true); -// auto turboOCI_mock = new UnTar(tar_idx, target, TAR_IGNORE_CRC, 4096, imgfile, true, true); -// auto ret = turboOCI_mock->extract_all(); -// delete turboOCI_mock; -// delete target; - -// ASSERT_EQ(0, ret); -// EXPECT_EQ(0, do_verify(verify_dev, imgfile)); -// delete tar_idx; -// delete tar; -// } - -// } +TEST_F(TarTest, gz_tarmeta_e2e) { + // set_log_output_level(0); + std::vector filelist { + "https://dadi-shared.oss-cn-beijing.aliyuncs.com/cri-containerd-cni-1.5.2-linux-amd64.tar.gz", + "https://dadi-shared.oss-cn-beijing.aliyuncs.com/containerd-1.4.4-linux-amd64.tar.gz", + "https://dadi-shared.oss-cn-beijing.aliyuncs.com/go1.17.6.linux-amd64.tar.gz" + }; + for (auto file : filelist){ + ASSERT_EQ(0, download(file.c_str())); + auto fn = std::string(basename(file.c_str())); + auto gzip_file = fs->open(fn.c_str(), O_RDONLY, 0600); + auto gzfile = open_gzfile_adaptor((workdir + "/" + fn).c_str()); + auto fn_idx = (workdir + "/" + fn + ".gz_idx"); + ASSERT_EQ(create_gz_index(gzip_file, fn_idx.c_str()), 0); + auto gz_idx = fs->open((fn + ".gz_idx").c_str(), O_RDONLY, 0644); + gzip_file->lseek(0, SEEK_SET); + auto src_file = new_gzfile(gzip_file, gz_idx, true); + ASSERT_NE(nullptr, src_file); + auto verify_dev = createDevice((fn + ".verify").c_str(), src_file); + make_extfs(verify_dev); + auto verify_ext4fs = new_extfs(verify_dev, false); + auto verifyfs = new_subfs(verify_ext4fs, "/", true); + // gzfile->lseek(0, SEEK_SET); + auto turboOCI_verify = new UnTar(gzfile, verifyfs, 0, 4096, verify_dev, true); + ASSERT_EQ(0, turboOCI_verify->extract_all()); + verify_ext4fs->sync(); + + // src_file->lseek(0, 0); + auto tar_idx = fs->open((fn + ".tar.meta").c_str(), O_TRUNC | O_CREAT | O_RDWR, 0644); + auto stream_src = fs->open(fn.c_str(), O_RDONLY, 0600); + auto streamfile = open_gzstream_file(stream_src, 0); + auto tar = new UnTar(streamfile, nullptr, 0, 4096, nullptr, true); + auto obj_count = tar->dump_tar_headers(tar_idx); + EXPECT_NE(-1, obj_count); + LOG_INFO("objects count: `", obj_count); + + auto fn_test_idx = streamfile->save_index(); + LOG_INFO("gzip index of [`]: `", fn, fn_test_idx); + auto test_gz_idx = open_localfile_adaptor(fn_test_idx.c_str(), O_RDONLY); + ASSERT_NE(test_gz_idx, nullptr); + auto test_gzfile = fs->open(fn.c_str(), O_RDONLY, 0600); + ASSERT_NE(test_gzfile, nullptr); + auto gz_target = new_gzfile(test_gzfile,test_gz_idx, true); + auto imgfile = createDevice((fn + ".mock").c_str(), gz_target); + + tar_idx->lseek(0,0); + + make_extfs(imgfile); + auto extfs = new_extfs(imgfile, false); + auto target = new_subfs(extfs, "/", true); + auto turboOCI_mock = new UnTar(tar_idx, target, TAR_IGNORE_CRC, 4096, imgfile, true, true); + auto ret = turboOCI_mock->extract_all(); + extfs->sync(); + + ASSERT_EQ(0, ret); + EXPECT_EQ(0, do_verify(verify_dev, imgfile)); + + delete turboOCI_mock; + delete target; + delete src_file; + delete gzfile; + delete turboOCI_verify; + delete verifyfs; + delete tar_idx; + delete stream_src; + delete streamfile; + delete tar; + + delete verify_dev; + delete imgfile; + } + +} TEST_F(TarTest, tar_header_check) { auto fn = "data"; diff --git a/src/tools/sha256file.h b/src/tools/sha256file.h index c56611ae..cd8bf2fd 100644 --- a/src/tools/sha256file.h +++ b/src/tools/sha256file.h @@ -11,4 +11,4 @@ class SHA256File : public photon::fs::VirtualReadOnlyFile { SHA256File *new_sha256_file(photon::fs::IFile *file, bool ownership); -std::string sha256sum(const char *fn); \ No newline at end of file +std::string sha256sum(const char *fn);