support save image to file

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "ocipack"
-version = "0.5.0"
+version = "0.6.0"
 edition = "2021"
 
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
@@ -37,3 +37,4 @@ zstd = "0.12"
 fantasy-util = "0.1.8"
 ubyte = "0.10"
 colored = "2"
+tempfile = "3"

src/adapter/registry.rs

@@ -6,13 +6,13 @@ use crate::adapter::{ImageInfo, TargetImageAdapter, TargetInfo};
 use crate::config::cmd::{BaseAuth, TargetFormat};
 use crate::config::RegAuthType;
 use crate::const_data::DEFAULT_IMAGE_HOST;
-use crate::progress::manager::ProcessorManager;
-use crate::progress::ProcessResult;
-use crate::progress::Processor;
 use crate::container::http::upload::UploadResult;
 use crate::container::manifest::Manifest;
 use crate::container::proxy::ProxyInfo;
 use crate::container::{ConfigBlobSerialize, Reference, RegDigest, Registry, RegistryCreateInfo};
+use crate::progress::manager::ProcessorManager;
+use crate::progress::ProcessResult;
+use crate::progress::Processor;
 use crate::GLOBAL_CONFIG;
 
 pub struct RegistryTargetAdapter {
@@ -86,8 +86,11 @@ impl RegistryTargetAdapter {
         let mut reg_uploader_vec = Vec::<Box<dyn Processor<UploadResult>>>::new();
         for manifest_layer in target_manifest.layers() {
             let layer_digest = RegDigest::new_with_digest(manifest_layer.digest.to_string());
-            let local_layer =
-                home_dir.cache.blobs.local_layer(&layer_digest).ok_or_else(|| anyhow!("local file not found {}",layer_digest.digest))?;
+            let local_layer = home_dir
+                .cache
+                .blobs
+                .local_layer(&layer_digest)
+                .ok_or_else(|| anyhow!("local file not found {}", layer_digest.digest))?;
             let layer_path = local_layer.layer_path();
             let reg_uploader = manager.layer_blob_upload(&target_info.image_info.image_name, &layer_digest, &layer_path)?;
             reg_uploader_vec.push(Box::new(reg_uploader))

src/adapter/tar.rs

@@ -1,20 +1,19 @@
-use std::fs::File;
-use std::io::{Cursor, Read, Write};
-use std::path::PathBuf;
+use std::fs::{create_dir_all, File};
+use std::io;
+use std::io::{Cursor, Write};
+use std::path::{Path, PathBuf};
 
 use anyhow::{anyhow, Result};
-use bytes::Buf;
-use flate2::Compression;
 use flate2::read::GzDecoder;
 use flate2::write::GzEncoder;
+use flate2::Compression;
+use log::info;
 use serde::{Deserialize, Serialize};
 use tar::{Builder, Header};
 
-use crate::adapter::{ImageInfo, TargetImageAdapter, TargetInfo};
-use crate::config::cmd::TargetFormat;
-use crate::container::{CompressType, ConfigBlobSerialize, RegDigest};
 use crate::container::manifest::{CommonManifestConfig, Manifest};
-use crate::container::oci::OciManifest;
+use crate::container::{CompressType, ConfigBlobSerialize, RegContentType, RegDigest};
+use crate::util::sha::bytes_sha256;
 use crate::GLOBAL_CONFIG;
 
 #[derive(Default, Debug, Clone, PartialEq, Serialize, Deserialize)]
@@ -33,104 +32,109 @@ pub struct TarManifestJson {
 }
 
 pub struct TarTargetAdapter {
-    info: TargetInfo,
-    target_manifest: Manifest,
-    target_config_blob_serialize: ConfigBlobSerialize,
-    save_path: PathBuf,
-    use_gzip: bool,
-}
-
-impl TargetImageAdapter for TarTargetAdapter {
-    fn info(&self) -> &TargetInfo {
-        &self.info
-    }
+    pub image_raw_name: String,
+    pub target_manifest: Manifest,
+    pub manifest_raw: String,
+    pub target_config_blob_serialize: ConfigBlobSerialize,
+    pub save_path: PathBuf,
+    pub use_gzip: bool,
 }
 
-
 impl TarTargetAdapter {
     pub fn save(self) -> Result<()> {
+        info!("start saving image as file");
+        if self.save_path.exists() {
+            return Err(anyhow!("file already exists: {:?}", self.save_path));
+        }
+        if let Some(parent) = self.save_path.parent() {
+            create_dir_all(parent)?;
+        }
         let output_file = File::create(self.save_path)?;
         let write_box: Box<dyn Write> = match self.use_gzip {
             true => Box::new(GzEncoder::new(output_file, Compression::fast())),
             false => Box::new(output_file),
         };
         let mut builder = Builder::new(write_box);
 
-        let (manifest_config, layers) = match self.target_manifest {
-            Manifest::OciV1(oci) => (oci.config.clone(), oci.layers.clone()),
-            Manifest::DockerV2S2(dockerv2s2) => (dockerv2s2.config.clone(), dockerv2s2.layers.clone()),
+        let (manifest_media_type, layers) = match self.target_manifest {
+            Manifest::OciV1(oci) => (RegContentType::OCI_MANIFEST.0.to_string(), oci.layers.clone()),
+            Manifest::DockerV2S2(dockerv2s2) => (RegContentType::DOCKER_MANIFEST.0.to_string(), dockerv2s2.layers.clone()),
         };
-        let image_index = ImageIndex {
-            schema_version: 2,
-            manifests: vec![manifest_config],
-        };
-        // TODO
 
         // 解析所有layer为tar
-        let mut layer_reader_vec = layers.iter().map(|comm_layer| {
-            let digest = RegDigest::new_with_digest(comm_layer.digest.clone());
-            let layer = GLOBAL_CONFIG.home_dir.cache.blobs.local_layer(&digest)
-                .ok_or_else(|| anyhow!("can not found this layer: {}",digest.sha256))?;
-            let layer_file = File::open(layer.layer_file_path)?;
-            let layer_reader: Box<dyn Read> = match layer.compress_type {
-                CompressType::Tar => Box::new(layer_file),
-                CompressType::Tgz => Box::new(GzDecoder::new(layer_file)),
-                CompressType::Zstd => Box::new(zstd::stream::Decoder::new(layer_file)?),
-            };
-            Ok((layer.diff_layer_sha, layer_reader))
-        }).collect::<Result<Vec<_>>>()?;
-        // 将config blob 也加入
+        let layer_sha_vec: Vec<String> = layers
+            .iter()
+            .map(|comm_layer| {
+                let digest = RegDigest::new_with_digest(comm_layer.digest.clone());
+                let layer = GLOBAL_CONFIG.home_dir.cache.blobs.local_layer(&digest)
+                    .ok_or_else(|| anyhow!("can not found this layer: {}", digest.sha256))?;
+                let layer_file = File::open(layer.layer_file_path)?;
+                let (size, layer_reader) = match layer.compress_type {
+                    CompressType::Tar => (layer_file.metadata()?.len(), Box::new(layer_file)),
+                    CompressType::Tgz => {
+                        let mut temp_file = tempfile::Builder::new().tempfile_in(&GLOBAL_CONFIG.home_dir.cache.temp_dir)?;
+                        let size = io::copy(&mut GzDecoder::new(layer_file), &mut temp_file)?;
+                        (size, Box::new(temp_file.reopen()?))
+                    }
+                    CompressType::Zstd => {
+                        let mut temp_file = tempfile::Builder::new().tempfile_in(&GLOBAL_CONFIG.home_dir.cache.temp_dir)?;
+                        let size = io::copy(&mut zstd::stream::Decoder::new(layer_file)?, &mut temp_file)?;
+                        (size, Box::new(temp_file.reopen()?))
+                    }
+                };
+                let mut header = Header::new_gnu();
+                let layer_path = format!("blobs/sha256/{}", layer.diff_layer_sha);
+                header.set_path(&layer_path)?;
+                header.set_size(size);
+                header.set_mode(0o644);
+                header.set_cksum();
+                builder.append(&header, layer_reader)?;
+                Ok(layer_path)
+            })
+            .collect::<Result<Vec<_>>>()?;
+
+        // 将config blob 也加入到layer中
         let config_blob = self.target_config_blob_serialize;
-        layer_reader_vec.push((config_blob.digest.digest, Box::new(Cursor::new(config_blob.json_str))));
+        //layer_sha_vec.push(config_blob.digest.sha256.clone());
+        let config_blob_path = format!("blobs/sha256/{}", config_blob.digest.sha256);
+        write_string_to_builder(config_blob.json_str, config_blob_path.clone(), &mut builder)?;
+        // 写入 index.json
 
-        let image_raw_name = self.info.image_info.image_raw_name
-            .ok_or_else(|| anyhow!("must set a image raw name"))?;
-        let layer_sha_vec = layer_reader_vec.into_iter().map(|(layer_sha, layer_reader)| {
-            let mut header = Header::new_gnu();
-            header.set_path(format!("blobs/sha256/{}", layer_sha))?;
-            builder.append(&header, layer_reader)?;
-            Ok(layer_sha)
-        }).collect::<Result<Vec<_>>>()?;
-        TarManifestJson {
-            config: format!("blobs/sha256/{}", config_blob.digest.sha256),
-            repo_tags: vec![image_raw_name],
+        let manifest_digest = RegDigest::new_with_sha256(bytes_sha256(self.manifest_raw.as_bytes()));
+        let image_index = ImageIndex {
+            schema_version: 2,
+            manifests: vec![CommonManifestConfig {
+                media_type: manifest_media_type.clone(),
+                size: manifest_media_type.as_bytes().len() as u64,
+                digest: manifest_digest.digest,
+            }],
+        };
+        write_string_to_builder(serde_json::to_string(&image_index)?, "index.json", &mut builder)?;
+        // 写入 manifest.json
+        let manifest_json = TarManifestJson {
+            config: config_blob_path,
+            repo_tags: vec![self.image_raw_name.clone()],
             layers: layer_sha_vec,
         };
-        // TODO
+        write_string_to_builder(serde_json::to_string(&vec![manifest_json])?, "manifest.json", &mut builder)?;
+        // 写入 oci-layout
+        write_string_to_builder(r#"{"imageLayoutVersion":"1.0.0"}"#.to_string(), "oci-layout", &mut builder)?;
+        // 写入manifest
+        let manifest_path = format!("blobs/sha256/{}", manifest_digest.sha256);
+        write_string_to_builder(self.manifest_raw, manifest_path, &mut builder)?;
+        builder.finish()?;
         Ok(())
     }
 }
 
-#[test]
-fn it_works() -> Result<()> {
-    let adapter = TarTargetAdapter {
-        info: TargetInfo {
-            image_info: ImageInfo {
-                image_raw_name: None,
-                image_host: "".to_string(),
-                image_name: "".to_string(),
-                reference: "".to_string(),
-            },
-            format: TargetFormat::Docker,
-        },
-        target_manifest: Manifest::OciV1(OciManifest {
-            schema_version: 0,
-            media_type: None,
-            config: CommonManifestConfig {
-                media_type: "".to_string(),
-                size: 0,
-                digest: "".to_string(),
-            },
-            layers: vec![],
-        }),
-
-        target_config_blob_serialize: ConfigBlobSerialize {
-            json_str: "".to_string(),
-            digest: RegDigest { sha256: "".to_string(), digest: "".to_string() },
-            size: 0,
-        },
-        save_path: Default::default(),
-        use_gzip: false,
-    };
-    adapter.save()
-}
+fn write_string_to_builder<P: AsRef<Path>>(data: String, path: P, builder: &mut Builder<Box<dyn Write>>) -> Result<()> {
+    let size = data.as_bytes().len() as u64;
+    let image_index_cursor = Cursor::new(data);
+    let mut header = Header::new_gnu();
+    header.set_path(path)?;
+    header.set_size(size);
+    header.set_mode(0o644);
+    header.set_cksum();
+    builder.append(&header, image_index_cursor)?;
+    Ok(())
+}

src/config/cmd.rs

@@ -90,8 +90,8 @@ pub struct BuildCmdArgs {
     pub source_proxy: Option<ProxyInfo>,
 
     /// Target type.
-    /// Support 'registry'
-    /// Example:'registry:my.container.com/target/image:1.1'
+    /// Support registry/tar/tgz'.
+    /// Example:'registry:my.container.com/target/image:1.1','tgz:image.tgz'
     #[clap(long, short)]
     pub target: TargetType,
 
@@ -144,8 +144,8 @@ pub struct TransformCmdArgs {
     pub source_proxy: Option<ProxyInfo>,
 
     /// Target type.
-    /// Support 'registry'
-    /// Example:'registry:my.container.com/target/image:1.1'
+    /// Support 'registry','tar','tgz'
+    /// Example:'registry:my.container.com/target/image:1.1', 'tgz:./image.tgz'
     #[clap(long, short)]
     pub target: TargetType,
 
@@ -252,9 +252,16 @@ impl FromStr for ProxyInfo {
     }
 }
 
-#[derive(Clone)]
+#[derive(Clone, Debug)]
 pub enum TargetType {
     Registry(String),
+    Tar(TarArg),
+}
+
+#[derive(Clone, Debug)]
+pub struct TarArg {
+    pub path: String,
+    pub usb_gzip: bool,
 }
 
 impl FromStr for TargetType {
@@ -263,8 +270,17 @@ impl FromStr for TargetType {
     fn from_str(arg: &str) -> Result<Self, Self::Err> {
         let potion = arg.chars().position(|c| c == ':').ok_or_else(|| anyhow!("error source"))?;
         let target_type = &arg[..potion];
+        let second = arg[potion + 1..].to_string();
         Ok(match target_type {
-            "registry" => TargetType::Registry(arg[potion + 1..].to_string()),
+            "registry" => TargetType::Registry(second),
+            "tar" => TargetType::Tar(TarArg {
+                path: second,
+                usb_gzip: false,
+            }),
+            "tgz" => TargetType::Tar(TarArg {
+                path: second,
+                usb_gzip: true,
+            }),
             _ => return Err(anyhow!("unknown target type: {}", target_type)),
         })
     }

src/container/http/download.rs

@@ -10,9 +10,9 @@ use anyhow::{anyhow, Result};
 use reqwest::blocking::{Client, Response};
 use reqwest::Method;
 
-use crate::progress::{CoreStatus, ProcessResult, Processor, ProcessorAsync, ProgressStatus};
 use crate::container::http::{do_request_raw, get_header, HttpAuth};
 use crate::container::BlobConfig;
+use crate::progress::{CoreStatus, ProcessResult, Processor, ProcessorAsync, ProgressStatus};
 
 pub struct RegDownloader {
     finished: bool,

src/container/http/upload.rs

@@ -10,9 +10,9 @@ use anyhow::{anyhow, Result};
 use reqwest::blocking::Client;
 use reqwest::Method;
 
-use crate::progress::{CoreStatus, ProcessResult, Processor, ProcessorAsync, ProgressStatus};
 use crate::container::http::{do_request_raw_read, HttpAuth};
 use crate::container::BlobConfig;
+use crate::progress::{CoreStatus, ProcessResult, Processor, ProcessorAsync, ProgressStatus};
 
 pub struct RegUploader {
     reg_uploader_enum: RegUploaderEnum,

src/container/manifest.rs

@@ -108,7 +108,7 @@ impl Manifest {
                         CompressType::Tgz => RegContentType::OCI_LAYER_TGZ.val(),
                         CompressType::Zstd => RegContentType::OCI_LAYER_ZSTD.val(),
                     }
-                        .to_string(),
+                    .to_string(),
                     size,
                     digest: reg_digest.digest,
                 },