Several improvements to build speed and handling of symlinks

src/main/scala/sbtdocker/DockerBuilder.scala

@@ -30,6 +30,7 @@ object DockerBuilder {
 
     IO.write(stageDir / "Dockerfile", dockerFile.mkString)
     copyFiles(dockerFile.stagedFiles, stageDir, log)
+    copyFiles(dockerFile.stagedArchives.map(_.outputFile), stageDir, log)
   }
 
   def copyFiles(files: Seq[StageFile], stageDir: File, log: Logger) = {

src/main/scala/sbtdocker/DockerKeys.scala

@@ -4,6 +4,8 @@ import sbt._
 
 object DockerKeys {
   val docker = taskKey[ImageId]("Build a Docker image.")
+  val dockerNoCache = taskKey[ImageId]("Build a Docker image avoiding sbt's cache.")
+  val baseDockerImage = taskKey[ImageId]("Build the base Docker image.")
   val dockerBuildAndPush = taskKey[ImageId]("Build a Docker image and pushes it to a registry.")
   val dockerPush = taskKey[Unit]("Push a already built Docker image to a registry.")
 

src/main/scala/sbtdocker/DockerSettings.scala

@@ -1,24 +1,15 @@
 package sbtdocker
 
-import sbt.Keys.target
+import sbt.Keys._
 import sbt._
 import sbtdocker.DockerKeys._
+import sbtdocker.Instructions.{From, Add}
 
 object DockerSettings {
   lazy val baseDockerSettings = Seq(
-    docker := {
-      val log = Keys.streams.value.log
-      val dockerCmd = (DockerKeys.dockerCmd in docker).value
-      val buildOptions = (DockerKeys.buildOptions in docker).value
-      val stageDir = (target in docker).value
-      val dockerfile = (DockerKeys.dockerfile in docker).value
-      val imageName = (DockerKeys.imageName in docker).value
-
-      log.debug("Dockerfile:")
-      log.debug(dockerfile.mkString)
-
-      DockerBuilder(dockerCmd, buildOptions, imageName, dockerfile, stageDir, log)
-    },
+    docker <<= buildDockerImage(docker, cache = true),
+    dockerNoCache <<= buildDockerImage(docker, cache = false),
+    baseDockerImage <<= buildDockerImage(baseDockerImage, cache = true),
     dockerPush := {
       val log = Keys.streams.value.log
       val dockerCmd = (DockerKeys.dockerCmd in docker).value
@@ -42,16 +33,96 @@ object DockerSettings {
           |}
         """.stripMargin)
     },
+    dockerfile in baseDockerImage := {
+      sys.error(
+        """A Dockerfile is not defined. Please define it with `dockerfile in docker`
+          |
+          |Example:
+          |dockerfile in docker := new Dockerfile {
+          | from("ubuntu")
+          | ...
+          |}
+        """.stripMargin)
+    },
     target in docker := target.value / "docker",
     imageName in docker := {
       val organisation = Option(Keys.organization.value).filter(_.nonEmpty)
       val name = Keys.normalizedName.value
       ImageName(namespace = organisation, repository = name)
     },
-    dockerCmd in docker := sys.env.get("DOCKER").filter(_.nonEmpty).getOrElse("docker"),
-    buildOptions in docker := BuildOptions()
+    imageName in baseDockerImage := {
+      val organisation = Option(Keys.organization.value).filter(_.nonEmpty)
+      val name = Keys.normalizedName.value + "-base"
+      ImageName(namespace = organisation, repository = name)
+    },
+    dockerCmd := sys.env.get("DOCKER").filter(_.nonEmpty).getOrElse("docker"),
+    buildOptions in docker := BuildOptions(),
+    buildOptions in baseDockerImage := BuildOptions()
   )
 
+  def buildDockerImage(dockerTask: TaskKey[_], cache: Boolean): Def.Initialize[Task[ImageId]] = Def.task {
+    val log = Keys.streams.value.log
+    val dockerCmd = DockerKeys.dockerCmd.value
+    val buildOptions = (DockerKeys.buildOptions in dockerTask).value
+    val stageDir = (target in docker).value
+    val dockerfile = (DockerKeys.dockerfile in dockerTask).value
+    val imageName = (DockerKeys.imageName in dockerTask).value
+    val cacheDir = target.value / s"docker-image-cache" / imageName.toString
+    val cacheFile = cacheDir/ name.value
+
+    log.debug("Dockerfile:")
+    log.debug(dockerfile.mkString)
+    if (cache) {
+      /* Wrap actual function in a function that provides basic caching . */
+      val cachedFun = FileFunction.cached(cacheDir, FilesInfo.lastModified, FilesInfo.exists) {
+        (inFiles: Set[File]) => {
+          val imageId = DockerBuilder(dockerCmd, buildOptions, imageName, dockerfile, stageDir, log)
+          IO.write(cacheFile, imageId.id)
+          Set(cacheFile)
+        }
+      }
+
+      /** Recursively list all directories so caching will detect changes in subdirectories */
+      def findDirs(file: File): Seq[File] = file.isDirectory match {
+        case true => file +: file.listFiles().flatMap(findDirs)
+        case false => Nil
+      }
+
+      // Normalize instructions that include our temporary tgz files since these change every run.
+      // Changes to these files will be detected by changes to StagedArchive directories.
+      // Also, convert names of the base image to ids to detect changes better.
+      val userInstructions = dockerfile.instructions.map {
+        case Add(src, dest) if src.contains("dockerbuild") => Add("archive", dest)
+        case From(baseImageName) => From(s"$dockerCmd history -q $baseImageName".!!.split("\n").head)
+        case other => other
+      }
+
+      // To detect changes in the docker file itself we create a fake file based on the hash
+      // of the docker file description.
+      // TODO: use a better hash function here...
+      val fakeDockerFile = target.value / (userInstructions.hashCode.toString + ".docker.hash")
+      log.debug(s"dockerfile caching hashcode: $fakeDockerFile")
+
+      val stagedArchives = dockerfile.stagedArchives.flatMap(d => findDirs(d.file))
+      val stagedFiles = dockerfile.stagedFiles.map(_.source)
+      val fileDependencies = (Seq(fakeDockerFile) ++ stagedArchives ++ stagedFiles).toSet
+
+      val cachedId = IO.read(cachedFun(fileDependencies).head)
+      val imageExists = try { s"docker history $cachedId".!!; true} catch {
+        case _: Exception => false
+      }
+
+      if (imageExists) {
+        // The cached image stil exists, return it.
+        ImageId(cachedId)
+      } else {
+        DockerBuilder(dockerCmd, buildOptions, imageName, dockerfile, stageDir, log)
+      }
+    } else {
+      DockerBuilder(dockerCmd, buildOptions, imageName, dockerfile, stageDir, log)
+    }
+  }
+
   def packageDockerSettings(fromImage: String, exposePorts: Seq[Int]) = Seq(
     docker <<= docker.dependsOn(Keys.`package`.in(Compile, Keys.packageBin)),
     Keys.mainClass in docker <<= Keys.mainClass in docker or Keys.mainClass.in(Compile, Keys.packageBin),

src/main/scala/sbtdocker/DockerfileCommands.scala

@@ -1,5 +1,7 @@
 package sbtdocker
 
+import java.io.File
+
 import sbt._
 import sbtdocker.Instructions._
 import sbtdocker.Utils._
@@ -24,13 +26,37 @@ object StageFile {
  */
 case class StageFile(source: File, target: File)
 
+/**
+ * Container for a directory that should be copied to a path in the stage directory in the form
+ * of a tar.gz file.
+ *
+ * @param file The path that should be compressed.
+ * @param dest Path in the stage directory.
+ */
+case class StagedArchive(file: File, dest: String) {
+  val tempFile = java.io.File.createTempFile("dockerbuild", ".tgz")
+
+  // We lazily build the actual tar ball, only when the SBT caching detects that the contents have
+  // changed.
+  lazy val outputFile = {
+    val parentDir = file.getParent
+    val command = s"tar czvf $tempFile -C $parentDir ${file.getName}"
+    println(command)
+    command.!!
+
+    StageFile(tempFile, expandPath(tempFile, "/"))
+  }
+}
+
 trait DockerfileLike extends DockerfileCommands {
   type T <: DockerfileLike
 
   def instructions: Seq[Instruction]
 
   def stagedFiles: Seq[StageFile]
 
+  def stagedArchives: Seq[StagedArchive]
+
   def mkString = instructions.mkString("\n")
 }
 
@@ -80,6 +106,14 @@ trait DockerfileCommands {
     stageFile(copy)
   }
 
+  /**
+   * Adds an directory that will be transmitted to the docker daemon in the form of a tar.gz file.
+   * This format is convenient when you want to preserve symlinks.
+   * @param archive
+   */
+  def stageArchive(archive: StagedArchive): T
+
+
   /**
    * Stage multiple files.
    */
@@ -117,6 +151,12 @@ trait DockerfileCommands {
       .stageFile(source, destination)
   }
 
+  def addCompressed(file: File, dest: String): T = {
+    val archive = StagedArchive(file, dest)
+    stageArchive(archive)
+    addInstruction(Add(archive.tempFile.getName, dest))
+  }
+
   def add(source: URL, destination: String) = addInstruction(Add(source.toString, destination))
 
   def add(source: URL, destination: File) = addInstruction(Add(source.toString, destination.toString))

src/main/scala/sbtdocker/immutable/Dockerfile.scala

@@ -1,6 +1,6 @@
 package sbtdocker.immutable
 
-import sbtdocker.{StageFile, DockerfileLike, Instruction}
+import sbtdocker.{StagedArchive, StageFile, DockerfileLike, Instruction}
 
 object Dockerfile {
   def empty = Dockerfile()
@@ -23,12 +23,19 @@ object Dockerfile {
  * @param stagedFiles Files and directories that should be copied to the stage directory.
  */
 case class Dockerfile(instructions: Seq[Instruction] = Seq.empty,
-                      stagedFiles: Seq[StageFile] = Seq.empty) extends DockerfileLike {
+                      stagedFiles: Seq[StageFile] = Seq.empty,
+                      stagedArchives: Seq[StagedArchive] = Seq.empty) extends DockerfileLike {
   type T = Dockerfile
 
-  def addInstruction(instruction: Instruction) = Dockerfile(instructions :+ instruction, stagedFiles)
+  def addInstruction(instruction: Instruction) =
+    Dockerfile(instructions :+ instruction, stagedFiles, stagedArchives)
 
-  def stageFile(file: StageFile) = Dockerfile(instructions, stagedFiles :+ file)
+  def stageFile(file: StageFile) =
+    Dockerfile(instructions, stagedFiles :+ file, stagedArchives)
 
-  def stageFiles(files: TraversableOnce[StageFile]) = Dockerfile(instructions, stagedFiles ++ files)
+  def stageFiles(files: TraversableOnce[StageFile]) =
+    Dockerfile(instructions, stagedFiles ++ files, stagedArchives)
+
+  def stageArchive(archive: StagedArchive) =
+    Dockerfile(instructions, stagedFiles, archive +: stagedArchives )
 }

src/main/scala/sbtdocker/mutable/Dockerfile.scala

@@ -1,6 +1,6 @@
 package sbtdocker.mutable
 
-import sbtdocker.{StageFile, DockerfileLike, Instruction}
+import sbtdocker.{StagedArchive, StageFile, DockerfileLike, Instruction}
 
 /**
  * Mutable Dockerfile.
@@ -20,7 +20,8 @@ import sbtdocker.{StageFile, DockerfileLike, Instruction}
  * @param stagedFiles Files and directories that should be copied to the stage directory.
  */
 case class Dockerfile(var instructions: Seq[Instruction] = Seq.empty,
-                      var stagedFiles: Seq[StageFile] = Seq.empty) extends DockerfileLike {
+                      var stagedFiles: Seq[StageFile] = Seq.empty,
+                      var stagedArchives: Seq[StagedArchive] = Seq.empty) extends DockerfileLike {
   type T = Dockerfile
 
   def addInstruction(instruction: Instruction) = {
@@ -37,4 +38,9 @@ case class Dockerfile(var instructions: Seq[Instruction] = Seq.empty,
     stagedFiles ++= files
     this
   }
+
+  def stageArchive(archive: StagedArchive) = {
+    stagedArchives :+= archive
+    this
+  }
 }