diff --git a/src/main/scala/sbtdocker/DockerBuilder.scala b/src/main/scala/sbtdocker/DockerBuilder.scala index 0ec8d13..1be11b9 100644 --- a/src/main/scala/sbtdocker/DockerBuilder.scala +++ b/src/main/scala/sbtdocker/DockerBuilder.scala @@ -30,6 +30,7 @@ object DockerBuilder { IO.write(stageDir / "Dockerfile", dockerFile.mkString) copyFiles(dockerFile.stagedFiles, stageDir, log) + copyFiles(dockerFile.stagedArchives.map(_.outputFile), stageDir, log) } def copyFiles(files: Seq[StageFile], stageDir: File, log: Logger) = { diff --git a/src/main/scala/sbtdocker/DockerKeys.scala b/src/main/scala/sbtdocker/DockerKeys.scala index e692fcb..700c4a7 100644 --- a/src/main/scala/sbtdocker/DockerKeys.scala +++ b/src/main/scala/sbtdocker/DockerKeys.scala @@ -4,6 +4,8 @@ import sbt._ object DockerKeys { val docker = taskKey[ImageId]("Build a Docker image.") + val dockerNoCache = taskKey[ImageId]("Build a Docker image avoiding sbt's cache.") + val baseDockerImage = taskKey[ImageId]("Build the base Docker image.") val dockerBuildAndPush = taskKey[ImageId]("Build a Docker image and pushes it to a registry.") val dockerPush = taskKey[Unit]("Push a already built Docker image to a registry.") diff --git a/src/main/scala/sbtdocker/DockerSettings.scala b/src/main/scala/sbtdocker/DockerSettings.scala index ae8ab86..221ad32 100644 --- a/src/main/scala/sbtdocker/DockerSettings.scala +++ b/src/main/scala/sbtdocker/DockerSettings.scala @@ -1,24 +1,15 @@ package sbtdocker -import sbt.Keys.target +import sbt.Keys._ import sbt._ import sbtdocker.DockerKeys._ +import sbtdocker.Instructions.{From, Add} object DockerSettings { lazy val baseDockerSettings = Seq( - docker := { - val log = Keys.streams.value.log - val dockerCmd = (DockerKeys.dockerCmd in docker).value - val buildOptions = (DockerKeys.buildOptions in docker).value - val stageDir = (target in docker).value - val dockerfile = (DockerKeys.dockerfile in docker).value - val imageName = (DockerKeys.imageName in docker).value - - log.debug("Dockerfile:") - log.debug(dockerfile.mkString) - - DockerBuilder(dockerCmd, buildOptions, imageName, dockerfile, stageDir, log) - }, + docker <<= buildDockerImage(docker, cache = true), + dockerNoCache <<= buildDockerImage(docker, cache = false), + baseDockerImage <<= buildDockerImage(baseDockerImage, cache = true), dockerPush := { val log = Keys.streams.value.log val dockerCmd = (DockerKeys.dockerCmd in docker).value @@ -42,16 +33,96 @@ object DockerSettings { |} """.stripMargin) }, + dockerfile in baseDockerImage := { + sys.error( + """A Dockerfile is not defined. Please define it with `dockerfile in docker` + | + |Example: + |dockerfile in docker := new Dockerfile { + | from("ubuntu") + | ... + |} + """.stripMargin) + }, target in docker := target.value / "docker", imageName in docker := { val organisation = Option(Keys.organization.value).filter(_.nonEmpty) val name = Keys.normalizedName.value ImageName(namespace = organisation, repository = name) }, - dockerCmd in docker := sys.env.get("DOCKER").filter(_.nonEmpty).getOrElse("docker"), - buildOptions in docker := BuildOptions() + imageName in baseDockerImage := { + val organisation = Option(Keys.organization.value).filter(_.nonEmpty) + val name = Keys.normalizedName.value + "-base" + ImageName(namespace = organisation, repository = name) + }, + dockerCmd := sys.env.get("DOCKER").filter(_.nonEmpty).getOrElse("docker"), + buildOptions in docker := BuildOptions(), + buildOptions in baseDockerImage := BuildOptions() ) + def buildDockerImage(dockerTask: TaskKey[_], cache: Boolean): Def.Initialize[Task[ImageId]] = Def.task { + val log = Keys.streams.value.log + val dockerCmd = DockerKeys.dockerCmd.value + val buildOptions = (DockerKeys.buildOptions in dockerTask).value + val stageDir = (target in docker).value + val dockerfile = (DockerKeys.dockerfile in dockerTask).value + val imageName = (DockerKeys.imageName in dockerTask).value + val cacheDir = target.value / s"docker-image-cache" / imageName.toString + val cacheFile = cacheDir/ name.value + + log.debug("Dockerfile:") + log.debug(dockerfile.mkString) + if (cache) { + /* Wrap actual function in a function that provides basic caching . */ + val cachedFun = FileFunction.cached(cacheDir, FilesInfo.lastModified, FilesInfo.exists) { + (inFiles: Set[File]) => { + val imageId = DockerBuilder(dockerCmd, buildOptions, imageName, dockerfile, stageDir, log) + IO.write(cacheFile, imageId.id) + Set(cacheFile) + } + } + + /** Recursively list all directories so caching will detect changes in subdirectories */ + def findDirs(file: File): Seq[File] = file.isDirectory match { + case true => file +: file.listFiles().flatMap(findDirs) + case false => Nil + } + + // Normalize instructions that include our temporary tgz files since these change every run. + // Changes to these files will be detected by changes to StagedArchive directories. + // Also, convert names of the base image to ids to detect changes better. + val userInstructions = dockerfile.instructions.map { + case Add(src, dest) if src.contains("dockerbuild") => Add("archive", dest) + case From(baseImageName) => From(s"$dockerCmd history -q $baseImageName".!!.split("\n").head) + case other => other + } + + // To detect changes in the docker file itself we create a fake file based on the hash + // of the docker file description. + // TODO: use a better hash function here... + val fakeDockerFile = target.value / (userInstructions.hashCode.toString + ".docker.hash") + log.debug(s"dockerfile caching hashcode: $fakeDockerFile") + + val stagedArchives = dockerfile.stagedArchives.flatMap(d => findDirs(d.file)) + val stagedFiles = dockerfile.stagedFiles.map(_.source) + val fileDependencies = (Seq(fakeDockerFile) ++ stagedArchives ++ stagedFiles).toSet + + val cachedId = IO.read(cachedFun(fileDependencies).head) + val imageExists = try { s"docker history $cachedId".!!; true} catch { + case _: Exception => false + } + + if (imageExists) { + // The cached image stil exists, return it. + ImageId(cachedId) + } else { + DockerBuilder(dockerCmd, buildOptions, imageName, dockerfile, stageDir, log) + } + } else { + DockerBuilder(dockerCmd, buildOptions, imageName, dockerfile, stageDir, log) + } + } + def packageDockerSettings(fromImage: String, exposePorts: Seq[Int]) = Seq( docker <<= docker.dependsOn(Keys.`package`.in(Compile, Keys.packageBin)), Keys.mainClass in docker <<= Keys.mainClass in docker or Keys.mainClass.in(Compile, Keys.packageBin), diff --git a/src/main/scala/sbtdocker/DockerfileCommands.scala b/src/main/scala/sbtdocker/DockerfileCommands.scala index 0ccfd06..7d31e21 100644 --- a/src/main/scala/sbtdocker/DockerfileCommands.scala +++ b/src/main/scala/sbtdocker/DockerfileCommands.scala @@ -1,5 +1,7 @@ package sbtdocker +import java.io.File + import sbt._ import sbtdocker.Instructions._ import sbtdocker.Utils._ @@ -24,6 +26,28 @@ object StageFile { */ case class StageFile(source: File, target: File) +/** + * Container for a directory that should be copied to a path in the stage directory in the form + * of a tar.gz file. + * + * @param file The path that should be compressed. + * @param dest Path in the stage directory. + */ +case class StagedArchive(file: File, dest: String) { + val tempFile = java.io.File.createTempFile("dockerbuild", ".tgz") + + // We lazily build the actual tar ball, only when the SBT caching detects that the contents have + // changed. + lazy val outputFile = { + val parentDir = file.getParent + val command = s"tar czvf $tempFile -C $parentDir ${file.getName}" + println(command) + command.!! + + StageFile(tempFile, expandPath(tempFile, "/")) + } +} + trait DockerfileLike extends DockerfileCommands { type T <: DockerfileLike @@ -31,6 +55,8 @@ trait DockerfileLike extends DockerfileCommands { def stagedFiles: Seq[StageFile] + def stagedArchives: Seq[StagedArchive] + def mkString = instructions.mkString("\n") } @@ -80,6 +106,14 @@ trait DockerfileCommands { stageFile(copy) } + /** + * Adds an directory that will be transmitted to the docker daemon in the form of a tar.gz file. + * This format is convenient when you want to preserve symlinks. + * @param archive + */ + def stageArchive(archive: StagedArchive): T + + /** * Stage multiple files. */ @@ -117,6 +151,12 @@ trait DockerfileCommands { .stageFile(source, destination) } + def addCompressed(file: File, dest: String): T = { + val archive = StagedArchive(file, dest) + stageArchive(archive) + addInstruction(Add(archive.tempFile.getName, dest)) + } + def add(source: URL, destination: String) = addInstruction(Add(source.toString, destination)) def add(source: URL, destination: File) = addInstruction(Add(source.toString, destination.toString)) diff --git a/src/main/scala/sbtdocker/immutable/Dockerfile.scala b/src/main/scala/sbtdocker/immutable/Dockerfile.scala index 2ab6c65..f35221a 100644 --- a/src/main/scala/sbtdocker/immutable/Dockerfile.scala +++ b/src/main/scala/sbtdocker/immutable/Dockerfile.scala @@ -1,6 +1,6 @@ package sbtdocker.immutable -import sbtdocker.{StageFile, DockerfileLike, Instruction} +import sbtdocker.{StagedArchive, StageFile, DockerfileLike, Instruction} object Dockerfile { def empty = Dockerfile() @@ -23,12 +23,19 @@ object Dockerfile { * @param stagedFiles Files and directories that should be copied to the stage directory. */ case class Dockerfile(instructions: Seq[Instruction] = Seq.empty, - stagedFiles: Seq[StageFile] = Seq.empty) extends DockerfileLike { + stagedFiles: Seq[StageFile] = Seq.empty, + stagedArchives: Seq[StagedArchive] = Seq.empty) extends DockerfileLike { type T = Dockerfile - def addInstruction(instruction: Instruction) = Dockerfile(instructions :+ instruction, stagedFiles) + def addInstruction(instruction: Instruction) = + Dockerfile(instructions :+ instruction, stagedFiles, stagedArchives) - def stageFile(file: StageFile) = Dockerfile(instructions, stagedFiles :+ file) + def stageFile(file: StageFile) = + Dockerfile(instructions, stagedFiles :+ file, stagedArchives) - def stageFiles(files: TraversableOnce[StageFile]) = Dockerfile(instructions, stagedFiles ++ files) + def stageFiles(files: TraversableOnce[StageFile]) = + Dockerfile(instructions, stagedFiles ++ files, stagedArchives) + + def stageArchive(archive: StagedArchive) = + Dockerfile(instructions, stagedFiles, archive +: stagedArchives ) } diff --git a/src/main/scala/sbtdocker/mutable/Dockerfile.scala b/src/main/scala/sbtdocker/mutable/Dockerfile.scala index 13233b7..262bdbf 100644 --- a/src/main/scala/sbtdocker/mutable/Dockerfile.scala +++ b/src/main/scala/sbtdocker/mutable/Dockerfile.scala @@ -1,6 +1,6 @@ package sbtdocker.mutable -import sbtdocker.{StageFile, DockerfileLike, Instruction} +import sbtdocker.{StagedArchive, StageFile, DockerfileLike, Instruction} /** * Mutable Dockerfile. @@ -20,7 +20,8 @@ import sbtdocker.{StageFile, DockerfileLike, Instruction} * @param stagedFiles Files and directories that should be copied to the stage directory. */ case class Dockerfile(var instructions: Seq[Instruction] = Seq.empty, - var stagedFiles: Seq[StageFile] = Seq.empty) extends DockerfileLike { + var stagedFiles: Seq[StageFile] = Seq.empty, + var stagedArchives: Seq[StagedArchive] = Seq.empty) extends DockerfileLike { type T = Dockerfile def addInstruction(instruction: Instruction) = { @@ -37,4 +38,9 @@ case class Dockerfile(var instructions: Seq[Instruction] = Seq.empty, stagedFiles ++= files this } + + def stageArchive(archive: StagedArchive) = { + stagedArchives :+= archive + this + } }