From 67cd586b30ba796b0a7d5b6ca37ad724fbf9ae07 Mon Sep 17 00:00:00 2001 From: Forest Fang Date: Wed, 24 Jun 2015 11:27:14 -0400 Subject: [PATCH] add convenience constructor --- README.md | 49 ++++++++++++------- .../project/SparkSubmit.scala | 10 ++-- .../project/SparkSubmit.scala | 6 +-- .../sbtsparksubmit/SparkSubmitPlugin.scala | 13 ++--- .../multi-main/project/SparkSubmit.scala | 27 +++++----- .../multi-project/project/SparkSubmit.scala | 12 ++--- .../sparkpi/project/SparkSubmit.scala | 9 +--- 7 files changed, 59 insertions(+), 67 deletions(-) diff --git a/README.md b/README.md index e08849b..f89e32c 100644 --- a/README.md +++ b/README.md @@ -135,29 +135,26 @@ import sbtsparksubmit.SparkSubmitPlugin.autoImport._ object SparkSubmit { lazy val settings = - SparkSubmitSetting("sparkPi"). - setting( - sparkSubmitSparkArgs, - "--class", "SparkPi" - ) + SparkSubmitSetting("sparkPi", + Seq("--class", "SparkPi") + ) } ``` Here we created a single `SparkSubmitSetting` object and fuses it with additional settings. To create multiple tasks, you can wrap them with `SparkSubmitSetting` again like this: ```scala -lazy val settings = SparkSubmitSetting( - SparkSubmitSetting("spark2"). - setting(sparkSubmitSparkArgs, - "--class", "Main2" - ), - SparkSubmitSetting("spark2Other"). - setting(sparkSubmitSparkArgs, - "--class", "Main2" - ). - setting(sparkSubmitAppArgs, - "hello.txt" - ) + lazy val settings = SparkSubmitSetting( + SparkSubmitSetting("spark1", + Seq("--class", "Main1") + ), + SparkSubmitSetting("spark2", + Seq("--class", "Main2") + ), + SparkSubmitSetting("spark2Other", + Seq("--class", "Main2"), + Seq("hello.txt") + ) ) ``` @@ -170,7 +167,23 @@ sbt spark2Other ``` would be equivalent. +`SparkSubmitSetting` has three `apply` functions: +```scala +def apply(name: String): SparkSubmitSetting +def apply(name: String, sparkArgs: Seq[String] = Seq(), appArgs: Seq[String] = Seq()): SparkSubmitSetting +def apply(sparkSubmitSettings: SparkSubmitSetting*): Seq[Def.Setting[_]] +``` +The first creates a simple `SparkSubmitSetting` object with a custom task name. The object itself has `setting` function +that allows you to blend in additional settings that is specific to this task. + +Because the most common use case of custom task is to provide custom default Spark and Application arguments, +the second variant allow you provide those directly. +There is already an implicit conversion from `SparkSubmitSetting` to `Seq[Def.Setting[_]]` which allows you to +append itself to your project. When there are multiple settings, the third variant allows you to aggregate all +of them without additional type hinting for implicit to work. + +See `src/sbt-test/sbt-spark-submit/multi-main` for examples. ## Multi-project builds @@ -186,6 +199,8 @@ select any specific project. Of course, `sparkB` task won't even trigger a build on `A` unless `B` depends on `A` thanks to the magic of sbt. +See `src/sbt-test/sbt-spark-submit/multi-project` for examples. + ## Resources For more information and working examples, see projects under `examples` and `src/sbt-test`. \ No newline at end of file diff --git a/examples/sbt-assembly-on-yarn/project/SparkSubmit.scala b/examples/sbt-assembly-on-yarn/project/SparkSubmit.scala index e305f06..6388351 100644 --- a/examples/sbt-assembly-on-yarn/project/SparkSubmit.scala +++ b/examples/sbt-assembly-on-yarn/project/SparkSubmit.scala @@ -2,10 +2,10 @@ import sbtsparksubmit.SparkSubmitPlugin.autoImport._ object SparkSubmit { lazy val settings: Seq[sbt.Def.Setting[_]] = - SparkSubmitSetting("sparkPi"). - setting( - sparkSubmitSparkArgs, - "--class", "SparkPi", - "--num-executors", "1000" + SparkSubmitSetting("sparkPi", + Seq( + "--class", "SparkPi", + "--num-executors", "1000" + ) ) } diff --git a/examples/spark-submit-plugin-test/project/SparkSubmit.scala b/examples/spark-submit-plugin-test/project/SparkSubmit.scala index 0b80bb9..0943a4e 100644 --- a/examples/spark-submit-plugin-test/project/SparkSubmit.scala +++ b/examples/spark-submit-plugin-test/project/SparkSubmit.scala @@ -2,9 +2,5 @@ import sbtsparksubmit.SparkSubmitPlugin.autoImport._ object SparkSubmit { lazy val settings = - SparkSubmitSetting("sparkPi"). - setting( - sparkSubmitSparkArgs, - "--class", "SparkPi" - ) + SparkSubmitSetting("sparkPi", Seq("--class", "SparkPi")) } diff --git a/src/main/scala/sbtsparksubmit/SparkSubmitPlugin.scala b/src/main/scala/sbtsparksubmit/SparkSubmitPlugin.scala index 99cb5b0..d3fef8e 100644 --- a/src/main/scala/sbtsparksubmit/SparkSubmitPlugin.scala +++ b/src/main/scala/sbtsparksubmit/SparkSubmitPlugin.scala @@ -41,18 +41,10 @@ object SparkSubmitPlugin extends AutoPlugin { this.settings +:= (taskKey in this := value) this } - def setting[T](taskKey: TaskKey[Seq[T]], value: T*): this.type = { - this.settings +:= (taskKey in this := value) - this - } def setting[T](settingKey: SettingKey[T], value: T): this.type = { this.settings +:= (settingKey in this := value) this } - def setting[T](settingKey: SettingKey[Seq[T]], value: T*): this.type = { - this.settings +:= (settingKey in this := value) - this - } lazy val defaultSettings = Seq( @@ -100,6 +92,11 @@ object SparkSubmitPlugin extends AutoPlugin { object SparkSubmitSetting { def apply(name: String): SparkSubmitSetting = new SparkSubmitSetting(name) + def apply(name: String, sparkArgs: Seq[String] = Seq(), appArgs: Seq[String] = Seq()): SparkSubmitSetting = { + new SparkSubmitSetting(name). + setting(sparkSubmitSparkArgs, sparkArgs). + setting(sparkSubmitAppArgs, appArgs) + } def apply(sparkSubmitSettings: SparkSubmitSetting*): Seq[Def.Setting[_]] = { sparkSubmitSettings.map(_.toSettings).reduce(_ ++ _) } diff --git a/src/sbt-test/sbt-spark-submit/multi-main/project/SparkSubmit.scala b/src/sbt-test/sbt-spark-submit/multi-main/project/SparkSubmit.scala index 3f67cba..923aebc 100644 --- a/src/sbt-test/sbt-spark-submit/multi-main/project/SparkSubmit.scala +++ b/src/sbt-test/sbt-spark-submit/multi-main/project/SparkSubmit.scala @@ -3,20 +3,15 @@ import sbt._ object SparkSubmit { lazy val settings = SparkSubmitSetting( - SparkSubmitSetting("spark1"). - setting(sparkSubmitSparkArgs, - "--class", "Main1" - ), - SparkSubmitSetting("spark2"). - setting(sparkSubmitSparkArgs, - "--class", "Main2" - ), - SparkSubmitSetting("spark2Other"). - setting(sparkSubmitSparkArgs, - "--class", "Main2" - ). - setting(sparkSubmitAppArgs, - "hello.txt" - ) + SparkSubmitSetting("spark1", + Seq("--class", "Main1") + ), + SparkSubmitSetting("spark2", + Seq("--class", "Main2") + ), + SparkSubmitSetting("spark2Other", + Seq("--class", "Main2"), + Seq("hello.txt") + ) ) -} \ No newline at end of file +} diff --git a/src/sbt-test/sbt-spark-submit/multi-project/project/SparkSubmit.scala b/src/sbt-test/sbt-spark-submit/multi-project/project/SparkSubmit.scala index 497795c..4f14d00 100644 --- a/src/sbt-test/sbt-spark-submit/multi-project/project/SparkSubmit.scala +++ b/src/sbt-test/sbt-spark-submit/multi-project/project/SparkSubmit.scala @@ -3,15 +3,9 @@ import sbt._ object SparkSubmit { lazy val settingsFoo = SparkSubmitSetting( - SparkSubmitSetting("sparkFoo"). - setting(sparkSubmitSparkArgs, - "--class", "Main" - ) + SparkSubmitSetting("sparkFoo", Seq("--class", "Main")) ) lazy val settingsBar = SparkSubmitSetting( - SparkSubmitSetting("sparkBar"). - setting(sparkSubmitSparkArgs, - "--class", "Main" - ) + SparkSubmitSetting("sparkBar", Seq("--class", "Main")) ) -} \ No newline at end of file +} diff --git a/src/sbt-test/sbt-spark-submit/sparkpi/project/SparkSubmit.scala b/src/sbt-test/sbt-spark-submit/sparkpi/project/SparkSubmit.scala index d26ab44..078e4b2 100644 --- a/src/sbt-test/sbt-spark-submit/sparkpi/project/SparkSubmit.scala +++ b/src/sbt-test/sbt-spark-submit/sparkpi/project/SparkSubmit.scala @@ -1,10 +1,5 @@ import sbtsparksubmit.SparkSubmitPlugin.autoImport._ object SparkSubmit { - lazy val settings = - SparkSubmitSetting("sparkPi"). - setting( - sparkSubmitSparkArgs, - "--class", "SparkPi" - ) -} \ No newline at end of file + lazy val settings = SparkSubmitSetting("sparkPi", Seq("--class", "SparkPi")) +}