Skip to content

Commit

Permalink
add convenience constructor
Browse files Browse the repository at this point in the history
  • Loading branch information
saurfang committed Jun 24, 2015
1 parent 6b853fc commit 67cd586
Show file tree
Hide file tree
Showing 7 changed files with 59 additions and 67 deletions.
49 changes: 32 additions & 17 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -135,29 +135,26 @@ import sbtsparksubmit.SparkSubmitPlugin.autoImport._

object SparkSubmit {
lazy val settings =
SparkSubmitSetting("sparkPi").
setting(
sparkSubmitSparkArgs,
"--class", "SparkPi"
)
SparkSubmitSetting("sparkPi",
Seq("--class", "SparkPi")
)
}
```
Here we created a single `SparkSubmitSetting` object and fuses it with additional settings.

To create multiple tasks, you can wrap them with `SparkSubmitSetting` again like this:
```scala
lazy val settings = SparkSubmitSetting(
SparkSubmitSetting("spark2").
setting(sparkSubmitSparkArgs,
"--class", "Main2"
),
SparkSubmitSetting("spark2Other").
setting(sparkSubmitSparkArgs,
"--class", "Main2"
).
setting(sparkSubmitAppArgs,
"hello.txt"
)
lazy val settings = SparkSubmitSetting(
SparkSubmitSetting("spark1",
Seq("--class", "Main1")
),
SparkSubmitSetting("spark2",
Seq("--class", "Main2")
),
SparkSubmitSetting("spark2Other",
Seq("--class", "Main2"),
Seq("hello.txt")
)
)
```

Expand All @@ -170,7 +167,23 @@ sbt spark2Other
```
would be equivalent.

`SparkSubmitSetting` has three `apply` functions:
```scala
def apply(name: String): SparkSubmitSetting
def apply(name: String, sparkArgs: Seq[String] = Seq(), appArgs: Seq[String] = Seq()): SparkSubmitSetting
def apply(sparkSubmitSettings: SparkSubmitSetting*): Seq[Def.Setting[_]]
```
The first creates a simple `SparkSubmitSetting` object with a custom task name. The object itself has `setting` function
that allows you to blend in additional settings that is specific to this task.

Because the most common use case of custom task is to provide custom default Spark and Application arguments,
the second variant allow you provide those directly.

There is already an implicit conversion from `SparkSubmitSetting` to `Seq[Def.Setting[_]]` which allows you to
append itself to your project. When there are multiple settings, the third variant allows you to aggregate all
of them without additional type hinting for implicit to work.

See `src/sbt-test/sbt-spark-submit/multi-main` for examples.

## Multi-project builds

Expand All @@ -186,6 +199,8 @@ select any specific project.

Of course, `sparkB` task won't even trigger a build on `A` unless `B` depends on `A` thanks to the magic of sbt.

See `src/sbt-test/sbt-spark-submit/multi-project` for examples.

## Resources

For more information and working examples, see projects under `examples` and `src/sbt-test`.
10 changes: 5 additions & 5 deletions examples/sbt-assembly-on-yarn/project/SparkSubmit.scala
Original file line number Diff line number Diff line change
Expand Up @@ -2,10 +2,10 @@ import sbtsparksubmit.SparkSubmitPlugin.autoImport._

object SparkSubmit {
lazy val settings: Seq[sbt.Def.Setting[_]] =
SparkSubmitSetting("sparkPi").
setting(
sparkSubmitSparkArgs,
"--class", "SparkPi",
"--num-executors", "1000"
SparkSubmitSetting("sparkPi",
Seq(
"--class", "SparkPi",
"--num-executors", "1000"
)
)
}
6 changes: 1 addition & 5 deletions examples/spark-submit-plugin-test/project/SparkSubmit.scala
Original file line number Diff line number Diff line change
Expand Up @@ -2,9 +2,5 @@ import sbtsparksubmit.SparkSubmitPlugin.autoImport._

object SparkSubmit {
lazy val settings =
SparkSubmitSetting("sparkPi").
setting(
sparkSubmitSparkArgs,
"--class", "SparkPi"
)
SparkSubmitSetting("sparkPi", Seq("--class", "SparkPi"))
}
13 changes: 5 additions & 8 deletions src/main/scala/sbtsparksubmit/SparkSubmitPlugin.scala
Original file line number Diff line number Diff line change
Expand Up @@ -41,18 +41,10 @@ object SparkSubmitPlugin extends AutoPlugin {
this.settings +:= (taskKey in this := value)
this
}
def setting[T](taskKey: TaskKey[Seq[T]], value: T*): this.type = {
this.settings +:= (taskKey in this := value)
this
}
def setting[T](settingKey: SettingKey[T], value: T): this.type = {
this.settings +:= (settingKey in this := value)
this
}
def setting[T](settingKey: SettingKey[Seq[T]], value: T*): this.type = {
this.settings +:= (settingKey in this := value)
this
}


lazy val defaultSettings = Seq(
Expand Down Expand Up @@ -100,6 +92,11 @@ object SparkSubmitPlugin extends AutoPlugin {

object SparkSubmitSetting {
def apply(name: String): SparkSubmitSetting = new SparkSubmitSetting(name)
def apply(name: String, sparkArgs: Seq[String] = Seq(), appArgs: Seq[String] = Seq()): SparkSubmitSetting = {
new SparkSubmitSetting(name).
setting(sparkSubmitSparkArgs, sparkArgs).
setting(sparkSubmitAppArgs, appArgs)
}
def apply(sparkSubmitSettings: SparkSubmitSetting*): Seq[Def.Setting[_]] = {
sparkSubmitSettings.map(_.toSettings).reduce(_ ++ _)
}
Expand Down
27 changes: 11 additions & 16 deletions src/sbt-test/sbt-spark-submit/multi-main/project/SparkSubmit.scala
Original file line number Diff line number Diff line change
Expand Up @@ -3,20 +3,15 @@ import sbt._

object SparkSubmit {
lazy val settings = SparkSubmitSetting(
SparkSubmitSetting("spark1").
setting(sparkSubmitSparkArgs,
"--class", "Main1"
),
SparkSubmitSetting("spark2").
setting(sparkSubmitSparkArgs,
"--class", "Main2"
),
SparkSubmitSetting("spark2Other").
setting(sparkSubmitSparkArgs,
"--class", "Main2"
).
setting(sparkSubmitAppArgs,
"hello.txt"
)
SparkSubmitSetting("spark1",
Seq("--class", "Main1")
),
SparkSubmitSetting("spark2",
Seq("--class", "Main2")
),
SparkSubmitSetting("spark2Other",
Seq("--class", "Main2"),
Seq("hello.txt")
)
)
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -3,15 +3,9 @@ import sbt._

object SparkSubmit {
lazy val settingsFoo = SparkSubmitSetting(
SparkSubmitSetting("sparkFoo").
setting(sparkSubmitSparkArgs,
"--class", "Main"
)
SparkSubmitSetting("sparkFoo", Seq("--class", "Main"))
)
lazy val settingsBar = SparkSubmitSetting(
SparkSubmitSetting("sparkBar").
setting(sparkSubmitSparkArgs,
"--class", "Main"
)
SparkSubmitSetting("sparkBar", Seq("--class", "Main"))
)
}
}
Original file line number Diff line number Diff line change
@@ -1,10 +1,5 @@
import sbtsparksubmit.SparkSubmitPlugin.autoImport._

object SparkSubmit {
lazy val settings =
SparkSubmitSetting("sparkPi").
setting(
sparkSubmitSparkArgs,
"--class", "SparkPi"
)
}
lazy val settings = SparkSubmitSetting("sparkPi", Seq("--class", "SparkPi"))
}

0 comments on commit 67cd586

Please sign in to comment.