Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Reformat with 0.2.13 #4

Open
wants to merge 1 commit into
base: 0.2.11
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
The table of contents is too big for display.
Diff view
Diff view
  •  
  •  
  •  
The diff you're trying to view is too large. We only load the first 3000 changed files.
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@ package io.prediction.controller
import io.prediction.annotation.Experimental
import io.prediction.core.BaseServing

/** Base class of serving.
/** Base class of serving.
*
* @tparam Q Input query class.
* @tparam P Output prediction class.
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -195,7 +195,7 @@ abstract class OptionStdevMetric[EI, Q, P, A]
}
}

/** Returns the sum of the score returned by the calculate method.
/** Returns the sum of the score returned by the calculate method.
*
* @tparam EI Evaluation information
* @tparam Q Query
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -70,24 +70,24 @@ private[prediction] case class FakeEvalResult() extends BaseEvaluatorResult {
}

/** FakeRun allows user to implement custom function under the exact enviroment
* as other PredictionIO workflow.
* as other PredictionIO workflow.
*
* Useful for developing new features. Only need to extend this trait and
* implement a function: (SparkContext => Unit). For example, the code below
* Useful for developing new features. Only need to extend this trait and
* implement a function: (SparkContext => Unit). For example, the code below
* can be run with `pio eval HelloWorld`.
*
* {{{
* object HelloWorld extends FakeRun {
* // func defines the function pio runs, must have signature (SparkContext => Unit).
* func = f
*
*
* def f(sc: SparkContext): Unit {
* val logger = Logger[this.type]
* logger.info("HelloWorld")
* }
* }
* }}}
*
* }}}
*
*/
@Experimental
trait FakeRun extends Evaluation with EngineParamsGenerator {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -39,10 +39,8 @@ object Workflow {
evaluatorParams: Params,
env: Map[String, String] = WorkflowUtils.pioEnvVars,
params: WorkflowParams = WorkflowParams()) {

implicit lazy val formats = Utils.json4sDefaultFormats +
new NameParamsSerializer

val engineInstance = EngineInstance(
id = "",
status = "INIT",
Expand All @@ -64,7 +62,6 @@ object Workflow {
evaluatorResults = "",
evaluatorResultsHTML = "",
evaluatorResultsJSON = "")

CoreWorkflow.runEval(
engine = engine,
engineParams = engineParams,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -43,21 +43,18 @@ object TestEvaluator {

/*
class Evaluator0 extends Evaluator[EvalInfo, Query, Prediction, Actual,
(Query, Prediction, Actual),
(Query, Prediction, Actual),
(EvalInfo, Seq[(Query, Prediction, Actual)]),
Seq[(EvalInfo, (EvalInfo, Seq[(Query, Prediction, Actual)]))]
] {

def evaluateUnit(q: Query, p: Prediction, a: Actual)
: (Query, Prediction, Actual) = (q, p, a)

def evaluateSet(
evalInfo: EvalInfo,
evalInfo: EvalInfo,
eus: Seq[(Query, Prediction, Actual)])
: (EvalInfo, Seq[(Query, Prediction, Actual)]) = (evalInfo, eus)

def evaluateAll(
input: Seq[(EvalInfo, (EvalInfo, Seq[(Query, Prediction, Actual)]))])
input: Seq[(EvalInfo, (EvalInfo, Seq[(Query, Prediction, Actual)]))])
= input
}
*/
Expand All @@ -67,25 +64,19 @@ object TestEvaluator {
class EvaluatorSuite
extends FunSuite with Inside with SharedSparkContext {
import io.prediction.controller.TestEvaluator._
@transient lazy val logger = Logger[this.type]

@transient lazy val logger = Logger[this.type]
test("Evaluator.evaluate") {
val engine = new FakeEngine(1, 3, 10)
val evaluator = new Evaluator0()

val evalDataSet = engine.eval(sc, null.asInstanceOf[EngineParams])
val er: Seq[(EvalInfo, (EvalInfo, Seq[(Query, Prediction, Actual)]))] =
evaluator.evaluateBase(sc, evalDataSet)

evalDataSet.zip(er).map { case (input, output) => {
val (inputEvalInfo, inputQpaRDD) = input
val (outputEvalInfo, (outputEvalInfo2, outputQpaSeq)) = output

inputEvalInfo shouldBe outputEvalInfo
inputEvalInfo shouldBe outputEvalInfo2

val inputQpaSeq: Array[(Query, Prediction, Actual)] = inputQpaRDD.collect

inputQpaSeq.size should be (outputQpaSeq.size)
// TODO. match inputQpa and outputQpa content.
}}
Expand Down
Original file line number Diff line number Diff line change
@@ -1,7 +1,9 @@
//package org.apache.spark
package io.prediction.workflow

import _root_.io.netty.util.internal.logging.{Slf4JLoggerFactory, InternalLoggerFactory}
import _root_.io.netty.util.internal.logging.{
Slf4JLoggerFactory, InternalLoggerFactory
}
import org.scalatest.BeforeAndAfterAll
import org.scalatest.BeforeAndAfterEach
import org.scalatest.Suite
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@

Original file line number Diff line number Diff line change
Expand Up @@ -129,7 +129,7 @@ object BiMap {
new BiMap(HashMap(ki: _*))
}

/** Create a BiMap[String, Double] from a set of String. The Double index
/** Create a BiMap[String, Double] from a set of String. The Double index
* starts from 0.
* @param keys a set of String
* @return a String to Double BiMap
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -84,7 +84,7 @@ object LEventStore {
}

/** Reads events generically. If entityType or entityId is not specified, it
* results in table scan.
* results in table scan.
*
* @param appName return events of this app
* @param entityType return events of this entityType
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -165,21 +165,18 @@ class LBatchView(val appId: Int,
predicate: Event => Boolean,
init: T,
op: (T, Event) => T): Map[String, T] = {

_events
.filter( predicate(_) )
.groupBy( _.entityId )
.mapValues( _.sortBy(_.eventTime.getMillis).foldLeft[T](init)(op))
.toMap

}
*/

/*
def groupByEntityOrdered[T](
predicate: Event => Boolean,
map: Event => T): Map[String, Seq[T]] = {

_events
.filter( predicate(_) )
.groupBy( _.entityId )
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -75,7 +75,7 @@ private[prediction] object MailChimpConnector extends FormConnector {
"data[merges][LNAME]": "API",
"data[merges][INTERESTS]": "Group1,Group2",
"data[ip_opt]": "10.20.10.30",
"data[ip_signup]": "10.20.10.30"
"data[ip_signup]": "10.20.10.30"
*/

// convert to ISO8601 format
Expand Down Expand Up @@ -109,7 +109,6 @@ private[prediction] object MailChimpConnector extends FormConnector {
/*
"action" will either be "unsub" or "delete".
The reason will be "manual" unless caused by a spam complaint - then it will be "abuse"

"type": "unsubscribe",
"fired_at": "2009-03-26 21:40:57",
"data[action]": "unsub",
Expand Down Expand Up @@ -196,11 +195,11 @@ private[prediction] object MailChimpConnector extends FormConnector {
import org.json4s.JsonDSL._

/*
"type": "upemail",
"fired_at": "2009-03-26 22:15:09",
"type": "upemail",
"fired_at": "2009-03-26 22:15:09",
"data[list_id]": "a6b5da1054",
"data[new_id]": "51da8c3259",
"data[new_email]": "[email protected]",
"data[new_id]": "51da8c3259",
"data[new_email]": "[email protected]",
"data[old_email]": "[email protected]"
*/

Expand All @@ -225,8 +224,8 @@ private[prediction] object MailChimpConnector extends FormConnector {

/*
Reason will be one of "hard" (for hard bounces) or "abuse"
"type": "cleaned",
"fired_at": "2009-03-26 22:01:00",
"type": "cleaned",
"fired_at": "2009-03-26 22:01:00",
"data[list_id]": "a6b5da1054",
"data[campaign_id]": "4fjk2ma9xd",
"data[reason]": "hard",
Expand All @@ -253,12 +252,12 @@ private[prediction] object MailChimpConnector extends FormConnector {
import org.json4s.JsonDSL._

/*
"type": "campaign",
"fired_at": "2009-03-26 21:31:21",
"type": "campaign",
"fired_at": "2009-03-26 21:31:21",
"data[id]": "5aa2102003",
"data[subject]": "Test Campaign Subject",
"data[subject]": "Test Campaign Subject",
"data[status]": "sent",
"data[reason]": "",
"data[reason]": "",
"data[list_id]": "a6b5da1054"
*/

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -83,7 +83,7 @@ class MailChimpConnectorSpec extends Specification with ConnectorTestUtil {
"data[merges][EMAIL]" -> "[email protected]",
"data[merges][FNAME]" -> "MailChimp",
"data[merges][LNAME]" -> "API",
"data[merges][INTERESTS]" -> "Group1,Group2", //optional
"data[merges][INTERESTS]" -> "Group1,Group2", //optional
"data[ip_opt]" -> "10.20.10.30",
"data[campaign_id]" -> "cb398d21d2"
)
Expand Down Expand Up @@ -116,7 +116,7 @@ class MailChimpConnectorSpec extends Specification with ConnectorTestUtil {
check(MailChimpConnector, unsubscribe, expected)
}

//check profile update to event Json
//check profile update to event Json
"convert profile update to event JSON" in {

val profileUpdate = Map(
Expand Down Expand Up @@ -158,7 +158,7 @@ class MailChimpConnectorSpec extends Specification with ConnectorTestUtil {
check(MailChimpConnector, profileUpdate, expected)
}

//check email update to event Json
//check email update to event Json
"convert email update to event JSON" in {

val emailUpdate = Map(
Expand Down Expand Up @@ -188,7 +188,7 @@ class MailChimpConnectorSpec extends Specification with ConnectorTestUtil {
check(MailChimpConnector, emailUpdate, expected)
}

//check cleaned email to event Json
//check cleaned email to event Json
"convert cleaned email to event JSON" in {

val cleanedEmail = Map(
Expand Down Expand Up @@ -217,7 +217,7 @@ class MailChimpConnectorSpec extends Specification with ConnectorTestUtil {
check(MailChimpConnector, cleanedEmail, expected)
}

//check campaign sending status to event Json
//check campaign sending status to event Json
"convert campaign sending status to event JSON" in {

val campaign = Map(
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -10,12 +10,10 @@ import org.apache.spark.rdd.RDD
/*
class Preparator
extends PPreparator[TrainingData, PreparedData] {

def prepare(sc: SparkContext, trainingData: TrainingData): PreparedData = {
new PreparedData(events = trainingData.events)
}
}

class PreparedData(
val events: RDD[Event]
) extends Serializable
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@ class FriendRecommendationDataSource(
readRelationship(dsp.userActionFilePath, userKeyword.size, userIdMap)
// Originally for the purpose of training an acceptance threshold
// Commented out here due to the high time and space complexity of training
// val trainingRecord = readTrainingRecord(dsp.trainingRecordFilePath,
// val trainingRecord = readTrainingRecord(dsp.trainingRecordFilePath,
// userIdMap, itemIdMap)
val trainingRecord = null
new FriendRecommendationTrainingData(userIdMap,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ class FriendRecommendationTrainingData(
// Keyword array, internal id index, term-weight map item
val userKeyword: Array[HashMap[Int, Double]],
val itemKeyword: Array[HashMap[Int, Double]],
// User relationship array,
// User relationship array,
// src internal id index, dest-internal-id-weight list item
val socialAction: Array[List[(Int, Int)]],
// Training record for training purpose
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -44,10 +44,10 @@ case class LocalDataSource(val dsp: DataSourceParams)
Source.fromFile(dsp.filepath).getLines.toSeq.map(_.split(" ", 2))

// FIXME: Use different training / testing data.
val x = lines.map { _ (1).split(' ').map { _.toDouble } }.map { e =>
val x = lines.map { _(1).split(' ').map { _.toDouble } }.map { e =>
Vector(e: _*)
}
val y = lines.map { _ (0).toDouble }
val y = lines.map { _(0).toDouble }

val td = TrainingData(Vector(x: _*), Vector(y: _*))

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -10,12 +10,10 @@ import org.apache.spark.rdd.RDD
/*
class Preparator
extends PPreparator[TrainingData, PreparedData] {

def prepare(sc: SparkContext, trainingData: TrainingData): PreparedData = {
new PreparedData(events = trainingData.events)
}
}

class PreparedData(
val events: RDD[Event]
) extends Serializable
Expand Down
Original file line number Diff line number Diff line change
@@ -1,23 +1,17 @@
/*
package pio.refactor

import io.prediction.controller.PPreparator
import io.prediction.data.storage.Event

import org.apache.spark.SparkContext
import org.apache.spark.SparkContext._
import org.apache.spark.rdd.RDD

class Preparator
extends PPreparator[TrainingData, PreparedData] {

def prepare(sc: SparkContext, trainingData: TrainingData): PreparedData = {
new PreparedData(events = trainingData.events)
}
}

class PreparedData(
val events: RDD[Event]
) extends Serializable

*/
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@ import nak.regress.LinearRegression
*
* All indicators should be defined as classes that extend
* this base class. See RSIIndicator as an example. These indicators can then
* be instantiated and passed into a StockStrategy class. Refer to tutorial
* be instantiated and passed into a StockStrategy class. Refer to tutorial
* for further explanation (found in the README.md file).
*/
@SerialVersionUID(100L)
Expand All @@ -34,12 +34,12 @@ abstract class BaseIndicator extends Serializable {
*/
def getTraining(logPrice: Series[DateTime, Double]): Series[DateTime, Double]

/** Applies indicator on a window size of the value returned by
/** Applies indicator on a window size of the value returned by
* getMinWindowSize() and returns the last value in the resulting series to
* be used for prediction in RegressionStrategy.
*
* @param logPrice series of logarithm of all prices for a particular stock
* @return the last value in the resulting series from the feature
* @return the last value in the resulting series from the feature
* calculation
*/
def getOne(input: Series[DateTime, Double]): Double
Expand Down Expand Up @@ -102,7 +102,7 @@ class RSIIndicator(rsiPeriod: Int = 14) extends BaseIndicator {
/** Indicator that calcuate differences of closing prices
*
* @constructor create an instance of a ShiftsIndicator
* @param period number of days between any 2 closing prices to consider for
* @param period number of days between any 2 closing prices to consider for
* calculating a return
*/
class ShiftsIndicator(period: Int) extends BaseIndicator {
Expand Down
Loading