From c35588940b2124016eaa8a84879eddd3b7c08415 Mon Sep 17 00:00:00 2001 From: Joris Borgdorff Date: Wed, 27 Sep 2017 16:06:49 +0200 Subject: [PATCH 01/23] Use new radar-schemas structure --- README.md | 6 +-- build.gradle | 6 +-- .../E4AggregatedAccelerationMonitor.java | 6 +-- .../radarcns/integration/PhoneStreamTest.java | 18 ++++----- src/integrationTest/resources/mock_file.yml | 2 +- src/main/avro/temporary_package_key.avsc | 2 +- .../monitor/AbstractKafkaMonitor.java | 12 ++++-- .../radarcns/monitor/BatteryLevelMonitor.java | 8 ++-- .../radarcns/monitor/DisconnectMonitor.java | 8 ++-- .../stream/empatica/E4AccelerationStream.java | 14 +++---- .../stream/empatica/E4BatteryLevelStream.java | 14 +++---- .../empatica/E4BloodVolumePulseStream.java | 14 +++---- .../E4ElectroDermalActivityStream.java | 14 +++---- .../stream/empatica/E4HeartRateStream.java | 14 +++---- .../empatica/E4InterBeatIntervalStream.java | 14 +++---- .../stream/empatica/E4TemperatureStream.java | 14 +++---- .../stream/phone/PhoneAccelerationStream.java | 14 +++---- .../stream/phone/PhoneBatteryStream.java | 14 +++---- .../phone/PhoneUsageAggregationStream.java | 12 +++--- .../stream/phone/PhoneUsageCollector.java | 4 +- .../stream/phone/PhoneUsageStream.java | 10 ++--- .../stream/phone/TemporaryPackageKey.java | 32 ++++++++-------- .../radarcns/util/PersistentStateStore.java | 34 +++++++++++++---- .../org/radarcns/util/RadarUtilities.java | 28 +++++++------- .../org/radarcns/util/RadarUtilitiesImpl.java | 38 +++++++++---------- .../monitor/BatteryLevelMonitorTest.java | 6 ++- .../monitor/DisconnectMonitorTest.java | 10 +++-- .../empatica/E4AccelerationTest.java.bak | 10 ++--- .../util/PersistentStateStoreTest.java | 6 +-- .../org/radarcns/util/RadarUtilsTest.java | 12 +++--- .../util/serde/JsonDeserializerTest.java | 6 +-- .../util/serde/JsonSerializerTest.java | 8 ++-- 32 files changed, 218 insertions(+), 192 deletions(-) diff --git a/README.md b/README.md index 00a67355..fb4bbed9 100644 --- a/README.md +++ b/README.md @@ -135,8 +135,8 @@ To get email notifications for Empatica E4 battery status, an email server witho data: - topic: topic1 file: topic1.csv - key_schema: org.radarcns.key.MeasurementKey - value_schema: org.radarcns.empatica.EmpaticaE4Acceleration + key_schema: org.radarcns.kafka.ObservationKey + value_schema: org.radarcns.passive.empatica.EmpaticaE4Acceleration ``` Each value has a topic to send the data to, a file containing the data, a schema class for the key and a schema class for the value. Also create a CSV file for each of these entries: @@ -177,7 +177,7 @@ RADAR-Stream is a layer on top of Kafka streams. Topics are processed by streams KafkaStreams currently communicates using master-slave model. The [MasterAggregator][1] defines the stream-master, while [AggregatorWorker][2] represents the stream-slave. The master-stream creates, starts and stops a list of stream-slaves registered with the corresponding master. While the classical Kafka Consumer requires two implementations to support standalone and group executions, the AggregatorWorker provides both behaviors with one implementation. -To extend the RADAR-Stream API, follow these steps (see the `org.radarcns.empatica` package as an example): +To extend the RADAR-Stream API, follow these steps (see the `org.radarcns.passive.empatica` package as an example): - Create a stream group by overriding [GeneralStreamGroup][8]. Use its `createSensorStream` and `createStream` methods to create the stream definitions. - For each topic, create a [AggregatorWorker][2]. diff --git a/build.gradle b/build.gradle index 6c289c14..ca2ae1fa 100644 --- a/build.gradle +++ b/build.gradle @@ -41,7 +41,7 @@ ext.findbugVersion = '3.0.1' ext.commonsCliVersion = '1.2' ext.mockitoVersion = '2.2.29' ext.radarCommonsVersion = '0.5' -ext.radarSchemasVersion = '0.2-alpha.1' +ext.radarSchemasVersion = '0.2-alpha.3-SNAPSHOT' ext.subethamailVersion = '3.1.7' ext.jsoupVersion = '1.10.2' ext.slf4jVersion = '1.7.21' @@ -100,9 +100,7 @@ repositories { dependencies { compile group: 'org.radarcns', name: 'radar-commons', version: radarCommonsVersion compile group: 'org.apache.avro', name: 'avro', version: avroVersion - compile (group: 'org.radarcns', name: 'radar-commons-testing', version: radarCommonsVersion) { - exclude group: 'org.slf4j', module: 'slf4j-simple' - } + compile group: 'org.radarcns', name: 'radar-commons-testing', version: radarCommonsVersion compile group: 'org.radarcns', name: 'radar-schemas-commons', version: radarSchemasVersion // Kafka streaming API diff --git a/src/integrationTest/java/org/radarcns/integration/E4AggregatedAccelerationMonitor.java b/src/integrationTest/java/org/radarcns/integration/E4AggregatedAccelerationMonitor.java index 0237b7f5..502d9710 100644 --- a/src/integrationTest/java/org/radarcns/integration/E4AggregatedAccelerationMonitor.java +++ b/src/integrationTest/java/org/radarcns/integration/E4AggregatedAccelerationMonitor.java @@ -33,7 +33,7 @@ import org.apache.kafka.clients.consumer.ConsumerRecords; import org.radarcns.config.RadarPropertyHandler; import org.radarcns.config.RadarPropertyHandlerImpl; -import org.radarcns.key.MeasurementKey; +import org.radarcns.kafka.ObservationKey; import org.radarcns.monitor.AbstractKafkaMonitor; import org.radarcns.util.RadarSingletonFactory; import org.slf4j.Logger; @@ -73,11 +73,11 @@ protected void evaluateRecords(ConsumerRecords rec logger.error("Failed to process record {} without a key.", record); return; } - MeasurementKey measurementKey; + ObservationKey measurementKey; Schema keySchema = key.getSchema(); if (keySchema.getField("userId") != null && keySchema.getField("sourceId") != null) { - measurementKey = new MeasurementKey(key.get("userId").toString(), + measurementKey = new ObservationKey("test", key.get("userId").toString(), key.get("sourceId").toString()); assertNotNull(measurementKey); } else { diff --git a/src/integrationTest/java/org/radarcns/integration/PhoneStreamTest.java b/src/integrationTest/java/org/radarcns/integration/PhoneStreamTest.java index cd958d67..7434152f 100644 --- a/src/integrationTest/java/org/radarcns/integration/PhoneStreamTest.java +++ b/src/integrationTest/java/org/radarcns/integration/PhoneStreamTest.java @@ -49,11 +49,11 @@ import org.radarcns.config.ConfigRadar; import org.radarcns.config.RadarBackendOptions; import org.radarcns.config.RadarPropertyHandler; -import org.radarcns.key.MeasurementKey; +import org.radarcns.kafka.ObservationKey; import org.radarcns.monitor.AbstractKafkaMonitor; import org.radarcns.monitor.KafkaMonitor; -import org.radarcns.phone.PhoneUsageEvent; -import org.radarcns.phone.UsageEventType; +import org.radarcns.passive.phone.PhoneUsageEvent; +import org.radarcns.passive.phone.UsageEventType; import org.radarcns.producer.KafkaTopicSender; import org.radarcns.producer.direct.DirectSender; import org.radarcns.stream.phone.PhoneStreamMaster; @@ -127,16 +127,16 @@ public void testDirect() throws Exception { properties.put(SCHEMA_REGISTRY_CONFIG, config.getSchemaRegistry().get(0)); properties.put(BOOTSTRAP_SERVERS_CONFIG, config.getBrokerPaths()); - DirectSender sender = new DirectSender<>(properties); - AvroTopic topic = new AvroTopic<>( + DirectSender sender = new DirectSender<>(properties); + AvroTopic topic = new AvroTopic<>( "android_phone_usage_event", - MeasurementKey.getClassSchema(), PhoneUsageEvent.getClassSchema(), - MeasurementKey.class, PhoneUsageEvent.class); + ObservationKey.getClassSchema(), PhoneUsageEvent.getClassSchema(), + ObservationKey.class, PhoneUsageEvent.class); long offset = 0; double time = System.currentTimeMillis() / 1000d - 10d; - MeasurementKey key = new MeasurementKey("a", "c"); - try (KafkaTopicSender topicSender = sender.sender(topic)) { + ObservationKey key = new ObservationKey("test", "a", "c"); + try (KafkaTopicSender topicSender = sender.sender(topic)) { topicSender.send(offset++, key, new PhoneUsageEvent(time, time++, "com.whatsapp", null, null, UsageEventType.FOREGROUND)); topicSender.send(offset++, key, new PhoneUsageEvent(time, time++, "com.whatsapp", null, null, UsageEventType.BACKGROUND)); topicSender.send(offset++, key, new PhoneUsageEvent(time, time++, "nl.thehyve.transmartclient", null, null, UsageEventType.FOREGROUND)); diff --git a/src/integrationTest/resources/mock_file.yml b/src/integrationTest/resources/mock_file.yml index 7d9326a4..a32f9728 100644 --- a/src/integrationTest/resources/mock_file.yml +++ b/src/integrationTest/resources/mock_file.yml @@ -1,5 +1,5 @@ data: - file: integration_test.csv topic: integration_test - key_schema: org.radarcns.key.MeasurementKey + key_schema: org.radarcns.kafka.ObservationKey value_schema: org.radarcns.aggregator.DoubleArrayAggregator \ No newline at end of file diff --git a/src/main/avro/temporary_package_key.avsc b/src/main/avro/temporary_package_key.avsc index 0145fe27..eb506c61 100644 --- a/src/main/avro/temporary_package_key.avsc +++ b/src/main/avro/temporary_package_key.avsc @@ -3,7 +3,7 @@ "name": "TemporaryPackageKey", "type": "record", "fields": [ - {"name": "key", "type": "org.radarcns.key.MeasurementKey", "doc": "Observation key."}, + {"name": "key", "type": "org.radarcns.kafka.ObservationKey", "doc": "Observation key."}, {"name": "packageName", "type": "string", "doc": "Package name."} ] } \ No newline at end of file diff --git a/src/main/java/org/radarcns/monitor/AbstractKafkaMonitor.java b/src/main/java/org/radarcns/monitor/AbstractKafkaMonitor.java index 46380bec..3a3239a7 100644 --- a/src/main/java/org/radarcns/monitor/AbstractKafkaMonitor.java +++ b/src/main/java/org/radarcns/monitor/AbstractKafkaMonitor.java @@ -32,7 +32,7 @@ import org.apache.kafka.common.errors.WakeupException; import org.radarcns.config.ConfigRadar; import org.radarcns.config.RadarPropertyHandler; -import org.radarcns.key.MeasurementKey; +import org.radarcns.kafka.ObservationKey; import org.radarcns.util.PersistentStateStore; import org.radarcns.util.RollingTimeAverage; import org.slf4j.Logger; @@ -247,12 +247,17 @@ public void setPollTimeout(long pollTimeout) { this.pollTimeout.set(pollTimeout); } - protected MeasurementKey extractKey(ConsumerRecord record) { + protected ObservationKey extractKey(ConsumerRecord record) { GenericRecord key = record.key(); if (key == null) { throw new IllegalArgumentException("Failed to process record without a key."); } Schema keySchema = key.getSchema(); + Field projectIdField = keySchema.getField("projectId"); + if (projectIdField == null) { + throw new IllegalArgumentException("Failed to process record with key type " + + key.getSchema() + " without project ID."); + } Field userIdField = keySchema.getField("userId"); if (userIdField == null) { throw new IllegalArgumentException("Failed to process record with key type " @@ -263,7 +268,8 @@ protected MeasurementKey extractKey(ConsumerRecord record) { throw new IllegalArgumentException("Failed to process record with key type " + key.getSchema() + " without source ID."); } - return new MeasurementKey( + return new ObservationKey( + key.get(projectIdField.pos()).toString(), key.get(userIdField.pos()).toString(), key.get(sourceIdField.pos()).toString()); } diff --git a/src/main/java/org/radarcns/monitor/BatteryLevelMonitor.java b/src/main/java/org/radarcns/monitor/BatteryLevelMonitor.java index 3d140bc2..cd625de5 100644 --- a/src/main/java/org/radarcns/monitor/BatteryLevelMonitor.java +++ b/src/main/java/org/radarcns/monitor/BatteryLevelMonitor.java @@ -30,7 +30,7 @@ import org.apache.kafka.clients.consumer.ConsumerConfig; import org.apache.kafka.clients.consumer.ConsumerRecord; import org.radarcns.config.RadarPropertyHandler; -import org.radarcns.key.MeasurementKey; +import org.radarcns.kafka.ObservationKey; import org.radarcns.monitor.BatteryLevelMonitor.BatteryLevelState; import org.radarcns.util.EmailSender; import org.radarcns.util.RadarSingletonFactory; @@ -74,7 +74,7 @@ public BatteryLevelMonitor(RadarPropertyHandler radar, Collection topics @Override protected void evaluateRecord(ConsumerRecord record) { try { - MeasurementKey key = extractKey(record); + ObservationKey key = extractKey(record); float batteryLevel = extractBatteryLevel(record); float previousLevel = state.updateLevel(key, batteryLevel); @@ -106,7 +106,7 @@ protected void evaluateRecord(ConsumerRecord recor } } - private void updateStatus(MeasurementKey key, Status status) { + private void updateStatus(ObservationKey key, Status status) { if (sender == null) { return; } @@ -183,7 +183,7 @@ public void setLevels(Map levels) { } /** Update a single battery level. */ - public float updateLevel(MeasurementKey key, float level) { + public float updateLevel(ObservationKey key, float level) { Float previousLevel = levels.put(measurementKeyToString(key), level); return previousLevel == null ? 1.0f : previousLevel; } diff --git a/src/main/java/org/radarcns/monitor/DisconnectMonitor.java b/src/main/java/org/radarcns/monitor/DisconnectMonitor.java index a3544a9e..e58d9e5a 100644 --- a/src/main/java/org/radarcns/monitor/DisconnectMonitor.java +++ b/src/main/java/org/radarcns/monitor/DisconnectMonitor.java @@ -25,7 +25,7 @@ import org.apache.kafka.clients.consumer.ConsumerRecords; import org.radarcns.config.DisconnectMonitorConfig; import org.radarcns.config.RadarPropertyHandler; -import org.radarcns.key.MeasurementKey; +import org.radarcns.kafka.ObservationKey; import org.radarcns.monitor.DisconnectMonitor.DisconnectMonitorState; import org.radarcns.util.EmailSender; import org.radarcns.util.Monitor; @@ -141,7 +141,7 @@ protected void evaluateRecords(ConsumerRecords rec @Override protected void evaluateRecord(ConsumerRecord record) { - MeasurementKey key = extractKey(record); + ObservationKey key = extractKey(record); this.monitor.increment(); @@ -172,7 +172,7 @@ private void scheduleRepetition(final String key, final MissingRecordsReport rep } private void reportMissing(String keyString, MissingRecordsReport report) { - MeasurementKey key = stringToKey(keyString); + ObservationKey key = stringToKey(keyString); long timeout = report.getTimeout(); logger.info("Device {} timeout {} (message {} of {}). Reporting it missing.", key, timeout, report.messageNumber, numRepetitions); @@ -205,7 +205,7 @@ private void reportMissing(String keyString, MissingRecordsReport report) { } } - private void reportRecovered(MeasurementKey key, long reportedMissingTime) { + private void reportRecovered(ObservationKey key, long reportedMissingTime) { logger.info("Device {} seen again. Reporting it recovered.", key); try { Date reportedMissingDate = new Date(reportedMissingTime); diff --git a/src/main/java/org/radarcns/stream/empatica/E4AccelerationStream.java b/src/main/java/org/radarcns/stream/empatica/E4AccelerationStream.java index 343823d0..3a0db65f 100644 --- a/src/main/java/org/radarcns/stream/empatica/E4AccelerationStream.java +++ b/src/main/java/org/radarcns/stream/empatica/E4AccelerationStream.java @@ -19,11 +19,11 @@ import javax.annotation.Nonnull; import org.apache.kafka.streams.kstream.KStream; import org.apache.kafka.streams.kstream.TimeWindows; -import org.radarcns.aggregator.DoubleArrayAggregator; +import org.radarcns.stream.aggregator.DoubleArrayAggregation; import org.radarcns.config.KafkaProperty; -import org.radarcns.empatica.EmpaticaE4Acceleration; -import org.radarcns.key.MeasurementKey; -import org.radarcns.key.WindowedKey; +import org.radarcns.passive.empatica.EmpaticaE4Acceleration; +import org.radarcns.kafka.ObservationKey; +import org.radarcns.kafka.AggregateKey; import org.radarcns.stream.StreamMaster; import org.radarcns.stream.StreamWorker; import org.radarcns.stream.collector.DoubleArrayCollector; @@ -36,7 +36,7 @@ /** * Definition of Kafka Stream for aggregating data collected by Empatica E4 Accelerometer sensor. */ -public class E4AccelerationStream extends StreamWorker { +public class E4AccelerationStream extends StreamWorker { private static final Logger log = LoggerFactory.getLogger(E4AccelerationStream.class); private final RadarUtilities utilities = RadarSingletonFactory.getRadarUtilities(); @@ -47,8 +47,8 @@ public E4AccelerationStream(String clientId, int numThread, StreamMaster master, } @Override - protected KStream defineStream( - @Nonnull KStream kstream) { + protected KStream defineStream( + @Nonnull KStream kstream) { return kstream.groupByKey() .aggregate( DoubleArrayCollector::new, diff --git a/src/main/java/org/radarcns/stream/empatica/E4BatteryLevelStream.java b/src/main/java/org/radarcns/stream/empatica/E4BatteryLevelStream.java index 52c8c394..afb2fc29 100644 --- a/src/main/java/org/radarcns/stream/empatica/E4BatteryLevelStream.java +++ b/src/main/java/org/radarcns/stream/empatica/E4BatteryLevelStream.java @@ -19,13 +19,13 @@ import javax.annotation.Nonnull; import org.apache.kafka.streams.kstream.KStream; import org.apache.kafka.streams.kstream.TimeWindows; -import org.radarcns.aggregator.DoubleAggregator; import org.radarcns.config.KafkaProperty; -import org.radarcns.empatica.EmpaticaE4BatteryLevel; -import org.radarcns.key.MeasurementKey; -import org.radarcns.key.WindowedKey; +import org.radarcns.passive.empatica.EmpaticaE4BatteryLevel; +import org.radarcns.kafka.ObservationKey; +import org.radarcns.kafka.AggregateKey; import org.radarcns.stream.StreamMaster; import org.radarcns.stream.StreamWorker; +import org.radarcns.stream.aggregator.DoubleAggregation; import org.radarcns.stream.collector.DoubleValueCollector; import org.radarcns.util.RadarSingletonFactory; import org.radarcns.util.RadarUtilities; @@ -36,7 +36,7 @@ /** * Kafka Stream for aggregating data about Empatica E4 battery level. */ -public class E4BatteryLevelStream extends StreamWorker { +public class E4BatteryLevelStream extends StreamWorker { private static final Logger log = LoggerFactory.getLogger(E4BatteryLevelStream.class); private final RadarUtilities utilities = RadarSingletonFactory.getRadarUtilities(); @@ -47,8 +47,8 @@ public E4BatteryLevelStream(String clientId, int numThread, StreamMaster master, } @Override - protected KStream defineStream( - @Nonnull KStream kstream) { + protected KStream defineStream( + @Nonnull KStream kstream) { return kstream.groupByKey() .aggregate( DoubleValueCollector::new, diff --git a/src/main/java/org/radarcns/stream/empatica/E4BloodVolumePulseStream.java b/src/main/java/org/radarcns/stream/empatica/E4BloodVolumePulseStream.java index 762dbefb..99000f28 100644 --- a/src/main/java/org/radarcns/stream/empatica/E4BloodVolumePulseStream.java +++ b/src/main/java/org/radarcns/stream/empatica/E4BloodVolumePulseStream.java @@ -19,13 +19,13 @@ import javax.annotation.Nonnull; import org.apache.kafka.streams.kstream.KStream; import org.apache.kafka.streams.kstream.TimeWindows; -import org.radarcns.aggregator.DoubleAggregator; import org.radarcns.config.KafkaProperty; -import org.radarcns.empatica.EmpaticaE4BloodVolumePulse; -import org.radarcns.key.MeasurementKey; -import org.radarcns.key.WindowedKey; +import org.radarcns.passive.empatica.EmpaticaE4BloodVolumePulse; +import org.radarcns.kafka.ObservationKey; +import org.radarcns.kafka.AggregateKey; import org.radarcns.stream.StreamMaster; import org.radarcns.stream.StreamWorker; +import org.radarcns.stream.aggregator.DoubleAggregation; import org.radarcns.stream.collector.DoubleValueCollector; import org.radarcns.util.RadarSingletonFactory; import org.radarcns.util.RadarUtilities; @@ -37,7 +37,7 @@ * Kafka Stream for aggregating data about Blood Volume Pulse collected by Empatica E4. */ public class E4BloodVolumePulseStream extends - StreamWorker { + StreamWorker { private static final Logger log = LoggerFactory.getLogger(E4BloodVolumePulseStream.class); private final RadarUtilities utilities = RadarSingletonFactory.getRadarUtilities(); @@ -49,8 +49,8 @@ public E4BloodVolumePulseStream(String clientId, int numThread, StreamMaster mas } @Override - protected KStream defineStream( - @Nonnull KStream kstream) { + protected KStream defineStream( + @Nonnull KStream kstream) { return kstream.groupByKey() .aggregate( DoubleValueCollector::new, diff --git a/src/main/java/org/radarcns/stream/empatica/E4ElectroDermalActivityStream.java b/src/main/java/org/radarcns/stream/empatica/E4ElectroDermalActivityStream.java index 1a0d34b9..37caa724 100644 --- a/src/main/java/org/radarcns/stream/empatica/E4ElectroDermalActivityStream.java +++ b/src/main/java/org/radarcns/stream/empatica/E4ElectroDermalActivityStream.java @@ -19,13 +19,13 @@ import javax.annotation.Nonnull; import org.apache.kafka.streams.kstream.KStream; import org.apache.kafka.streams.kstream.TimeWindows; -import org.radarcns.aggregator.DoubleAggregator; import org.radarcns.config.KafkaProperty; -import org.radarcns.empatica.EmpaticaE4ElectroDermalActivity; -import org.radarcns.key.MeasurementKey; -import org.radarcns.key.WindowedKey; +import org.radarcns.passive.empatica.EmpaticaE4ElectroDermalActivity; +import org.radarcns.kafka.ObservationKey; +import org.radarcns.kafka.AggregateKey; import org.radarcns.stream.StreamMaster; import org.radarcns.stream.StreamWorker; +import org.radarcns.stream.aggregator.DoubleAggregation; import org.radarcns.stream.collector.DoubleValueCollector; import org.radarcns.util.RadarSingletonFactory; import org.radarcns.util.RadarUtilities; @@ -37,7 +37,7 @@ * Kafka Stream for aggregating data about electrodermal activity collected by Empatica E4. */ public class E4ElectroDermalActivityStream extends - StreamWorker { + StreamWorker { private static final Logger log = LoggerFactory.getLogger(E4ElectroDermalActivityStream.class); private final RadarUtilities utilities = RadarSingletonFactory.getRadarUtilities(); @@ -49,8 +49,8 @@ public E4ElectroDermalActivityStream(String clientId, int numThread, StreamMaste } @Override - protected KStream defineStream( - @Nonnull KStream kstream) { + protected KStream defineStream( + @Nonnull KStream kstream) { return kstream.groupByKey() .aggregate( DoubleValueCollector::new, diff --git a/src/main/java/org/radarcns/stream/empatica/E4HeartRateStream.java b/src/main/java/org/radarcns/stream/empatica/E4HeartRateStream.java index e699118b..9ac447d4 100644 --- a/src/main/java/org/radarcns/stream/empatica/E4HeartRateStream.java +++ b/src/main/java/org/radarcns/stream/empatica/E4HeartRateStream.java @@ -19,13 +19,13 @@ import javax.annotation.Nonnull; import org.apache.kafka.streams.kstream.KStream; import org.apache.kafka.streams.kstream.TimeWindows; -import org.radarcns.aggregator.DoubleAggregator; import org.radarcns.config.KafkaProperty; -import org.radarcns.empatica.EmpaticaE4InterBeatInterval; -import org.radarcns.key.MeasurementKey; -import org.radarcns.key.WindowedKey; +import org.radarcns.passive.empatica.EmpaticaE4InterBeatInterval; +import org.radarcns.kafka.ObservationKey; +import org.radarcns.kafka.AggregateKey; import org.radarcns.stream.StreamMaster; import org.radarcns.stream.StreamWorker; +import org.radarcns.stream.aggregator.DoubleAggregation; import org.radarcns.stream.collector.DoubleValueCollector; import org.radarcns.util.RadarSingletonFactory; import org.radarcns.util.RadarUtilities; @@ -36,7 +36,7 @@ /** * Kafka Stream for computing and aggregating Heart Rate values collected by Empatica E4. */ -public class E4HeartRateStream extends StreamWorker { +public class E4HeartRateStream extends StreamWorker { private static final Logger log = LoggerFactory.getLogger(E4HeartRateStream.class); private final RadarUtilities utilities = RadarSingletonFactory.getRadarUtilities(); @@ -47,8 +47,8 @@ public E4HeartRateStream(String clientId, int numThread, StreamMaster master, } @Override - protected KStream defineStream( - @Nonnull KStream kstream) { + protected KStream defineStream( + @Nonnull KStream kstream) { return kstream.groupByKey() .aggregate( DoubleValueCollector::new, diff --git a/src/main/java/org/radarcns/stream/empatica/E4InterBeatIntervalStream.java b/src/main/java/org/radarcns/stream/empatica/E4InterBeatIntervalStream.java index 0fc1e0d2..836e8386 100644 --- a/src/main/java/org/radarcns/stream/empatica/E4InterBeatIntervalStream.java +++ b/src/main/java/org/radarcns/stream/empatica/E4InterBeatIntervalStream.java @@ -19,13 +19,13 @@ import javax.annotation.Nonnull; import org.apache.kafka.streams.kstream.KStream; import org.apache.kafka.streams.kstream.TimeWindows; -import org.radarcns.aggregator.DoubleAggregator; import org.radarcns.config.KafkaProperty; -import org.radarcns.empatica.EmpaticaE4InterBeatInterval; -import org.radarcns.key.MeasurementKey; -import org.radarcns.key.WindowedKey; +import org.radarcns.passive.empatica.EmpaticaE4InterBeatInterval; +import org.radarcns.kafka.ObservationKey; +import org.radarcns.kafka.AggregateKey; import org.radarcns.stream.StreamMaster; import org.radarcns.stream.StreamWorker; +import org.radarcns.stream.aggregator.DoubleAggregation; import org.radarcns.stream.collector.DoubleValueCollector; import org.radarcns.util.RadarSingletonFactory; import org.radarcns.util.RadarUtilities; @@ -37,7 +37,7 @@ * Definition of Kafka Stream for aggregating Inter Beat Interval values collected by Empatica E4. */ public class E4InterBeatIntervalStream extends - StreamWorker { + StreamWorker { private static final Logger log = LoggerFactory.getLogger(E4InterBeatIntervalStream.class); private final RadarUtilities utilities = RadarSingletonFactory.getRadarUtilities(); @@ -48,8 +48,8 @@ public E4InterBeatIntervalStream(String clientId, int numThread, StreamMaster ma } @Override - protected KStream defineStream( - @Nonnull KStream kstream) { + protected KStream defineStream( + @Nonnull KStream kstream) { return kstream.groupByKey() .aggregate( DoubleValueCollector::new, diff --git a/src/main/java/org/radarcns/stream/empatica/E4TemperatureStream.java b/src/main/java/org/radarcns/stream/empatica/E4TemperatureStream.java index b2cf2f8c..91aed8e6 100644 --- a/src/main/java/org/radarcns/stream/empatica/E4TemperatureStream.java +++ b/src/main/java/org/radarcns/stream/empatica/E4TemperatureStream.java @@ -19,13 +19,13 @@ import javax.annotation.Nonnull; import org.apache.kafka.streams.kstream.KStream; import org.apache.kafka.streams.kstream.TimeWindows; -import org.radarcns.aggregator.DoubleAggregator; import org.radarcns.config.KafkaProperty; -import org.radarcns.empatica.EmpaticaE4Temperature; -import org.radarcns.key.MeasurementKey; -import org.radarcns.key.WindowedKey; +import org.radarcns.passive.empatica.EmpaticaE4Temperature; +import org.radarcns.kafka.ObservationKey; +import org.radarcns.kafka.AggregateKey; import org.radarcns.stream.StreamMaster; import org.radarcns.stream.StreamWorker; +import org.radarcns.stream.aggregator.DoubleAggregation; import org.radarcns.stream.collector.DoubleValueCollector; import org.radarcns.util.RadarSingletonFactory; import org.radarcns.util.RadarUtilities; @@ -36,7 +36,7 @@ /** * Definition of Kafka Stream for aggregating temperature values collected by Empatica E4. */ -public class E4TemperatureStream extends StreamWorker { +public class E4TemperatureStream extends StreamWorker { private final RadarUtilities utilities = RadarSingletonFactory.getRadarUtilities(); private static final Logger log = LoggerFactory.getLogger(E4TemperatureStream.class); @@ -47,8 +47,8 @@ public E4TemperatureStream(String clientId, int numThread, StreamMaster master, } @Override - protected KStream defineStream( - @Nonnull KStream kstream) { + protected KStream defineStream( + @Nonnull KStream kstream) { return kstream.groupByKey() .aggregate( DoubleValueCollector::new, diff --git a/src/main/java/org/radarcns/stream/phone/PhoneAccelerationStream.java b/src/main/java/org/radarcns/stream/phone/PhoneAccelerationStream.java index d83af76f..e6edb1d1 100644 --- a/src/main/java/org/radarcns/stream/phone/PhoneAccelerationStream.java +++ b/src/main/java/org/radarcns/stream/phone/PhoneAccelerationStream.java @@ -2,11 +2,11 @@ import org.apache.kafka.streams.kstream.KStream; import org.apache.kafka.streams.kstream.TimeWindows; -import org.radarcns.aggregator.DoubleArrayAggregator; +import org.radarcns.stream.aggregator.DoubleArrayAggregation; import org.radarcns.config.KafkaProperty; -import org.radarcns.key.MeasurementKey; -import org.radarcns.key.WindowedKey; -import org.radarcns.phone.PhoneAcceleration; +import org.radarcns.kafka.ObservationKey; +import org.radarcns.kafka.AggregateKey; +import org.radarcns.passive.phone.PhoneAcceleration; import org.radarcns.stream.StreamMaster; import org.radarcns.stream.StreamWorker; import org.radarcns.stream.collector.DoubleArrayCollector; @@ -20,7 +20,7 @@ import static org.radarcns.util.Serialization.floatToDouble; -public class PhoneAccelerationStream extends StreamWorker { +public class PhoneAccelerationStream extends StreamWorker { private static final Logger logger = LoggerFactory.getLogger(PhoneAccelerationStream.class); private final RadarUtilities utilities = RadarSingletonFactory.getRadarUtilities(); @@ -32,8 +32,8 @@ public PhoneAccelerationStream(String phoneAccelerationStream, int priority, } @Override - protected KStream defineStream( - @Nonnull KStream kstream) { + protected KStream defineStream( + @Nonnull KStream kstream) { return kstream.groupByKey() .aggregate( DoubleArrayCollector::new, diff --git a/src/main/java/org/radarcns/stream/phone/PhoneBatteryStream.java b/src/main/java/org/radarcns/stream/phone/PhoneBatteryStream.java index 2395cb32..36422f47 100644 --- a/src/main/java/org/radarcns/stream/phone/PhoneBatteryStream.java +++ b/src/main/java/org/radarcns/stream/phone/PhoneBatteryStream.java @@ -2,13 +2,13 @@ import org.apache.kafka.streams.kstream.KStream; import org.apache.kafka.streams.kstream.TimeWindows; -import org.radarcns.aggregator.DoubleAggregator; import org.radarcns.config.KafkaProperty; -import org.radarcns.key.MeasurementKey; -import org.radarcns.key.WindowedKey; -import org.radarcns.phone.PhoneBatteryLevel; +import org.radarcns.kafka.ObservationKey; +import org.radarcns.kafka.AggregateKey; +import org.radarcns.passive.phone.PhoneBatteryLevel; import org.radarcns.stream.StreamMaster; import org.radarcns.stream.StreamWorker; +import org.radarcns.stream.aggregator.DoubleAggregation; import org.radarcns.stream.collector.DoubleValueCollector; import org.radarcns.util.RadarSingletonFactory; import org.radarcns.util.RadarUtilities; @@ -18,7 +18,7 @@ import javax.annotation.Nonnull; -public class PhoneBatteryStream extends StreamWorker { +public class PhoneBatteryStream extends StreamWorker { private static final Logger logger = LoggerFactory.getLogger(PhoneBatteryStream.class); private final RadarUtilities utilities = RadarSingletonFactory.getRadarUtilities(); @@ -30,8 +30,8 @@ public PhoneBatteryStream(String phoneBatteryStream, int priority, StreamMaster } @Override - protected KStream defineStream( - @Nonnull KStream kstream) { + protected KStream defineStream( + @Nonnull KStream kstream) { return kstream.groupByKey() .aggregate( DoubleValueCollector::new, diff --git a/src/main/java/org/radarcns/stream/phone/PhoneUsageAggregationStream.java b/src/main/java/org/radarcns/stream/phone/PhoneUsageAggregationStream.java index 0ddbcf9e..c5c6569a 100644 --- a/src/main/java/org/radarcns/stream/phone/PhoneUsageAggregationStream.java +++ b/src/main/java/org/radarcns/stream/phone/PhoneUsageAggregationStream.java @@ -4,9 +4,9 @@ import org.apache.kafka.streams.kstream.TimeWindows; import org.radarcns.aggregator.PhoneUsageAggregator; import org.radarcns.config.KafkaProperty; -import org.radarcns.key.MeasurementKey; -import org.radarcns.key.WindowedKey; -import org.radarcns.phone.PhoneUsageEvent; +import org.radarcns.kafka.ObservationKey; +import org.radarcns.kafka.AggregateKey; +import org.radarcns.passive.phone.PhoneUsageEvent; import org.radarcns.stream.StreamMaster; import org.radarcns.stream.StreamWorker; import org.radarcns.util.RadarSingletonFactory; @@ -20,7 +20,7 @@ /** * Created by piotrzakrzewski on 26/07/2017. */ -public class PhoneUsageAggregationStream extends StreamWorker { +public class PhoneUsageAggregationStream extends StreamWorker { private static final Logger log = LoggerFactory.getLogger(PhoneUsageAggregationStream.class); private static final long DAY_IN_MS = 24 * 60 * 60 * 1000; @@ -35,8 +35,8 @@ public PhoneUsageAggregationStream(@Nonnull String clientId, } @Override - protected KStream defineStream( - @Nonnull KStream kstream) { + protected KStream defineStream( + @Nonnull KStream kstream) { return kstream.groupBy((k, v) -> new TemporaryPackageKey(k, v.getPackageName())) .aggregate( PhoneUsageCollector::new, diff --git a/src/main/java/org/radarcns/stream/phone/PhoneUsageCollector.java b/src/main/java/org/radarcns/stream/phone/PhoneUsageCollector.java index 870de7f0..8c7e1679 100644 --- a/src/main/java/org/radarcns/stream/phone/PhoneUsageCollector.java +++ b/src/main/java/org/radarcns/stream/phone/PhoneUsageCollector.java @@ -1,7 +1,7 @@ package org.radarcns.stream.phone; -import org.radarcns.phone.PhoneUsageEvent; -import org.radarcns.phone.UsageEventType; +import org.radarcns.passive.phone.PhoneUsageEvent; +import org.radarcns.passive.phone.UsageEventType; import java.math.BigDecimal; import java.math.MathContext; diff --git a/src/main/java/org/radarcns/stream/phone/PhoneUsageStream.java b/src/main/java/org/radarcns/stream/phone/PhoneUsageStream.java index 4f779409..83cee1a4 100644 --- a/src/main/java/org/radarcns/stream/phone/PhoneUsageStream.java +++ b/src/main/java/org/radarcns/stream/phone/PhoneUsageStream.java @@ -20,14 +20,14 @@ import org.apache.kafka.streams.KeyValue; import org.apache.kafka.streams.kstream.KStream; import org.radarcns.config.KafkaProperty; -import org.radarcns.key.MeasurementKey; -import org.radarcns.phone.PhoneUsageEvent; +import org.radarcns.kafka.ObservationKey; +import org.radarcns.passive.phone.PhoneUsageEvent; import org.radarcns.stream.StreamMaster; import org.radarcns.stream.StreamWorker; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -public class PhoneUsageStream extends StreamWorker { +public class PhoneUsageStream extends StreamWorker { private static final Logger log = LoggerFactory.getLogger(PhoneUsageStream.class); // 1 day until an item is refreshed @@ -46,8 +46,8 @@ public PhoneUsageStream(String clientId, int numThread, StreamMaster master, } @Override - protected KStream defineStream( - @Nonnull KStream kstream) { + protected KStream defineStream( + @Nonnull KStream kstream) { return kstream .map((key, value) -> { String packageName = value.getPackageName(); diff --git a/src/main/java/org/radarcns/stream/phone/TemporaryPackageKey.java b/src/main/java/org/radarcns/stream/phone/TemporaryPackageKey.java index f3f7e0f1..0da53d9b 100644 --- a/src/main/java/org/radarcns/stream/phone/TemporaryPackageKey.java +++ b/src/main/java/org/radarcns/stream/phone/TemporaryPackageKey.java @@ -14,7 +14,7 @@ @org.apache.avro.specific.AvroGenerated public class TemporaryPackageKey extends org.apache.avro.specific.SpecificRecordBase implements org.apache.avro.specific.SpecificRecord { private static final long serialVersionUID = -8584100260725184052L; - public static final org.apache.avro.Schema SCHEMA$ = new org.apache.avro.Schema.Parser().parse("{\"type\":\"record\",\"name\":\"TemporaryPackageKey\",\"namespace\":\"org.radarcns.stream.phone\",\"fields\":[{\"name\":\"key\",\"type\":{\"type\":\"record\",\"name\":\"MeasurementKey\",\"namespace\":\"org.radarcns.key\",\"doc\":\"Key of an observation.\",\"fields\":[{\"name\":\"userId\",\"type\":{\"type\":\"string\",\"avro.java.string\":\"String\"},\"doc\":\"User Identifier created during the enrolment.\"},{\"name\":\"sourceId\",\"type\":{\"type\":\"string\",\"avro.java.string\":\"String\"},\"doc\":\"Unique identifier associated with the source.\"}]},\"doc\":\"Observation key.\"},{\"name\":\"packageName\",\"type\":{\"type\":\"string\",\"avro.java.string\":\"String\"},\"doc\":\"Package name.\"}]}"); + public static final org.apache.avro.Schema SCHEMA$ = new org.apache.avro.Schema.Parser().parse("{\"type\":\"record\",\"name\":\"TemporaryPackageKey\",\"namespace\":\"org.radarcns.stream.phone\",\"fields\":[{\"name\":\"key\",\"type\":{\"type\":\"record\",\"name\":\"ObservationKey\",\"namespace\":\"org.radarcns.key\",\"doc\":\"Key of an observation.\",\"fields\":[{\"name\":\"userId\",\"type\":{\"type\":\"string\",\"avro.java.string\":\"String\"},\"doc\":\"User Identifier created during the enrolment.\"},{\"name\":\"sourceId\",\"type\":{\"type\":\"string\",\"avro.java.string\":\"String\"},\"doc\":\"Unique identifier associated with the source.\"}]},\"doc\":\"Observation key.\"},{\"name\":\"packageName\",\"type\":{\"type\":\"string\",\"avro.java.string\":\"String\"},\"doc\":\"Package name.\"}]}"); public static org.apache.avro.Schema getClassSchema() { return SCHEMA$; } private static SpecificData MODEL$ = new SpecificData(); @@ -52,7 +52,7 @@ public static TemporaryPackageKey fromByteBuffer( } /** Observation key. */ - @Deprecated public org.radarcns.key.MeasurementKey key; + @Deprecated public org.radarcns.kafka.ObservationKey key; /** Package name. */ @Deprecated public java.lang.String packageName; @@ -68,7 +68,7 @@ public TemporaryPackageKey() {} * @param key Observation key. * @param packageName Package name. */ - public TemporaryPackageKey(org.radarcns.key.MeasurementKey key, java.lang.String packageName) { + public TemporaryPackageKey(org.radarcns.kafka.ObservationKey key, java.lang.String packageName) { this.key = key; this.packageName = packageName; } @@ -87,7 +87,7 @@ public java.lang.Object get(int field$) { @SuppressWarnings(value="unchecked") public void put(int field$, java.lang.Object value$) { switch (field$) { - case 0: key = (org.radarcns.key.MeasurementKey)value$; break; + case 0: key = (org.radarcns.kafka.ObservationKey)value$; break; case 1: packageName = (java.lang.String)value$; break; default: throw new org.apache.avro.AvroRuntimeException("Bad index"); } @@ -97,7 +97,7 @@ public void put(int field$, java.lang.Object value$) { * Gets the value of the 'key' field. * @return Observation key. */ - public org.radarcns.key.MeasurementKey getKey() { + public org.radarcns.kafka.ObservationKey getKey() { return key; } @@ -106,7 +106,7 @@ public org.radarcns.key.MeasurementKey getKey() { * Observation key. * @param value the value to set. */ - public void setKey(org.radarcns.key.MeasurementKey value) { + public void setKey(org.radarcns.kafka.ObservationKey value) { this.key = value; } @@ -160,8 +160,8 @@ public static class Builder extends org.apache.avro.specific.SpecificRecordBuild implements org.apache.avro.data.RecordBuilder { /** Observation key. */ - private org.radarcns.key.MeasurementKey key; - private org.radarcns.key.MeasurementKey.Builder keyBuilder; + private org.radarcns.kafka.ObservationKey key; + private org.radarcns.kafka.ObservationKey.Builder keyBuilder; /** Package name. */ private java.lang.String packageName; @@ -181,7 +181,7 @@ private Builder(org.radarcns.stream.phone.TemporaryPackageKey.Builder other) { fieldSetFlags()[0] = true; } if (other.hasKeyBuilder()) { - this.keyBuilder = org.radarcns.key.MeasurementKey.newBuilder(other.getKeyBuilder()); + this.keyBuilder = org.radarcns.kafka.ObservationKey.newBuilder(other.getKeyBuilder()); } if (isValidValue(fields()[1], other.packageName)) { this.packageName = data().deepCopy(fields()[1].schema(), other.packageName); @@ -211,7 +211,7 @@ private Builder(org.radarcns.stream.phone.TemporaryPackageKey other) { * Observation key. * @return The value. */ - public org.radarcns.key.MeasurementKey getKey() { + public org.radarcns.kafka.ObservationKey getKey() { return key; } @@ -221,7 +221,7 @@ public org.radarcns.key.MeasurementKey getKey() { * @param value The value of 'key'. * @return This builder. */ - public org.radarcns.stream.phone.TemporaryPackageKey.Builder setKey(org.radarcns.key.MeasurementKey value) { + public org.radarcns.stream.phone.TemporaryPackageKey.Builder setKey(org.radarcns.kafka.ObservationKey value) { validate(fields()[0], value); this.keyBuilder = null; this.key = value; @@ -243,12 +243,12 @@ public boolean hasKey() { * Observation key. * @return This builder. */ - public org.radarcns.key.MeasurementKey.Builder getKeyBuilder() { + public org.radarcns.kafka.ObservationKey.Builder getKeyBuilder() { if (keyBuilder == null) { if (hasKey()) { - setKeyBuilder(org.radarcns.key.MeasurementKey.newBuilder(key)); + setKeyBuilder(org.radarcns.kafka.ObservationKey.newBuilder(key)); } else { - setKeyBuilder(org.radarcns.key.MeasurementKey.newBuilder()); + setKeyBuilder(org.radarcns.kafka.ObservationKey.newBuilder()); } } return keyBuilder; @@ -260,7 +260,7 @@ public org.radarcns.key.MeasurementKey.Builder getKeyBuilder() { * @param value The builder instance that must be set. * @return This builder. */ - public org.radarcns.stream.phone.TemporaryPackageKey.Builder setKeyBuilder(org.radarcns.key.MeasurementKey.Builder value) { + public org.radarcns.stream.phone.TemporaryPackageKey.Builder setKeyBuilder(org.radarcns.kafka.ObservationKey.Builder value) { clearKey(); keyBuilder = value; return this; @@ -338,7 +338,7 @@ public TemporaryPackageKey build() { if (keyBuilder != null) { record.key = this.keyBuilder.build(); } else { - record.key = fieldSetFlags()[0] ? this.key : (org.radarcns.key.MeasurementKey) defaultValue(fields()[0]); + record.key = fieldSetFlags()[0] ? this.key : (org.radarcns.kafka.ObservationKey) defaultValue(fields()[0]); } record.packageName = fieldSetFlags()[1] ? this.packageName : (java.lang.String) defaultValue(fields()[1]); return record; diff --git a/src/main/java/org/radarcns/util/PersistentStateStore.java b/src/main/java/org/radarcns/util/PersistentStateStore.java index 2070b5c5..f5a43aa3 100644 --- a/src/main/java/org/radarcns/util/PersistentStateStore.java +++ b/src/main/java/org/radarcns/util/PersistentStateStore.java @@ -20,7 +20,7 @@ import java.io.FileOutputStream; import java.io.IOException; import org.radarcns.config.YamlConfigLoader; -import org.radarcns.key.MeasurementKey; +import org.radarcns.kafka.ObservationKey; /** Store a state for a Kafka consumer. */ public class PersistentStateStore { @@ -94,10 +94,17 @@ private File getFile(String groupId, String clientId) { * @param key key to serialize * @return unique serialized form */ - public static String measurementKeyToString(MeasurementKey key) { + public static String measurementKeyToString(ObservationKey key) { + String projectId = key.getProjectId(); String userId = key.getUserId(); String sourceId = key.getSourceId(); - StringBuilder builder = new StringBuilder(userId.length() + 5 + sourceId.length()); + StringBuilder builder = new StringBuilder( + (projectId == null ? 0 : projectId.length()) + + userId.length() + 6 + sourceId.length()); + if (projectId != null) { + escape(projectId, builder); + } + builder.append(SEPARATOR); escape(userId, builder); builder.append(SEPARATOR); escape(sourceId, builder); @@ -118,15 +125,16 @@ private static void escape(String string, StringBuilder builder) { /** * Efficiently serializes a measurement key serialized with - * {@link #measurementKeyToString(MeasurementKey)}. + * {@link #measurementKeyToString(ObservationKey)}. * * @param string serialized form * @return original measurement key */ - public static MeasurementKey stringToKey(String string) { + public static ObservationKey stringToKey(String string) { StringBuilder builder = new StringBuilder(string.length()); - MeasurementKey key = new MeasurementKey(); + ObservationKey key = new ObservationKey(); boolean hasSlash = false; + int numFound = 0; for (char c : string.toCharArray()) { if (c == '\\') { if (hasSlash) { @@ -140,8 +148,18 @@ public static MeasurementKey stringToKey(String string) { builder.append(c); hasSlash = false; } else { - key.setUserId(builder.toString()); - builder.setLength(0); + if (numFound == 0) { + numFound++; + if (builder.length() == 0) { + key.setProjectId(null); + } else { + key.setProjectId(builder.toString()); + builder.setLength(0); + } + } else { + key.setUserId(builder.toString()); + builder.setLength(0); + } } } else { builder.append(c); diff --git a/src/main/java/org/radarcns/util/RadarUtilities.java b/src/main/java/org/radarcns/util/RadarUtilities.java index e7727b87..884aa5af 100644 --- a/src/main/java/org/radarcns/util/RadarUtilities.java +++ b/src/main/java/org/radarcns/util/RadarUtilities.java @@ -18,12 +18,12 @@ import org.apache.kafka.streams.KeyValue; import org.apache.kafka.streams.kstream.Windowed; -import org.radarcns.aggregator.DoubleAggregator; -import org.radarcns.aggregator.DoubleArrayAggregator; import org.radarcns.aggregator.PhoneUsageAggregator; -import org.radarcns.empatica.EmpaticaE4Acceleration; -import org.radarcns.key.MeasurementKey; -import org.radarcns.key.WindowedKey; +import org.radarcns.kafka.AggregateKey; +import org.radarcns.kafka.ObservationKey; +import org.radarcns.passive.empatica.EmpaticaE4Acceleration; +import org.radarcns.stream.aggregator.DoubleAggregation; +import org.radarcns.stream.aggregator.DoubleArrayAggregation; import org.radarcns.stream.collector.DoubleArrayCollector; import org.radarcns.stream.collector.DoubleValueCollector; import org.radarcns.stream.phone.PhoneUsageCollector; @@ -35,21 +35,21 @@ public interface RadarUtilities { /** - * Creates a WindowedKey for a window of MeasurementKey. + * Creates a AggregateKey for a window of ObservationKey. * @param window Windowed measurement keys - * @return relevant WindowedKey + * @return relevant AggregateKey */ - WindowedKey getWindowed(Windowed window); + AggregateKey getWindowed(Windowed window); - WindowedKey getWindowedTuple(Windowed window); + AggregateKey getWindowedTuple(Windowed window); - KeyValue collectorToAvro( - Windowed window, DoubleArrayCollector collector); + KeyValue collectorToAvro( + Windowed window, DoubleArrayCollector collector); - KeyValue collectorToAvro( - Windowed window, DoubleValueCollector collector); + KeyValue collectorToAvro( + Windowed window, DoubleValueCollector collector); - KeyValue collectorToAvro( + KeyValue collectorToAvro( Windowed window, PhoneUsageCollector collector); double floatToDouble(float input); diff --git a/src/main/java/org/radarcns/util/RadarUtilitiesImpl.java b/src/main/java/org/radarcns/util/RadarUtilitiesImpl.java index 81461fd2..c92bce42 100644 --- a/src/main/java/org/radarcns/util/RadarUtilitiesImpl.java +++ b/src/main/java/org/radarcns/util/RadarUtilitiesImpl.java @@ -18,12 +18,12 @@ import org.apache.kafka.streams.KeyValue; import org.apache.kafka.streams.kstream.Windowed; -import org.radarcns.aggregator.DoubleAggregator; -import org.radarcns.aggregator.DoubleArrayAggregator; import org.radarcns.aggregator.PhoneUsageAggregator; -import org.radarcns.empatica.EmpaticaE4Acceleration; -import org.radarcns.key.MeasurementKey; -import org.radarcns.key.WindowedKey; +import org.radarcns.passive.empatica.EmpaticaE4Acceleration; +import org.radarcns.kafka.ObservationKey; +import org.radarcns.kafka.AggregateKey; +import org.radarcns.stream.aggregator.DoubleAggregation; +import org.radarcns.stream.aggregator.DoubleArrayAggregation; import org.radarcns.stream.collector.DoubleArrayCollector; import org.radarcns.stream.collector.DoubleValueCollector; import org.radarcns.stream.phone.PhoneUsageCollector; @@ -41,16 +41,16 @@ protected RadarUtilitiesImpl() { } @Override - public WindowedKey getWindowed(Windowed window) { - return new WindowedKey(window.key().getUserId(), window.key().getSourceId(), - window.window().start(), window.window().end()); + public AggregateKey getWindowed(Windowed window) { + return new AggregateKey(window.key().getProjectId(), window.key().getUserId(), + window.key().getSourceId(), window.window().start(), window.window().end()); } @Override - public WindowedKey getWindowedTuple(Windowed window) { - MeasurementKey measurementKey = window.key().getKey(); - return new WindowedKey(measurementKey.getUserId(), measurementKey.getSourceId(), - window.window().start(), window.window().end()); + public AggregateKey getWindowedTuple(Windowed window) { + ObservationKey measurementKey = window.key().getKey(); + return new AggregateKey(measurementKey.getProjectId(), measurementKey.getUserId(), + measurementKey.getSourceId(), window.window().start(), window.window().end()); } @Override @@ -60,7 +60,7 @@ public double floatToDouble(float input) { @Override - public KeyValue collectorToAvro( + public KeyValue collectorToAvro( Windowed window, PhoneUsageCollector collector ) { return new KeyValue<>(getWindowedTuple(window) , new PhoneUsageAggregator( @@ -73,8 +73,8 @@ public KeyValue collectorToAvro( } @Override - public KeyValue collectorToAvro( - Windowed window, DoubleArrayCollector collector) { + public KeyValue collectorToAvro( + Windowed window, DoubleArrayCollector collector) { List subcollectors = collector.getCollectors(); int len = subcollectors.size(); List min = new ArrayList<>(len); @@ -96,14 +96,14 @@ public KeyValue collectorToAvro( } return new KeyValue<>(getWindowed(window), - new DoubleArrayAggregator(min, max, sum, count, avg, quartile, iqr)); + new DoubleArrayAggregation(min, max, sum, count, avg, quartile, iqr)); } @Override - public KeyValue collectorToAvro( - Windowed window, DoubleValueCollector collector) { + public KeyValue collectorToAvro( + Windowed window, DoubleValueCollector collector) { return new KeyValue<>(getWindowed(window), - new DoubleAggregator(collector.getMin(), collector.getMax(), collector.getSum(), + new DoubleAggregation(collector.getMin(), collector.getMax(), collector.getSum(), collector.getCount(), collector.getAvg(), collector.getQuartile(), collector.getIqr())); } diff --git a/src/test/java/org/radarcns/monitor/BatteryLevelMonitorTest.java b/src/test/java/org/radarcns/monitor/BatteryLevelMonitorTest.java index 4bbc8854..d295dc29 100644 --- a/src/test/java/org/radarcns/monitor/BatteryLevelMonitorTest.java +++ b/src/test/java/org/radarcns/monitor/BatteryLevelMonitorTest.java @@ -39,7 +39,7 @@ import org.junit.rules.TemporaryFolder; import org.radarcns.config.ConfigRadar; import org.radarcns.config.RadarPropertyHandler; -import org.radarcns.key.MeasurementKey; +import org.radarcns.kafka.ObservationKey; import org.radarcns.monitor.BatteryLevelMonitor.BatteryLevelState; import org.radarcns.util.EmailSender; import org.radarcns.util.PersistentStateStore; @@ -60,6 +60,7 @@ public class BatteryLevelMonitorTest { public void setUp() { Parser parser = new Parser(); keySchema = parser.parse("{\"name\": \"key\", \"type\": \"record\", \"fields\": [" + + "{\"name\": \"projectId\", \"type\": [\"null\", \"string\"]}," + "{\"name\": \"userId\", \"type\": \"string\"}," + "{\"name\": \"sourceId\", \"type\": \"string\"}" + "]}"); @@ -102,6 +103,7 @@ public void evaluateRecord() throws Exception { private void sendMessage(BatteryLevelMonitor monitor, float batteryLevel, boolean sentMessage) throws MessagingException { Record key = new Record(keySchema); + key.put("projectId", "test"); key.put("sourceId", "1"); key.put("userId", "me"); @@ -121,7 +123,7 @@ public void retrieveState() throws Exception { File base = folder.newFolder(); PersistentStateStore stateStore = new PersistentStateStore(base); BatteryLevelState state = new BatteryLevelState(); - MeasurementKey key1 = new MeasurementKey("a", "b"); + ObservationKey key1 = new ObservationKey("test", "a", "b"); state.updateLevel(key1, 0.1f); stateStore.storeState("one", "two", state); diff --git a/src/test/java/org/radarcns/monitor/DisconnectMonitorTest.java b/src/test/java/org/radarcns/monitor/DisconnectMonitorTest.java index f85b8f99..8dcb7770 100644 --- a/src/test/java/org/radarcns/monitor/DisconnectMonitorTest.java +++ b/src/test/java/org/radarcns/monitor/DisconnectMonitorTest.java @@ -30,7 +30,7 @@ import org.radarcns.config.ConfigRadar; import org.radarcns.config.DisconnectMonitorConfig; import org.radarcns.config.RadarPropertyHandler; -import org.radarcns.key.MeasurementKey; +import org.radarcns.kafka.ObservationKey; import org.radarcns.monitor.DisconnectMonitor.DisconnectMonitorState; import org.radarcns.monitor.DisconnectMonitor.MissingRecordsReport; import org.radarcns.util.EmailSender; @@ -68,6 +68,7 @@ public class DisconnectMonitorTest { public void setUp() { Parser parser = new Parser(); keySchema = parser.parse("{\"name\": \"key\", \"type\": \"record\", \"fields\": [" + + "{\"name\": \"projectId\", \"type\": [\"null\", \"string\"]}," + "{\"name\": \"userId\", \"type\": \"string\"}," + "{\"name\": \"sourceId\", \"type\": \"string\"}" + "]}"); @@ -135,6 +136,7 @@ public void evaluateRecordsWithScheduledAlerts() throws Exception { private void sendMessage(DisconnectMonitor monitor, String source, int sentMessages) throws MessagingException { Record key = new Record(keySchema); + key.put("projectId", "test"); key.put("sourceId", source); key.put("userId", "me"); @@ -154,9 +156,9 @@ public void retrieveState() throws Exception { File base = folder.newFolder(); PersistentStateStore stateStore = new PersistentStateStore(base); DisconnectMonitorState state = new DisconnectMonitorState(); - MeasurementKey key1 = new MeasurementKey("a", "b"); - MeasurementKey key2 = new MeasurementKey("b", "c"); - MeasurementKey key3 = new MeasurementKey("c", "d"); + ObservationKey key1 = new ObservationKey("test", "a", "b"); + ObservationKey key2 = new ObservationKey("test", "b", "c"); + ObservationKey key3 = new ObservationKey("test", "c", "d"); long now = System.currentTimeMillis(); state.getLastSeen().put(measurementKeyToString(key1), now); state.getLastSeen().put(measurementKeyToString(key2), now + 1L); diff --git a/src/test/java/org/radarcns/stream/empatica/E4AccelerationTest.java.bak b/src/test/java/org/radarcns/stream/empatica/E4AccelerationTest.java.bak index 38690470..d91d783d 100644 --- a/src/test/java/org/radarcns/stream/empatica/E4AccelerationTest.java.bak +++ b/src/test/java/org/radarcns/stream/empatica/E4AccelerationTest.java.bak @@ -1,4 +1,4 @@ -package org.radarcns.empatica.streams; +package org.radarcns.passive.empatica.streams; import org.apache.kafka.streams.kstream.KStream; import org.apache.kafka.streams.kstream.KStreamBuilder; @@ -10,10 +10,10 @@ import org.mockito.Mockito; import org.mortbay.util.IO; import org.radarcns.config.ConfigRadar; import org.radarcns.config.PropertiesRadar; -import org.radarcns.empatica.EmpaticaE4Acceleration; -import org.radarcns.empatica.topic.E4SensorStreams; +import org.radarcns.passive.empatica.EmpaticaE4Acceleration; +import org.radarcns.passive.empatica.topic.E4SensorStreams; import E4Streams; -import org.radarcns.key.MeasurementKey; +import org.radarcns.kafka.ObservationKey; import org.radarcns.stream.StreamWorker; import org.radarcns.stream.StreamMaster; import org.radarcns.topic.SensorTopic; @@ -49,7 +49,7 @@ public class E4AccelerationTest { when(PropertiesRadar.getInstance()).thenReturn(propertiesRadar); KStreamBuilder streamBuilder = new KStreamBuilder(); - KStream stream = streamBuilder.stream("E4Acceleration"); + KStream stream = streamBuilder.stream("E4Acceleration"); SensorTopic sensorTopic = E4Streams.getInstance().getSensorTopics().getAccelerationStream(); e4Acceleration.setStream(stream, sensorTopic); diff --git a/src/test/java/org/radarcns/util/PersistentStateStoreTest.java b/src/test/java/org/radarcns/util/PersistentStateStoreTest.java index 71a8fd6b..b222ce19 100644 --- a/src/test/java/org/radarcns/util/PersistentStateStoreTest.java +++ b/src/test/java/org/radarcns/util/PersistentStateStoreTest.java @@ -28,7 +28,7 @@ import org.junit.Rule; import org.junit.Test; import org.junit.rules.TemporaryFolder; -import org.radarcns.key.MeasurementKey; +import org.radarcns.kafka.ObservationKey; import org.radarcns.monitor.BatteryLevelMonitor.BatteryLevelState; public class PersistentStateStoreTest { @@ -40,14 +40,14 @@ public void retrieveState() throws Exception { File base = folder.newFolder(); PersistentStateStore stateStore = new PersistentStateStore(base); BatteryLevelState state = new BatteryLevelState(); - MeasurementKey key1 = new MeasurementKey("a", "b"); + ObservationKey key1 = new ObservationKey("test", "a", "b"); state.updateLevel(key1, 0.1f); stateStore.storeState("one", "two", state); File outputFile = new File(base, "one_two.yml"); assertThat(outputFile.exists(), is(true)); String rawFile = new String(Files.readAllBytes(outputFile.toPath())); - assertThat(rawFile, equalTo("---\nlevels:\n a#b: 0.1\n")); + assertThat(rawFile, equalTo("---\nlevels:\n test#a#b: 0.1\n")); PersistentStateStore stateStore2 = new PersistentStateStore(base); BatteryLevelState state2 = stateStore2.retrieveState("one", "two", new BatteryLevelState()); diff --git a/src/test/java/org/radarcns/util/RadarUtilsTest.java b/src/test/java/org/radarcns/util/RadarUtilsTest.java index f9dd4190..fe8bad7b 100644 --- a/src/test/java/org/radarcns/util/RadarUtilsTest.java +++ b/src/test/java/org/radarcns/util/RadarUtilsTest.java @@ -23,9 +23,9 @@ import org.apache.kafka.streams.kstream.internals.TimeWindow; import org.junit.Before; import org.junit.Test; -import org.radarcns.empatica.EmpaticaE4Acceleration; -import org.radarcns.key.MeasurementKey; -import org.radarcns.key.WindowedKey; +import org.radarcns.passive.empatica.EmpaticaE4Acceleration; +import org.radarcns.kafka.ObservationKey; +import org.radarcns.kafka.AggregateKey; public class RadarUtilsTest { @@ -41,14 +41,14 @@ public void getWindowed() { String userId = "userId"; String sourceId = "sourceId"; - MeasurementKey measurementKey = new MeasurementKey(); + ObservationKey measurementKey = new ObservationKey(); measurementKey.setUserId(userId); measurementKey.setSourceId(sourceId); Window window = new TimeWindow(1, 4); - Windowed measurementKeyWindowed = new Windowed<>(measurementKey, window); + Windowed measurementKeyWindowed = new Windowed<>(measurementKey, window); - WindowedKey windowedKey = radarUtilities.getWindowed(measurementKeyWindowed); + AggregateKey windowedKey = radarUtilities.getWindowed(measurementKeyWindowed); assertEquals(windowedKey.getUserId(), userId); assertEquals(windowedKey.getSourceId(), sourceId); diff --git a/src/test/java/org/radarcns/util/serde/JsonDeserializerTest.java b/src/test/java/org/radarcns/util/serde/JsonDeserializerTest.java index 019625c6..0f6815ac 100644 --- a/src/test/java/org/radarcns/util/serde/JsonDeserializerTest.java +++ b/src/test/java/org/radarcns/util/serde/JsonDeserializerTest.java @@ -18,14 +18,14 @@ import java.nio.charset.Charset; import junit.framework.TestCase; -import org.radarcns.key.MeasurementKey; +import org.radarcns.kafka.ObservationKey; public class JsonDeserializerTest extends TestCase { public void testSerialize() throws Exception { byte[] json = "{\"userId\":\"user\",\"sourceId\":\"source\"}" .getBytes(Charset.forName("UTF-8")); - JsonDeserializer serializer = new JsonDeserializer<>(MeasurementKey.class); - MeasurementKey key = serializer.deserialize("mytest", json); + JsonDeserializer serializer = new JsonDeserializer<>(ObservationKey.class); + ObservationKey key = serializer.deserialize("mytest", json); assertEquals("user", key.getUserId()); assertEquals("source", key.getSourceId()); } diff --git a/src/test/java/org/radarcns/util/serde/JsonSerializerTest.java b/src/test/java/org/radarcns/util/serde/JsonSerializerTest.java index aea2ad7b..aa9132d0 100644 --- a/src/test/java/org/radarcns/util/serde/JsonSerializerTest.java +++ b/src/test/java/org/radarcns/util/serde/JsonSerializerTest.java @@ -17,13 +17,13 @@ package org.radarcns.util.serde; import junit.framework.TestCase; -import org.radarcns.key.MeasurementKey; +import org.radarcns.kafka.ObservationKey; public class JsonSerializerTest extends TestCase { public void testSerialize() throws Exception { - JsonSerializer serializer = new JsonSerializer<>(); - MeasurementKey key = new MeasurementKey("user", "source"); + JsonSerializer serializer = new JsonSerializer<>(); + ObservationKey key = new ObservationKey("test", "user", "source"); String result = new String(serializer.serialize("mytest", key)); - assertEquals("{\"userId\":\"user\",\"sourceId\":\"source\"}", result); + assertEquals("{\"projectId\":\"test\",\"userId\":\"user\",\"sourceId\":\"source\"}", result); } } \ No newline at end of file From 526564b6e80339cf6c823a29fddd2d9ed508dd83 Mon Sep 17 00:00:00 2001 From: Joris Borgdorff Date: Thu, 28 Sep 2017 12:22:05 +0200 Subject: [PATCH 02/23] Added multiple time windows per stream worker --- .../integration/DirectProducerTest.java | 2 +- .../org/radarcns/config/KafkaProperty.java | 8 +- .../radarcns/stream/CombinedStreamMaster.java | 2 +- .../radarcns/stream/GeneralStreamGroup.java | 70 ++++++- .../org/radarcns/stream/StreamDefinition.java | 83 +++++++- .../java/org/radarcns/stream/StreamGroup.java | 3 +- .../org/radarcns/stream/StreamMaster.java | 5 +- .../org/radarcns/stream/StreamWorker.java | 184 ++++++++++++------ .../stream/empatica/E4AccelerationStream.java | 44 ++--- .../stream/empatica/E4BatteryLevelStream.java | 36 ++-- .../empatica/E4BloodVolumePulseStream.java | 37 ++-- .../E4ElectroDermalActivityStream.java | 39 ++-- .../stream/empatica/E4HeartRateStream.java | 42 ++-- .../empatica/E4InterBeatIntervalStream.java | 37 ++-- .../stream/empatica/E4StreamMaster.java | 15 +- .../radarcns/stream/empatica/E4Streams.java | 53 +++-- .../stream/empatica/E4TemperatureStream.java | 36 ++-- .../stream/phone/PhoneAccelerationStream.java | 41 ++-- .../stream/phone/PhoneBatteryStream.java | 30 +-- .../stream/phone/PhoneStreamMaster.java | 14 +- .../radarcns/stream/phone/PhoneStreams.java | 22 ++- .../phone/PhoneUsageAggregationStream.java | 28 ++- .../stream/phone/PhoneUsageStream.java | 14 +- .../java/org/radarcns/util/Comparison.java | 20 ++ .../java/org/radarcns/util/StreamUtil.java | 45 +++++ .../radarcns/stream/StreamDefinitionTest.java | 9 +- .../org/radarcns/stream/StreamWorkerTest.java | 25 ++- .../stream/empatica/E4StreamsTest.java | 19 +- 28 files changed, 564 insertions(+), 399 deletions(-) create mode 100644 src/main/java/org/radarcns/util/Comparison.java create mode 100644 src/main/java/org/radarcns/util/StreamUtil.java diff --git a/src/integrationTest/java/org/radarcns/integration/DirectProducerTest.java b/src/integrationTest/java/org/radarcns/integration/DirectProducerTest.java index d56744ba..facd221f 100644 --- a/src/integrationTest/java/org/radarcns/integration/DirectProducerTest.java +++ b/src/integrationTest/java/org/radarcns/integration/DirectProducerTest.java @@ -78,7 +78,7 @@ public void testDirect() throws Exception { String clientId = "someclinet"; E4AggregatedAccelerationMonitor monitor = new E4AggregatedAccelerationMonitor( RadarSingletonFactory.getRadarPropertyHandler(), - "android_empatica_e4_acceleration_output", clientId); + "android_empatica_e4_acceleration_10sec", clientId); monitor.setPollTimeout(280_000L); monitor.start(); } diff --git a/src/main/java/org/radarcns/config/KafkaProperty.java b/src/main/java/org/radarcns/config/KafkaProperty.java index 21c378d9..d415fa52 100644 --- a/src/main/java/org/radarcns/config/KafkaProperty.java +++ b/src/main/java/org/radarcns/config/KafkaProperty.java @@ -29,7 +29,7 @@ public class KafkaProperty { private final ConfigRadar configRadar; - protected KafkaProperty(ConfigRadar configRadar) { + public KafkaProperty(ConfigRadar configRadar) { this.configRadar = configRadar; } @@ -38,7 +38,7 @@ protected KafkaProperty(ConfigRadar configRadar) { * @param numThread number of threads to execute stream processing * @return Properties for a Kafka Stream */ - public Properties getStream(@Nonnull String clientId, @Nonnull int numThread) { + public Properties getStreamProperties(@Nonnull String clientId, @Nonnull int numThread) { Properties props = new Properties(); props.put(StreamsConfig.APPLICATION_ID_CONFIG, clientId); @@ -60,9 +60,9 @@ public Properties getStream(@Nonnull String clientId, @Nonnull int numThread) { * @param timestampExtractor custom timestamp extract that overrides the out-of-the-box * @return Properties for a Kafka Stream */ - public Properties getStream(@Nonnull String clientId, @Nonnull int numThread, + public Properties getStreamProperties(@Nonnull String clientId, @Nonnull int numThread, @Nonnull Class timestampExtractor) { - Properties props = getStream(clientId, numThread); + Properties props = getStreamProperties(clientId, numThread); props.put(StreamsConfig.TIMESTAMP_EXTRACTOR_CLASS_CONFIG, timestampExtractor.getName()); diff --git a/src/main/java/org/radarcns/stream/CombinedStreamMaster.java b/src/main/java/org/radarcns/stream/CombinedStreamMaster.java index 5fde2586..55ca9685 100644 --- a/src/main/java/org/radarcns/stream/CombinedStreamMaster.java +++ b/src/main/java/org/radarcns/stream/CombinedStreamMaster.java @@ -72,7 +72,7 @@ public List getTopicNames() { } @Override - public StreamDefinition getStreamDefinition(String inputTopic) { + public Collection getStreamDefinition(String inputTopic) { for (StreamMaster master : streamMasters) { if (master.getStreamGroup().getTopicNames().contains(inputTopic)) { return master.getStreamGroup().getStreamDefinition(inputTopic); diff --git a/src/main/java/org/radarcns/stream/GeneralStreamGroup.java b/src/main/java/org/radarcns/stream/GeneralStreamGroup.java index dd95e111..744e64ff 100644 --- a/src/main/java/org/radarcns/stream/GeneralStreamGroup.java +++ b/src/main/java/org/radarcns/stream/GeneralStreamGroup.java @@ -16,13 +16,18 @@ package org.radarcns.stream; +import org.radarcns.topic.KafkaTopic; + import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; import java.util.HashMap; -import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Set; -import org.radarcns.topic.KafkaTopic; +import java.util.TreeSet; +import java.util.stream.Collectors; /** * Implementation of a {@link StreamGroup}. Override to create specific streams for a given @@ -35,12 +40,23 @@ public class GeneralStreamGroup implements StreamGroup { public static final String OUTPUT_LABEL = "_output"; - private final Map topicMap; + private final Map> topicMap; private final Set topicNames; public GeneralStreamGroup() { topicMap = new HashMap<>(); - topicNames = new HashSet<>(); + topicNames = new TreeSet<>(); + } + + /** + * Create a stream from input to output topic. By using this method, {@link #getTopicNames()} + * and {@link #getStreamDefinition(String)} automatically get updated. + * @param input input topic name + * @param output output topic name + * @return stream definition. + */ + protected Collection createStream(String input, String output) { + return createStream(input, output, 0L); } /** @@ -48,10 +64,12 @@ public GeneralStreamGroup() { * and {@link #getStreamDefinition(String)} automatically get updated. * @param input input topic name * @param output output topic name + * @param window time windows size in milliseconds, 0 if none. * @return stream definition. */ - protected StreamDefinition createStream(String input, String output) { - StreamDefinition ret = new StreamDefinition(new KafkaTopic(input), new KafkaTopic(output)); + protected Collection createStream(String input, String output, long window) { + Collection ret = Collections.singleton( + new StreamDefinition(new KafkaTopic(input), new KafkaTopic(output), window)); topicMap.put(input, ret); topicNames.add(input); topicNames.add(output); @@ -64,13 +82,45 @@ protected StreamDefinition createStream(String input, String output) { * @param input input topic name * @return sensor stream definition */ - protected StreamDefinition createSensorStream(String input) { - return createStream(input, input + OUTPUT_LABEL); + protected Collection createSensorStream(String input) { + return createStream(input, input + OUTPUT_LABEL, 0L); + } + + protected Collection createAggregateStream(String input, long window) { + return createStream(input, input + OUTPUT_LABEL, window); + } + + protected Collection createWindowedSensorStream(String input) { + return createWindowedSensorStream(input, input); + } + + protected Collection createWindowedSensorStream(String input, + String outputBase) { + + topicNames.add(input); + Collection streams = new TreeSet<>( + Arrays.stream(TimeWindowMetadata.values()) + .map(w -> new StreamDefinition(new KafkaTopic(input), + new KafkaTopic(w.getTopicLabel(outputBase)), + w.getIntervalInMilliSec())) + .collect(Collectors.toList())); + + topicNames.addAll(streams.stream() + .map(t -> t.getOutputTopic().getName()) + .collect(Collectors.toList())); + + topicMap.merge(input, streams, (v1, v2) -> { + Set newSet = new TreeSet<>(v1); + newSet.addAll(v2); + return newSet; + }); + + return streams; } @Override - public StreamDefinition getStreamDefinition(String inputTopic) { - StreamDefinition topic = topicMap.get(inputTopic); + public Collection getStreamDefinition(String inputTopic) { + Collection topic = topicMap.get(inputTopic); if (topic == null) { throw new IllegalArgumentException("Topic " + inputTopic + " unknown"); } diff --git a/src/main/java/org/radarcns/stream/StreamDefinition.java b/src/main/java/org/radarcns/stream/StreamDefinition.java index edc4227d..9b38c249 100644 --- a/src/main/java/org/radarcns/stream/StreamDefinition.java +++ b/src/main/java/org/radarcns/stream/StreamDefinition.java @@ -16,35 +16,64 @@ package org.radarcns.stream; +import org.apache.kafka.streams.kstream.TimeWindows; import org.radarcns.topic.KafkaTopic; -public class StreamDefinition { +import javax.annotation.Nonnull; +import javax.annotation.Nullable; +import java.util.Objects; - public static final String FROM_LABEL = "From-"; - public static final String TO_LABEL = "-To-"; +import static org.radarcns.util.Comparison.compare; +public class StreamDefinition implements Comparable { private final KafkaTopic inputTopic; private final KafkaTopic outputTopic; + private final TimeWindows window; + + /** + * Constructor. It takes in input the topic name to be consumed and to topic name where the + * related stream will write the computed values. Default 0 window is used. + * @param input source {@link KafkaTopic} + * @param output output {@link KafkaTopic} + */ + public StreamDefinition(@Nonnull KafkaTopic input, @Nonnull KafkaTopic output) { + this(input, output, 0L); + } /** * Constructor. It takes in input the topic name to be consumed and to topic name where the * related stream will write the computed values. - * * @param input source {@link KafkaTopic} * @param output output {@link KafkaTopic} + * @param window time window for aggregation. */ - public StreamDefinition(KafkaTopic input, KafkaTopic output) { - if (input == null || output == null) { - throw new IllegalArgumentException("Input and output topic may not be null"); - } + public StreamDefinition(@Nonnull KafkaTopic input, @Nonnull KafkaTopic output, long window) { + this(input, output, window == 0 ? null : TimeWindows.of(window)); + } + + /** + * Constructor. It takes in input the topic name to be consumed and to topic name where the + * related stream will write the computed values. + * @param input source {@link KafkaTopic} + * @param output output {@link KafkaTopic} + * @param window time window for aggregation. + */ + public StreamDefinition(@Nonnull KafkaTopic input, @Nonnull KafkaTopic output, + @Nullable TimeWindows window) { + Objects.requireNonNull(input); + Objects.requireNonNull(output); + this.inputTopic = input; this.outputTopic = output; + this.window = window; } + @Nonnull public KafkaTopic getInputTopic() { return inputTopic; } + @Nonnull public KafkaTopic getOutputTopic() { return outputTopic; } @@ -56,7 +85,43 @@ public KafkaTopic getOutputTopic() { * * @return {@code String} representing the changelog topic name */ + @Nonnull public String getStateStoreName() { - return FROM_LABEL + getInputTopic().getName() + TO_LABEL + getOutputTopic().getName(); + return "From-" + getInputTopic().getName() + "-To-" + getOutputTopic().getName(); + } + + @Nullable + public TimeWindows getTimeWindows() { + return window; + } + + + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + StreamDefinition that = (StreamDefinition) o; + return Objects.equals(inputTopic, that.inputTopic) + && Objects.equals(outputTopic, that.outputTopic) + && Objects.equals(window, that.window); + } + + @Override + public int hashCode() { + return Objects.hash(inputTopic, outputTopic, window); + } + + @Override + public int compareTo(@Nonnull StreamDefinition o) { + return compare((StreamDefinition d) -> d.getInputTopic().getName()) + .then(d -> d.getOutputTopic().getName()) + .then(d -> d.getTimeWindows() == null ? 0 : d.getTimeWindows().sizeMs) + .then(d -> d.getTimeWindows() == null ? 0 : d.getTimeWindows().advanceMs) + .apply(this, o); } } diff --git a/src/main/java/org/radarcns/stream/StreamGroup.java b/src/main/java/org/radarcns/stream/StreamGroup.java index 70385f7c..7bf65575 100644 --- a/src/main/java/org/radarcns/stream/StreamGroup.java +++ b/src/main/java/org/radarcns/stream/StreamGroup.java @@ -16,6 +16,7 @@ package org.radarcns.stream; +import java.util.Collection; import java.util.List; /** @@ -35,5 +36,5 @@ public interface StreamGroup { * @return stream definition of given input topic * @throws IllegalArgumentException if the input topic is not found. */ - StreamDefinition getStreamDefinition(String inputTopic); + Collection getStreamDefinition(String inputTopic); } diff --git a/src/main/java/org/radarcns/stream/StreamMaster.java b/src/main/java/org/radarcns/stream/StreamMaster.java index 41227519..cfe200f9 100644 --- a/src/main/java/org/radarcns/stream/StreamMaster.java +++ b/src/main/java/org/radarcns/stream/StreamMaster.java @@ -28,6 +28,7 @@ import java.util.concurrent.Executors; import java.util.concurrent.RejectedExecutionException; import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.ScheduledFuture; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; @@ -183,8 +184,8 @@ protected void announceTopics() { protected abstract StreamGroup getStreamGroup(); /** Add a monitor to the master. It will run every 30 seconds. */ - void addMonitor(Monitor monitor) { - executor.scheduleAtFixedRate(monitor, 0, 30, TimeUnit.SECONDS); + ScheduledFuture addMonitor(Monitor monitor) { + return executor.scheduleAtFixedRate(monitor, 0, 30, TimeUnit.SECONDS); } protected synchronized int lowPriority() { diff --git a/src/main/java/org/radarcns/stream/StreamWorker.java b/src/main/java/org/radarcns/stream/StreamWorker.java index da387bdf..24c8aa41 100644 --- a/src/main/java/org/radarcns/stream/StreamWorker.java +++ b/src/main/java/org/radarcns/stream/StreamWorker.java @@ -16,19 +16,40 @@ package org.radarcns.stream; -import java.io.IOException; -import javax.annotation.Nonnull; import org.apache.avro.specific.SpecificRecord; import org.apache.kafka.streams.KafkaStreams; import org.apache.kafka.streams.KeyValue; import org.apache.kafka.streams.errors.StreamsException; import org.apache.kafka.streams.kstream.KStream; import org.apache.kafka.streams.kstream.KStreamBuilder; +import org.apache.kafka.streams.kstream.TimeWindows; import org.radarcns.config.KafkaProperty; +import org.radarcns.kafka.AggregateKey; +import org.radarcns.kafka.ObservationKey; +import org.radarcns.stream.aggregator.DoubleAggregation; +import org.radarcns.stream.aggregator.DoubleArrayAggregation; +import org.radarcns.stream.collector.DoubleArrayCollector; +import org.radarcns.stream.collector.DoubleValueCollector; import org.radarcns.util.Monitor; +import org.radarcns.util.RadarSingletonFactory; +import org.radarcns.util.RadarUtilities; +import org.radarcns.util.serde.RadarSerdes; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import javax.annotation.Nonnull; +import java.util.Collection; +import java.util.List; +import java.util.Properties; +import java.util.concurrent.ScheduledFuture; +import java.util.function.Function; +import java.util.stream.Collectors; + +import static org.apache.kafka.streams.KeyValue.pair; +import static org.radarcns.util.Serialization.floatToDouble; +import static org.radarcns.util.StreamUtil.first; +import static org.radarcns.util.StreamUtil.second; + /** * Abstraction of a Kafka Stream. */ @@ -36,63 +57,82 @@ public abstract class StreamWorker streamDefinitions; + private Collection> monitors; private final KafkaProperty kafkaProperty; - private KafkaStreams streams; + protected final RadarUtilities utilities = RadarSingletonFactory.getRadarUtilities(); + + private List streams; - public StreamWorker(@Nonnull StreamDefinition streamDefinition, @Nonnull String clientId, + public StreamWorker(@Nonnull Collection streamDefinitions, int numThreads, @Nonnull StreamMaster aggregator, KafkaProperty kafkaProperty, Logger monitorLog) { if (numThreads < 1) { throw new IllegalStateException( "The number of concurrent threads must be at least 1"); } - this.clientId = clientId; this.master = aggregator; - this.streamDefinition = streamDefinition; + this.streamDefinitions = streamDefinitions; this.numThreads = numThreads; this.kafkaProperty = kafkaProperty; this.streams = null; - - if (monitorLog == null) { - this.monitor = null; - } else { - this.monitor = new Monitor(monitorLog, "records have been read from " - + streamDefinition.getInputTopic()); - } + this.monitors = null; + this.monitorLog = monitorLog; } /** * Create a Kafka Stream builder. This implementation will create a stream from given * input topic to given output topic. It monitors the amount of messages that are read. */ - protected KStreamBuilder createBuilder() throws IOException { - KStreamBuilder builder = new KStreamBuilder(); + protected KeyValue, KafkaStreams> createBuilder(StreamDefinition def) { + Monitor monitor; + ScheduledFuture future = null; + if (monitorLog != null) { + monitor = new Monitor(monitorLog, "records have been read from " + + def.getInputTopic() + " to " + def.getOutputTopic()); + future = master.addMonitor(monitor); + } else { + monitor = null; + } - StreamDefinition definition = getStreamDefinition(); - String inputTopic = definition.getInputTopic().getName(); - String outputTopic = definition.getOutputTopic().getName(); + KStreamBuilder builder = new KStreamBuilder(); - KStream inputStream = builder.stream(inputTopic) - .map((k, v) -> { - incrementMonitor(); - return new KeyValue<>(k, v); - }); + implementStream(def, + builder.stream(def.getInputTopic().getName()) + .map((k, v) -> { + if (monitor != null) { + monitor.increment(); + } + return pair(k, v); + }) + ).to(def.getOutputTopic().getName()); + + return pair(future, new KafkaStreams(builder, getStreamProperties(def))); + } - defineStream(inputStream).to(outputTopic); + /** + * @return Properties for a Kafka Stream + */ + protected Properties getStreamProperties(@Nonnull StreamDefinition definition) { + String localClientId = getClass().getName(); + TimeWindows window = definition.getTimeWindows(); + if (window != null) { + localClientId += '-' + window.sizeMs + '-' + window.advanceMs; + } - return builder; + return kafkaProperty.getStreamProperties(localClientId, numThreads, + DeviceTimestampExtractor.class); } /** * Defines the stream computation. */ - protected abstract KStream defineStream(@Nonnull KStream kstream); + protected abstract KStream implementStream(StreamDefinition definition, + @Nonnull KStream kstream); /** * Starts the stream and notify the StreamMaster. @@ -102,30 +142,30 @@ public void start() { throw new IllegalStateException("Streams already started. Cannot start them again."); } - log.info("Creating the stream {} from topic {} to topic {}", - clientId, streamDefinition.getInputTopic(), - streamDefinition.getOutputTopic()); - - try { - if (monitor != null) { - master.addMonitor(monitor); - } - streams = new KafkaStreams(createBuilder(), - kafkaProperty.getStream(clientId, numThreads, DeviceTimestampExtractor.class)); - streams.setUncaughtExceptionHandler(this); - streams.start(); - - master.notifyStartedStream(this); - } catch (IOException ex) { - uncaughtException(Thread.currentThread(), ex); - } + List, KafkaStreams>> streamBuilders = getStreamDefinitions() + .parallelStream() + .map(this::createBuilder) + .collect(Collectors.toList()); + + monitors = streamBuilders.stream() + .map(first()) + .collect(Collectors.toList()); + + streams = streamBuilders.stream() + .map(second()) + .collect(Collectors.toList()); + + streams.forEach(stream -> { + stream.setUncaughtExceptionHandler(this); + stream.start(); + }); } /** * Close the stream and notify the StreamMaster. */ public void shutdown() { - log.info("Shutting down {} stream", clientId); + log.info("Shutting down {} stream", getClass().getSimpleName()); closeStreams(); @@ -145,28 +185,62 @@ public void uncaughtException(Thread t, Throwable e) { if (e instanceof StreamsException) { master.restartStream(this); } else { - master.notifyCrashedStream(clientId); + master.notifyCrashedStream(getClass().getSimpleName()); } } private void closeStreams() { if (streams != null) { - streams.close(); + streams.forEach(KafkaStreams::close); streams = null; } + + if (monitors != null) { + monitors.forEach(f -> f.cancel(false)); + monitors = null; + } + } + + protected Collection getStreamDefinitions() { + return streamDefinitions; + } + + protected final KStream aggregateFloat( + StreamDefinition definition, + @Nonnull KStream kstream, Function field) { + return aggregateDouble(definition, kstream, v -> floatToDouble(field.apply(v))); } - protected StreamDefinition getStreamDefinition() { - return streamDefinition; + protected final KStream aggregateDouble( + StreamDefinition definition, + @Nonnull KStream kstream, Function field) { + return kstream.groupByKey() + .aggregate( + DoubleValueCollector::new, + (k, v, valueCollector) -> valueCollector.add(field.apply(v)), + definition.getTimeWindows(), + RadarSerdes.getInstance().getDoubleCollector(), + definition.getStateStoreName()) + .toStream() + .map(utilities::collectorToAvro); } - /** Increment the number of messages processed. */ - protected void incrementMonitor() { - monitor.increment(); + protected final KStream aggregateDoubleArray( + StreamDefinition definition, + @Nonnull KStream kstream, Function field) { + return kstream.groupByKey() + .aggregate( + DoubleArrayCollector::new, + (k, v, valueCollector) -> valueCollector.add(field.apply(v)), + definition.getTimeWindows(), + RadarSerdes.getInstance().getDoubleArrayCollector(), + definition.getStateStoreName()) + .toStream() + .map(utilities::collectorToAvro); } @Override public String toString() { - return getClass().getSimpleName() + '<' + clientId + '>'; + return getClass().getSimpleName(); } } diff --git a/src/main/java/org/radarcns/stream/empatica/E4AccelerationStream.java b/src/main/java/org/radarcns/stream/empatica/E4AccelerationStream.java index 3a0db65f..cb18af70 100644 --- a/src/main/java/org/radarcns/stream/empatica/E4AccelerationStream.java +++ b/src/main/java/org/radarcns/stream/empatica/E4AccelerationStream.java @@ -16,47 +16,41 @@ package org.radarcns.stream.empatica; -import javax.annotation.Nonnull; import org.apache.kafka.streams.kstream.KStream; -import org.apache.kafka.streams.kstream.TimeWindows; -import org.radarcns.stream.aggregator.DoubleArrayAggregation; import org.radarcns.config.KafkaProperty; -import org.radarcns.passive.empatica.EmpaticaE4Acceleration; -import org.radarcns.kafka.ObservationKey; import org.radarcns.kafka.AggregateKey; +import org.radarcns.kafka.ObservationKey; +import org.radarcns.passive.empatica.EmpaticaE4Acceleration; +import org.radarcns.stream.StreamDefinition; import org.radarcns.stream.StreamMaster; import org.radarcns.stream.StreamWorker; -import org.radarcns.stream.collector.DoubleArrayCollector; -import org.radarcns.util.RadarSingletonFactory; -import org.radarcns.util.RadarUtilities; -import org.radarcns.util.serde.RadarSerdes; +import org.radarcns.stream.aggregator.DoubleArrayAggregation; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import javax.annotation.Nonnull; +import java.util.Collection; + +import static org.radarcns.util.Serialization.floatToDouble; + /** * Definition of Kafka Stream for aggregating data collected by Empatica E4 Accelerometer sensor. */ public class E4AccelerationStream extends StreamWorker { - private static final Logger log = LoggerFactory.getLogger(E4AccelerationStream.class); - private final RadarUtilities utilities = RadarSingletonFactory.getRadarUtilities(); + private static final Logger logger = LoggerFactory.getLogger(E4AccelerationStream.class); - public E4AccelerationStream(String clientId, int numThread, StreamMaster master, - KafkaProperty kafkaProperties) { - super(E4Streams.getInstance().getAccelerationStream(), clientId, - numThread, master, kafkaProperties, log); + public E4AccelerationStream(Collection definitions, int numThread, + StreamMaster master, KafkaProperty kafkaProperties) { + super(definitions, numThread, master, kafkaProperties, logger); } @Override - protected KStream defineStream( + protected KStream implementStream( + StreamDefinition definition, @Nonnull KStream kstream) { - return kstream.groupByKey() - .aggregate( - DoubleArrayCollector::new, - (k, v, valueCollector) -> valueCollector.add(utilities.accelerationToArray(v)), - TimeWindows.of(10 * 1000L), - RadarSerdes.getInstance().getDoubleArrayCollector(), - getStreamDefinition().getStateStoreName()) - .toStream() - .map(utilities::collectorToAvro); + return aggregateDoubleArray(definition, kstream, v -> new double[] { + floatToDouble(v.getX()), + floatToDouble(v.getY()), + floatToDouble(v.getZ())}); } } diff --git a/src/main/java/org/radarcns/stream/empatica/E4BatteryLevelStream.java b/src/main/java/org/radarcns/stream/empatica/E4BatteryLevelStream.java index afb2fc29..88b8b3d2 100644 --- a/src/main/java/org/radarcns/stream/empatica/E4BatteryLevelStream.java +++ b/src/main/java/org/radarcns/stream/empatica/E4BatteryLevelStream.java @@ -16,47 +16,35 @@ package org.radarcns.stream.empatica; -import javax.annotation.Nonnull; import org.apache.kafka.streams.kstream.KStream; -import org.apache.kafka.streams.kstream.TimeWindows; import org.radarcns.config.KafkaProperty; -import org.radarcns.passive.empatica.EmpaticaE4BatteryLevel; -import org.radarcns.kafka.ObservationKey; import org.radarcns.kafka.AggregateKey; +import org.radarcns.kafka.ObservationKey; +import org.radarcns.passive.empatica.EmpaticaE4BatteryLevel; +import org.radarcns.stream.StreamDefinition; import org.radarcns.stream.StreamMaster; import org.radarcns.stream.StreamWorker; import org.radarcns.stream.aggregator.DoubleAggregation; -import org.radarcns.stream.collector.DoubleValueCollector; -import org.radarcns.util.RadarSingletonFactory; -import org.radarcns.util.RadarUtilities; -import org.radarcns.util.serde.RadarSerdes; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import javax.annotation.Nonnull; +import java.util.Collection; + /** * Kafka Stream for aggregating data about Empatica E4 battery level. */ public class E4BatteryLevelStream extends StreamWorker { - private static final Logger log = LoggerFactory.getLogger(E4BatteryLevelStream.class); - private final RadarUtilities utilities = RadarSingletonFactory.getRadarUtilities(); + private static final Logger logger = LoggerFactory.getLogger(E4BatteryLevelStream.class); - public E4BatteryLevelStream(String clientId, int numThread, StreamMaster master, - KafkaProperty kafkaProperties) { - super(E4Streams.getInstance().getBatteryLevelStream(), clientId, - numThread, master, kafkaProperties, log); + public E4BatteryLevelStream(Collection definitions, int numThread, + StreamMaster master, KafkaProperty kafkaProperties) { + super(definitions, numThread, master, kafkaProperties, logger); } @Override - protected KStream defineStream( + protected KStream implementStream(StreamDefinition definition, @Nonnull KStream kstream) { - return kstream.groupByKey() - .aggregate( - DoubleValueCollector::new, - (k, v, valueCollector) -> valueCollector.add(v.getBatteryLevel()), - TimeWindows.of(10 * 1000L), - RadarSerdes.getInstance().getDoubleCollector(), - getStreamDefinition().getStateStoreName()) - .toStream() - .map(utilities::collectorToAvro); + return aggregateFloat(definition, kstream, EmpaticaE4BatteryLevel::getBatteryLevel); } } diff --git a/src/main/java/org/radarcns/stream/empatica/E4BloodVolumePulseStream.java b/src/main/java/org/radarcns/stream/empatica/E4BloodVolumePulseStream.java index 99000f28..d7d4d152 100644 --- a/src/main/java/org/radarcns/stream/empatica/E4BloodVolumePulseStream.java +++ b/src/main/java/org/radarcns/stream/empatica/E4BloodVolumePulseStream.java @@ -16,49 +16,36 @@ package org.radarcns.stream.empatica; -import javax.annotation.Nonnull; import org.apache.kafka.streams.kstream.KStream; -import org.apache.kafka.streams.kstream.TimeWindows; import org.radarcns.config.KafkaProperty; -import org.radarcns.passive.empatica.EmpaticaE4BloodVolumePulse; -import org.radarcns.kafka.ObservationKey; import org.radarcns.kafka.AggregateKey; +import org.radarcns.kafka.ObservationKey; +import org.radarcns.passive.empatica.EmpaticaE4BloodVolumePulse; +import org.radarcns.stream.StreamDefinition; import org.radarcns.stream.StreamMaster; import org.radarcns.stream.StreamWorker; import org.radarcns.stream.aggregator.DoubleAggregation; -import org.radarcns.stream.collector.DoubleValueCollector; -import org.radarcns.util.RadarSingletonFactory; -import org.radarcns.util.RadarUtilities; -import org.radarcns.util.serde.RadarSerdes; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import javax.annotation.Nonnull; +import java.util.Collection; + /** * Kafka Stream for aggregating data about Blood Volume Pulse collected by Empatica E4. */ public class E4BloodVolumePulseStream extends StreamWorker { - private static final Logger log = LoggerFactory.getLogger(E4BloodVolumePulseStream.class); - - private final RadarUtilities utilities = RadarSingletonFactory.getRadarUtilities(); + private static final Logger logger = LoggerFactory.getLogger(E4BloodVolumePulseStream.class); - public E4BloodVolumePulseStream(String clientId, int numThread, StreamMaster master, - KafkaProperty kafkaProperties) { - super(E4Streams.getInstance().getBloodVolumePulseStream(), - clientId, numThread, master, kafkaProperties, log); + public E4BloodVolumePulseStream(Collection definitions, int numThread, + StreamMaster master, KafkaProperty kafkaProperties) { + super(definitions, numThread, master, kafkaProperties, logger); } @Override - protected KStream defineStream( + protected KStream implementStream(StreamDefinition definition, @Nonnull KStream kstream) { - return kstream.groupByKey() - .aggregate( - DoubleValueCollector::new, - (k, v, valueCollector) -> valueCollector.add(v.getBloodVolumePulse()), - TimeWindows.of(10 * 1000L), - RadarSerdes.getInstance().getDoubleCollector(), - getStreamDefinition().getStateStoreName()) - .toStream() - .map(utilities::collectorToAvro); + return aggregateFloat(definition, kstream, EmpaticaE4BloodVolumePulse::getBloodVolumePulse); } } diff --git a/src/main/java/org/radarcns/stream/empatica/E4ElectroDermalActivityStream.java b/src/main/java/org/radarcns/stream/empatica/E4ElectroDermalActivityStream.java index 37caa724..18671be9 100644 --- a/src/main/java/org/radarcns/stream/empatica/E4ElectroDermalActivityStream.java +++ b/src/main/java/org/radarcns/stream/empatica/E4ElectroDermalActivityStream.java @@ -16,49 +16,38 @@ package org.radarcns.stream.empatica; -import javax.annotation.Nonnull; import org.apache.kafka.streams.kstream.KStream; -import org.apache.kafka.streams.kstream.TimeWindows; import org.radarcns.config.KafkaProperty; -import org.radarcns.passive.empatica.EmpaticaE4ElectroDermalActivity; -import org.radarcns.kafka.ObservationKey; import org.radarcns.kafka.AggregateKey; +import org.radarcns.kafka.ObservationKey; +import org.radarcns.passive.empatica.EmpaticaE4ElectroDermalActivity; +import org.radarcns.stream.StreamDefinition; import org.radarcns.stream.StreamMaster; import org.radarcns.stream.StreamWorker; import org.radarcns.stream.aggregator.DoubleAggregation; -import org.radarcns.stream.collector.DoubleValueCollector; -import org.radarcns.util.RadarSingletonFactory; -import org.radarcns.util.RadarUtilities; -import org.radarcns.util.serde.RadarSerdes; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import javax.annotation.Nonnull; +import java.util.Collection; + /** * Kafka Stream for aggregating data about electrodermal activity collected by Empatica E4. */ public class E4ElectroDermalActivityStream extends StreamWorker { - private static final Logger log = LoggerFactory.getLogger(E4ElectroDermalActivityStream.class); - - private final RadarUtilities utilities = RadarSingletonFactory.getRadarUtilities(); + private static final Logger logger = LoggerFactory.getLogger( + E4ElectroDermalActivityStream.class); - public E4ElectroDermalActivityStream(String clientId, int numThread, StreamMaster master, - KafkaProperty kafkaProperties) { - super(E4Streams.getInstance().getElectroDermalActivityStream(), - clientId, numThread, master, kafkaProperties, log); + public E4ElectroDermalActivityStream(Collection definitions, int numThread, + StreamMaster master, KafkaProperty kafkaProperties) { + super(definitions, numThread, master, kafkaProperties, logger); } @Override - protected KStream defineStream( + protected KStream implementStream(StreamDefinition definition, @Nonnull KStream kstream) { - return kstream.groupByKey() - .aggregate( - DoubleValueCollector::new, - (k, v, valueCollector) -> valueCollector.add(v.getElectroDermalActivity()), - TimeWindows.of(10 * 1000L), - RadarSerdes.getInstance().getDoubleCollector(), - getStreamDefinition().getStateStoreName()) - .toStream() - .map(utilities::collectorToAvro); + return aggregateFloat(definition, kstream, + EmpaticaE4ElectroDermalActivity::getElectroDermalActivity); } } diff --git a/src/main/java/org/radarcns/stream/empatica/E4HeartRateStream.java b/src/main/java/org/radarcns/stream/empatica/E4HeartRateStream.java index 9ac447d4..36b5d71c 100644 --- a/src/main/java/org/radarcns/stream/empatica/E4HeartRateStream.java +++ b/src/main/java/org/radarcns/stream/empatica/E4HeartRateStream.java @@ -16,50 +16,36 @@ package org.radarcns.stream.empatica; -import javax.annotation.Nonnull; import org.apache.kafka.streams.kstream.KStream; -import org.apache.kafka.streams.kstream.TimeWindows; import org.radarcns.config.KafkaProperty; -import org.radarcns.passive.empatica.EmpaticaE4InterBeatInterval; -import org.radarcns.kafka.ObservationKey; import org.radarcns.kafka.AggregateKey; +import org.radarcns.kafka.ObservationKey; +import org.radarcns.passive.empatica.EmpaticaE4InterBeatInterval; +import org.radarcns.stream.StreamDefinition; import org.radarcns.stream.StreamMaster; import org.radarcns.stream.StreamWorker; import org.radarcns.stream.aggregator.DoubleAggregation; -import org.radarcns.stream.collector.DoubleValueCollector; -import org.radarcns.util.RadarSingletonFactory; -import org.radarcns.util.RadarUtilities; -import org.radarcns.util.serde.RadarSerdes; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import javax.annotation.Nonnull; +import java.util.Collection; + /** * Kafka Stream for computing and aggregating Heart Rate values collected by Empatica E4. */ public class E4HeartRateStream extends StreamWorker { - private static final Logger log = LoggerFactory.getLogger(E4HeartRateStream.class); - private final RadarUtilities utilities = RadarSingletonFactory.getRadarUtilities(); + private static final Logger logger = LoggerFactory.getLogger(E4HeartRateStream.class); - public E4HeartRateStream(String clientId, int numThread, StreamMaster master, - KafkaProperty kafkaProperties) { - super(E4Streams.getInstance().getHeartRateStream(), clientId, - numThread, master, kafkaProperties, log); + public E4HeartRateStream(Collection definitions, int numThread, + StreamMaster master, KafkaProperty kafkaProperties) { + super(definitions, numThread, master, kafkaProperties, logger); } - @Override - protected KStream defineStream( + protected KStream implementStream( + StreamDefinition definition, @Nonnull KStream kstream) { - return kstream.groupByKey() - .aggregate( - DoubleValueCollector::new, - (k, v, valueCollector) -> { - double value = utilities.ibiToHeartRate(v.getInterBeatInterval()); - return valueCollector.add(value); - }, - TimeWindows.of(10 * 1000L), - RadarSerdes.getInstance().getDoubleCollector(), - getStreamDefinition().getStateStoreName()) - .toStream() - .map(utilities::collectorToAvro); + return aggregateDouble(definition, kstream, + v -> utilities.ibiToHeartRate(v.getInterBeatInterval())); } } diff --git a/src/main/java/org/radarcns/stream/empatica/E4InterBeatIntervalStream.java b/src/main/java/org/radarcns/stream/empatica/E4InterBeatIntervalStream.java index 836e8386..757547e6 100644 --- a/src/main/java/org/radarcns/stream/empatica/E4InterBeatIntervalStream.java +++ b/src/main/java/org/radarcns/stream/empatica/E4InterBeatIntervalStream.java @@ -16,48 +16,37 @@ package org.radarcns.stream.empatica; -import javax.annotation.Nonnull; import org.apache.kafka.streams.kstream.KStream; -import org.apache.kafka.streams.kstream.TimeWindows; import org.radarcns.config.KafkaProperty; -import org.radarcns.passive.empatica.EmpaticaE4InterBeatInterval; -import org.radarcns.kafka.ObservationKey; import org.radarcns.kafka.AggregateKey; +import org.radarcns.kafka.ObservationKey; +import org.radarcns.passive.empatica.EmpaticaE4InterBeatInterval; +import org.radarcns.stream.StreamDefinition; import org.radarcns.stream.StreamMaster; import org.radarcns.stream.StreamWorker; import org.radarcns.stream.aggregator.DoubleAggregation; -import org.radarcns.stream.collector.DoubleValueCollector; -import org.radarcns.util.RadarSingletonFactory; -import org.radarcns.util.RadarUtilities; -import org.radarcns.util.serde.RadarSerdes; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import javax.annotation.Nonnull; +import java.util.Collection; + /** * Definition of Kafka Stream for aggregating Inter Beat Interval values collected by Empatica E4. */ public class E4InterBeatIntervalStream extends StreamWorker { - private static final Logger log = LoggerFactory.getLogger(E4InterBeatIntervalStream.class); - private final RadarUtilities utilities = RadarSingletonFactory.getRadarUtilities(); + private static final Logger logger = LoggerFactory.getLogger(E4InterBeatIntervalStream.class); - public E4InterBeatIntervalStream(String clientId, int numThread, StreamMaster master, - KafkaProperty kafkaProperties) { - super(E4Streams.getInstance().getInterBeatIntervalStream(), - clientId, numThread, master, kafkaProperties, log); + public E4InterBeatIntervalStream(Collection definitions, int numThread, + StreamMaster master, KafkaProperty kafkaProperties) { + super(definitions, numThread, master, kafkaProperties, logger); } @Override - protected KStream defineStream( + protected KStream implementStream(StreamDefinition definition, @Nonnull KStream kstream) { - return kstream.groupByKey() - .aggregate( - DoubleValueCollector::new, - (k, v, valueCollector) -> valueCollector.add(v.getInterBeatInterval()), - TimeWindows.of(10 * 1000L), - RadarSerdes.getInstance().getDoubleCollector(), - getStreamDefinition().getStateStoreName()) - .toStream() - .map(utilities::collectorToAvro); + return aggregateFloat(definition, kstream, + EmpaticaE4InterBeatInterval::getInterBeatInterval); } } diff --git a/src/main/java/org/radarcns/stream/empatica/E4StreamMaster.java b/src/main/java/org/radarcns/stream/empatica/E4StreamMaster.java index a2013dea..7152e14f 100644 --- a/src/main/java/org/radarcns/stream/empatica/E4StreamMaster.java +++ b/src/main/java/org/radarcns/stream/empatica/E4StreamMaster.java @@ -38,19 +38,20 @@ protected StreamGroup getStreamGroup() { protected void createWorkers(List> list, StreamMaster master) { RadarPropertyHandler propertyHandler = RadarSingletonFactory.getRadarPropertyHandler(); KafkaProperty kafkaProperty = propertyHandler.getKafkaProperties(); + E4Streams defs = E4Streams.getInstance(); list.add(new E4AccelerationStream( - "E4AccelerationStream", highPriority(), master, kafkaProperty)); + defs.getAccelerationStream(), highPriority(), master, kafkaProperty)); list.add(new E4BatteryLevelStream( - "E4BatteryLevelStream", lowPriority(), master, kafkaProperty)); + defs.getBatteryLevelStream(), lowPriority(), master, kafkaProperty)); list.add(new E4BloodVolumePulseStream( - "E4BloodVolumePulseStream", highPriority(), master, kafkaProperty)); + defs.getBloodVolumePulseStream(), highPriority(), master, kafkaProperty)); list.add(new E4ElectroDermalActivityStream( - "E4ElectroDermalActivityStream", normalPriority(), master, kafkaProperty)); + defs.getElectroDermalActivityStream(), normalPriority(), master, kafkaProperty)); list.add(new E4HeartRateStream( - "E4HeartRateStream", highPriority(), master, kafkaProperty)); + defs.getHeartRateStream(), lowPriority(), master, kafkaProperty)); list.add(new E4InterBeatIntervalStream( - "E4InterBeatIntervalStream", highPriority(), master, kafkaProperty)); + defs.getInterBeatIntervalStream(), lowPriority(), master, kafkaProperty)); list.add(new E4TemperatureStream( - "E4TemperatureStream", highPriority(), master, kafkaProperty)); + defs.getTemperatureStream(), lowPriority(), master, kafkaProperty)); } } diff --git a/src/main/java/org/radarcns/stream/empatica/E4Streams.java b/src/main/java/org/radarcns/stream/empatica/E4Streams.java index 42ef06c7..6e404259 100644 --- a/src/main/java/org/radarcns/stream/empatica/E4Streams.java +++ b/src/main/java/org/radarcns/stream/empatica/E4Streams.java @@ -20,6 +20,8 @@ import org.radarcns.stream.GeneralStreamGroup; import org.radarcns.stream.StreamDefinition; +import java.util.Collection; + /** * Singleton class representing the list of Empatica E4 topics */ @@ -27,71 +29,64 @@ public final class E4Streams extends GeneralStreamGroup { private static final E4Streams INSTANCE = new E4Streams(); //All sensor topics - private final StreamDefinition accelerationStream; - private final StreamDefinition batteryLevelStream; - private final StreamDefinition bloodVolumePulseStream; - private final StreamDefinition electroDermalActivityStream; - private final StreamDefinition interBeatIntervalStream; - private final StreamDefinition sensorStatusStream; - private final StreamDefinition temperatureStream; + private final Collection accelerationStream; + private final Collection batteryLevelStream; + private final Collection bloodVolumePulseStream; + private final Collection electroDermalActivityStream; + private final Collection interBeatIntervalStream; + private final Collection temperatureStream; // Internal topics - private final StreamDefinition heartRateStream; + private final Collection heartRateStream; public static E4Streams getInstance() { return INSTANCE; } private E4Streams() { - accelerationStream = createSensorStream( + accelerationStream = createWindowedSensorStream( "android_empatica_e4_acceleration"); - batteryLevelStream = createSensorStream( + batteryLevelStream = createWindowedSensorStream( "android_empatica_e4_battery_level"); - bloodVolumePulseStream = createSensorStream( + bloodVolumePulseStream = createWindowedSensorStream( "android_empatica_e4_blood_volume_pulse"); - electroDermalActivityStream = createSensorStream( + electroDermalActivityStream = createWindowedSensorStream( "android_empatica_e4_electrodermal_activity"); - interBeatIntervalStream = createSensorStream( + interBeatIntervalStream = createWindowedSensorStream( "android_empatica_e4_inter_beat_interval"); - sensorStatusStream = createSensorStream( - "android_empatica_e4_sensor_status"); - temperatureStream = createSensorStream( + temperatureStream = createWindowedSensorStream( "android_empatica_e4_temperature"); - heartRateStream = createStream( + heartRateStream = createWindowedSensorStream( "android_empatica_e4_inter_beat_interval", - "android_empatica_e4_heartrate"); + "android_empatica_e4_heart_rate"); } - public StreamDefinition getAccelerationStream() { + public Collection getAccelerationStream() { return accelerationStream; } - public StreamDefinition getBatteryLevelStream() { + public Collection getBatteryLevelStream() { return batteryLevelStream; } - public StreamDefinition getBloodVolumePulseStream() { + public Collection getBloodVolumePulseStream() { return bloodVolumePulseStream; } - public StreamDefinition getElectroDermalActivityStream() { + public Collection getElectroDermalActivityStream() { return electroDermalActivityStream; } - public StreamDefinition getInterBeatIntervalStream() { + public Collection getInterBeatIntervalStream() { return interBeatIntervalStream; } - public StreamDefinition getSensorStatusStream() { - return sensorStatusStream; - } - - public StreamDefinition getTemperatureStream() { + public Collection getTemperatureStream() { return temperatureStream; } - public StreamDefinition getHeartRateStream() { + public Collection getHeartRateStream() { return heartRateStream; } } diff --git a/src/main/java/org/radarcns/stream/empatica/E4TemperatureStream.java b/src/main/java/org/radarcns/stream/empatica/E4TemperatureStream.java index 91aed8e6..015e407c 100644 --- a/src/main/java/org/radarcns/stream/empatica/E4TemperatureStream.java +++ b/src/main/java/org/radarcns/stream/empatica/E4TemperatureStream.java @@ -16,47 +16,35 @@ package org.radarcns.stream.empatica; -import javax.annotation.Nonnull; import org.apache.kafka.streams.kstream.KStream; -import org.apache.kafka.streams.kstream.TimeWindows; import org.radarcns.config.KafkaProperty; -import org.radarcns.passive.empatica.EmpaticaE4Temperature; -import org.radarcns.kafka.ObservationKey; import org.radarcns.kafka.AggregateKey; +import org.radarcns.kafka.ObservationKey; +import org.radarcns.passive.empatica.EmpaticaE4Temperature; +import org.radarcns.stream.StreamDefinition; import org.radarcns.stream.StreamMaster; import org.radarcns.stream.StreamWorker; import org.radarcns.stream.aggregator.DoubleAggregation; -import org.radarcns.stream.collector.DoubleValueCollector; -import org.radarcns.util.RadarSingletonFactory; -import org.radarcns.util.RadarUtilities; -import org.radarcns.util.serde.RadarSerdes; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import javax.annotation.Nonnull; +import java.util.Collection; + /** * Definition of Kafka Stream for aggregating temperature values collected by Empatica E4. */ public class E4TemperatureStream extends StreamWorker { - private final RadarUtilities utilities = RadarSingletonFactory.getRadarUtilities(); - private static final Logger log = LoggerFactory.getLogger(E4TemperatureStream.class); + private static final Logger logger = LoggerFactory.getLogger(E4TemperatureStream.class); - public E4TemperatureStream(String clientId, int numThread, StreamMaster master, - KafkaProperty kafkaProperties) { - super(E4Streams.getInstance().getTemperatureStream(), clientId, - numThread, master, kafkaProperties, log); + public E4TemperatureStream(Collection definitions, int numThread, + StreamMaster master, KafkaProperty kafkaProperties) { + super(definitions, numThread, master, kafkaProperties, logger); } @Override - protected KStream defineStream( + protected KStream implementStream(StreamDefinition definition, @Nonnull KStream kstream) { - return kstream.groupByKey() - .aggregate( - DoubleValueCollector::new, - (k, v, valueCollector) -> valueCollector.add(v.getTemperature()), - TimeWindows.of(10 * 1000L), - RadarSerdes.getInstance().getDoubleCollector(), - getStreamDefinition().getStateStoreName()) - .toStream() - .map(utilities::collectorToAvro); + return aggregateFloat(definition, kstream, EmpaticaE4Temperature::getTemperature); } } diff --git a/src/main/java/org/radarcns/stream/phone/PhoneAccelerationStream.java b/src/main/java/org/radarcns/stream/phone/PhoneAccelerationStream.java index e6edb1d1..d705ba49 100644 --- a/src/main/java/org/radarcns/stream/phone/PhoneAccelerationStream.java +++ b/src/main/java/org/radarcns/stream/phone/PhoneAccelerationStream.java @@ -1,51 +1,38 @@ package org.radarcns.stream.phone; import org.apache.kafka.streams.kstream.KStream; -import org.apache.kafka.streams.kstream.TimeWindows; -import org.radarcns.stream.aggregator.DoubleArrayAggregation; import org.radarcns.config.KafkaProperty; -import org.radarcns.kafka.ObservationKey; import org.radarcns.kafka.AggregateKey; +import org.radarcns.kafka.ObservationKey; import org.radarcns.passive.phone.PhoneAcceleration; +import org.radarcns.stream.StreamDefinition; import org.radarcns.stream.StreamMaster; import org.radarcns.stream.StreamWorker; -import org.radarcns.stream.collector.DoubleArrayCollector; -import org.radarcns.util.RadarSingletonFactory; -import org.radarcns.util.RadarUtilities; -import org.radarcns.util.serde.RadarSerdes; +import org.radarcns.stream.aggregator.DoubleArrayAggregation; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import javax.annotation.Nonnull; +import java.util.Collection; import static org.radarcns.util.Serialization.floatToDouble; public class PhoneAccelerationStream extends StreamWorker { private static final Logger logger = LoggerFactory.getLogger(PhoneAccelerationStream.class); - private final RadarUtilities utilities = RadarSingletonFactory.getRadarUtilities(); - public PhoneAccelerationStream(String phoneAccelerationStream, int priority, - StreamMaster phoneStreamMaster, KafkaProperty kafkaProperty) { - super(PhoneStreams.getInstance().getAccelerationStream(), phoneAccelerationStream, - priority, phoneStreamMaster, - kafkaProperty, logger); + public PhoneAccelerationStream(Collection definitions, int numThread, + StreamMaster master, KafkaProperty kafkaProperties) { + super(definitions, numThread, master, kafkaProperties, logger); } @Override - protected KStream defineStream( + protected KStream implementStream( + StreamDefinition definition, @Nonnull KStream kstream) { - return kstream.groupByKey() - .aggregate( - DoubleArrayCollector::new, - (k, v, valueCollector) -> valueCollector.add(new double[] { - floatToDouble(v.getX()), - floatToDouble(v.getY()), - floatToDouble(v.getZ()) - }), - TimeWindows.of(10 * 1000L), - RadarSerdes.getInstance().getDoubleArrayCollector(), - getStreamDefinition().getStateStoreName()) - .toStream() - .map(utilities::collectorToAvro); + return aggregateDoubleArray(definition, kstream, v -> new double[] { + floatToDouble(v.getX()), + floatToDouble(v.getY()), + floatToDouble(v.getZ()) + }); } } diff --git a/src/main/java/org/radarcns/stream/phone/PhoneBatteryStream.java b/src/main/java/org/radarcns/stream/phone/PhoneBatteryStream.java index 36422f47..545de756 100644 --- a/src/main/java/org/radarcns/stream/phone/PhoneBatteryStream.java +++ b/src/main/java/org/radarcns/stream/phone/PhoneBatteryStream.java @@ -1,45 +1,31 @@ package org.radarcns.stream.phone; import org.apache.kafka.streams.kstream.KStream; -import org.apache.kafka.streams.kstream.TimeWindows; import org.radarcns.config.KafkaProperty; -import org.radarcns.kafka.ObservationKey; import org.radarcns.kafka.AggregateKey; +import org.radarcns.kafka.ObservationKey; import org.radarcns.passive.phone.PhoneBatteryLevel; +import org.radarcns.stream.StreamDefinition; import org.radarcns.stream.StreamMaster; import org.radarcns.stream.StreamWorker; import org.radarcns.stream.aggregator.DoubleAggregation; -import org.radarcns.stream.collector.DoubleValueCollector; -import org.radarcns.util.RadarSingletonFactory; -import org.radarcns.util.RadarUtilities; -import org.radarcns.util.serde.RadarSerdes; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import javax.annotation.Nonnull; +import java.util.Collection; public class PhoneBatteryStream extends StreamWorker { private static final Logger logger = LoggerFactory.getLogger(PhoneBatteryStream.class); - private final RadarUtilities utilities = RadarSingletonFactory.getRadarUtilities(); - public PhoneBatteryStream(String phoneBatteryStream, int priority, StreamMaster master, - KafkaProperty kafkaProperty) { - super(PhoneStreams.getInstance().getBatteryStream(), phoneBatteryStream, - priority, master, - kafkaProperty, logger); + public PhoneBatteryStream(Collection definitions, int numThread, + StreamMaster master, KafkaProperty kafkaProperties) { + super(definitions, numThread, master, kafkaProperties, logger); } @Override - protected KStream defineStream( + protected KStream implementStream(StreamDefinition definition, @Nonnull KStream kstream) { - return kstream.groupByKey() - .aggregate( - DoubleValueCollector::new, - (k, v, valueCollector) -> valueCollector.add(v.getBatteryLevel()), - TimeWindows.of(10 * 1000L), - RadarSerdes.getInstance().getDoubleCollector(), - getStreamDefinition().getStateStoreName()) - .toStream() - .map(utilities::collectorToAvro); + return aggregateFloat(definition, kstream, PhoneBatteryLevel::getBatteryLevel); } } diff --git a/src/main/java/org/radarcns/stream/phone/PhoneStreamMaster.java b/src/main/java/org/radarcns/stream/phone/PhoneStreamMaster.java index e0d9d75e..248e1d97 100644 --- a/src/main/java/org/radarcns/stream/phone/PhoneStreamMaster.java +++ b/src/main/java/org/radarcns/stream/phone/PhoneStreamMaster.java @@ -38,12 +38,14 @@ protected StreamGroup getStreamGroup() { protected void createWorkers(List> list, StreamMaster master) { RadarPropertyHandler propertyHandler = RadarSingletonFactory.getRadarPropertyHandler(); KafkaProperty kafkaProperty = propertyHandler.getKafkaProperties(); - list.add(new PhoneUsageStream("PhoneUsageStream", lowPriority(), master, kafkaProperty)); + PhoneStreams defs = PhoneStreams.getInstance(); + list.add(new PhoneUsageStream( + defs.getUsageStream(), lowPriority(), master, kafkaProperty)); list.add(new PhoneUsageAggregationStream( - "PhoneUsageAggregationStream", lowPriority(), master, kafkaProperty)); - list.add(new PhoneBatteryStream("PhoneBatteryStream", lowPriority(), master, - kafkaProperty)); - list.add(new PhoneAccelerationStream("PhoneAccelerationStream", normalPriority(), master, - kafkaProperty)); + defs.getUsageEventAggregationStream(), lowPriority(), master, kafkaProperty)); + list.add(new PhoneBatteryStream( + defs.getBatteryStream(), lowPriority(), master, kafkaProperty)); + list.add(new PhoneAccelerationStream( + defs.getAccelerationStream(), normalPriority(), master, kafkaProperty)); } } diff --git a/src/main/java/org/radarcns/stream/phone/PhoneStreams.java b/src/main/java/org/radarcns/stream/phone/PhoneStreams.java index 26df8d68..8a9ca58d 100644 --- a/src/main/java/org/radarcns/stream/phone/PhoneStreams.java +++ b/src/main/java/org/radarcns/stream/phone/PhoneStreams.java @@ -19,13 +19,16 @@ import org.radarcns.stream.GeneralStreamGroup; import org.radarcns.stream.StreamDefinition; +import java.util.Collection; +import java.util.concurrent.TimeUnit; + public final class PhoneStreams extends GeneralStreamGroup { private static final PhoneStreams INSTANCE = new PhoneStreams(); - private final StreamDefinition usageEventStream; - private final StreamDefinition usageEventAggregationStream; - private final StreamDefinition accelerationStream; - private final StreamDefinition batteryStream; + private final Collection usageEventStream; + private final Collection usageEventAggregationStream; + private final Collection accelerationStream; + private final Collection batteryStream; public static PhoneStreams getInstance() { return INSTANCE; @@ -36,24 +39,25 @@ private PhoneStreams() { "android_phone_usage_event"); usageEventAggregationStream = createStream( "android_phone_usage_event_output", - "android_phone_usage_event_aggregated"); + "android_phone_usage_event_aggregated", + TimeUnit.DAYS.toMillis(1)); accelerationStream = createSensorStream("android_phone_acceleration"); batteryStream = createSensorStream("android_phone_battery_level"); } - public StreamDefinition getUsageStream() { + public Collection getUsageStream() { return usageEventStream; } - public StreamDefinition getUsageEventAggregationStream() { + public Collection getUsageEventAggregationStream() { return usageEventAggregationStream; } - public StreamDefinition getAccelerationStream() { + public Collection getAccelerationStream() { return accelerationStream; } - public StreamDefinition getBatteryStream() { + public Collection getBatteryStream() { return batteryStream; } } diff --git a/src/main/java/org/radarcns/stream/phone/PhoneUsageAggregationStream.java b/src/main/java/org/radarcns/stream/phone/PhoneUsageAggregationStream.java index c5c6569a..aec730d4 100644 --- a/src/main/java/org/radarcns/stream/phone/PhoneUsageAggregationStream.java +++ b/src/main/java/org/radarcns/stream/phone/PhoneUsageAggregationStream.java @@ -1,49 +1,43 @@ package org.radarcns.stream.phone; import org.apache.kafka.streams.kstream.KStream; -import org.apache.kafka.streams.kstream.TimeWindows; import org.radarcns.aggregator.PhoneUsageAggregator; import org.radarcns.config.KafkaProperty; -import org.radarcns.kafka.ObservationKey; import org.radarcns.kafka.AggregateKey; +import org.radarcns.kafka.ObservationKey; import org.radarcns.passive.phone.PhoneUsageEvent; +import org.radarcns.stream.StreamDefinition; import org.radarcns.stream.StreamMaster; import org.radarcns.stream.StreamWorker; -import org.radarcns.util.RadarSingletonFactory; -import org.radarcns.util.RadarUtilities; import org.radarcns.util.serde.RadarSerdes; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import javax.annotation.Nonnull; +import java.util.Collection; /** * Created by piotrzakrzewski on 26/07/2017. */ public class PhoneUsageAggregationStream extends StreamWorker { + private static final Logger logger = LoggerFactory.getLogger(PhoneUsageAggregationStream.class); - private static final Logger log = LoggerFactory.getLogger(PhoneUsageAggregationStream.class); - private static final long DAY_IN_MS = 24 * 60 * 60 * 1000; - private final RadarUtilities utilities = RadarSingletonFactory.getRadarUtilities(); - - public PhoneUsageAggregationStream(@Nonnull String clientId, - int numThreads, - @Nonnull StreamMaster master, - KafkaProperty kafkaProperties) { - super(PhoneStreams.getInstance().getUsageEventAggregationStream(), clientId, - numThreads, master, kafkaProperties, log); + public PhoneUsageAggregationStream(Collection definitions, int numThread, + StreamMaster master, KafkaProperty kafkaProperties) { + super(definitions, numThread, master, kafkaProperties, logger); } @Override - protected KStream defineStream( + protected KStream implementStream( + StreamDefinition definition, @Nonnull KStream kstream) { return kstream.groupBy((k, v) -> new TemporaryPackageKey(k, v.getPackageName())) .aggregate( PhoneUsageCollector::new, (k, v, valueCollector) -> valueCollector.update(v), - TimeWindows.of(DAY_IN_MS), + definition.getTimeWindows(), RadarSerdes.getInstance().getPhoneUsageCollector(), - getStreamDefinition().getStateStoreName()) + definition.getStateStoreName()) .toStream() .map(utilities::collectorToAvro); } diff --git a/src/main/java/org/radarcns/stream/phone/PhoneUsageStream.java b/src/main/java/org/radarcns/stream/phone/PhoneUsageStream.java index 83cee1a4..0e5695e4 100644 --- a/src/main/java/org/radarcns/stream/phone/PhoneUsageStream.java +++ b/src/main/java/org/radarcns/stream/phone/PhoneUsageStream.java @@ -22,13 +22,16 @@ import org.radarcns.config.KafkaProperty; import org.radarcns.kafka.ObservationKey; import org.radarcns.passive.phone.PhoneUsageEvent; +import org.radarcns.stream.StreamDefinition; import org.radarcns.stream.StreamMaster; import org.radarcns.stream.StreamWorker; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.util.Collection; + public class PhoneUsageStream extends StreamWorker { - private static final Logger log = LoggerFactory.getLogger(PhoneUsageStream.class); + private static final Logger logger = LoggerFactory.getLogger(PhoneUsageStream.class); // 1 day until an item is refreshed private static final int CACHE_TIMEOUT = 24 * 3600; @@ -38,15 +41,14 @@ public class PhoneUsageStream extends StreamWorker definitions, int numThread, + StreamMaster master, KafkaProperty kafkaProperties) { + super(definitions, numThread, master, kafkaProperties, logger); playStoreLookup = new PlayStoreLookup(CACHE_TIMEOUT, MAX_CACHE_SIZE); } @Override - protected KStream defineStream( + protected KStream implementStream(StreamDefinition definition, @Nonnull KStream kstream) { return kstream .map((key, value) -> { diff --git a/src/main/java/org/radarcns/util/Comparison.java b/src/main/java/org/radarcns/util/Comparison.java new file mode 100644 index 00000000..b185c045 --- /dev/null +++ b/src/main/java/org/radarcns/util/Comparison.java @@ -0,0 +1,20 @@ +package org.radarcns.util; + +import java.util.function.BiFunction; +import java.util.function.Function; + +public interface Comparison extends BiFunction { + static > Comparison compare(Function property) { + return (a, b) -> property.apply(a).compareTo(property.apply(b)); + } + + default > Comparison then(Function property) { + return (a, b) -> { + int ret = apply(a, b); + if (ret != 0) { + return ret; + } + return compare(property).apply(a, b); + }; + } +} diff --git a/src/main/java/org/radarcns/util/StreamUtil.java b/src/main/java/org/radarcns/util/StreamUtil.java new file mode 100644 index 00000000..ea982a22 --- /dev/null +++ b/src/main/java/org/radarcns/util/StreamUtil.java @@ -0,0 +1,45 @@ +package org.radarcns.util; + +import org.apache.kafka.streams.KeyValue; + +import java.util.function.BiFunction; +import java.util.function.BiPredicate; +import java.util.function.Function; +import java.util.function.Predicate; +import java.util.stream.Stream; + +public final class StreamUtil { + private StreamUtil() { + // utility class + } + + public static Predicate> test(BiPredicate bip) { + return entry -> bip.test(entry.key, entry.value); + } + + public static Function, R> apply( + BiFunction bif) { + return entry -> bif.apply(entry.key, entry.value); + } + + public static Function, K> first() { + return e -> e.key; + } + + public static Function, V> second() { + return e -> e.value; + } + + @FunctionalInterface + public interface StreamSupplier { + Stream get(); + + default StreamSupplier concat(StreamSupplier other) { + return () -> Stream.concat(get(), other.get()); + } + + static StreamSupplier supply(StreamSupplier supplier) { + return supplier; + } + } +} diff --git a/src/test/java/org/radarcns/stream/StreamDefinitionTest.java b/src/test/java/org/radarcns/stream/StreamDefinitionTest.java index 4979ff1b..e897daa9 100644 --- a/src/test/java/org/radarcns/stream/StreamDefinitionTest.java +++ b/src/test/java/org/radarcns/stream/StreamDefinitionTest.java @@ -1,5 +1,3 @@ -package org.radarcns.stream; - /* * Copyright 2017 King's College London and The Hyve * @@ -16,9 +14,10 @@ * limitations under the License. */ +package org.radarcns.stream; + import static org.junit.Assert.assertEquals; -import org.apache.kafka.common.errors.InvalidTopicException; import org.junit.Test; import org.radarcns.topic.KafkaTopic; @@ -36,8 +35,8 @@ public void nameValidation() { kafka.common.Topic.validate(definition.getStateStoreName()); - assertEquals("From-" + "android_empatica_e4_blood_volume_pulse" + "-To-" + - "android_empatica_e4_blood_volume_pulse" + "_output", + assertEquals("From-android_empatica_e4_blood_volume_pulse" + + "-To-android_empatica_e4_blood_volume_pulse_output", definition.getStateStoreName()); } diff --git a/src/test/java/org/radarcns/stream/StreamWorkerTest.java b/src/test/java/org/radarcns/stream/StreamWorkerTest.java index d1611415..636a853a 100644 --- a/src/test/java/org/radarcns/stream/StreamWorkerTest.java +++ b/src/test/java/org/radarcns/stream/StreamWorkerTest.java @@ -16,6 +16,7 @@ package org.radarcns.stream; +import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.any; import static org.mockito.Mockito.doCallRealMethod; import static org.mockito.Mockito.mock; @@ -24,10 +25,18 @@ import static org.mockito.Mockito.when; import java.io.IOException; +import java.util.Collections; +import java.util.Properties; + import org.apache.kafka.streams.kstream.KStream; import org.junit.Before; import org.junit.Test; +import org.radarcns.config.ConfigRadar; +import org.radarcns.config.KafkaProperty; +import org.radarcns.config.RadarPropertyHandler; import org.radarcns.topic.KafkaTopic; +import org.radarcns.util.RadarSingletonFactory; + /** * Created by nivethika on 20-12-16. */ @@ -43,11 +52,17 @@ public void setUp() { public void getBuilder() throws IOException { String topicName = "TESTTopic"; StreamDefinition sensorTopic = new StreamDefinition(new KafkaTopic(topicName), new KafkaTopic(topicName + "_output")); - when(aggregator.getStreamDefinition()).thenReturn(sensorTopic); - when(aggregator.defineStream(any())).thenReturn(mock(KStream.class)); - doCallRealMethod().when(aggregator).createBuilder(); - aggregator.createBuilder(); + when(aggregator.getStreamDefinitions()).thenReturn(Collections.singleton(sensorTopic)); + + RadarPropertyHandler propertyHandler = RadarSingletonFactory.getRadarPropertyHandler(); + propertyHandler.load("src/test/resources/config/radar.yml"); + KafkaProperty kafkaProperty = propertyHandler.getKafkaProperties(); + when(aggregator.getStreamProperties(eq(sensorTopic))).thenReturn( + kafkaProperty.getStreamProperties("test", 1, DeviceTimestampExtractor.class)); + when(aggregator.implementStream(eq(sensorTopic), any())).thenReturn(mock(KStream.class)); + doCallRealMethod().when(aggregator).createBuilder(sensorTopic); + aggregator.createBuilder(sensorTopic); - verify(aggregator, times(1)).defineStream(any()); + verify(aggregator, times(1)).implementStream(eq(sensorTopic), any()); } } diff --git a/src/test/java/org/radarcns/stream/empatica/E4StreamsTest.java b/src/test/java/org/radarcns/stream/empatica/E4StreamsTest.java index f771cb3a..669667dd 100644 --- a/src/test/java/org/radarcns/stream/empatica/E4StreamsTest.java +++ b/src/test/java/org/radarcns/stream/empatica/E4StreamsTest.java @@ -19,12 +19,14 @@ import static org.junit.Assert.assertEquals; import java.util.Arrays; +import java.util.Collection; import java.util.List; import org.junit.Before; import org.junit.Rule; import org.junit.Test; import org.junit.rules.ExpectedException; import org.radarcns.stream.StreamDefinition; +import org.radarcns.stream.TimeWindowMetadata; import org.radarcns.stream.empatica.E4Streams; /** @@ -49,32 +51,33 @@ public void getTopic() { "android_empatica_e4_blood_volume_pulse", "android_empatica_e4_electrodermal_activity", "android_empatica_e4_inter_beat_interval", - "android_empatica_e4_sensor_status", "android_empatica_e4_temperature"); for(String topicName : topicNames) { - StreamDefinition topic = this.e4Streams.getStreamDefinition(topicName); - assertEquals(topic.getInputTopic().getName(), topicName); + Collection topic = this.e4Streams.getStreamDefinition(topicName); + assertEquals(topic.iterator().next().getInputTopic().getName(), topicName); } } @Test public void getInternalTopic() { - StreamDefinition topic = this.e4Streams.getStreamDefinition("android_empatica_e4_inter_beat_interval"); + StreamDefinition topic = this.e4Streams.getStreamDefinition("android_empatica_e4_inter_beat_interval").iterator().next(); assertEquals("android_empatica_e4_inter_beat_interval", topic.getInputTopic().getName()); - assertEquals("android_empatica_e4_heartrate", topic.getOutputTopic().getName()); - assertEquals("From-android_empatica_e4_inter_beat_interval-To-android_empatica_e4_heartrate", topic.getStateStoreName()); + assertEquals("android_empatica_e4_heart_rate_10min", topic.getOutputTopic().getName()); + assertEquals("From-android_empatica_e4_inter_beat_interval-To-android_empatica_e4_heart_rate_10min", topic.getStateStoreName()); } @Test public void getInvalidTopic() { exception.expect(IllegalArgumentException.class); exception.expectMessage("Topic something unknown"); - StreamDefinition topic = this.e4Streams.getStreamDefinition("something"); + this.e4Streams.getStreamDefinition("something"); } @Test public void getTopicNames() { - assertEquals(15, this.e4Streams.getTopicNames().size()); // sort removes the redundant + System.out.println(this.e4Streams.getTopicNames()); + assertEquals(8 * TimeWindowMetadata.values().length, + this.e4Streams.getTopicNames().size()); // sort removes the redundant } } From c850afd18ff30539c5a32261a89e228da16397f7 Mon Sep 17 00:00:00 2001 From: Joris Borgdorff Date: Thu, 28 Sep 2017 13:22:59 +0200 Subject: [PATCH 03/23] Make sure that the phone event stream is initialized --- .../radarcns/integration/PhoneStreamTest.java | 45 ++++++++++++++----- .../radarcns/stream/phone/PhoneStreams.java | 4 +- 2 files changed, 35 insertions(+), 14 deletions(-) diff --git a/src/integrationTest/java/org/radarcns/integration/PhoneStreamTest.java b/src/integrationTest/java/org/radarcns/integration/PhoneStreamTest.java index 7434152f..e4ae0276 100644 --- a/src/integrationTest/java/org/radarcns/integration/PhoneStreamTest.java +++ b/src/integrationTest/java/org/radarcns/integration/PhoneStreamTest.java @@ -29,8 +29,10 @@ import static org.radarcns.util.serde.AbstractKafkaAvroSerde.SCHEMA_REGISTRY_CONFIG; import java.io.IOException; +import java.util.Arrays; import java.util.Collections; import java.util.HashMap; +import java.util.List; import java.util.Map; import java.util.Properties; import java.util.UUID; @@ -128,26 +130,45 @@ public void testDirect() throws Exception { properties.put(BOOTSTRAP_SERVERS_CONFIG, config.getBrokerPaths()); DirectSender sender = new DirectSender<>(properties); + + long offset = 0; + double time = System.currentTimeMillis() / 1000d - 10d; + ObservationKey key = new ObservationKey("test", "a", "c"); + + List events = Arrays.asList( + new PhoneUsageEvent(time, time++, "com.whatsapp", null, null, UsageEventType.FOREGROUND), + new PhoneUsageEvent(time, time++, "com.whatsapp", null, null, UsageEventType.BACKGROUND), + new PhoneUsageEvent(time, time++, "nl.thehyve.transmartclient", null, null, UsageEventType.FOREGROUND), + new PhoneUsageEvent(time, time++, "nl.thehyve.transmartclient", null, null, UsageEventType.BACKGROUND), + new PhoneUsageEvent(time, time++, "com.strava", null, null, UsageEventType.FOREGROUND), + new PhoneUsageEvent(time, time++, "com.strava", null, null, UsageEventType.BACKGROUND), + new PhoneUsageEvent(time, time++, "com.android.systemui", null, null, UsageEventType.FOREGROUND), + new PhoneUsageEvent(time, time, "com.android.systemui", null, null, UsageEventType.BACKGROUND)); + AvroTopic topic = new AvroTopic<>( "android_phone_usage_event", ObservationKey.getClassSchema(), PhoneUsageEvent.getClassSchema(), ObservationKey.class, PhoneUsageEvent.class); - long offset = 0; - double time = System.currentTimeMillis() / 1000d - 10d; - ObservationKey key = new ObservationKey("test", "a", "c"); try (KafkaTopicSender topicSender = sender.sender(topic)) { - topicSender.send(offset++, key, new PhoneUsageEvent(time, time++, "com.whatsapp", null, null, UsageEventType.FOREGROUND)); - topicSender.send(offset++, key, new PhoneUsageEvent(time, time++, "com.whatsapp", null, null, UsageEventType.BACKGROUND)); - topicSender.send(offset++, key, new PhoneUsageEvent(time, time++, "nl.thehyve.transmartclient", null, null, UsageEventType.FOREGROUND)); - topicSender.send(offset++, key, new PhoneUsageEvent(time, time++, "nl.thehyve.transmartclient", null, null, UsageEventType.BACKGROUND)); - topicSender.send(offset++, key, new PhoneUsageEvent(time, time++, "com.strava", null, null, UsageEventType.FOREGROUND)); - topicSender.send(offset++, key, new PhoneUsageEvent(time, time++, "com.strava", null, null, UsageEventType.BACKGROUND)); - topicSender.send(offset++, key, new PhoneUsageEvent(time, time++, "com.android.systemui", null, null, UsageEventType.FOREGROUND)); - topicSender.send(offset++, key, new PhoneUsageEvent(time, time, "com.android.systemui", null, null, UsageEventType.BACKGROUND)); + for (PhoneUsageEvent event : events) { + topicSender.send(offset++, key, event); + } + } + + AvroTopic outputTopic = new AvroTopic<>( + "android_phone_usage_event_output", + ObservationKey.getClassSchema(), PhoneUsageEvent.getClassSchema(), + ObservationKey.class, PhoneUsageEvent.class); + + try (KafkaTopicSender topicSender = sender.sender(outputTopic)) { + for (PhoneUsageEvent event : events) { + topicSender.send(offset++, key, event); + } } + sender.close(); - consumePhone(offset); + consumePhone(offset / 2); consumeAggregated(offset / 2); } diff --git a/src/main/java/org/radarcns/stream/phone/PhoneStreams.java b/src/main/java/org/radarcns/stream/phone/PhoneStreams.java index 8a9ca58d..cd0628d0 100644 --- a/src/main/java/org/radarcns/stream/phone/PhoneStreams.java +++ b/src/main/java/org/radarcns/stream/phone/PhoneStreams.java @@ -41,8 +41,8 @@ private PhoneStreams() { "android_phone_usage_event_output", "android_phone_usage_event_aggregated", TimeUnit.DAYS.toMillis(1)); - accelerationStream = createSensorStream("android_phone_acceleration"); - batteryStream = createSensorStream("android_phone_battery_level"); + accelerationStream = createWindowedSensorStream("android_phone_acceleration"); + batteryStream = createWindowedSensorStream("android_phone_battery_level"); } public Collection getUsageStream() { From e86174109784f80312d32b9ff24b12b82a1c81a6 Mon Sep 17 00:00:00 2001 From: Joris Borgdorff Date: Wed, 11 Oct 2017 14:18:19 +0200 Subject: [PATCH 04/23] Fixed kafka properties and use latest kafka version --- .idea/.name | 1 - .idea/compiler.xml | 31 ---------- .idea/modules.xml | 11 ---- Dockerfile | 2 +- build.gradle | 14 ++--- docker-compose.yml | 15 +++-- radar.yml | 4 +- .../integration/DirectProducerTest.java | 1 + .../resources/org/radarcns/kafka/radar.yml | 10 ++- .../org/radarcns/config/KafkaProperty.java | 14 +++-- .../monitor/AbstractKafkaMonitor.java | 6 +- .../util/serde/GenericAvroDeserializer.java | 50 --------------- .../radarcns/util/serde/GenericAvroSerde.java | 61 ------------------- .../util/serde/GenericAvroSerializer.java | 55 ----------------- .../util/serde/SpecificAvroDeserializer.java | 54 ---------------- .../util/serde/SpecificAvroSerde.java | 57 ----------------- .../util/serde/SpecificAvroSerializer.java | 55 ----------------- .../radarcns/stream/StreamDefinitionTest.java | 11 ++-- .../util/serde/GenericAvroSerdeTest.java | 54 ---------------- .../util/serde/SpecificAvroSerdeTest.java | 48 --------------- src/test/radar.yml | 4 +- src/test/resources/config/radar.yml | 4 +- 22 files changed, 54 insertions(+), 508 deletions(-) delete mode 100644 .idea/.name delete mode 100644 .idea/compiler.xml delete mode 100644 .idea/modules.xml delete mode 100644 src/main/java/org/radarcns/util/serde/GenericAvroDeserializer.java delete mode 100644 src/main/java/org/radarcns/util/serde/GenericAvroSerde.java delete mode 100644 src/main/java/org/radarcns/util/serde/GenericAvroSerializer.java delete mode 100644 src/main/java/org/radarcns/util/serde/SpecificAvroDeserializer.java delete mode 100644 src/main/java/org/radarcns/util/serde/SpecificAvroSerde.java delete mode 100644 src/main/java/org/radarcns/util/serde/SpecificAvroSerializer.java delete mode 100644 src/test/java/org/radarcns/util/serde/GenericAvroSerdeTest.java delete mode 100644 src/test/java/org/radarcns/util/serde/SpecificAvroSerdeTest.java diff --git a/.idea/.name b/.idea/.name deleted file mode 100644 index 1a452025..00000000 --- a/.idea/.name +++ /dev/null @@ -1 +0,0 @@ -radar-backend \ No newline at end of file diff --git a/.idea/compiler.xml b/.idea/compiler.xml deleted file mode 100644 index 6e0a691c..00000000 --- a/.idea/compiler.xml +++ /dev/null @@ -1,31 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/.idea/modules.xml b/.idea/modules.xml deleted file mode 100644 index ad7f0f8e..00000000 --- a/.idea/modules.xml +++ /dev/null @@ -1,11 +0,0 @@ - - - - - - - - - - - \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index f402cabd..68192f2d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM confluentinc/cp-base:3.2.1-5 +FROM confluentinc/cp-base:3.3.0 ENV TERM=dumb diff --git a/build.gradle b/build.gradle index ca2ae1fa..e1656985 100644 --- a/build.gradle +++ b/build.gradle @@ -30,9 +30,9 @@ sourceCompatibility = '1.8' ext.boundaryVersion = '1.0.6' ext.codacyVersion = '1.0.10' -ext.confluentVersion = '3.2.1' +ext.confluentVersion = '3.3.0' ext.hamcrestVersion = '1.3' -ext.kafkaVersion = '0.10.2.1' +ext.kafkaVersion = '0.11.0.1' ext.jacksonVersion='2.8.5' ext.javaMailVersion = '1.5.6' ext.junitVersion = '4.12' @@ -40,7 +40,7 @@ ext.mathVersion = '3.0' ext.findbugVersion = '3.0.1' ext.commonsCliVersion = '1.2' ext.mockitoVersion = '2.2.29' -ext.radarCommonsVersion = '0.5' +ext.radarCommonsVersion = '0.6-SNAPSHOT' ext.radarSchemasVersion = '0.2-alpha.3-SNAPSHOT' ext.subethamailVersion = '3.1.7' ext.jsoupVersion = '1.10.2' @@ -81,9 +81,6 @@ configurations.all { // In this section you declare where to find the dependencies of your project repositories { - // For working with dev-branch commons - maven { url "${rootProject.projectDir}/libs" } - // Use 'jcenter' for resolving your dependencies. // You can declare any Maven/Ivy/file repository here. jcenter() @@ -105,8 +102,11 @@ dependencies { // Kafka streaming API compile group: 'org.apache.kafka', name: 'kafka-streams', version: kafkaVersion - compile (group: 'io.confluent', name: 'kafka-avro-serializer', version: confluentVersion) { + compile (group: 'io.confluent', name: 'kafka-streams-avro-serde', version: confluentVersion) { exclude group: 'org.slf4j', module: 'slf4j-log4j12' + exclude group: 'log4j', module: 'log4j' + exclude group: 'jline', module: 'jline' + exclude group: 'io.netty', module: 'netty' } // Nonnull annotation diff --git a/docker-compose.yml b/docker-compose.yml index 770e60f9..35de5f84 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -6,7 +6,7 @@ services: # Zookeeper Cluster # #---------------------------------------------------------------------------# zookeeper-1: - image: confluentinc/cp-zookeeper:3.2.1 + image: confluentinc/cp-zookeeper:3.3.0 environment: ZOOKEEPER_SERVER_ID: 1 ZOOKEEPER_CLIENT_PORT: 2181 @@ -19,7 +19,7 @@ services: # Kafka Cluster # #---------------------------------------------------------------------------# kafka-1: - image: confluentinc/cp-kafka:3.2.1 + image: confluentinc/cp-kafka:3.3.0 depends_on: - zookeeper-1 environment: @@ -28,12 +28,19 @@ services: KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka-1:9092 KAFKA_AUTO_CREATE_TOPICS_ENABLE: "true" KAFKA_GROUP_MIN_SESSION_TIMEOUT_MS: 5000 + KAFKA_INTER_BROKER_PROTOCOL_VERSION: 0.11.0 + KAFKA_LOG_MESSAGE_FORMAT_VERSION: 0.11.0 + KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 0 + KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1 + KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1 + KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1 + #---------------------------------------------------------------------------# # Schema Registry # #---------------------------------------------------------------------------# schema-registry-1: - image: confluentinc/cp-schema-registry:3.2.1 + image: confluentinc/cp-schema-registry:3.3.0 depends_on: - zookeeper-1 - kafka-1 @@ -50,7 +57,7 @@ services: # REST proxy # #---------------------------------------------------------------------------# rest-proxy-1: - image: confluentinc/cp-kafka-rest:3.2.1 + image: confluentinc/cp-kafka-rest:3.3.0 depends_on: - kafka-1 - schema-registry-1 diff --git a/radar.yml b/radar.yml index 293910e9..50ea150f 100644 --- a/radar.yml +++ b/radar.yml @@ -19,8 +19,8 @@ broker: #Kafka internal parameters stream_properties: - auto_commit_interval_ms: 1000 - session_timeout_ms: 10000 + auto.commit.interval.ms: 1000 + session.timeout.ms: 10000 #============================ Kafka Streams ============================# #The number of threads that a stream must be run according is priority diff --git a/src/integrationTest/java/org/radarcns/integration/DirectProducerTest.java b/src/integrationTest/java/org/radarcns/integration/DirectProducerTest.java index facd221f..6907e7b6 100644 --- a/src/integrationTest/java/org/radarcns/integration/DirectProducerTest.java +++ b/src/integrationTest/java/org/radarcns/integration/DirectProducerTest.java @@ -20,6 +20,7 @@ import java.io.IOException; import java.util.Arrays; +import io.confluent.kafka.serializers.KafkaAvroDeserializer; import org.apache.commons.cli.ParseException; import org.junit.After; import org.junit.Before; diff --git a/src/integrationTest/resources/org/radarcns/kafka/radar.yml b/src/integrationTest/resources/org/radarcns/kafka/radar.yml index 6e44eeea..d5a22539 100644 --- a/src/integrationTest/resources/org/radarcns/kafka/radar.yml +++ b/src/integrationTest/resources/org/radarcns/kafka/radar.yml @@ -20,8 +20,14 @@ broker: #Kafka internal parameters stream_properties: - auto_commit_interval_ms: 1000 - session_timeout_ms: 10000 + auto.commit.interval.ms: 1000 + session.timeout.ms: 10000 + buffered.records.per.partition: 1 + request.timeout.ms: 1000 + batch.size: 1 + metadata.max.age.ms: 10000 + metadata.fetch.timeout.ms: 5000 + timeout.ms: 5000 #============================ Kafka Streams ============================# #The number of threads that a stream must be run according is priority diff --git a/src/main/java/org/radarcns/config/KafkaProperty.java b/src/main/java/org/radarcns/config/KafkaProperty.java index d415fa52..ceacd323 100644 --- a/src/main/java/org/radarcns/config/KafkaProperty.java +++ b/src/main/java/org/radarcns/config/KafkaProperty.java @@ -19,10 +19,11 @@ import io.confluent.kafka.serializers.AbstractKafkaAvroSerDeConfig; import java.util.Properties; import javax.annotation.Nonnull; + +import io.confluent.kafka.streams.serdes.avro.SpecificAvroSerde; import org.apache.kafka.clients.consumer.ConsumerConfig; import org.apache.kafka.streams.StreamsConfig; import org.apache.kafka.streams.processor.TimestampExtractor; -import org.radarcns.util.serde.SpecificAvroSerde; // TODO this class should substitute org.radarcns.util.RadarConfig public class KafkaProperty { @@ -38,15 +39,15 @@ public KafkaProperty(ConfigRadar configRadar) { * @param numThread number of threads to execute stream processing * @return Properties for a Kafka Stream */ - public Properties getStreamProperties(@Nonnull String clientId, @Nonnull int numThread) { + public Properties getStreamProperties(@Nonnull String clientId, int numThread) { Properties props = new Properties(); props.put(StreamsConfig.APPLICATION_ID_CONFIG, clientId); props.put(StreamsConfig.BOOTSTRAP_SERVERS_CONFIG, configRadar.getBrokerPaths()); props.put(AbstractKafkaAvroSerDeConfig.SCHEMA_REGISTRY_URL_CONFIG, configRadar.getSchemaRegistryPaths()); - props.put(StreamsConfig.KEY_SERDE_CLASS_CONFIG, SpecificAvroSerde.class); - props.put(StreamsConfig.VALUE_SERDE_CLASS_CONFIG, SpecificAvroSerde.class); + props.put(StreamsConfig.DEFAULT_KEY_SERDE_CLASS_CONFIG, SpecificAvroSerde.class); + props.put(StreamsConfig.DEFAULT_VALUE_SERDE_CLASS_CONFIG, SpecificAvroSerde.class); props.put(StreamsConfig.NUM_STREAM_THREADS_CONFIG, numThread); props.putAll(configRadar.getStreamProperties()); props.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest"); @@ -60,11 +61,12 @@ public Properties getStreamProperties(@Nonnull String clientId, @Nonnull int num * @param timestampExtractor custom timestamp extract that overrides the out-of-the-box * @return Properties for a Kafka Stream */ - public Properties getStreamProperties(@Nonnull String clientId, @Nonnull int numThread, + public Properties getStreamProperties(@Nonnull String clientId, int numThread, @Nonnull Class timestampExtractor) { Properties props = getStreamProperties(clientId, numThread); - props.put(StreamsConfig.TIMESTAMP_EXTRACTOR_CLASS_CONFIG, timestampExtractor.getName()); + props.put(StreamsConfig.DEFAULT_TIMESTAMP_EXTRACTOR_CLASS_CONFIG, + timestampExtractor.getName()); return props; } diff --git a/src/main/java/org/radarcns/monitor/AbstractKafkaMonitor.java b/src/main/java/org/radarcns/monitor/AbstractKafkaMonitor.java index 3a3239a7..8ec1cbbd 100644 --- a/src/main/java/org/radarcns/monitor/AbstractKafkaMonitor.java +++ b/src/main/java/org/radarcns/monitor/AbstractKafkaMonitor.java @@ -164,8 +164,11 @@ public void start() { evaluateRecords(records); } catch (SerializationException ex) { handleSerializationException(); - } catch (InterruptException | WakeupException ex) { + } catch (WakeupException ex) { logger.info("Consumer woke up"); + } catch (InterruptException ex) { + logger.info("Consumer was interrupted"); + shutdown(); } catch (KafkaException ex) { logger.error("Kafka consumer gave exception", ex); } @@ -235,6 +238,7 @@ public synchronized boolean isShutdown() { @Override public synchronized void shutdown() { + logger.info("Shutting down monitor {}", getClass().getSimpleName()); this.done = true; this.consumer.wakeup(); } diff --git a/src/main/java/org/radarcns/util/serde/GenericAvroDeserializer.java b/src/main/java/org/radarcns/util/serde/GenericAvroDeserializer.java deleted file mode 100644 index 3e8208a5..00000000 --- a/src/main/java/org/radarcns/util/serde/GenericAvroDeserializer.java +++ /dev/null @@ -1,50 +0,0 @@ -/* - * Copyright 2017 King's College London and The Hyve - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.radarcns.util.serde; - -import io.confluent.kafka.serializers.KafkaAvroDeserializer; -import org.apache.avro.generic.GenericRecord; -import org.apache.kafka.common.serialization.Deserializer; - -import java.util.Map; - -public class GenericAvroDeserializer implements Deserializer { - - private final KafkaAvroDeserializer inner; - - /** - * Constructor used by Kafka Streams. - */ - public GenericAvroDeserializer() { - inner = new KafkaAvroDeserializer(); - } - - @Override - public void configure(Map configs, boolean isKey) { - inner.configure(configs, isKey); - } - - @Override - public GenericRecord deserialize(String s, byte[] bytes) { - return (GenericRecord) inner.deserialize(s, bytes); - } - - @Override - public void close() { - inner.close(); - } -} diff --git a/src/main/java/org/radarcns/util/serde/GenericAvroSerde.java b/src/main/java/org/radarcns/util/serde/GenericAvroSerde.java deleted file mode 100644 index 0a5ce631..00000000 --- a/src/main/java/org/radarcns/util/serde/GenericAvroSerde.java +++ /dev/null @@ -1,61 +0,0 @@ -/* - * Copyright 2017 King's College London and The Hyve - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.radarcns.util.serde; - -import java.util.Map; -import org.apache.avro.generic.GenericRecord; -import org.apache.kafka.common.serialization.Deserializer; -import org.apache.kafka.common.serialization.Serde; -import org.apache.kafka.common.serialization.Serdes; -import org.apache.kafka.common.serialization.Serializer; - -/** - * Created by Francesco Nobilia on 12/10/2016. - */ -public class GenericAvroSerde implements Serde { - - private final Serde inner; - - /** - * Constructor used by Kafka Streams. - */ - public GenericAvroSerde() { - inner = Serdes.serdeFrom(new GenericAvroSerializer(), new GenericAvroDeserializer()); - } - - @Override - public Serializer serializer() { - return inner.serializer(); - } - - @Override - public Deserializer deserializer() { - return inner.deserializer(); - } - - @Override - public void configure(Map configs, boolean isKey) { - inner.serializer().configure(configs, isKey); - inner.deserializer().configure(configs, isKey); - } - - @Override - public void close() { - inner.serializer().close(); - inner.deserializer().close(); - } -} diff --git a/src/main/java/org/radarcns/util/serde/GenericAvroSerializer.java b/src/main/java/org/radarcns/util/serde/GenericAvroSerializer.java deleted file mode 100644 index b0e0e97a..00000000 --- a/src/main/java/org/radarcns/util/serde/GenericAvroSerializer.java +++ /dev/null @@ -1,55 +0,0 @@ -/* - * Copyright 2017 King's College London and The Hyve - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.radarcns.util.serde; - -/** - * Created by Francesco Nobilia on 12/10/2016. - */ - -import io.confluent.kafka.serializers.KafkaAvroSerializer; -import org.apache.avro.generic.GenericRecord; -import org.apache.kafka.common.serialization.Serializer; - -import java.util.Map; - - -public class GenericAvroSerializer implements Serializer { - - private final KafkaAvroSerializer inner; - - /** - * Constructor used by Kafka Streams. - */ - public GenericAvroSerializer() { - inner = new KafkaAvroSerializer(); - } - - @Override - public void configure(Map configs, boolean isKey) { - inner.configure(configs, isKey); - } - - @Override - public byte[] serialize(String topic, GenericRecord record) { - return inner.serialize(topic, record); - } - - @Override - public void close() { - inner.close(); - } -} diff --git a/src/main/java/org/radarcns/util/serde/SpecificAvroDeserializer.java b/src/main/java/org/radarcns/util/serde/SpecificAvroDeserializer.java deleted file mode 100644 index 44740035..00000000 --- a/src/main/java/org/radarcns/util/serde/SpecificAvroDeserializer.java +++ /dev/null @@ -1,54 +0,0 @@ -/* - * Copyright 2017 King's College London and The Hyve - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.radarcns.util.serde; - -import static io.confluent.kafka.serializers.KafkaAvroDeserializerConfig.SPECIFIC_AVRO_READER_CONFIG; - -import io.confluent.kafka.serializers.KafkaAvroDeserializer; -import java.util.HashMap; -import java.util.Map; -import org.apache.avro.specific.SpecificRecord; -import org.apache.kafka.common.serialization.Deserializer; - -public class SpecificAvroDeserializer implements Deserializer { - private final KafkaAvroDeserializer inner; - - /** - * Constructor used by Kafka Streams. - */ - public SpecificAvroDeserializer() { - inner = new KafkaAvroDeserializer(); - } - - @Override - public void configure(Map configs, boolean isKey) { - Map effectiveConfigs = new HashMap<>(configs); - effectiveConfigs.put(SPECIFIC_AVRO_READER_CONFIG, true); - inner.configure(effectiveConfigs, isKey); - } - - @SuppressWarnings("unchecked") - @Override - public T deserialize(String s, byte[] bytes) { - return (T) inner.deserialize(s, bytes); - } - - @Override - public void close() { - inner.close(); - } -} diff --git a/src/main/java/org/radarcns/util/serde/SpecificAvroSerde.java b/src/main/java/org/radarcns/util/serde/SpecificAvroSerde.java deleted file mode 100644 index 8377e8a9..00000000 --- a/src/main/java/org/radarcns/util/serde/SpecificAvroSerde.java +++ /dev/null @@ -1,57 +0,0 @@ -/* - * Copyright 2017 King's College London and The Hyve - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.radarcns.util.serde; - -import java.util.Map; -import org.apache.avro.specific.SpecificRecord; -import org.apache.kafka.common.serialization.Deserializer; -import org.apache.kafka.common.serialization.Serde; -import org.apache.kafka.common.serialization.Serdes; -import org.apache.kafka.common.serialization.Serializer; - -public class SpecificAvroSerde implements Serde { - private final Serde inner; - - /** - * Constructor used by Kafka Streams. - */ - public SpecificAvroSerde() { - inner = Serdes.serdeFrom(new SpecificAvroSerializer<>(), new SpecificAvroDeserializer<>()); - } - - @Override - public Serializer serializer() { - return inner.serializer(); - } - - @Override - public Deserializer deserializer() { - return inner.deserializer(); - } - - @Override - public void configure(Map configs, boolean isKey) { - inner.serializer().configure(configs, isKey); - inner.deserializer().configure(configs, isKey); - } - - @Override - public void close() { - inner.serializer().close(); - inner.deserializer().close(); - } -} \ No newline at end of file diff --git a/src/main/java/org/radarcns/util/serde/SpecificAvroSerializer.java b/src/main/java/org/radarcns/util/serde/SpecificAvroSerializer.java deleted file mode 100644 index 6a4ea11d..00000000 --- a/src/main/java/org/radarcns/util/serde/SpecificAvroSerializer.java +++ /dev/null @@ -1,55 +0,0 @@ -/* - * Copyright 2017 King's College London and The Hyve - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.radarcns.util.serde; - -import static io.confluent.kafka.serializers.KafkaAvroDeserializerConfig.SPECIFIC_AVRO_READER_CONFIG; - -import io.confluent.kafka.serializers.KafkaAvroSerializer; -import java.util.HashMap; -import java.util.Map; -import org.apache.avro.specific.SpecificRecord; -import org.apache.kafka.common.serialization.Serializer; - -public class SpecificAvroSerializer implements Serializer { - - private final KafkaAvroSerializer inner; - - /** - * Constructor used by Kafka Streams. - */ - public SpecificAvroSerializer() { - inner = new KafkaAvroSerializer(); - } - - @Override - @SuppressWarnings("unchecked") - public void configure(Map configs, boolean isKey) { - Map effectiveConfigs = new HashMap<>(configs); - effectiveConfigs.put(SPECIFIC_AVRO_READER_CONFIG, true); - inner.configure(effectiveConfigs, isKey); - } - - @Override - public byte[] serialize(String topic, T record) { - return inner.serialize(topic, record); - } - - @Override - public void close() { - inner.close(); - } -} diff --git a/src/test/java/org/radarcns/stream/StreamDefinitionTest.java b/src/test/java/org/radarcns/stream/StreamDefinitionTest.java index e897daa9..1486162d 100644 --- a/src/test/java/org/radarcns/stream/StreamDefinitionTest.java +++ b/src/test/java/org/radarcns/stream/StreamDefinitionTest.java @@ -17,11 +17,16 @@ package org.radarcns.stream; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; import org.junit.Test; import org.radarcns.topic.KafkaTopic; +import java.util.regex.Pattern; + public class StreamDefinitionTest { + private static final Pattern TOPIC_PATTERN = Pattern.compile("^[A-Za-z0-9_-]+$"); private static final String INPUT = "android_empatica_e4_blood_volume_pulse"; private static final String OUTPUT = INPUT + GeneralStreamGroup.OUTPUT_LABEL; @@ -33,8 +38,7 @@ public void nameValidation() { StreamDefinition definition = new StreamDefinition(inputTopic, outputTopic); - kafka.common.Topic.validate(definition.getStateStoreName()); - + assertTrue(TOPIC_PATTERN.matcher(definition.getStateStoreName()).matches()); assertEquals("From-android_empatica_e4_blood_volume_pulse" + "-To-android_empatica_e4_blood_volume_pulse_output", definition.getStateStoreName()); @@ -46,7 +50,6 @@ public void faultyNameValidation() { KafkaTopic outputTopic = new KafkaTopic(OUTPUT); StreamDefinition definition = new StreamDefinition(inputTopic, outputTopic); - - kafka.common.Topic.validate(definition.getStateStoreName()); + assertFalse(TOPIC_PATTERN.matcher(definition.getStateStoreName()).matches()); } } diff --git a/src/test/java/org/radarcns/util/serde/GenericAvroSerdeTest.java b/src/test/java/org/radarcns/util/serde/GenericAvroSerdeTest.java deleted file mode 100644 index a6ee3306..00000000 --- a/src/test/java/org/radarcns/util/serde/GenericAvroSerdeTest.java +++ /dev/null @@ -1,54 +0,0 @@ -/* - * Copyright 2017 King's College London and The Hyve - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.radarcns.util.serde; - -import static org.junit.Assert.assertEquals; - -import java.util.HashMap; -import java.util.Map; -import org.junit.Before; -import org.junit.Test; - -/** - * Created by nivethika on 21-12-16. - */ -public class GenericAvroSerdeTest { - - private GenericAvroSerde genericAvroSerde; - - @Before - public void setUp() { - this.genericAvroSerde = new GenericAvroSerde(); - } - - @Test - public void serializer() { - assertEquals(this.genericAvroSerde.serializer().getClass(), GenericAvroSerializer.class); - } - - @Test - public void deserializer() { - assertEquals(this.genericAvroSerde.deserializer().getClass(), GenericAvroDeserializer.class); - } - - @Test - public void configure() { - Map map = new HashMap<>(); - map.put("schema.registry.url", "testvalue"); - this.genericAvroSerde.configure(map, false); - } -} diff --git a/src/test/java/org/radarcns/util/serde/SpecificAvroSerdeTest.java b/src/test/java/org/radarcns/util/serde/SpecificAvroSerdeTest.java deleted file mode 100644 index 629309ca..00000000 --- a/src/test/java/org/radarcns/util/serde/SpecificAvroSerdeTest.java +++ /dev/null @@ -1,48 +0,0 @@ -/* - * Copyright 2017 King's College London and The Hyve - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.radarcns.util.serde; - -import static org.junit.Assert.assertEquals; - -import org.junit.Before; -import org.junit.Test; - -/** - * Created by nivethika on 21-12-16. - */ -public class SpecificAvroSerdeTest { - - private SpecificAvroSerde specificAvroSerde ; - - @Before - public void setUp() { - this.specificAvroSerde = new SpecificAvroSerde(); - } - - @Test - public void serializer() { - assertEquals(this.specificAvroSerde.serializer().getClass(), SpecificAvroSerializer.class ); - } - - @Test - public void deserializer() { - assertEquals(this.specificAvroSerde.deserializer().getClass(), SpecificAvroDeserializer.class ); - } - - - -} diff --git a/src/test/radar.yml b/src/test/radar.yml index 20ae4760..5f7a3dec 100644 --- a/src/test/radar.yml +++ b/src/test/radar.yml @@ -20,8 +20,8 @@ broker: #Kafka internal parameters stream_properties: - auto_commit_interval_ms: 1000 - session_timeout_ms: 10000 + auto.commit.interval.ms: 1000 + session.timeout.ms: 10000 #============================ Kafka Streams ============================# #The number of threads that a stream must be run according is priority diff --git a/src/test/resources/config/radar.yml b/src/test/resources/config/radar.yml index 7c1c0aa6..fe4cd9c6 100644 --- a/src/test/resources/config/radar.yml +++ b/src/test/resources/config/radar.yml @@ -19,8 +19,8 @@ broker: #Kafka internal parameters stream_properties: - auto_commit_interval_ms: 1000 - session_timeout_ms: 10000 + auto.commit.interval.ms: 1000 + session.timeout.ms: 10000 #============================ Kafka Streams ============================# #The number of threads that a stream must be run according is priority From d99e3ca5cde4b0afb6d80a4312322700ce298c3c Mon Sep 17 00:00:00 2001 From: Joris Borgdorff Date: Wed, 11 Oct 2017 16:47:14 +0200 Subject: [PATCH 05/23] Fixed temporary key serialization --- .../radarcns/integration/PhoneStreamTest.java | 55 ++-- .../resources/org/radarcns/kafka/radar.yml | 7 +- src/main/avro/temporary_package_key.avsc | 4 +- .../phone/PhoneUsageAggregationStream.java | 7 +- .../stream/phone/PhoneUsageStream.java | 7 +- .../stream/phone/TemporaryPackageKey.java | 289 ++++++++++++------ .../org/radarcns/util/RadarUtilitiesImpl.java | 4 +- 7 files changed, 242 insertions(+), 131 deletions(-) diff --git a/src/integrationTest/java/org/radarcns/integration/PhoneStreamTest.java b/src/integrationTest/java/org/radarcns/integration/PhoneStreamTest.java index e4ae0276..83ead2fe 100644 --- a/src/integrationTest/java/org/radarcns/integration/PhoneStreamTest.java +++ b/src/integrationTest/java/org/radarcns/integration/PhoneStreamTest.java @@ -16,27 +16,6 @@ package org.radarcns.integration; -import static org.apache.kafka.clients.consumer.ConsumerConfig.AUTO_COMMIT_INTERVAL_MS_CONFIG; -import static org.apache.kafka.clients.consumer.ConsumerConfig.HEARTBEAT_INTERVAL_MS_CONFIG; -import static org.apache.kafka.clients.consumer.ConsumerConfig.SESSION_TIMEOUT_MS_CONFIG; -import static org.apache.kafka.clients.producer.ProducerConfig.BOOTSTRAP_SERVERS_CONFIG; -import static org.apache.kafka.clients.producer.ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG; -import static org.apache.kafka.clients.producer.ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertNull; -import static org.junit.Assert.assertTrue; -import static org.radarcns.util.serde.AbstractKafkaAvroSerde.SCHEMA_REGISTRY_CONFIG; - -import java.io.IOException; -import java.util.Arrays; -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Properties; -import java.util.UUID; - import org.apache.avro.generic.GenericRecord; import org.apache.avro.specific.SpecificRecord; import org.apache.commons.cli.ParseException; @@ -65,6 +44,27 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.io.IOException; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Properties; +import java.util.UUID; + +import static org.apache.kafka.clients.consumer.ConsumerConfig.AUTO_COMMIT_INTERVAL_MS_CONFIG; +import static org.apache.kafka.clients.consumer.ConsumerConfig.HEARTBEAT_INTERVAL_MS_CONFIG; +import static org.apache.kafka.clients.consumer.ConsumerConfig.SESSION_TIMEOUT_MS_CONFIG; +import static org.apache.kafka.clients.producer.ProducerConfig.BOOTSTRAP_SERVERS_CONFIG; +import static org.apache.kafka.clients.producer.ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG; +import static org.apache.kafka.clients.producer.ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; +import static org.radarcns.util.serde.AbstractKafkaAvroSerde.SCHEMA_REGISTRY_CONFIG; + public class PhoneStreamTest { private static final Logger logger = LoggerFactory.getLogger(PhoneStreamTest.class); private static final Map CATEGORIES = new HashMap<>(); @@ -156,19 +156,8 @@ public void testDirect() throws Exception { } } - AvroTopic outputTopic = new AvroTopic<>( - "android_phone_usage_event_output", - ObservationKey.getClassSchema(), PhoneUsageEvent.getClassSchema(), - ObservationKey.class, PhoneUsageEvent.class); - - try (KafkaTopicSender topicSender = sender.sender(outputTopic)) { - for (PhoneUsageEvent event : events) { - topicSender.send(offset++, key, event); - } - } - sender.close(); - consumePhone(offset / 2); + consumePhone(offset); consumeAggregated(offset / 2); } diff --git a/src/integrationTest/resources/org/radarcns/kafka/radar.yml b/src/integrationTest/resources/org/radarcns/kafka/radar.yml index d5a22539..a5f91c5b 100644 --- a/src/integrationTest/resources/org/radarcns/kafka/radar.yml +++ b/src/integrationTest/resources/org/radarcns/kafka/radar.yml @@ -21,13 +21,16 @@ broker: #Kafka internal parameters stream_properties: auto.commit.interval.ms: 1000 - session.timeout.ms: 10000 + session.timeout.ms: 4000 buffered.records.per.partition: 1 - request.timeout.ms: 1000 + request.timeout.ms: 70000 + fetch.timeout.ms: 3000 batch.size: 1 metadata.max.age.ms: 10000 metadata.fetch.timeout.ms: 5000 timeout.ms: 5000 + commit.interval.ms: 1000 + buffered.records.per.partition: 1 #============================ Kafka Streams ============================# #The number of threads that a stream must be run according is priority diff --git a/src/main/avro/temporary_package_key.avsc b/src/main/avro/temporary_package_key.avsc index eb506c61..99b0d1df 100644 --- a/src/main/avro/temporary_package_key.avsc +++ b/src/main/avro/temporary_package_key.avsc @@ -3,7 +3,9 @@ "name": "TemporaryPackageKey", "type": "record", "fields": [ - {"name": "key", "type": "org.radarcns.kafka.ObservationKey", "doc": "Observation key."}, + {"name": "projectId", "type": ["null", "string"], "doc": "Project ID."}, + {"name": "userId", "type": "string", "doc": "User ID."}, + {"name": "sourceId", "type": "string", "doc": "Source ID."}, {"name": "packageName", "type": "string", "doc": "Package name."} ] } \ No newline at end of file diff --git a/src/main/java/org/radarcns/stream/phone/PhoneUsageAggregationStream.java b/src/main/java/org/radarcns/stream/phone/PhoneUsageAggregationStream.java index aec730d4..1c2edd88 100644 --- a/src/main/java/org/radarcns/stream/phone/PhoneUsageAggregationStream.java +++ b/src/main/java/org/radarcns/stream/phone/PhoneUsageAggregationStream.java @@ -31,7 +31,7 @@ public PhoneUsageAggregationStream(Collection definitions, int protected KStream implementStream( StreamDefinition definition, @Nonnull KStream kstream) { - return kstream.groupBy((k, v) -> new TemporaryPackageKey(k, v.getPackageName())) + return kstream.groupBy(PhoneUsageAggregationStream::temporaryKey) .aggregate( PhoneUsageCollector::new, (k, v, valueCollector) -> valueCollector.update(v), @@ -41,4 +41,9 @@ protected KStream implementStream( .toStream() .map(utilities::collectorToAvro); } + + private static TemporaryPackageKey temporaryKey(ObservationKey key, PhoneUsageEvent value) { + return new TemporaryPackageKey(key.getProjectId(), key.getUserId(), key.getSourceId(), + value.getPackageName()); + } } diff --git a/src/main/java/org/radarcns/stream/phone/PhoneUsageStream.java b/src/main/java/org/radarcns/stream/phone/PhoneUsageStream.java index 0e5695e4..7c4752b8 100644 --- a/src/main/java/org/radarcns/stream/phone/PhoneUsageStream.java +++ b/src/main/java/org/radarcns/stream/phone/PhoneUsageStream.java @@ -30,6 +30,8 @@ import java.util.Collection; +import static org.apache.kafka.streams.KeyValue.pair; + public class PhoneUsageStream extends StreamWorker { private static final Logger logger = LoggerFactory.getLogger(PhoneUsageStream.class); @@ -44,7 +46,7 @@ public class PhoneUsageStream extends StreamWorker definitions, int numThread, StreamMaster master, KafkaProperty kafkaProperties) { super(definitions, numThread, master, kafkaProperties, logger); - playStoreLookup = new PlayStoreLookup(CACHE_TIMEOUT, MAX_CACHE_SIZE); + playStoreLookup = new PlayStoreLookup(CACHE_TIMEOUT, MAX_CACHE_SIZE); } @Override @@ -54,9 +56,10 @@ protected KStream implementStream(StreamDefinit .map((key, value) -> { String packageName = value.getPackageName(); PlayStoreLookup.AppCategory category = playStoreLookup.lookupCategory(packageName); + logger.info("Looked up {}: {}", packageName, category.getCategoryName()); value.setCategoryName(category.getCategoryName()); value.setCategoryNameFetchTime(category.getFetchTimeStamp()); - return new KeyValue<>(key, value); + return pair(key, value); }); } } diff --git a/src/main/java/org/radarcns/stream/phone/TemporaryPackageKey.java b/src/main/java/org/radarcns/stream/phone/TemporaryPackageKey.java index 0da53d9b..cd76ebf4 100644 --- a/src/main/java/org/radarcns/stream/phone/TemporaryPackageKey.java +++ b/src/main/java/org/radarcns/stream/phone/TemporaryPackageKey.java @@ -13,8 +13,8 @@ @SuppressWarnings("all") @org.apache.avro.specific.AvroGenerated public class TemporaryPackageKey extends org.apache.avro.specific.SpecificRecordBase implements org.apache.avro.specific.SpecificRecord { - private static final long serialVersionUID = -8584100260725184052L; - public static final org.apache.avro.Schema SCHEMA$ = new org.apache.avro.Schema.Parser().parse("{\"type\":\"record\",\"name\":\"TemporaryPackageKey\",\"namespace\":\"org.radarcns.stream.phone\",\"fields\":[{\"name\":\"key\",\"type\":{\"type\":\"record\",\"name\":\"ObservationKey\",\"namespace\":\"org.radarcns.key\",\"doc\":\"Key of an observation.\",\"fields\":[{\"name\":\"userId\",\"type\":{\"type\":\"string\",\"avro.java.string\":\"String\"},\"doc\":\"User Identifier created during the enrolment.\"},{\"name\":\"sourceId\",\"type\":{\"type\":\"string\",\"avro.java.string\":\"String\"},\"doc\":\"Unique identifier associated with the source.\"}]},\"doc\":\"Observation key.\"},{\"name\":\"packageName\",\"type\":{\"type\":\"string\",\"avro.java.string\":\"String\"},\"doc\":\"Package name.\"}]}"); + private static final long serialVersionUID = 5825691401492875312L; + public static final org.apache.avro.Schema SCHEMA$ = new org.apache.avro.Schema.Parser().parse("{\"type\":\"record\",\"name\":\"TemporaryPackageKey\",\"namespace\":\"org.radarcns.stream.phone\",\"fields\":[{\"name\":\"projectId\",\"type\":[\"null\",{\"type\":\"string\",\"avro.java.string\":\"String\"}],\"doc\":\"Project ID.\"},{\"name\":\"userId\",\"type\":{\"type\":\"string\",\"avro.java.string\":\"String\"},\"doc\":\"User ID.\"},{\"name\":\"sourceId\",\"type\":{\"type\":\"string\",\"avro.java.string\":\"String\"},\"doc\":\"Source ID.\"},{\"name\":\"packageName\",\"type\":{\"type\":\"string\",\"avro.java.string\":\"String\"},\"doc\":\"Package name.\"}]}"); public static org.apache.avro.Schema getClassSchema() { return SCHEMA$; } private static SpecificData MODEL$ = new SpecificData(); @@ -51,8 +51,12 @@ public static TemporaryPackageKey fromByteBuffer( return DECODER.decode(b); } - /** Observation key. */ - @Deprecated public org.radarcns.kafka.ObservationKey key; + /** Project ID. */ + @Deprecated public java.lang.String projectId; + /** User ID. */ + @Deprecated public java.lang.String userId; + /** Source ID. */ + @Deprecated public java.lang.String sourceId; /** Package name. */ @Deprecated public java.lang.String packageName; @@ -65,11 +69,15 @@ public TemporaryPackageKey() {} /** * All-args constructor. - * @param key Observation key. + * @param projectId Project ID. + * @param userId User ID. + * @param sourceId Source ID. * @param packageName Package name. */ - public TemporaryPackageKey(org.radarcns.kafka.ObservationKey key, java.lang.String packageName) { - this.key = key; + public TemporaryPackageKey(java.lang.String projectId, java.lang.String userId, java.lang.String sourceId, java.lang.String packageName) { + this.projectId = projectId; + this.userId = userId; + this.sourceId = sourceId; this.packageName = packageName; } @@ -77,8 +85,10 @@ public TemporaryPackageKey(org.radarcns.kafka.ObservationKey key, java.lang.Stri // Used by DatumWriter. Applications should not call. public java.lang.Object get(int field$) { switch (field$) { - case 0: return key; - case 1: return packageName; + case 0: return projectId; + case 1: return userId; + case 2: return sourceId; + case 3: return packageName; default: throw new org.apache.avro.AvroRuntimeException("Bad index"); } } @@ -87,27 +97,63 @@ public java.lang.Object get(int field$) { @SuppressWarnings(value="unchecked") public void put(int field$, java.lang.Object value$) { switch (field$) { - case 0: key = (org.radarcns.kafka.ObservationKey)value$; break; - case 1: packageName = (java.lang.String)value$; break; + case 0: projectId = (java.lang.String)value$; break; + case 1: userId = (java.lang.String)value$; break; + case 2: sourceId = (java.lang.String)value$; break; + case 3: packageName = (java.lang.String)value$; break; default: throw new org.apache.avro.AvroRuntimeException("Bad index"); } } /** - * Gets the value of the 'key' field. - * @return Observation key. + * Gets the value of the 'projectId' field. + * @return Project ID. */ - public org.radarcns.kafka.ObservationKey getKey() { - return key; + public java.lang.String getProjectId() { + return projectId; } /** - * Sets the value of the 'key' field. - * Observation key. + * Sets the value of the 'projectId' field. + * Project ID. * @param value the value to set. */ - public void setKey(org.radarcns.kafka.ObservationKey value) { - this.key = value; + public void setProjectId(java.lang.String value) { + this.projectId = value; + } + + /** + * Gets the value of the 'userId' field. + * @return User ID. + */ + public java.lang.String getUserId() { + return userId; + } + + /** + * Sets the value of the 'userId' field. + * User ID. + * @param value the value to set. + */ + public void setUserId(java.lang.String value) { + this.userId = value; + } + + /** + * Gets the value of the 'sourceId' field. + * @return Source ID. + */ + public java.lang.String getSourceId() { + return sourceId; + } + + /** + * Sets the value of the 'sourceId' field. + * Source ID. + * @param value the value to set. + */ + public void setSourceId(java.lang.String value) { + this.sourceId = value; } /** @@ -159,9 +205,12 @@ public static org.radarcns.stream.phone.TemporaryPackageKey.Builder newBuilder(o public static class Builder extends org.apache.avro.specific.SpecificRecordBuilderBase implements org.apache.avro.data.RecordBuilder { - /** Observation key. */ - private org.radarcns.kafka.ObservationKey key; - private org.radarcns.kafka.ObservationKey.Builder keyBuilder; + /** Project ID. */ + private java.lang.String projectId; + /** User ID. */ + private java.lang.String userId; + /** Source ID. */ + private java.lang.String sourceId; /** Package name. */ private java.lang.String packageName; @@ -176,17 +225,22 @@ private Builder() { */ private Builder(org.radarcns.stream.phone.TemporaryPackageKey.Builder other) { super(other); - if (isValidValue(fields()[0], other.key)) { - this.key = data().deepCopy(fields()[0].schema(), other.key); + if (isValidValue(fields()[0], other.projectId)) { + this.projectId = data().deepCopy(fields()[0].schema(), other.projectId); fieldSetFlags()[0] = true; } - if (other.hasKeyBuilder()) { - this.keyBuilder = org.radarcns.kafka.ObservationKey.newBuilder(other.getKeyBuilder()); - } - if (isValidValue(fields()[1], other.packageName)) { - this.packageName = data().deepCopy(fields()[1].schema(), other.packageName); + if (isValidValue(fields()[1], other.userId)) { + this.userId = data().deepCopy(fields()[1].schema(), other.userId); fieldSetFlags()[1] = true; } + if (isValidValue(fields()[2], other.sourceId)) { + this.sourceId = data().deepCopy(fields()[2].schema(), other.sourceId); + fieldSetFlags()[2] = true; + } + if (isValidValue(fields()[3], other.packageName)) { + this.packageName = data().deepCopy(fields()[3].schema(), other.packageName); + fieldSetFlags()[3] = true; + } } /** @@ -195,95 +249,150 @@ private Builder(org.radarcns.stream.phone.TemporaryPackageKey.Builder other) { */ private Builder(org.radarcns.stream.phone.TemporaryPackageKey other) { super(SCHEMA$); - if (isValidValue(fields()[0], other.key)) { - this.key = data().deepCopy(fields()[0].schema(), other.key); + if (isValidValue(fields()[0], other.projectId)) { + this.projectId = data().deepCopy(fields()[0].schema(), other.projectId); fieldSetFlags()[0] = true; } - this.keyBuilder = null; - if (isValidValue(fields()[1], other.packageName)) { - this.packageName = data().deepCopy(fields()[1].schema(), other.packageName); + if (isValidValue(fields()[1], other.userId)) { + this.userId = data().deepCopy(fields()[1].schema(), other.userId); fieldSetFlags()[1] = true; } + if (isValidValue(fields()[2], other.sourceId)) { + this.sourceId = data().deepCopy(fields()[2].schema(), other.sourceId); + fieldSetFlags()[2] = true; + } + if (isValidValue(fields()[3], other.packageName)) { + this.packageName = data().deepCopy(fields()[3].schema(), other.packageName); + fieldSetFlags()[3] = true; + } } /** - * Gets the value of the 'key' field. - * Observation key. + * Gets the value of the 'projectId' field. + * Project ID. * @return The value. */ - public org.radarcns.kafka.ObservationKey getKey() { - return key; + public java.lang.String getProjectId() { + return projectId; } /** - * Sets the value of the 'key' field. - * Observation key. - * @param value The value of 'key'. + * Sets the value of the 'projectId' field. + * Project ID. + * @param value The value of 'projectId'. * @return This builder. */ - public org.radarcns.stream.phone.TemporaryPackageKey.Builder setKey(org.radarcns.kafka.ObservationKey value) { + public org.radarcns.stream.phone.TemporaryPackageKey.Builder setProjectId(java.lang.String value) { validate(fields()[0], value); - this.keyBuilder = null; - this.key = value; + this.projectId = value; fieldSetFlags()[0] = true; return this; } /** - * Checks whether the 'key' field has been set. - * Observation key. - * @return True if the 'key' field has been set, false otherwise. + * Checks whether the 'projectId' field has been set. + * Project ID. + * @return True if the 'projectId' field has been set, false otherwise. */ - public boolean hasKey() { + public boolean hasProjectId() { return fieldSetFlags()[0]; } + /** - * Gets the Builder instance for the 'key' field and creates one if it doesn't exist yet. - * Observation key. - * @return This builder. - */ - public org.radarcns.kafka.ObservationKey.Builder getKeyBuilder() { - if (keyBuilder == null) { - if (hasKey()) { - setKeyBuilder(org.radarcns.kafka.ObservationKey.newBuilder(key)); - } else { - setKeyBuilder(org.radarcns.kafka.ObservationKey.newBuilder()); - } - } - return keyBuilder; + * Clears the value of the 'projectId' field. + * Project ID. + * @return This builder. + */ + public org.radarcns.stream.phone.TemporaryPackageKey.Builder clearProjectId() { + projectId = null; + fieldSetFlags()[0] = false; + return this; } /** - * Sets the Builder instance for the 'key' field - * Observation key. - * @param value The builder instance that must be set. - * @return This builder. - */ - public org.radarcns.stream.phone.TemporaryPackageKey.Builder setKeyBuilder(org.radarcns.kafka.ObservationKey.Builder value) { - clearKey(); - keyBuilder = value; + * Gets the value of the 'userId' field. + * User ID. + * @return The value. + */ + public java.lang.String getUserId() { + return userId; + } + + /** + * Sets the value of the 'userId' field. + * User ID. + * @param value The value of 'userId'. + * @return This builder. + */ + public org.radarcns.stream.phone.TemporaryPackageKey.Builder setUserId(java.lang.String value) { + validate(fields()[1], value); + this.userId = value; + fieldSetFlags()[1] = true; return this; } /** - * Checks whether the 'key' field has an active Builder instance - * Observation key. - * @return True if the 'key' field has an active Builder instance - */ - public boolean hasKeyBuilder() { - return keyBuilder != null; + * Checks whether the 'userId' field has been set. + * User ID. + * @return True if the 'userId' field has been set, false otherwise. + */ + public boolean hasUserId() { + return fieldSetFlags()[1]; } + /** - * Clears the value of the 'key' field. - * Observation key. + * Clears the value of the 'userId' field. + * User ID. * @return This builder. */ - public org.radarcns.stream.phone.TemporaryPackageKey.Builder clearKey() { - key = null; - keyBuilder = null; - fieldSetFlags()[0] = false; + public org.radarcns.stream.phone.TemporaryPackageKey.Builder clearUserId() { + userId = null; + fieldSetFlags()[1] = false; + return this; + } + + /** + * Gets the value of the 'sourceId' field. + * Source ID. + * @return The value. + */ + public java.lang.String getSourceId() { + return sourceId; + } + + /** + * Sets the value of the 'sourceId' field. + * Source ID. + * @param value The value of 'sourceId'. + * @return This builder. + */ + public org.radarcns.stream.phone.TemporaryPackageKey.Builder setSourceId(java.lang.String value) { + validate(fields()[2], value); + this.sourceId = value; + fieldSetFlags()[2] = true; + return this; + } + + /** + * Checks whether the 'sourceId' field has been set. + * Source ID. + * @return True if the 'sourceId' field has been set, false otherwise. + */ + public boolean hasSourceId() { + return fieldSetFlags()[2]; + } + + + /** + * Clears the value of the 'sourceId' field. + * Source ID. + * @return This builder. + */ + public org.radarcns.stream.phone.TemporaryPackageKey.Builder clearSourceId() { + sourceId = null; + fieldSetFlags()[2] = false; return this; } @@ -303,9 +412,9 @@ public java.lang.String getPackageName() { * @return This builder. */ public org.radarcns.stream.phone.TemporaryPackageKey.Builder setPackageName(java.lang.String value) { - validate(fields()[1], value); + validate(fields()[3], value); this.packageName = value; - fieldSetFlags()[1] = true; + fieldSetFlags()[3] = true; return this; } @@ -315,7 +424,7 @@ public org.radarcns.stream.phone.TemporaryPackageKey.Builder setPackageName(java * @return True if the 'packageName' field has been set, false otherwise. */ public boolean hasPackageName() { - return fieldSetFlags()[1]; + return fieldSetFlags()[3]; } @@ -326,7 +435,7 @@ public boolean hasPackageName() { */ public org.radarcns.stream.phone.TemporaryPackageKey.Builder clearPackageName() { packageName = null; - fieldSetFlags()[1] = false; + fieldSetFlags()[3] = false; return this; } @@ -335,12 +444,10 @@ public org.radarcns.stream.phone.TemporaryPackageKey.Builder clearPackageName() public TemporaryPackageKey build() { try { TemporaryPackageKey record = new TemporaryPackageKey(); - if (keyBuilder != null) { - record.key = this.keyBuilder.build(); - } else { - record.key = fieldSetFlags()[0] ? this.key : (org.radarcns.kafka.ObservationKey) defaultValue(fields()[0]); - } - record.packageName = fieldSetFlags()[1] ? this.packageName : (java.lang.String) defaultValue(fields()[1]); + record.projectId = fieldSetFlags()[0] ? this.projectId : (java.lang.String) defaultValue(fields()[0]); + record.userId = fieldSetFlags()[1] ? this.userId : (java.lang.String) defaultValue(fields()[1]); + record.sourceId = fieldSetFlags()[2] ? this.sourceId : (java.lang.String) defaultValue(fields()[2]); + record.packageName = fieldSetFlags()[3] ? this.packageName : (java.lang.String) defaultValue(fields()[3]); return record; } catch (java.lang.Exception e) { throw new org.apache.avro.AvroRuntimeException(e); diff --git a/src/main/java/org/radarcns/util/RadarUtilitiesImpl.java b/src/main/java/org/radarcns/util/RadarUtilitiesImpl.java index c92bce42..a3b26468 100644 --- a/src/main/java/org/radarcns/util/RadarUtilitiesImpl.java +++ b/src/main/java/org/radarcns/util/RadarUtilitiesImpl.java @@ -48,7 +48,9 @@ public AggregateKey getWindowed(Windowed window) { @Override public AggregateKey getWindowedTuple(Windowed window) { - ObservationKey measurementKey = window.key().getKey(); + TemporaryPackageKey k = window.key(); + ObservationKey measurementKey = new ObservationKey(k.getProjectId(), k.getUserId(), + k.getSourceId()); return new AggregateKey(measurementKey.getProjectId(), measurementKey.getUserId(), measurementKey.getSourceId(), window.window().start(), window.window().end()); } From 8ff0ea1dcf89b9570bcb912c122cdcd4869f47da Mon Sep 17 00:00:00 2001 From: Joris Borgdorff Date: Wed, 11 Oct 2017 16:58:18 +0200 Subject: [PATCH 06/23] Use mapValues in phone usage stream to avoid repartitioning --- src/main/java/org/radarcns/stream/phone/PhoneUsageStream.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/radarcns/stream/phone/PhoneUsageStream.java b/src/main/java/org/radarcns/stream/phone/PhoneUsageStream.java index 7c4752b8..69594da1 100644 --- a/src/main/java/org/radarcns/stream/phone/PhoneUsageStream.java +++ b/src/main/java/org/radarcns/stream/phone/PhoneUsageStream.java @@ -53,13 +53,13 @@ public PhoneUsageStream(Collection definitions, int numThread, protected KStream implementStream(StreamDefinition definition, @Nonnull KStream kstream) { return kstream - .map((key, value) -> { + .mapValues(value -> { String packageName = value.getPackageName(); PlayStoreLookup.AppCategory category = playStoreLookup.lookupCategory(packageName); logger.info("Looked up {}: {}", packageName, category.getCategoryName()); value.setCategoryName(category.getCategoryName()); value.setCategoryNameFetchTime(category.getFetchTimeStamp()); - return pair(key, value); + return value; }); } } From e489604017ea5f2c158ca7b733efc6062218be58 Mon Sep 17 00:00:00 2001 From: Joris Borgdorff Date: Wed, 11 Oct 2017 17:05:29 +0200 Subject: [PATCH 07/23] Fixed code style and settings --- src/integrationTest/resources/org/radarcns/kafka/radar.yml | 4 ++-- .../java/org/radarcns/stream/phone/PhoneUsageStream.java | 5 +---- src/main/java/org/radarcns/util/RadarUtilitiesImpl.java | 6 +++--- 3 files changed, 6 insertions(+), 9 deletions(-) diff --git a/src/integrationTest/resources/org/radarcns/kafka/radar.yml b/src/integrationTest/resources/org/radarcns/kafka/radar.yml index a5f91c5b..fc6ed194 100644 --- a/src/integrationTest/resources/org/radarcns/kafka/radar.yml +++ b/src/integrationTest/resources/org/radarcns/kafka/radar.yml @@ -21,9 +21,9 @@ broker: #Kafka internal parameters stream_properties: auto.commit.interval.ms: 1000 - session.timeout.ms: 4000 + session.timeout.ms: 10000 buffered.records.per.partition: 1 - request.timeout.ms: 70000 + request.timeout.ms: 11000 fetch.timeout.ms: 3000 batch.size: 1 metadata.max.age.ms: 10000 diff --git a/src/main/java/org/radarcns/stream/phone/PhoneUsageStream.java b/src/main/java/org/radarcns/stream/phone/PhoneUsageStream.java index 69594da1..74054c56 100644 --- a/src/main/java/org/radarcns/stream/phone/PhoneUsageStream.java +++ b/src/main/java/org/radarcns/stream/phone/PhoneUsageStream.java @@ -16,8 +16,6 @@ package org.radarcns.stream.phone; -import javax.annotation.Nonnull; -import org.apache.kafka.streams.KeyValue; import org.apache.kafka.streams.kstream.KStream; import org.radarcns.config.KafkaProperty; import org.radarcns.kafka.ObservationKey; @@ -28,10 +26,9 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import javax.annotation.Nonnull; import java.util.Collection; -import static org.apache.kafka.streams.KeyValue.pair; - public class PhoneUsageStream extends StreamWorker { private static final Logger logger = LoggerFactory.getLogger(PhoneUsageStream.class); diff --git a/src/main/java/org/radarcns/util/RadarUtilitiesImpl.java b/src/main/java/org/radarcns/util/RadarUtilitiesImpl.java index a3b26468..b5fa11c4 100644 --- a/src/main/java/org/radarcns/util/RadarUtilitiesImpl.java +++ b/src/main/java/org/radarcns/util/RadarUtilitiesImpl.java @@ -48,9 +48,9 @@ public AggregateKey getWindowed(Windowed window) { @Override public AggregateKey getWindowedTuple(Windowed window) { - TemporaryPackageKey k = window.key(); - ObservationKey measurementKey = new ObservationKey(k.getProjectId(), k.getUserId(), - k.getSourceId()); + TemporaryPackageKey temp = window.key(); + ObservationKey measurementKey = new ObservationKey(temp.getProjectId(), temp.getUserId(), + temp.getSourceId()); return new AggregateKey(measurementKey.getProjectId(), measurementKey.getUserId(), measurementKey.getSourceId(), window.window().start(), window.window().end()); } From abb3e9ffa3f8dd28190800df1ea1da9dd70fa7c9 Mon Sep 17 00:00:00 2001 From: Joris Borgdorff Date: Thu, 12 Oct 2017 09:16:43 +0200 Subject: [PATCH 08/23] Add build version to Stream ID --- build.gradle | 4 ++++ .../java/org/radarcns/config/ConfigRadar.java | 11 +++++++++++ .../config/RadarPropertyHandlerImpl.java | 13 +++++++++++++ .../org/radarcns/stream/StreamWorker.java | 9 ++++++--- .../stream/empatica/E4AccelerationStream.java | 5 +++-- .../stream/empatica/E4BatteryLevelStream.java | 6 +++--- .../empatica/E4BloodVolumePulseStream.java | 6 +++--- .../E4ElectroDermalActivityStream.java | 6 +++--- .../stream/empatica/E4HeartRateStream.java | 6 +++--- .../empatica/E4InterBeatIntervalStream.java | 6 +++--- .../stream/empatica/E4StreamMaster.java | 19 +++++++++---------- .../stream/empatica/E4TemperatureStream.java | 6 +++--- .../stream/phone/PhoneAccelerationStream.java | 6 +++--- .../stream/phone/PhoneBatteryStream.java | 6 +++--- .../stream/phone/PhoneStreamMaster.java | 13 ++++++------- .../phone/PhoneUsageAggregationStream.java | 6 +++--- .../stream/phone/PhoneUsageStream.java | 6 +++--- src/main/resources/build.properties | 1 + 18 files changed, 83 insertions(+), 52 deletions(-) create mode 100644 src/main/resources/build.properties diff --git a/build.gradle b/build.gradle index e1656985..866ccac3 100644 --- a/build.gradle +++ b/build.gradle @@ -298,6 +298,10 @@ ext.sharedManifest = manifest { // Packaging // //---------------------------------------------------------------------------// +processResources { + expand(version: version) +} + jar { manifest { from sharedManifest diff --git a/src/main/java/org/radarcns/config/ConfigRadar.java b/src/main/java/org/radarcns/config/ConfigRadar.java index 1b58727f..cc3c4dca 100644 --- a/src/main/java/org/radarcns/config/ConfigRadar.java +++ b/src/main/java/org/radarcns/config/ConfigRadar.java @@ -53,6 +53,9 @@ public class ConfigRadar { @JsonProperty("stream_properties") private Map streamProperties = new HashMap<>(); + @JsonProperty("build_version") + private String buildVersion; + public Date getReleased() { return released; } @@ -243,4 +246,12 @@ public void setExtras(Map extras) { public String toString() { return new YamlConfigLoader().prettyString(this); } + + public String getBuildVersion() { + return buildVersion; + } + + public void setBuildVersion(String buildVersion) { + this.buildVersion = buildVersion; + } } diff --git a/src/main/java/org/radarcns/config/RadarPropertyHandlerImpl.java b/src/main/java/org/radarcns/config/RadarPropertyHandlerImpl.java index 630b51f0..d6ad364a 100644 --- a/src/main/java/org/radarcns/config/RadarPropertyHandlerImpl.java +++ b/src/main/java/org/radarcns/config/RadarPropertyHandlerImpl.java @@ -23,8 +23,10 @@ import java.io.File; import java.io.IOException; +import java.io.InputStream; import java.net.URISyntaxException; import java.net.URL; +import java.util.Properties; import static org.radarcns.util.Strings.isNullOrEmpty; @@ -76,6 +78,17 @@ public void load(String pathFile) throws IOException { } properties = new YamlConfigLoader().load(file, ConfigRadar.class); + + Properties buildProperties = new Properties(); + try (InputStream in = getClass().getResourceAsStream("/build.properties")) { + if (in != null) { + buildProperties.load(in); + } + } + String version = buildProperties.getProperty("version"); + if (version != null) { + properties.setBuildVersion(version); + } } private File getDefaultFile() throws IOException { diff --git a/src/main/java/org/radarcns/stream/StreamWorker.java b/src/main/java/org/radarcns/stream/StreamWorker.java index 24c8aa41..17f9061b 100644 --- a/src/main/java/org/radarcns/stream/StreamWorker.java +++ b/src/main/java/org/radarcns/stream/StreamWorker.java @@ -24,6 +24,7 @@ import org.apache.kafka.streams.kstream.KStreamBuilder; import org.apache.kafka.streams.kstream.TimeWindows; import org.radarcns.config.KafkaProperty; +import org.radarcns.config.RadarPropertyHandler; import org.radarcns.kafka.AggregateKey; import org.radarcns.kafka.ObservationKey; import org.radarcns.stream.aggregator.DoubleAggregation; @@ -61,6 +62,7 @@ public abstract class StreamWorker streamDefinitions; + private final String buildVersion; private Collection> monitors; private final KafkaProperty kafkaProperty; @@ -69,7 +71,7 @@ public abstract class StreamWorker streams; public StreamWorker(@Nonnull Collection streamDefinitions, - int numThreads, @Nonnull StreamMaster aggregator, KafkaProperty kafkaProperty, + int numThreads, @Nonnull StreamMaster aggregator, RadarPropertyHandler properties, Logger monitorLog) { if (numThreads < 1) { throw new IllegalStateException( @@ -78,7 +80,8 @@ public StreamWorker(@Nonnull Collection streamDefinitions, this.master = aggregator; this.streamDefinitions = streamDefinitions; this.numThreads = numThreads; - this.kafkaProperty = kafkaProperty; + this.buildVersion = properties.getRadarProperties().getBuildVersion(); + this.kafkaProperty = properties.getKafkaProperties(); this.streams = null; this.monitors = null; this.monitorLog = monitorLog; @@ -118,7 +121,7 @@ protected KeyValue, KafkaStreams> createBuilder(StreamDefinit * @return Properties for a Kafka Stream */ protected Properties getStreamProperties(@Nonnull StreamDefinition definition) { - String localClientId = getClass().getName(); + String localClientId = getClass().getName() + "-" + buildVersion; TimeWindows window = definition.getTimeWindows(); if (window != null) { localClientId += '-' + window.sizeMs + '-' + window.advanceMs; diff --git a/src/main/java/org/radarcns/stream/empatica/E4AccelerationStream.java b/src/main/java/org/radarcns/stream/empatica/E4AccelerationStream.java index cb18af70..7540fbb2 100644 --- a/src/main/java/org/radarcns/stream/empatica/E4AccelerationStream.java +++ b/src/main/java/org/radarcns/stream/empatica/E4AccelerationStream.java @@ -18,6 +18,7 @@ import org.apache.kafka.streams.kstream.KStream; import org.radarcns.config.KafkaProperty; +import org.radarcns.config.RadarPropertyHandler; import org.radarcns.kafka.AggregateKey; import org.radarcns.kafka.ObservationKey; import org.radarcns.passive.empatica.EmpaticaE4Acceleration; @@ -40,8 +41,8 @@ public class E4AccelerationStream extends StreamWorker definitions, int numThread, - StreamMaster master, KafkaProperty kafkaProperties) { - super(definitions, numThread, master, kafkaProperties, logger); + StreamMaster master, RadarPropertyHandler properties) { + super(definitions, numThread, master, properties, logger); } @Override diff --git a/src/main/java/org/radarcns/stream/empatica/E4BatteryLevelStream.java b/src/main/java/org/radarcns/stream/empatica/E4BatteryLevelStream.java index 88b8b3d2..727489b4 100644 --- a/src/main/java/org/radarcns/stream/empatica/E4BatteryLevelStream.java +++ b/src/main/java/org/radarcns/stream/empatica/E4BatteryLevelStream.java @@ -17,7 +17,7 @@ package org.radarcns.stream.empatica; import org.apache.kafka.streams.kstream.KStream; -import org.radarcns.config.KafkaProperty; +import org.radarcns.config.RadarPropertyHandler; import org.radarcns.kafka.AggregateKey; import org.radarcns.kafka.ObservationKey; import org.radarcns.passive.empatica.EmpaticaE4BatteryLevel; @@ -38,8 +38,8 @@ public class E4BatteryLevelStream extends StreamWorker definitions, int numThread, - StreamMaster master, KafkaProperty kafkaProperties) { - super(definitions, numThread, master, kafkaProperties, logger); + StreamMaster master, RadarPropertyHandler properties) { + super(definitions, numThread, master, properties, logger); } @Override diff --git a/src/main/java/org/radarcns/stream/empatica/E4BloodVolumePulseStream.java b/src/main/java/org/radarcns/stream/empatica/E4BloodVolumePulseStream.java index d7d4d152..81af643d 100644 --- a/src/main/java/org/radarcns/stream/empatica/E4BloodVolumePulseStream.java +++ b/src/main/java/org/radarcns/stream/empatica/E4BloodVolumePulseStream.java @@ -17,7 +17,7 @@ package org.radarcns.stream.empatica; import org.apache.kafka.streams.kstream.KStream; -import org.radarcns.config.KafkaProperty; +import org.radarcns.config.RadarPropertyHandler; import org.radarcns.kafka.AggregateKey; import org.radarcns.kafka.ObservationKey; import org.radarcns.passive.empatica.EmpaticaE4BloodVolumePulse; @@ -39,8 +39,8 @@ public class E4BloodVolumePulseStream extends private static final Logger logger = LoggerFactory.getLogger(E4BloodVolumePulseStream.class); public E4BloodVolumePulseStream(Collection definitions, int numThread, - StreamMaster master, KafkaProperty kafkaProperties) { - super(definitions, numThread, master, kafkaProperties, logger); + StreamMaster master, RadarPropertyHandler properties) { + super(definitions, numThread, master, properties, logger); } @Override diff --git a/src/main/java/org/radarcns/stream/empatica/E4ElectroDermalActivityStream.java b/src/main/java/org/radarcns/stream/empatica/E4ElectroDermalActivityStream.java index 18671be9..5aeca80a 100644 --- a/src/main/java/org/radarcns/stream/empatica/E4ElectroDermalActivityStream.java +++ b/src/main/java/org/radarcns/stream/empatica/E4ElectroDermalActivityStream.java @@ -17,7 +17,7 @@ package org.radarcns.stream.empatica; import org.apache.kafka.streams.kstream.KStream; -import org.radarcns.config.KafkaProperty; +import org.radarcns.config.RadarPropertyHandler; import org.radarcns.kafka.AggregateKey; import org.radarcns.kafka.ObservationKey; import org.radarcns.passive.empatica.EmpaticaE4ElectroDermalActivity; @@ -40,8 +40,8 @@ public class E4ElectroDermalActivityStream extends E4ElectroDermalActivityStream.class); public E4ElectroDermalActivityStream(Collection definitions, int numThread, - StreamMaster master, KafkaProperty kafkaProperties) { - super(definitions, numThread, master, kafkaProperties, logger); + StreamMaster master, RadarPropertyHandler properties) { + super(definitions, numThread, master, properties, logger); } @Override diff --git a/src/main/java/org/radarcns/stream/empatica/E4HeartRateStream.java b/src/main/java/org/radarcns/stream/empatica/E4HeartRateStream.java index 36b5d71c..50f01790 100644 --- a/src/main/java/org/radarcns/stream/empatica/E4HeartRateStream.java +++ b/src/main/java/org/radarcns/stream/empatica/E4HeartRateStream.java @@ -17,7 +17,7 @@ package org.radarcns.stream.empatica; import org.apache.kafka.streams.kstream.KStream; -import org.radarcns.config.KafkaProperty; +import org.radarcns.config.RadarPropertyHandler; import org.radarcns.kafka.AggregateKey; import org.radarcns.kafka.ObservationKey; import org.radarcns.passive.empatica.EmpaticaE4InterBeatInterval; @@ -38,8 +38,8 @@ public class E4HeartRateStream extends StreamWorker definitions, int numThread, - StreamMaster master, KafkaProperty kafkaProperties) { - super(definitions, numThread, master, kafkaProperties, logger); + StreamMaster master, RadarPropertyHandler properties) { + super(definitions, numThread, master, properties, logger); } protected KStream implementStream( diff --git a/src/main/java/org/radarcns/stream/empatica/E4InterBeatIntervalStream.java b/src/main/java/org/radarcns/stream/empatica/E4InterBeatIntervalStream.java index 757547e6..66ff8701 100644 --- a/src/main/java/org/radarcns/stream/empatica/E4InterBeatIntervalStream.java +++ b/src/main/java/org/radarcns/stream/empatica/E4InterBeatIntervalStream.java @@ -17,7 +17,7 @@ package org.radarcns.stream.empatica; import org.apache.kafka.streams.kstream.KStream; -import org.radarcns.config.KafkaProperty; +import org.radarcns.config.RadarPropertyHandler; import org.radarcns.kafka.AggregateKey; import org.radarcns.kafka.ObservationKey; import org.radarcns.passive.empatica.EmpaticaE4InterBeatInterval; @@ -39,8 +39,8 @@ public class E4InterBeatIntervalStream extends private static final Logger logger = LoggerFactory.getLogger(E4InterBeatIntervalStream.class); public E4InterBeatIntervalStream(Collection definitions, int numThread, - StreamMaster master, KafkaProperty kafkaProperties) { - super(definitions, numThread, master, kafkaProperties, logger); + StreamMaster master, RadarPropertyHandler properties) { + super(definitions, numThread, master, properties, logger); } @Override diff --git a/src/main/java/org/radarcns/stream/empatica/E4StreamMaster.java b/src/main/java/org/radarcns/stream/empatica/E4StreamMaster.java index 7152e14f..335ba294 100644 --- a/src/main/java/org/radarcns/stream/empatica/E4StreamMaster.java +++ b/src/main/java/org/radarcns/stream/empatica/E4StreamMaster.java @@ -16,14 +16,14 @@ package org.radarcns.stream.empatica; -import java.util.List; -import org.radarcns.config.KafkaProperty; import org.radarcns.config.RadarPropertyHandler; import org.radarcns.stream.StreamGroup; import org.radarcns.stream.StreamMaster; import org.radarcns.stream.StreamWorker; import org.radarcns.util.RadarSingletonFactory; +import java.util.List; + /** * Singleton StreamMaster for Empatica E4. * @see StreamMaster @@ -37,21 +37,20 @@ protected StreamGroup getStreamGroup() { @Override protected void createWorkers(List> list, StreamMaster master) { RadarPropertyHandler propertyHandler = RadarSingletonFactory.getRadarPropertyHandler(); - KafkaProperty kafkaProperty = propertyHandler.getKafkaProperties(); E4Streams defs = E4Streams.getInstance(); list.add(new E4AccelerationStream( - defs.getAccelerationStream(), highPriority(), master, kafkaProperty)); + defs.getAccelerationStream(), highPriority(), master, propertyHandler)); list.add(new E4BatteryLevelStream( - defs.getBatteryLevelStream(), lowPriority(), master, kafkaProperty)); + defs.getBatteryLevelStream(), lowPriority(), master, propertyHandler)); list.add(new E4BloodVolumePulseStream( - defs.getBloodVolumePulseStream(), highPriority(), master, kafkaProperty)); + defs.getBloodVolumePulseStream(), highPriority(), master, propertyHandler)); list.add(new E4ElectroDermalActivityStream( - defs.getElectroDermalActivityStream(), normalPriority(), master, kafkaProperty)); + defs.getElectroDermalActivityStream(), normalPriority(), master, propertyHandler)); list.add(new E4HeartRateStream( - defs.getHeartRateStream(), lowPriority(), master, kafkaProperty)); + defs.getHeartRateStream(), lowPriority(), master, propertyHandler)); list.add(new E4InterBeatIntervalStream( - defs.getInterBeatIntervalStream(), lowPriority(), master, kafkaProperty)); + defs.getInterBeatIntervalStream(), lowPriority(), master, propertyHandler)); list.add(new E4TemperatureStream( - defs.getTemperatureStream(), lowPriority(), master, kafkaProperty)); + defs.getTemperatureStream(), lowPriority(), master, propertyHandler)); } } diff --git a/src/main/java/org/radarcns/stream/empatica/E4TemperatureStream.java b/src/main/java/org/radarcns/stream/empatica/E4TemperatureStream.java index 015e407c..5662adb0 100644 --- a/src/main/java/org/radarcns/stream/empatica/E4TemperatureStream.java +++ b/src/main/java/org/radarcns/stream/empatica/E4TemperatureStream.java @@ -17,7 +17,7 @@ package org.radarcns.stream.empatica; import org.apache.kafka.streams.kstream.KStream; -import org.radarcns.config.KafkaProperty; +import org.radarcns.config.RadarPropertyHandler; import org.radarcns.kafka.AggregateKey; import org.radarcns.kafka.ObservationKey; import org.radarcns.passive.empatica.EmpaticaE4Temperature; @@ -38,8 +38,8 @@ public class E4TemperatureStream extends StreamWorker definitions, int numThread, - StreamMaster master, KafkaProperty kafkaProperties) { - super(definitions, numThread, master, kafkaProperties, logger); + StreamMaster master, RadarPropertyHandler properties) { + super(definitions, numThread, master, properties, logger); } @Override diff --git a/src/main/java/org/radarcns/stream/phone/PhoneAccelerationStream.java b/src/main/java/org/radarcns/stream/phone/PhoneAccelerationStream.java index d705ba49..0dccb43b 100644 --- a/src/main/java/org/radarcns/stream/phone/PhoneAccelerationStream.java +++ b/src/main/java/org/radarcns/stream/phone/PhoneAccelerationStream.java @@ -1,7 +1,7 @@ package org.radarcns.stream.phone; import org.apache.kafka.streams.kstream.KStream; -import org.radarcns.config.KafkaProperty; +import org.radarcns.config.RadarPropertyHandler; import org.radarcns.kafka.AggregateKey; import org.radarcns.kafka.ObservationKey; import org.radarcns.passive.phone.PhoneAcceleration; @@ -21,8 +21,8 @@ public class PhoneAccelerationStream extends StreamWorker definitions, int numThread, - StreamMaster master, KafkaProperty kafkaProperties) { - super(definitions, numThread, master, kafkaProperties, logger); + StreamMaster master, RadarPropertyHandler properties) { + super(definitions, numThread, master, properties, logger); } @Override diff --git a/src/main/java/org/radarcns/stream/phone/PhoneBatteryStream.java b/src/main/java/org/radarcns/stream/phone/PhoneBatteryStream.java index 545de756..43198faf 100644 --- a/src/main/java/org/radarcns/stream/phone/PhoneBatteryStream.java +++ b/src/main/java/org/radarcns/stream/phone/PhoneBatteryStream.java @@ -1,7 +1,7 @@ package org.radarcns.stream.phone; import org.apache.kafka.streams.kstream.KStream; -import org.radarcns.config.KafkaProperty; +import org.radarcns.config.RadarPropertyHandler; import org.radarcns.kafka.AggregateKey; import org.radarcns.kafka.ObservationKey; import org.radarcns.passive.phone.PhoneBatteryLevel; @@ -19,8 +19,8 @@ public class PhoneBatteryStream extends StreamWorker definitions, int numThread, - StreamMaster master, KafkaProperty kafkaProperties) { - super(definitions, numThread, master, kafkaProperties, logger); + StreamMaster master, RadarPropertyHandler properties) { + super(definitions, numThread, master, properties, logger); } @Override diff --git a/src/main/java/org/radarcns/stream/phone/PhoneStreamMaster.java b/src/main/java/org/radarcns/stream/phone/PhoneStreamMaster.java index 248e1d97..48e260c5 100644 --- a/src/main/java/org/radarcns/stream/phone/PhoneStreamMaster.java +++ b/src/main/java/org/radarcns/stream/phone/PhoneStreamMaster.java @@ -16,14 +16,14 @@ package org.radarcns.stream.phone; -import java.util.List; -import org.radarcns.config.KafkaProperty; import org.radarcns.config.RadarPropertyHandler; import org.radarcns.stream.StreamGroup; import org.radarcns.stream.StreamMaster; import org.radarcns.stream.StreamWorker; import org.radarcns.util.RadarSingletonFactory; +import java.util.List; + /** * Singleton StreamMaster for Phone * @see StreamMaster @@ -37,15 +37,14 @@ protected StreamGroup getStreamGroup() { @Override protected void createWorkers(List> list, StreamMaster master) { RadarPropertyHandler propertyHandler = RadarSingletonFactory.getRadarPropertyHandler(); - KafkaProperty kafkaProperty = propertyHandler.getKafkaProperties(); PhoneStreams defs = PhoneStreams.getInstance(); list.add(new PhoneUsageStream( - defs.getUsageStream(), lowPriority(), master, kafkaProperty)); + defs.getUsageStream(), lowPriority(), master, propertyHandler)); list.add(new PhoneUsageAggregationStream( - defs.getUsageEventAggregationStream(), lowPriority(), master, kafkaProperty)); + defs.getUsageEventAggregationStream(), lowPriority(), master, propertyHandler)); list.add(new PhoneBatteryStream( - defs.getBatteryStream(), lowPriority(), master, kafkaProperty)); + defs.getBatteryStream(), lowPriority(), master, propertyHandler)); list.add(new PhoneAccelerationStream( - defs.getAccelerationStream(), normalPriority(), master, kafkaProperty)); + defs.getAccelerationStream(), normalPriority(), master, propertyHandler)); } } diff --git a/src/main/java/org/radarcns/stream/phone/PhoneUsageAggregationStream.java b/src/main/java/org/radarcns/stream/phone/PhoneUsageAggregationStream.java index 1c2edd88..c12d7acd 100644 --- a/src/main/java/org/radarcns/stream/phone/PhoneUsageAggregationStream.java +++ b/src/main/java/org/radarcns/stream/phone/PhoneUsageAggregationStream.java @@ -2,7 +2,7 @@ import org.apache.kafka.streams.kstream.KStream; import org.radarcns.aggregator.PhoneUsageAggregator; -import org.radarcns.config.KafkaProperty; +import org.radarcns.config.RadarPropertyHandler; import org.radarcns.kafka.AggregateKey; import org.radarcns.kafka.ObservationKey; import org.radarcns.passive.phone.PhoneUsageEvent; @@ -23,8 +23,8 @@ public class PhoneUsageAggregationStream extends StreamWorker definitions, int numThread, - StreamMaster master, KafkaProperty kafkaProperties) { - super(definitions, numThread, master, kafkaProperties, logger); + StreamMaster master, RadarPropertyHandler properties) { + super(definitions, numThread, master, properties, logger); } @Override diff --git a/src/main/java/org/radarcns/stream/phone/PhoneUsageStream.java b/src/main/java/org/radarcns/stream/phone/PhoneUsageStream.java index 74054c56..09592c8d 100644 --- a/src/main/java/org/radarcns/stream/phone/PhoneUsageStream.java +++ b/src/main/java/org/radarcns/stream/phone/PhoneUsageStream.java @@ -17,7 +17,7 @@ package org.radarcns.stream.phone; import org.apache.kafka.streams.kstream.KStream; -import org.radarcns.config.KafkaProperty; +import org.radarcns.config.RadarPropertyHandler; import org.radarcns.kafka.ObservationKey; import org.radarcns.passive.phone.PhoneUsageEvent; import org.radarcns.stream.StreamDefinition; @@ -41,8 +41,8 @@ public class PhoneUsageStream extends StreamWorker definitions, int numThread, - StreamMaster master, KafkaProperty kafkaProperties) { - super(definitions, numThread, master, kafkaProperties, logger); + StreamMaster master, RadarPropertyHandler properties) { + super(definitions, numThread, master, properties, logger); playStoreLookup = new PlayStoreLookup(CACHE_TIMEOUT, MAX_CACHE_SIZE); } diff --git a/src/main/resources/build.properties b/src/main/resources/build.properties new file mode 100644 index 00000000..75ac2d20 --- /dev/null +++ b/src/main/resources/build.properties @@ -0,0 +1 @@ +version=${version} \ No newline at end of file From b45186e735535051dace2e964015b9660780ebf5 Mon Sep 17 00:00:00 2001 From: Joris Borgdorff Date: Thu, 12 Oct 2017 09:19:27 +0200 Subject: [PATCH 09/23] Merged PhoneStreamTest and DirectProducerTest --- .../integration/DirectProducerTest.java | 86 ------------------- .../radarcns/integration/PhoneStreamTest.java | 37 ++++++-- .../stream/empatica/E4AccelerationStream.java | 1 - 3 files changed, 31 insertions(+), 93 deletions(-) delete mode 100644 src/integrationTest/java/org/radarcns/integration/DirectProducerTest.java diff --git a/src/integrationTest/java/org/radarcns/integration/DirectProducerTest.java b/src/integrationTest/java/org/radarcns/integration/DirectProducerTest.java deleted file mode 100644 index 6907e7b6..00000000 --- a/src/integrationTest/java/org/radarcns/integration/DirectProducerTest.java +++ /dev/null @@ -1,86 +0,0 @@ -/* - * Copyright 2017 King's College London and The Hyve - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.radarcns.integration; - -import java.io.File; -import java.io.IOException; -import java.util.Arrays; - -import io.confluent.kafka.serializers.KafkaAvroDeserializer; -import org.apache.commons.cli.ParseException; -import org.junit.After; -import org.junit.Before; -import org.junit.Rule; -import org.junit.Test; -import org.junit.rules.ExpectedException; -import org.radarcns.RadarBackend; -import org.radarcns.config.RadarBackendOptions; -import org.radarcns.config.RadarPropertyHandler; -import org.radarcns.config.YamlConfigLoader; -import org.radarcns.mock.MockProducer; -import org.radarcns.mock.config.BasicMockConfig; -import org.radarcns.stream.empatica.E4StreamMaster; -import org.radarcns.stream.phone.PhoneStreamMaster; -import org.radarcns.util.RadarSingletonFactory; - -public class DirectProducerTest { - @Rule - public ExpectedException exception = ExpectedException.none(); - private RadarBackend backend; - - @Before - public void setUp() throws IOException, ParseException, InterruptedException { - String propertiesPath = "src/integrationTest/resources/org/radarcns/kafka/radar.yml"; - RadarPropertyHandler propHandler = RadarSingletonFactory.getRadarPropertyHandler(); - if (!propHandler.isLoaded()) { - propHandler.load(propertiesPath); - } - - String[] args = {"-c", propertiesPath, "stream"}; - - RadarBackendOptions opts = RadarBackendOptions.parse(args); - propHandler.getRadarProperties().setStreamMasters(Arrays.asList( - E4StreamMaster.class.getName(), - PhoneStreamMaster.class.getName() - )); - backend = new RadarBackend(opts, propHandler); - backend.start(); - } - - @After - public void tearDown() throws IOException, InterruptedException { - backend.shutdown(); - } - - @Test(timeout = 300_000L) - public void testDirect() throws Exception { - File file = new File(getClass().getResource("/mock_devices.yml").getFile()); - BasicMockConfig mockConfig = new YamlConfigLoader().load(file, BasicMockConfig.class); - - MockProducer mockProducer = new MockProducer(mockConfig); - mockProducer.start(); - Thread.sleep(mockConfig.getDuration()); - mockProducer.shutdown(); - - String clientId = "someclinet"; - E4AggregatedAccelerationMonitor monitor = new E4AggregatedAccelerationMonitor( - RadarSingletonFactory.getRadarPropertyHandler(), - "android_empatica_e4_acceleration_10sec", clientId); - monitor.setPollTimeout(280_000L); - monitor.start(); - } -} diff --git a/src/integrationTest/java/org/radarcns/integration/PhoneStreamTest.java b/src/integrationTest/java/org/radarcns/integration/PhoneStreamTest.java index 83ead2fe..b2ead5f9 100644 --- a/src/integrationTest/java/org/radarcns/integration/PhoneStreamTest.java +++ b/src/integrationTest/java/org/radarcns/integration/PhoneStreamTest.java @@ -30,13 +30,17 @@ import org.radarcns.config.ConfigRadar; import org.radarcns.config.RadarBackendOptions; import org.radarcns.config.RadarPropertyHandler; +import org.radarcns.config.YamlConfigLoader; import org.radarcns.kafka.ObservationKey; +import org.radarcns.mock.MockProducer; +import org.radarcns.mock.config.BasicMockConfig; import org.radarcns.monitor.AbstractKafkaMonitor; import org.radarcns.monitor.KafkaMonitor; import org.radarcns.passive.phone.PhoneUsageEvent; import org.radarcns.passive.phone.UsageEventType; import org.radarcns.producer.KafkaTopicSender; import org.radarcns.producer.direct.DirectSender; +import org.radarcns.stream.empatica.E4StreamMaster; import org.radarcns.stream.phone.PhoneStreamMaster; import org.radarcns.topic.AvroTopic; import org.radarcns.util.RadarSingletonFactory; @@ -44,6 +48,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.io.File; import java.io.IOException; import java.util.Arrays; import java.util.Collections; @@ -101,8 +106,9 @@ public void setUp() throws IOException, ParseException, InterruptedException { String[] args = {"-c", propertiesPath, "stream"}; RadarBackendOptions opts = RadarBackendOptions.parse(args); - propHandler.getRadarProperties().setStreamMasters( - Collections.singletonList(PhoneStreamMaster.class.getName())); + propHandler.getRadarProperties().setStreamMasters(Arrays.asList( + PhoneStreamMaster.class.getName(), + E4StreamMaster.class.getName())); Map streamProps = new HashMap<>(); streamProps.put(AUTO_COMMIT_INTERVAL_MS_CONFIG, String.valueOf(1_000)); @@ -157,12 +163,31 @@ public void testDirect() throws Exception { } sender.close(); + + File file = new File(getClass().getResource("/mock_devices.yml").getFile()); + BasicMockConfig mockConfig = new YamlConfigLoader().load(file, BasicMockConfig.class); + + MockProducer mockProducer = new MockProducer(mockConfig); + mockProducer.start(); + Thread.sleep(mockConfig.getDuration()); + mockProducer.shutdown(); + consumePhone(offset); consumeAggregated(offset / 2); + consumeE4(); + } + + private void consumeE4() throws IOException { + String clientId = "consumeE4"; + E4AggregatedAccelerationMonitor monitor = new E4AggregatedAccelerationMonitor( + RadarSingletonFactory.getRadarPropertyHandler(), + "android_empatica_e4_acceleration_10sec", clientId); + monitor.setPollTimeout(280_000L); + monitor.start(); } private void consumePhone(final long numRecordsExpected) throws IOException, InterruptedException { - String clientId = "someclinet"; + String clientId = "consumePhone"; KafkaMonitor monitor = new PhoneOutputMonitor(RadarSingletonFactory.getRadarPropertyHandler(), clientId, numRecordsExpected); monitor.setPollTimeout(280_000L); @@ -170,7 +195,7 @@ private void consumePhone(final long numRecordsExpected) throws IOException, Int } private void consumeAggregated(final long numRecordsExpected) throws IOException, InterruptedException { - String clientId = "someclient"; + String clientId = "consumeAggregated"; KafkaMonitor monitor = new PhoneAggregateMonitor(RadarSingletonFactory.getRadarPropertyHandler(), clientId, numRecordsExpected); monitor.setPollTimeout(280_000L); @@ -183,7 +208,7 @@ private static class PhoneOutputMonitor extends AbstractKafkaMonitor Date: Thu, 12 Oct 2017 09:50:16 +0200 Subject: [PATCH 10/23] Fixed E4 integration test --- .../E4AggregatedAccelerationMonitor.java | 26 ++++--------------- 1 file changed, 5 insertions(+), 21 deletions(-) diff --git a/src/integrationTest/java/org/radarcns/integration/E4AggregatedAccelerationMonitor.java b/src/integrationTest/java/org/radarcns/integration/E4AggregatedAccelerationMonitor.java index 502d9710..d805678e 100644 --- a/src/integrationTest/java/org/radarcns/integration/E4AggregatedAccelerationMonitor.java +++ b/src/integrationTest/java/org/radarcns/integration/E4AggregatedAccelerationMonitor.java @@ -73,36 +73,20 @@ protected void evaluateRecords(ConsumerRecords rec logger.error("Failed to process record {} without a key.", record); return; } - ObservationKey measurementKey; Schema keySchema = key.getSchema(); if (keySchema.getField("userId") != null && keySchema.getField("sourceId") != null) { - measurementKey = new ObservationKey("test", key.get("userId").toString(), - key.get("sourceId").toString()); - assertNotNull(measurementKey); + assertNotNull(key.get("userId")); + assertNotNull(key.get("sourceId"); } else { logger.error("Failed to process record {} with wrong key type {}.", record, key.getSchema()); return; } GenericRecord value = record.value(); - Schema recordSchema = value.getSchema(); - - int minFieldId = recordSchema.getField("min").pos(); - - GenericData.Array min = (GenericData.Array) value.get(minFieldId); - assertNotNull(min); - assertEquals(15.0d, (double)min.get(0), 0.0); - assertEquals(-15.0d, (double)min.get(1), 0.0); - assertEquals(64.0d, (double)min.get(2), 0.0); - - int maxFieldId = recordSchema.getField("max").pos(); - - GenericData.Array max = (GenericData.Array) value.get(maxFieldId); - assertNotNull(max); - assertEquals(15.0d, (double)max.get(0), 0.0); - assertEquals(Double.MIN_VALUE, (double)max.get(1), 0.0d); - assertEquals(64.0d, (double)max.get(2), 0.0); + GenericData.Array count = (GenericData.Array) value.get("count"); + logger.info("Received [{}, {}, {}] E4 messages", + count.get(0), count.get(1), count.get(2)); } } } From 6363d3b9b77c3d6446330e4d96548d8c04ae1cca Mon Sep 17 00:00:00 2001 From: Joris Borgdorff Date: Thu, 12 Oct 2017 09:51:18 +0200 Subject: [PATCH 11/23] Fixed code error --- .../radarcns/integration/E4AggregatedAccelerationMonitor.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/integrationTest/java/org/radarcns/integration/E4AggregatedAccelerationMonitor.java b/src/integrationTest/java/org/radarcns/integration/E4AggregatedAccelerationMonitor.java index d805678e..b6ddf55b 100644 --- a/src/integrationTest/java/org/radarcns/integration/E4AggregatedAccelerationMonitor.java +++ b/src/integrationTest/java/org/radarcns/integration/E4AggregatedAccelerationMonitor.java @@ -77,7 +77,7 @@ protected void evaluateRecords(ConsumerRecords rec if (keySchema.getField("userId") != null && keySchema.getField("sourceId") != null) { assertNotNull(key.get("userId")); - assertNotNull(key.get("sourceId"); + assertNotNull(key.get("sourceId")); } else { logger.error("Failed to process record {} with wrong key type {}.", record, key.getSchema()); From 458e412082173704a8edc46d9c6f004202280d68 Mon Sep 17 00:00:00 2001 From: Joris Borgdorff Date: Thu, 12 Oct 2017 09:51:54 +0200 Subject: [PATCH 12/23] Updated imports --- .../E4AggregatedAccelerationMonitor.java | 19 +++++++------------ .../radarcns/integration/PhoneStreamTest.java | 1 - 2 files changed, 7 insertions(+), 13 deletions(-) diff --git a/src/integrationTest/java/org/radarcns/integration/E4AggregatedAccelerationMonitor.java b/src/integrationTest/java/org/radarcns/integration/E4AggregatedAccelerationMonitor.java index b6ddf55b..d9395bb5 100644 --- a/src/integrationTest/java/org/radarcns/integration/E4AggregatedAccelerationMonitor.java +++ b/src/integrationTest/java/org/radarcns/integration/E4AggregatedAccelerationMonitor.java @@ -16,15 +16,6 @@ package org.radarcns.integration; -import static org.apache.kafka.clients.consumer.ConsumerConfig.HEARTBEAT_INTERVAL_MS_CONFIG; -import static org.apache.kafka.clients.consumer.ConsumerConfig.SESSION_TIMEOUT_MS_CONFIG; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertTrue; - -import java.io.IOException; -import java.util.Collections; -import java.util.Properties; import org.apache.avro.Schema; import org.apache.avro.generic.GenericData; import org.apache.avro.generic.GenericRecord; @@ -32,13 +23,17 @@ import org.apache.kafka.clients.consumer.ConsumerRecord; import org.apache.kafka.clients.consumer.ConsumerRecords; import org.radarcns.config.RadarPropertyHandler; -import org.radarcns.config.RadarPropertyHandlerImpl; -import org.radarcns.kafka.ObservationKey; import org.radarcns.monitor.AbstractKafkaMonitor; -import org.radarcns.util.RadarSingletonFactory; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.io.IOException; +import java.util.Collections; +import java.util.Properties; + +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; + /** * Consumer for Aggregated Acceleration Stream */ diff --git a/src/integrationTest/java/org/radarcns/integration/PhoneStreamTest.java b/src/integrationTest/java/org/radarcns/integration/PhoneStreamTest.java index b2ead5f9..1e20b815 100644 --- a/src/integrationTest/java/org/radarcns/integration/PhoneStreamTest.java +++ b/src/integrationTest/java/org/radarcns/integration/PhoneStreamTest.java @@ -56,7 +56,6 @@ import java.util.List; import java.util.Map; import java.util.Properties; -import java.util.UUID; import static org.apache.kafka.clients.consumer.ConsumerConfig.AUTO_COMMIT_INTERVAL_MS_CONFIG; import static org.apache.kafka.clients.consumer.ConsumerConfig.HEARTBEAT_INTERVAL_MS_CONFIG; From 232f58f20379064f89fa998738da62cc53b4d566 Mon Sep 17 00:00:00 2001 From: Joris Borgdorff Date: Thu, 12 Oct 2017 10:30:09 +0200 Subject: [PATCH 13/23] Wait for streams to finish: a segfault is the result otherwise --- .../integration/E4AggregatedAccelerationMonitor.java | 8 ++++---- .../java/org/radarcns/integration/PhoneStreamTest.java | 2 +- .../resources/org/radarcns/kafka/radar.yml | 6 +++++- src/main/java/org/radarcns/stream/StreamMaster.java | 3 ++- 4 files changed, 12 insertions(+), 7 deletions(-) diff --git a/src/integrationTest/java/org/radarcns/integration/E4AggregatedAccelerationMonitor.java b/src/integrationTest/java/org/radarcns/integration/E4AggregatedAccelerationMonitor.java index d9395bb5..6eb9671d 100644 --- a/src/integrationTest/java/org/radarcns/integration/E4AggregatedAccelerationMonitor.java +++ b/src/integrationTest/java/org/radarcns/integration/E4AggregatedAccelerationMonitor.java @@ -56,10 +56,6 @@ protected void evaluateRecord(ConsumerRecord recor @Override protected void evaluateRecords(ConsumerRecords records) { - if (records.isEmpty()) { - shutdown(); - return; - } assertTrue(records.count() > 0); for (ConsumerRecord record : records) { @@ -82,6 +78,10 @@ protected void evaluateRecords(ConsumerRecords rec GenericData.Array count = (GenericData.Array) value.get("count"); logger.info("Received [{}, {}, {}] E4 messages", count.get(0), count.get(1), count.get(2)); + + if ((Double)count.get(0) > 200) { + shutdown(); + } } } } diff --git a/src/integrationTest/java/org/radarcns/integration/PhoneStreamTest.java b/src/integrationTest/java/org/radarcns/integration/PhoneStreamTest.java index 1e20b815..6de31a15 100644 --- a/src/integrationTest/java/org/radarcns/integration/PhoneStreamTest.java +++ b/src/integrationTest/java/org/radarcns/integration/PhoneStreamTest.java @@ -114,7 +114,7 @@ public void setUp() throws IOException, ParseException, InterruptedException { streamProps.put(SESSION_TIMEOUT_MS_CONFIG, String.valueOf(5_000)); streamProps.put(HEARTBEAT_INTERVAL_MS_CONFIG, String.valueOf(1_000)); - propHandler.getRadarProperties().setStreamProperties(streamProps); + propHandler.getRadarProperties().getStreamProperties().putAll(streamProps); backend = new RadarBackend(opts, propHandler); backend.start(); } diff --git a/src/integrationTest/resources/org/radarcns/kafka/radar.yml b/src/integrationTest/resources/org/radarcns/kafka/radar.yml index fc6ed194..2207213c 100644 --- a/src/integrationTest/resources/org/radarcns/kafka/radar.yml +++ b/src/integrationTest/resources/org/radarcns/kafka/radar.yml @@ -20,6 +20,7 @@ broker: #Kafka internal parameters stream_properties: + max.poll.interval.ms: 10000 auto.commit.interval.ms: 1000 session.timeout.ms: 10000 buffered.records.per.partition: 1 @@ -30,7 +31,10 @@ stream_properties: metadata.fetch.timeout.ms: 5000 timeout.ms: 5000 commit.interval.ms: 1000 - buffered.records.per.partition: 1 + buffered.records.per.pasrtition: 1 + metadata.max.age.ms: 10000 + state.cleanup.delay.ms: 15000 + cache.max.bytes.buffering: 1 #============================ Kafka Streams ============================# #The number of threads that a stream must be run according is priority diff --git a/src/main/java/org/radarcns/stream/StreamMaster.java b/src/main/java/org/radarcns/stream/StreamMaster.java index cfe200f9..63fca3a9 100644 --- a/src/main/java/org/radarcns/stream/StreamMaster.java +++ b/src/main/java/org/radarcns/stream/StreamMaster.java @@ -111,7 +111,7 @@ public void start() { * Signal all workers to shut down. This does not wait for the workers to shut down. */ @Override - public void shutdown() { + public void shutdown() throws InterruptedException { log.info("Shutting down all streams for {}", nameSensor); while (!streamWorkers.isEmpty()) { @@ -120,6 +120,7 @@ public void shutdown() { } executor.shutdown(); + executor.awaitTermination(30, TimeUnit.SECONDS); } /** From b93f7a266081f24dc1fd3cbac6c3407288003068 Mon Sep 17 00:00:00 2001 From: Joris Borgdorff Date: Thu, 12 Oct 2017 10:37:50 +0200 Subject: [PATCH 14/23] Shutdown on closed stream. --- src/main/java/org/radarcns/stream/StreamMaster.java | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/radarcns/stream/StreamMaster.java b/src/main/java/org/radarcns/stream/StreamMaster.java index 63fca3a9..18c84606 100644 --- a/src/main/java/org/radarcns/stream/StreamMaster.java +++ b/src/main/java/org/radarcns/stream/StreamMaster.java @@ -155,12 +155,16 @@ public void notifyClosedStream(@Nonnull StreamWorker stream) { * * @param stream the name of the stream that is crashed. Useful for debug purpose */ - public void notifyCrashedStream(@Nonnull String stream) { + public void notifyCrashedStream(@Nonnull String stream) throws InterruptedException { log.error("[{}] {} is crashed", nameSensor, stream); log.info("Forcing shutdown of {}", nameSensor); - //TODO implement forcing shutdown + try { + shutdown(); + } catch (InterruptedException ex) { + log.warn("Shutdown interrupted"); + } } public void restartStream(final StreamWorker worker) { From acff0ecd263af5bc4ca2ed3fa0f7d36b1def30ea Mon Sep 17 00:00:00 2001 From: Joris Borgdorff Date: Thu, 12 Oct 2017 10:42:08 +0200 Subject: [PATCH 15/23] Fixed stream interruptedexception build error --- src/main/java/org/radarcns/stream/StreamMaster.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/org/radarcns/stream/StreamMaster.java b/src/main/java/org/radarcns/stream/StreamMaster.java index 18c84606..9ab02233 100644 --- a/src/main/java/org/radarcns/stream/StreamMaster.java +++ b/src/main/java/org/radarcns/stream/StreamMaster.java @@ -155,7 +155,7 @@ public void notifyClosedStream(@Nonnull StreamWorker stream) { * * @param stream the name of the stream that is crashed. Useful for debug purpose */ - public void notifyCrashedStream(@Nonnull String stream) throws InterruptedException { + public void notifyCrashedStream(@Nonnull String stream) { log.error("[{}] {} is crashed", nameSensor, stream); log.info("Forcing shutdown of {}", nameSensor); From 74894e7cb0e0c6fd8bf7ad5ec3fdd043962a2c49 Mon Sep 17 00:00:00 2001 From: Joris Borgdorff Date: Thu, 12 Oct 2017 11:00:23 +0200 Subject: [PATCH 16/23] Updated travis build --- .travis.yml | 16 ++++++++++------ docker-compose.yml | 3 +++ 2 files changed, 13 insertions(+), 6 deletions(-) diff --git a/.travis.yml b/.travis.yml index 00db462a..c54f4615 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,18 +2,22 @@ language: java jdk: - oraclejdk8 sudo: required + services: - docker + env: - DOCKER_COMPOSE_VERSION: 1.9.0 + DOCKER_COMPOSE_VERSION: 1.16.1 TERM: dumb -before_cache: - - rm -f $HOME/.gradle/caches/modules-2/modules-2.lock - - rm -fr $HOME/.gradle/caches/*/plugin-resolution/ + cache: directories: - - $HOME/.gradle/caches/ - - $HOME/.gradle/wrapper/ + - $HOME/.gradle/caches/jars-1 + - $HOME/.gradle/caches/jars-2 + - $HOME/.gradle/caches/jars-3 + - $HOME/.gradle/caches/modules-2/files-2.1/ + - $HOME/.gradle/native + - $HOME/.gradle/wrapper before_install: - mkdir -p "$HOME/bin"; diff --git a/docker-compose.yml b/docker-compose.yml index 35de5f84..70b4f232 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -81,3 +81,6 @@ services: - schema-registry-1 command: - integrationTest + volumes: + - ./build/jacoco:/code/build/jacoco + - ./build/reports:/code/build/reports From 09119e65e8caeb74d524ab1a38c6fff256797ad7 Mon Sep 17 00:00:00 2001 From: Joris Borgdorff Date: Thu, 12 Oct 2017 12:19:43 +0200 Subject: [PATCH 17/23] Trying to improve the test reliability --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index c54f4615..1b216313 100644 --- a/.travis.yml +++ b/.travis.yml @@ -26,7 +26,7 @@ before_install: - chmod +x "$HOME/bin/docker-compose"; script: - ./gradlew check - - sudo docker-compose up -d zookeeper-1 kafka-1 schema-registry-1 && sleep 30 && sudo docker-compose run --rm integration-test + - sudo docker-compose up -d zookeeper-1 kafka-1 schema-registry-1 && sleep 40 && sudo docker-compose run --rm integration-test - sudo docker-compose down after_script: - ./gradlew sendCoverageToCodacy From f896021e05be963ea87664228d54c69592f2e5dc Mon Sep 17 00:00:00 2001 From: Joris Borgdorff Date: Thu, 12 Oct 2017 14:55:17 +0200 Subject: [PATCH 18/23] Reduced logging output --- build.gradle | 16 ++++++++++++--- .../resources/org/radarcns/kafka/radar.yml | 5 ++--- .../org/radarcns/stream/StreamWorker.java | 2 ++ src/main/resources/log4j.properties | 20 ++++--------------- 4 files changed, 21 insertions(+), 22 deletions(-) diff --git a/build.gradle b/build.gradle index 866ccac3..6e44480c 100644 --- a/build.gradle +++ b/build.gradle @@ -44,7 +44,8 @@ ext.radarCommonsVersion = '0.6-SNAPSHOT' ext.radarSchemasVersion = '0.2-alpha.3-SNAPSHOT' ext.subethamailVersion = '3.1.7' ext.jsoupVersion = '1.10.2' -ext.slf4jVersion = '1.7.21' +ext.slf4jVersion = '1.7.25' +ext.log4jVersion = '1.2.17' ext.avroVersion = '1.8.2' ext.githubUrl = 'https://github.com/' + githubRepoName + '.git' @@ -133,8 +134,12 @@ dependencies { // Mock mail server testCompile group: 'org.subethamail', name: 'subethasmtp', version: subethamailVersion - testImplementation group: 'org.slf4j', name: 'slf4j-simple', version: slf4jVersion - integrationTestImplementation group: 'org.slf4j', name: 'slf4j-simple', version: slf4jVersion + runtimeOnly group: 'log4j', name: 'log4j', version: log4jVersion + runtimeOnly group: 'org.slf4j', name: 'slf4j-log4j12', version: slf4jVersion + testImplementation group: 'log4j', name: 'log4j', version: log4jVersion + testImplementation group: 'org.slf4j', name: 'slf4j-log4j12', version: slf4jVersion + integrationTestImplementation group: 'log4j', name: 'log4j', version: log4jVersion + integrationTestImplementation group: 'org.slf4j', name: 'slf4j-log4j12', version: slf4jVersion // For Topic name validation based on Kafka classes testImplementation (group: 'org.apache.kafka', name: 'kafka_2.11', version: kafkaVersion) { @@ -347,6 +352,11 @@ publishing { root.appendNode('description', description) root.appendNode('name', rootProject.name) root.appendNode('url', githubUrl) + root.dependencies.'*'.findAll() { + it.artifactId.text() in ['log4j', 'slf4j-log4j12'] + }.each() { + it.parent().remove(it) + } root.children().last() + pomConfig } } diff --git a/src/integrationTest/resources/org/radarcns/kafka/radar.yml b/src/integrationTest/resources/org/radarcns/kafka/radar.yml index 2207213c..2cbbd6f1 100644 --- a/src/integrationTest/resources/org/radarcns/kafka/radar.yml +++ b/src/integrationTest/resources/org/radarcns/kafka/radar.yml @@ -27,14 +27,13 @@ stream_properties: request.timeout.ms: 11000 fetch.timeout.ms: 3000 batch.size: 1 - metadata.max.age.ms: 10000 + metadata.max.age.ms: 15000 metadata.fetch.timeout.ms: 5000 timeout.ms: 5000 commit.interval.ms: 1000 - buffered.records.per.pasrtition: 1 metadata.max.age.ms: 10000 state.cleanup.delay.ms: 15000 - cache.max.bytes.buffering: 1 + cache.max.bytes.buffering: 1000 #============================ Kafka Streams ============================# #The number of threads that a stream must be run according is priority diff --git a/src/main/java/org/radarcns/stream/StreamWorker.java b/src/main/java/org/radarcns/stream/StreamWorker.java index 17f9061b..22c4b389 100644 --- a/src/main/java/org/radarcns/stream/StreamWorker.java +++ b/src/main/java/org/radarcns/stream/StreamWorker.java @@ -162,6 +162,8 @@ public void start() { stream.setUncaughtExceptionHandler(this); stream.start(); }); + + master.notifyStartedStream(this); } /** diff --git a/src/main/resources/log4j.properties b/src/main/resources/log4j.properties index db6aa12b..1c05bf0a 100644 --- a/src/main/resources/log4j.properties +++ b/src/main/resources/log4j.properties @@ -15,15 +15,11 @@ # # Root logger option -log4j.rootLogger=INFO, stdout, file +log4j.rootLogger=INFO, stdout -log4j.logger.org.radarcns.RadarBackend=INFO - -log4j.logger.org.radarcns.stream.StreamMaster=INFO - -log4j.logger.org.radarcns.stream.aggregator.SensorAggregator=INFO -log4j.logger.org.radarcns.stream.aggregator.InternalAggregator=INFO -log4j.logger.org.radarcns.process.AbstractKafkaMonitor=INFO +log4j.logger.org.apache.kafka.streams.processor.internals=WARN +log4j.logger.org.apache.kafka.clients.consumer.internals=WARN +log4j.logger.org.apache.kafka.clients.producer.internals=WARN # Redirect log messages to console log4j.appender.stdout=org.apache.log4j.ConsoleAppender @@ -31,11 +27,3 @@ log4j.appender.stdout.Target=System.out log4j.appender.stdout.layout=org.apache.log4j.PatternLayout log4j.appender.stdout.layout.ConversionPattern=[%d{yyyy-MM-dd HH:mm:ss zzz}] %5p [%t] (%F:%L) - %m (%c)%n #log4j.appender.stdout.layout.ConversionPattern=[%d{yyyy-MM-dd HH:mm:ss zzz}] %-5p %t %C.%M:%L - %m%n - -# Redirect log messages to a log file, support file rolling. -log4j.appender.file=org.apache.log4j.RollingFileAppender -log4j.appender.file.File=./backend.log -log4j.appender.file.MaxFileSize=10MB -log4j.appender.file.MaxBackupIndex=10 -log4j.appender.file.layout=org.apache.log4j.PatternLayout -log4j.appender.file.layout.ConversionPattern=[%d{yyyy-MM-dd HH:mm:ss zzz}] %-5p %t %c{1}:%L - %m%n From 6a4e573bbba713dc134ab4a068164f9812846dca Mon Sep 17 00:00:00 2001 From: Joris Borgdorff Date: Thu, 12 Oct 2017 16:29:58 +0200 Subject: [PATCH 19/23] Reduced number of stream properties explicitly set --- .travis.yml | 2 +- .../org/radarcns/integration/PhoneStreamTest.java | 10 ---------- .../resources/org/radarcns/kafka/radar.yml | 13 ------------- 3 files changed, 1 insertion(+), 24 deletions(-) diff --git a/.travis.yml b/.travis.yml index 1b216313..c54f4615 100644 --- a/.travis.yml +++ b/.travis.yml @@ -26,7 +26,7 @@ before_install: - chmod +x "$HOME/bin/docker-compose"; script: - ./gradlew check - - sudo docker-compose up -d zookeeper-1 kafka-1 schema-registry-1 && sleep 40 && sudo docker-compose run --rm integration-test + - sudo docker-compose up -d zookeeper-1 kafka-1 schema-registry-1 && sleep 30 && sudo docker-compose run --rm integration-test - sudo docker-compose down after_script: - ./gradlew sendCoverageToCodacy diff --git a/src/integrationTest/java/org/radarcns/integration/PhoneStreamTest.java b/src/integrationTest/java/org/radarcns/integration/PhoneStreamTest.java index 6de31a15..aa3d8ab5 100644 --- a/src/integrationTest/java/org/radarcns/integration/PhoneStreamTest.java +++ b/src/integrationTest/java/org/radarcns/integration/PhoneStreamTest.java @@ -105,16 +105,6 @@ public void setUp() throws IOException, ParseException, InterruptedException { String[] args = {"-c", propertiesPath, "stream"}; RadarBackendOptions opts = RadarBackendOptions.parse(args); - propHandler.getRadarProperties().setStreamMasters(Arrays.asList( - PhoneStreamMaster.class.getName(), - E4StreamMaster.class.getName())); - - Map streamProps = new HashMap<>(); - streamProps.put(AUTO_COMMIT_INTERVAL_MS_CONFIG, String.valueOf(1_000)); - streamProps.put(SESSION_TIMEOUT_MS_CONFIG, String.valueOf(5_000)); - streamProps.put(HEARTBEAT_INTERVAL_MS_CONFIG, String.valueOf(1_000)); - - propHandler.getRadarProperties().getStreamProperties().putAll(streamProps); backend = new RadarBackend(opts, propHandler); backend.start(); } diff --git a/src/integrationTest/resources/org/radarcns/kafka/radar.yml b/src/integrationTest/resources/org/radarcns/kafka/radar.yml index 2cbbd6f1..b89db0d4 100644 --- a/src/integrationTest/resources/org/radarcns/kafka/radar.yml +++ b/src/integrationTest/resources/org/radarcns/kafka/radar.yml @@ -20,19 +20,6 @@ broker: #Kafka internal parameters stream_properties: - max.poll.interval.ms: 10000 - auto.commit.interval.ms: 1000 - session.timeout.ms: 10000 - buffered.records.per.partition: 1 - request.timeout.ms: 11000 - fetch.timeout.ms: 3000 - batch.size: 1 - metadata.max.age.ms: 15000 - metadata.fetch.timeout.ms: 5000 - timeout.ms: 5000 - commit.interval.ms: 1000 - metadata.max.age.ms: 10000 - state.cleanup.delay.ms: 15000 cache.max.bytes.buffering: 1000 #============================ Kafka Streams ============================# From 8cdc10a2478e8291337cfac516f6c0383a858d80 Mon Sep 17 00:00:00 2001 From: Joris Borgdorff Date: Tue, 5 Dec 2017 12:25:54 +0100 Subject: [PATCH 20/23] Use latest commons and schemas --- build.gradle | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/build.gradle b/build.gradle index 56151792..891cb197 100644 --- a/build.gradle +++ b/build.gradle @@ -40,8 +40,8 @@ ext.mathVersion = '3.0' ext.findbugVersion = '3.0.1' ext.commonsCliVersion = '1.2' ext.mockitoVersion = '2.2.29' -ext.radarCommonsVersion = '0.6-SNAPSHOT' -ext.radarSchemasVersion = '0.2-alpha.3-SNAPSHOT' +ext.radarCommonsVersion = '0.6.3' +ext.radarSchemasVersion = '0.2.2' ext.subethamailVersion = '3.1.7' ext.jsoupVersion = '1.10.2' ext.slf4jVersion = '1.7.25' From b68a1bbc91ed1e25919c7a64c34d609e419a1c95 Mon Sep 17 00:00:00 2001 From: Joris Borgdorff Date: Tue, 5 Dec 2017 15:02:56 +0100 Subject: [PATCH 21/23] Distribute full application --- .travis.yml | 5 ++++- build.gradle | 7 ++++++- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index c54f4615..12ce0e67 100644 --- a/.travis.yml +++ b/.travis.yml @@ -35,7 +35,10 @@ deploy: provider: releases api_key: ${GH_TOKEN} file_glob: true - file: "build/libs/*.jar" + file: + - build/libs/*.jar + - build/distributions/*.zip + - build/distributions/*.tar.gz skip_cleanup: true on: tags: true diff --git a/build.gradle b/build.gradle index 891cb197..62f8b1f5 100644 --- a/build.gradle +++ b/build.gradle @@ -319,6 +319,11 @@ shadowJar { exclude 'log4j.properties' } +tasks.withType(Tar){ + compression = Compression.GZIP + extension = 'tar.gz' +} + // custom tasks for creating source/javadoc jars task sourcesJar(type: Jar, dependsOn: classes) { classifier = 'sources' @@ -392,7 +397,7 @@ bintray { task wrapper(type: Wrapper) { gradleVersion = '4.1' - distributionUrl distributionUrl.replace("bin", "all") + distributionType 'all' } run { From b0d8e12b60c6faeccb4417e2e856e46072c2cd62 Mon Sep 17 00:00:00 2001 From: Joris Borgdorff Date: Tue, 5 Dec 2017 15:47:26 +0100 Subject: [PATCH 22/23] Add log4j properties back to shadowJar --- build.gradle | 1 - 1 file changed, 1 deletion(-) diff --git a/build.gradle b/build.gradle index 62f8b1f5..8ae06ab7 100644 --- a/build.gradle +++ b/build.gradle @@ -316,7 +316,6 @@ jar { shadowJar { mergeServiceFiles() - exclude 'log4j.properties' } tasks.withType(Tar){ From 940a93faecd5f2e277bf79e669ba07ea1981b1a7 Mon Sep 17 00:00:00 2001 From: Yatharth Ranjan Date: Tue, 5 Dec 2017 15:33:21 +0000 Subject: [PATCH 23/23] Update version to 0.1.1 --- build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index 8ae06ab7..fadb6124 100644 --- a/build.gradle +++ b/build.gradle @@ -17,7 +17,7 @@ plugins { //---------------------------------------------------------------------------// group = 'org.radarcns' -version = '0.1' +version = '0.1.1' ext.description = 'Kafka backend for processing device data.' mainClassName = 'org.radarcns.RadarBackend'