Skip to content

Commit

Permalink
#340 field type string array map does work as expected (#408)
Browse files Browse the repository at this point in the history 
#340 field_type_string-array-map_does_work_as_expected

---------

Co-authored-by: samuel.amado <[email protected]>
Co-authored-by: Antonio Mondaca <[email protected]>
  • Loading branch information
@jemacineiras @SamuelAmadoSngular @amondacaSN
3 people authored Sep 21, 2023
1 parent 1885173 commit 49897e6
Show file tree
Hide file tree
Showing 11 changed files with 118 additions and 99 deletions.
2 changes: 1 addition & 1 deletion pom-maven-central.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@

<artifactId>kloadgen</artifactId>

<version>5.6.6</version>
<version>5.6.7</version>

<name>KLoadGen</name>
<description>Load Generation Jmeter plugin for Kafka Cluster. Supporting AVRO, JSON Schema and Protobuf schema types. Generate Artificial
Expand Down
9 changes: 8 additions & 1 deletion pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@

<artifactId>kloadgen</artifactId>

<version>5.6.6</version>
<version>5.6.7</version>

<name>KLoadGen</name>
<description>Load Generation Jmeter plugin for Kafka Cluster. Supporting AVRO, JSON Schema and Protobuf schema types. Generate Artificial
Expand Down Expand Up @@ -426,6 +426,12 @@
<groupId>io.apicurio</groupId>
<artifactId>apicurio-registry-client</artifactId>
<version>${apicurio-registry.version}</version>
<exclusions>
<exclusion>
<groupId>org.jboss.slf4j</groupId>
<artifactId>slf4j-jboss-logmanager</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>io.apicurio</groupId>
Expand Down Expand Up @@ -467,6 +473,7 @@
<groupId>org.slf4j</groupId>
<artifactId>slf4j-api</artifactId>
<version>${slf4j-api.version}</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>org.apache.commons</groupId>
Expand Down
1 change: 0 additions & 1 deletion src/main/java/com/sngular/kloadgen/extractor/extractors/avro/AbstractAvroFileExtractor.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -358,7 +358,6 @@ private Set<String> extractSchemaNames(Schema schema) {
Set<String> schemaNames = new HashSet<>();
if (checkIfRecord(schema)) {
schemaNames.add(schema.getName());
schema.getFields().forEach(field -> schemaNames.addAll(extractSchemaNames(field.schema())));
} else if (checkIfArray(schema)) {
schemaNames.addAll(extractSchemaNames(schema.getElementType()));
} else if (checkIfUnion(schema)) {
Expand Down
35 changes: 25 additions & 10 deletions src/main/java/com/sngular/kloadgen/model/FieldValueMapping.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,12 +7,13 @@
package com.sngular.kloadgen.model;

import java.util.ArrayList;
import java.util.Arrays;
import java.util.EnumMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.DeserializationFeature;
Expand Down Expand Up @@ -107,28 +108,42 @@ public final List<String> getFieldValuesList() {
if (StringUtils.isNotBlank(inputFieldValueList) && !"[]".equalsIgnoreCase(inputFieldValueList)) {
try {
inputFieldValueAux = inputFieldValueList;
if (inputFieldValueAux.charAt(0) != "[".charAt(0)) {

if (inputFieldValueAux.charAt(0) != '[') {
inputFieldValueAux = "[" + inputFieldValueAux;
}
if (inputFieldValueAux.charAt(inputFieldValueAux.length() - 1) != "]".charAt(0)) {
if (inputFieldValueAux.charAt(inputFieldValueAux.length() - 1) != ']') {
inputFieldValueAux += "]";
}
final JsonNode nodes = OBJECT_MAPPER.readTree(inputFieldValueAux);
final Iterator<JsonNode> nodeElements = nodes.elements();
while (nodeElements.hasNext()) {
result.add(nodeElements.next().toString());

}
} catch (final JsonProcessingException ex) {
inputFieldValueAux = inputFieldValueList;
if (inputFieldValueAux.charAt(0) == "[".charAt(0)) {
inputFieldValueAux = inputFieldValueAux.substring(1);
// Warning: even though IntelliJ say that can be simplified, it can't be simplified!! (test fails)

if (inputFieldValueList.startsWith("[") && inputFieldValueList.endsWith("]")) {
final String pattern = "(?<=\\[?)((([À-ÿ\\p{Alnum}\\p{Punct}&&[^,\\[\\]]]+:([À-ÿ\\p{Alnum}\\p{Punct}&&[^,\\[\\]]]+|\\[([À-ÿ\\p{Alnum}\\p{Punct}&&[^,\\[\\]]]+,"
+ "[À-ÿ\\p{Alnum}\\p{Punct}&&[^,\\[\\]]]+|)*]))|[À-ÿ\\p{Alnum}\\p{Punct}&&[^,\\[\\]]]+)(?=[]|,]))";
final Pattern r = Pattern.compile(pattern);
final Matcher matcher = r.matcher(inputFieldValueList.trim());
while (matcher.find()) {
result.add(matcher.group(0));
}
} else {
final String pattern = "([À-ÿ\\p{Alnum}\\p{Punct}&&[^,\\[\\]]][À-ÿ\\s\\p{Alnum}\\p{Punct}&&[^,\\[\\]]]+)[^,\\s]?+";
final Pattern r = Pattern.compile(pattern);
final Matcher matcher = r.matcher(inputFieldValueList.trim());
while (matcher.find()) {
result.add(matcher.group(0));
}
}
if (inputFieldValueAux.charAt(inputFieldValueAux.length() - 1) == "]".charAt(0)) {
inputFieldValueAux = inputFieldValueAux.substring(0, inputFieldValueAux.length() - 1);
}
result.addAll(Arrays.asList(inputFieldValueAux.trim().split("\\s*,\\s*", -1)));

}
}

return result;
}

Expand Down
5 changes: 2 additions & 3 deletions src/main/java/com/sngular/kloadgen/property/editor/FileSubjectPropertyEditor.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,6 @@
import javax.swing.filechooser.FileSystemView;
import lombok.extern.slf4j.Slf4j;
import org.apache.avro.AvroRuntimeException;
import org.apache.commons.compress.utils.Lists;
import org.apache.jmeter.gui.ClearGui;
import org.apache.jmeter.gui.GuiPackage;
import org.apache.jmeter.testbeans.gui.GenericTestBeanCustomizer;
Expand Down Expand Up @@ -135,10 +134,10 @@ public final List<FieldValueMapping> getAttributeList(final ParsedSchema selecte
public final void actionPerformed(final ActionEvent event) {

if (subjectNameComboBox.getItemCount() != 0) {
final String schemaType = schemaTypeComboBox.getSelectedItem().toString();
final String schemaType = (String) schemaTypeComboBox.getSelectedItem();
final String selectedItem = (String) subjectNameComboBox.getSelectedItem();
final String selectedSchema = getSelectedSchema(selectedItem);
final List<FieldValueMapping> attributeList = Lists.newArrayList();
final List<FieldValueMapping> attributeList = SchemaExtractor.flatPropertiesList(selectedSchema).getValue();

if (!attributeList.isEmpty()) {
try {
Expand Down
2 changes: 1 addition & 1 deletion src/main/java/com/sngular/kloadgen/randomtool/generator/AvroGeneratorTool.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -248,7 +248,7 @@ public final Object generateMap(
}
}
} else {
value = (HashMap<Object, Object>) RANDOM_MAP.generateMap(fieldType, mapSize, parameterList, valueLength, mapSize, constraints);
value = (HashMap<Object, Object>) RANDOM_MAP.generateMap(fieldType, /*mapSize*/parameterList.size(), parameterList, valueLength, mapSize, constraints);
}

return value;
Expand Down
4 changes: 4 additions & 0 deletions src/main/java/com/sngular/kloadgen/randomtool/random/RandomArray.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,10 @@ public RandomArray() {
randomObject = new RandomObject();
}

protected static boolean isArray(final String type) {
return type.toLowerCase().endsWith("array");
}

public final Object generateArray(
final String fieldType, final Integer valueLength, final List<String> fieldValueList, final Integer arraySize,
final Map<ConstraintTypeEnum, String> constraints) {
Expand Down
110 changes: 60 additions & 50 deletions src/main/java/com/sngular/kloadgen/randomtool/random/RandomMap.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,6 @@

package com.sngular.kloadgen.randomtool.random;

import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
Expand All @@ -21,8 +20,11 @@ public class RandomMap {

private final RandomObject randomObject;

private final RandomArray randomArray;

public RandomMap() {
randomObject = new RandomObject();
randomArray = new RandomArray();
}

private static String[] getMapEntryValue(final List<String> fieldValueList) {
Expand Down Expand Up @@ -60,14 +62,15 @@ public final Object generateMap(
case ValidTypeConstants.BOOLEAN_MAP:
value = generate(ValidTypeConstants.BOOLEAN, mapSize, fieldValueList, mapSize, Collections.emptyMap());
break;
case ValidTypeConstants.STRING_ARRAY:
value = generate(ValidTypeConstants.STRING_ARRAY, mapSize, fieldValueList, mapSize, constraints);
break;
default:
value = fieldType;
break;
}

if (fieldType.endsWith("array")) {
value = generateRandomMapArray(fieldType, mapSize, fieldValueList, mapSize, arraySize, constraints);
} else if (fieldType.endsWith("map-map")) {
if (fieldType.endsWith("map-map")) {
value = generateMapOfMap(fieldType.replace("-map-map", "-map"), mapSize, mapSize, fieldValueList, arraySize, constraints);
}

Expand Down Expand Up @@ -104,37 +107,21 @@ public final Object generateMap(
case ValidTypeConstants.BOOLEAN_MAP:
value = generate(ValidTypeConstants.BOOLEAN, mapSize, fieldValueList, valueLength, Collections.emptyMap());
break;
case ValidTypeConstants.STRING_ARRAY:
value = generate(ValidTypeConstants.STRING_ARRAY, mapSize, fieldValueList, mapSize, constraints);
break;
default:
value = fieldType;
break;
}

if (fieldType.endsWith("array")) {
value = generateRandomMapArray(fieldType, mapSize, fieldValueList, valueLength, arraySize, constraints);
} else if (fieldType.endsWith("map-map")) {
if (fieldType.endsWith("map-map")) {
value = generateMapOfMap(fieldType.replace("-map-map", "-map"), mapSize, mapSize, fieldValueList, valueLength, constraints);
}

return value;
}

private Object generateRandomMapArray(
final String type, final Integer mapSize, final List<String> fieldValueList, final Integer valueLength, final Integer arraySize,
final Map<ConstraintTypeEnum, String> constraints) {

final List<Map<String, Object>> generatedMapArray = new ArrayList<>(arraySize);
int tempValueLength = valueLength;
if (valueLength == 0) {
tempValueLength = (int) Math.floor(Math.random() * (9 - 1 + 1) + 1);
}
final String newType = type.substring(0, type.length() - 6);
for (int i = 0; i < arraySize; i++) {
generatedMapArray.add((Map<String, Object>) generateMap(newType, mapSize, fieldValueList, tempValueLength, arraySize, constraints));
}

return generatedMapArray;
}

private Map<String, Object> generate(
final String type, final Integer mapSize, final List<String> fieldValueList, final int valueLength,
final Map<ConstraintTypeEnum, String> constraints) {
Expand All @@ -144,30 +131,37 @@ private Map<String, Object> generate(
while (map.size() < Math.min(size, fieldValueList.size())) {
final String[] tempValue = getMapEntryValue(fieldValueList);
if (tempValue.length > 1) {
switch (type) {
case ValidTypeConstants.INT:
map.put(tempValue[0], Integer.parseInt(tempValue[1]));
break;
case ValidTypeConstants.LONG:
map.put(tempValue[0], Long.parseLong(tempValue[1]));
break;
case ValidTypeConstants.FLOAT:
map.put(tempValue[0], Float.parseFloat(tempValue[1]));
break;
case ValidTypeConstants.DOUBLE:
map.put(tempValue[0], Double.parseDouble(tempValue[1]));
break;
case ValidTypeConstants.SHORT:
map.put(tempValue[0], Short.parseShort(tempValue[1]));
break;
case ValidTypeConstants.UUID:
map.put(tempValue[0], UUID.fromString(tempValue[1]));
break;
default:
map.put(tempValue[0], tempValue[1]);
break;
if (RandomArray.isArray(type)) {
final String[] array = tempValue[1].substring(tempValue[1].indexOf("[")).replaceAll("[^a-zA-Z\\s*,\\s*^0-9]", "").split("\\s*,\\s*", -1);
map.put(tempValue[0], List.of(array));
} else if (isMap(type)) {
final String[] fixMap = tempValue[1].substring(tempValue[1].indexOf("[")).replaceAll("[^a-zA-Z\\s*,\\s*^0-9]", "").split("\\s*,\\s*", -1);
map.put(tempValue[0], generateMap(type, fixMap.length, List.of(fixMap), fixMap.length, constraints));
} else {
switch (type) {
case ValidTypeConstants.INT:
map.put(tempValue[0], Integer.parseInt(tempValue[1]));
break;
case ValidTypeConstants.LONG:
map.put(tempValue[0], Long.parseLong(tempValue[1]));
break;
case ValidTypeConstants.FLOAT:
map.put(tempValue[0], Float.parseFloat(tempValue[1]));
break;
case ValidTypeConstants.DOUBLE:
map.put(tempValue[0], Double.parseDouble(tempValue[1]));
break;
case ValidTypeConstants.SHORT:
map.put(tempValue[0], Short.parseShort(tempValue[1]));
break;
case ValidTypeConstants.UUID:
map.put(tempValue[0], UUID.fromString(tempValue[1]));
break;
default:
map.put(tempValue[0], tempValue[1]);
break;
}
}

} else {
map.put(
tempValue[0],
Expand All @@ -178,17 +172,33 @@ private Map<String, Object> generate(
}

if (map.size() != mapSize) {
for (int i = 0; i <= Math.abs(map.size() - mapSize); i++) {
final int limit = Math.abs(map.size() - mapSize);
for (int i = 0; i < limit; i++) {
map.put(
(String) randomObject.generateRandom(ValidTypeConstants.STRING, valueLength, Collections.emptyList(), constraints),
randomObject.generateRandom(type, valueLength, Collections.emptyList(), constraints)
);
generateMapValue(type, valueLength, constraints));
}
}

return map;
}

private Object generateMapValue(final String type, final int valueLength, final Map<ConstraintTypeEnum, String> constraints) {
final Object value;
if (isMap(type)) {
value = generateMap(type, valueLength, Collections.emptyList(), valueLength, constraints);
} else if (RandomArray.isArray(type)) {
value = randomArray.generateArray(type, valueLength, Collections.emptyList(), valueLength, constraints);
} else {
value = randomObject.generateRandom(type, valueLength, Collections.emptyList(), constraints);
}
return value;
}

private static boolean isMap(final String type) {
return type.toLowerCase().endsWith("map");
}

private Map<String, Object> generateMapOfMap(
final String type, final Integer mapSize, final Integer innerMapSize, final List<String> fieldValueList, final int valueLength,
final Map<ConstraintTypeEnum, String> constraints) {
Expand Down
4 changes: 3 additions & 1 deletion src/main/java/com/sngular/kloadgen/randomtool/util/ValidTypeConstants.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -115,11 +115,13 @@ public class ValidTypeConstants {

public static final String INT_NANOS = "int_nanos";

public static final String STRING_MAP_ARRAY = "string_map_array";

public static final Set<String> VALID_OBJECT_TYPES = Set.of(
ARRAY, MAP, ENUM, STRING, INT, LONG, TIMESTAMP, STRING_TIMESTAMP, SHORT, DOUBLE, LONG_TIMESTAMP, UUID, BOOLEAN,
BYTES, INT_DATE, INT_TIME_MILLIS, LONG_TIME_MICROS, LONG_TIMESTAMP_MILLIS, LONG_TIMESTAMP_MICROS,
LONG_LOCAL_TIMESTAMP_MILLIS, LONG_LOCAL_TIMESTAMP_MICROS, STRING_UUID, BYTES_DECIMAL, FIXED_DECIMAL,
INT_YEAR, INT_MONTH, INT_DAY, INT_HOURS, INT_MINUTES, INT_SECONDS, INT_NANOS
INT_YEAR, INT_MONTH, INT_DAY, INT_HOURS, INT_MINUTES, INT_SECONDS, INT_NANOS, STRING_MAP_ARRAY
);

private ValidTypeConstants() {
Expand Down
1 change: 1 addition & 0 deletions src/test/java/com/sngular/kloadgen/randomtool/random/RandomArrayTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@ private static Stream<Arguments> parametersForGenerateArrayRandomValue() {
Arguments.of("long-array", 1, Collections.singletonList("1"), 1L),
Arguments.of("short-array", 1, Collections.singletonList("1"), (short) 1),
Arguments.of("double-array", 1, Collections.singletonList("1.0"), 1.0),
Arguments.of("map-array", 1, Collections.singletonList("testString:testString,testString:testString"), 2),
Arguments.of("float-array", 1, Collections.singletonList("1.0"), 1.0f),
Arguments.of("uuid-array", 1, Collections.singletonList("0177f035-e51c-4a46-8b82-5b157371c2a5"),
UUID.fromString("0177f035-e51c-4a46-8b82-5b157371c2a5"))
Expand Down
Loading

0 comments on commit 49897e6

Please sign in to comment.