diff --git a/pom-maven-central.xml b/pom-maven-central.xml index 17769b03..2137a7e0 100644 --- a/pom-maven-central.xml +++ b/pom-maven-central.xml @@ -7,7 +7,7 @@ kloadgen - 5.6.6 + 5.6.7 KLoadGen Load Generation Jmeter plugin for Kafka Cluster. Supporting AVRO, JSON Schema and Protobuf schema types. Generate Artificial diff --git a/pom.xml b/pom.xml index 841dab45..b9fad739 100644 --- a/pom.xml +++ b/pom.xml @@ -7,7 +7,7 @@ kloadgen - 5.6.6 + 5.6.7 KLoadGen Load Generation Jmeter plugin for Kafka Cluster. Supporting AVRO, JSON Schema and Protobuf schema types. Generate Artificial @@ -426,6 +426,12 @@ io.apicurio apicurio-registry-client ${apicurio-registry.version} + + + org.jboss.slf4j + slf4j-jboss-logmanager + + io.apicurio @@ -467,6 +473,7 @@ org.slf4j slf4j-api ${slf4j-api.version} + provided org.apache.commons diff --git a/src/main/java/com/sngular/kloadgen/extractor/extractors/avro/AbstractAvroFileExtractor.java b/src/main/java/com/sngular/kloadgen/extractor/extractors/avro/AbstractAvroFileExtractor.java index fbe9de28..882c068e 100644 --- a/src/main/java/com/sngular/kloadgen/extractor/extractors/avro/AbstractAvroFileExtractor.java +++ b/src/main/java/com/sngular/kloadgen/extractor/extractors/avro/AbstractAvroFileExtractor.java @@ -358,7 +358,6 @@ private Set extractSchemaNames(Schema schema) { Set schemaNames = new HashSet<>(); if (checkIfRecord(schema)) { schemaNames.add(schema.getName()); - schema.getFields().forEach(field -> schemaNames.addAll(extractSchemaNames(field.schema()))); } else if (checkIfArray(schema)) { schemaNames.addAll(extractSchemaNames(schema.getElementType())); } else if (checkIfUnion(schema)) { diff --git a/src/main/java/com/sngular/kloadgen/model/FieldValueMapping.java b/src/main/java/com/sngular/kloadgen/model/FieldValueMapping.java index c3f257a2..c7882fae 100644 --- a/src/main/java/com/sngular/kloadgen/model/FieldValueMapping.java +++ b/src/main/java/com/sngular/kloadgen/model/FieldValueMapping.java @@ -7,12 +7,13 @@ package com.sngular.kloadgen.model; import java.util.ArrayList; -import java.util.Arrays; import java.util.EnumMap; import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.Objects; +import java.util.regex.Matcher; +import java.util.regex.Pattern; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.DeserializationFeature; @@ -107,28 +108,42 @@ public final List getFieldValuesList() { if (StringUtils.isNotBlank(inputFieldValueList) && !"[]".equalsIgnoreCase(inputFieldValueList)) { try { inputFieldValueAux = inputFieldValueList; - if (inputFieldValueAux.charAt(0) != "[".charAt(0)) { + + if (inputFieldValueAux.charAt(0) != '[') { inputFieldValueAux = "[" + inputFieldValueAux; } - if (inputFieldValueAux.charAt(inputFieldValueAux.length() - 1) != "]".charAt(0)) { + if (inputFieldValueAux.charAt(inputFieldValueAux.length() - 1) != ']') { inputFieldValueAux += "]"; } final JsonNode nodes = OBJECT_MAPPER.readTree(inputFieldValueAux); final Iterator nodeElements = nodes.elements(); while (nodeElements.hasNext()) { result.add(nodeElements.next().toString()); + } } catch (final JsonProcessingException ex) { - inputFieldValueAux = inputFieldValueList; - if (inputFieldValueAux.charAt(0) == "[".charAt(0)) { - inputFieldValueAux = inputFieldValueAux.substring(1); + // Warning: even though IntelliJ say that can be simplified, it can't be simplified!! (test fails) + + if (inputFieldValueList.startsWith("[") && inputFieldValueList.endsWith("]")) { + final String pattern = "(?<=\\[?)((([À-ÿ\\p{Alnum}\\p{Punct}&&[^,\\[\\]]]+:([À-ÿ\\p{Alnum}\\p{Punct}&&[^,\\[\\]]]+|\\[([À-ÿ\\p{Alnum}\\p{Punct}&&[^,\\[\\]]]+," + + "[À-ÿ\\p{Alnum}\\p{Punct}&&[^,\\[\\]]]+|)*]))|[À-ÿ\\p{Alnum}\\p{Punct}&&[^,\\[\\]]]+)(?=[]|,]))"; + final Pattern r = Pattern.compile(pattern); + final Matcher matcher = r.matcher(inputFieldValueList.trim()); + while (matcher.find()) { + result.add(matcher.group(0)); + } + } else { + final String pattern = "([À-ÿ\\p{Alnum}\\p{Punct}&&[^,\\[\\]]][À-ÿ\\s\\p{Alnum}\\p{Punct}&&[^,\\[\\]]]+)[^,\\s]?+"; + final Pattern r = Pattern.compile(pattern); + final Matcher matcher = r.matcher(inputFieldValueList.trim()); + while (matcher.find()) { + result.add(matcher.group(0)); + } } - if (inputFieldValueAux.charAt(inputFieldValueAux.length() - 1) == "]".charAt(0)) { - inputFieldValueAux = inputFieldValueAux.substring(0, inputFieldValueAux.length() - 1); - } - result.addAll(Arrays.asList(inputFieldValueAux.trim().split("\\s*,\\s*", -1))); + } } + return result; } diff --git a/src/main/java/com/sngular/kloadgen/property/editor/FileSubjectPropertyEditor.java b/src/main/java/com/sngular/kloadgen/property/editor/FileSubjectPropertyEditor.java index d4883119..6db26c5b 100644 --- a/src/main/java/com/sngular/kloadgen/property/editor/FileSubjectPropertyEditor.java +++ b/src/main/java/com/sngular/kloadgen/property/editor/FileSubjectPropertyEditor.java @@ -38,7 +38,6 @@ import javax.swing.filechooser.FileSystemView; import lombok.extern.slf4j.Slf4j; import org.apache.avro.AvroRuntimeException; -import org.apache.commons.compress.utils.Lists; import org.apache.jmeter.gui.ClearGui; import org.apache.jmeter.gui.GuiPackage; import org.apache.jmeter.testbeans.gui.GenericTestBeanCustomizer; @@ -135,10 +134,10 @@ public final List getAttributeList(final ParsedSchema selecte public final void actionPerformed(final ActionEvent event) { if (subjectNameComboBox.getItemCount() != 0) { - final String schemaType = schemaTypeComboBox.getSelectedItem().toString(); + final String schemaType = (String) schemaTypeComboBox.getSelectedItem(); final String selectedItem = (String) subjectNameComboBox.getSelectedItem(); final String selectedSchema = getSelectedSchema(selectedItem); - final List attributeList = Lists.newArrayList(); + final List attributeList = SchemaExtractor.flatPropertiesList(selectedSchema).getValue(); if (!attributeList.isEmpty()) { try { diff --git a/src/main/java/com/sngular/kloadgen/randomtool/generator/AvroGeneratorTool.java b/src/main/java/com/sngular/kloadgen/randomtool/generator/AvroGeneratorTool.java index 0c45cfee..3f3aa8f6 100644 --- a/src/main/java/com/sngular/kloadgen/randomtool/generator/AvroGeneratorTool.java +++ b/src/main/java/com/sngular/kloadgen/randomtool/generator/AvroGeneratorTool.java @@ -248,7 +248,7 @@ public final Object generateMap( } } } else { - value = (HashMap) RANDOM_MAP.generateMap(fieldType, mapSize, parameterList, valueLength, mapSize, constraints); + value = (HashMap) RANDOM_MAP.generateMap(fieldType, /*mapSize*/parameterList.size(), parameterList, valueLength, mapSize, constraints); } return value; diff --git a/src/main/java/com/sngular/kloadgen/randomtool/random/RandomArray.java b/src/main/java/com/sngular/kloadgen/randomtool/random/RandomArray.java index f62b7081..a20c0062 100644 --- a/src/main/java/com/sngular/kloadgen/randomtool/random/RandomArray.java +++ b/src/main/java/com/sngular/kloadgen/randomtool/random/RandomArray.java @@ -23,6 +23,10 @@ public RandomArray() { randomObject = new RandomObject(); } + protected static boolean isArray(final String type) { + return type.toLowerCase().endsWith("array"); + } + public final Object generateArray( final String fieldType, final Integer valueLength, final List fieldValueList, final Integer arraySize, final Map constraints) { diff --git a/src/main/java/com/sngular/kloadgen/randomtool/random/RandomMap.java b/src/main/java/com/sngular/kloadgen/randomtool/random/RandomMap.java index e1be8a57..0507aa27 100644 --- a/src/main/java/com/sngular/kloadgen/randomtool/random/RandomMap.java +++ b/src/main/java/com/sngular/kloadgen/randomtool/random/RandomMap.java @@ -6,7 +6,6 @@ package com.sngular.kloadgen.randomtool.random; -import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; import java.util.List; @@ -21,8 +20,11 @@ public class RandomMap { private final RandomObject randomObject; + private final RandomArray randomArray; + public RandomMap() { randomObject = new RandomObject(); + randomArray = new RandomArray(); } private static String[] getMapEntryValue(final List fieldValueList) { @@ -60,14 +62,15 @@ public final Object generateMap( case ValidTypeConstants.BOOLEAN_MAP: value = generate(ValidTypeConstants.BOOLEAN, mapSize, fieldValueList, mapSize, Collections.emptyMap()); break; + case ValidTypeConstants.STRING_ARRAY: + value = generate(ValidTypeConstants.STRING_ARRAY, mapSize, fieldValueList, mapSize, constraints); + break; default: value = fieldType; break; } - if (fieldType.endsWith("array")) { - value = generateRandomMapArray(fieldType, mapSize, fieldValueList, mapSize, arraySize, constraints); - } else if (fieldType.endsWith("map-map")) { + if (fieldType.endsWith("map-map")) { value = generateMapOfMap(fieldType.replace("-map-map", "-map"), mapSize, mapSize, fieldValueList, arraySize, constraints); } @@ -104,37 +107,21 @@ public final Object generateMap( case ValidTypeConstants.BOOLEAN_MAP: value = generate(ValidTypeConstants.BOOLEAN, mapSize, fieldValueList, valueLength, Collections.emptyMap()); break; + case ValidTypeConstants.STRING_ARRAY: + value = generate(ValidTypeConstants.STRING_ARRAY, mapSize, fieldValueList, mapSize, constraints); + break; default: value = fieldType; break; } - if (fieldType.endsWith("array")) { - value = generateRandomMapArray(fieldType, mapSize, fieldValueList, valueLength, arraySize, constraints); - } else if (fieldType.endsWith("map-map")) { + if (fieldType.endsWith("map-map")) { value = generateMapOfMap(fieldType.replace("-map-map", "-map"), mapSize, mapSize, fieldValueList, valueLength, constraints); } return value; } - private Object generateRandomMapArray( - final String type, final Integer mapSize, final List fieldValueList, final Integer valueLength, final Integer arraySize, - final Map constraints) { - - final List> generatedMapArray = new ArrayList<>(arraySize); - int tempValueLength = valueLength; - if (valueLength == 0) { - tempValueLength = (int) Math.floor(Math.random() * (9 - 1 + 1) + 1); - } - final String newType = type.substring(0, type.length() - 6); - for (int i = 0; i < arraySize; i++) { - generatedMapArray.add((Map) generateMap(newType, mapSize, fieldValueList, tempValueLength, arraySize, constraints)); - } - - return generatedMapArray; - } - private Map generate( final String type, final Integer mapSize, final List fieldValueList, final int valueLength, final Map constraints) { @@ -144,30 +131,37 @@ private Map generate( while (map.size() < Math.min(size, fieldValueList.size())) { final String[] tempValue = getMapEntryValue(fieldValueList); if (tempValue.length > 1) { - switch (type) { - case ValidTypeConstants.INT: - map.put(tempValue[0], Integer.parseInt(tempValue[1])); - break; - case ValidTypeConstants.LONG: - map.put(tempValue[0], Long.parseLong(tempValue[1])); - break; - case ValidTypeConstants.FLOAT: - map.put(tempValue[0], Float.parseFloat(tempValue[1])); - break; - case ValidTypeConstants.DOUBLE: - map.put(tempValue[0], Double.parseDouble(tempValue[1])); - break; - case ValidTypeConstants.SHORT: - map.put(tempValue[0], Short.parseShort(tempValue[1])); - break; - case ValidTypeConstants.UUID: - map.put(tempValue[0], UUID.fromString(tempValue[1])); - break; - default: - map.put(tempValue[0], tempValue[1]); - break; + if (RandomArray.isArray(type)) { + final String[] array = tempValue[1].substring(tempValue[1].indexOf("[")).replaceAll("[^a-zA-Z\\s*,\\s*^0-9]", "").split("\\s*,\\s*", -1); + map.put(tempValue[0], List.of(array)); + } else if (isMap(type)) { + final String[] fixMap = tempValue[1].substring(tempValue[1].indexOf("[")).replaceAll("[^a-zA-Z\\s*,\\s*^0-9]", "").split("\\s*,\\s*", -1); + map.put(tempValue[0], generateMap(type, fixMap.length, List.of(fixMap), fixMap.length, constraints)); + } else { + switch (type) { + case ValidTypeConstants.INT: + map.put(tempValue[0], Integer.parseInt(tempValue[1])); + break; + case ValidTypeConstants.LONG: + map.put(tempValue[0], Long.parseLong(tempValue[1])); + break; + case ValidTypeConstants.FLOAT: + map.put(tempValue[0], Float.parseFloat(tempValue[1])); + break; + case ValidTypeConstants.DOUBLE: + map.put(tempValue[0], Double.parseDouble(tempValue[1])); + break; + case ValidTypeConstants.SHORT: + map.put(tempValue[0], Short.parseShort(tempValue[1])); + break; + case ValidTypeConstants.UUID: + map.put(tempValue[0], UUID.fromString(tempValue[1])); + break; + default: + map.put(tempValue[0], tempValue[1]); + break; + } } - } else { map.put( tempValue[0], @@ -178,17 +172,33 @@ private Map generate( } if (map.size() != mapSize) { - for (int i = 0; i <= Math.abs(map.size() - mapSize); i++) { + final int limit = Math.abs(map.size() - mapSize); + for (int i = 0; i < limit; i++) { map.put( (String) randomObject.generateRandom(ValidTypeConstants.STRING, valueLength, Collections.emptyList(), constraints), - randomObject.generateRandom(type, valueLength, Collections.emptyList(), constraints) - ); + generateMapValue(type, valueLength, constraints)); } } return map; } + private Object generateMapValue(final String type, final int valueLength, final Map constraints) { + final Object value; + if (isMap(type)) { + value = generateMap(type, valueLength, Collections.emptyList(), valueLength, constraints); + } else if (RandomArray.isArray(type)) { + value = randomArray.generateArray(type, valueLength, Collections.emptyList(), valueLength, constraints); + } else { + value = randomObject.generateRandom(type, valueLength, Collections.emptyList(), constraints); + } + return value; + } + + private static boolean isMap(final String type) { + return type.toLowerCase().endsWith("map"); + } + private Map generateMapOfMap( final String type, final Integer mapSize, final Integer innerMapSize, final List fieldValueList, final int valueLength, final Map constraints) { diff --git a/src/main/java/com/sngular/kloadgen/randomtool/util/ValidTypeConstants.java b/src/main/java/com/sngular/kloadgen/randomtool/util/ValidTypeConstants.java index 03667cf8..45ea40fa 100644 --- a/src/main/java/com/sngular/kloadgen/randomtool/util/ValidTypeConstants.java +++ b/src/main/java/com/sngular/kloadgen/randomtool/util/ValidTypeConstants.java @@ -115,11 +115,13 @@ public class ValidTypeConstants { public static final String INT_NANOS = "int_nanos"; + public static final String STRING_MAP_ARRAY = "string_map_array"; + public static final Set VALID_OBJECT_TYPES = Set.of( ARRAY, MAP, ENUM, STRING, INT, LONG, TIMESTAMP, STRING_TIMESTAMP, SHORT, DOUBLE, LONG_TIMESTAMP, UUID, BOOLEAN, BYTES, INT_DATE, INT_TIME_MILLIS, LONG_TIME_MICROS, LONG_TIMESTAMP_MILLIS, LONG_TIMESTAMP_MICROS, LONG_LOCAL_TIMESTAMP_MILLIS, LONG_LOCAL_TIMESTAMP_MICROS, STRING_UUID, BYTES_DECIMAL, FIXED_DECIMAL, - INT_YEAR, INT_MONTH, INT_DAY, INT_HOURS, INT_MINUTES, INT_SECONDS, INT_NANOS + INT_YEAR, INT_MONTH, INT_DAY, INT_HOURS, INT_MINUTES, INT_SECONDS, INT_NANOS, STRING_MAP_ARRAY ); private ValidTypeConstants() { diff --git a/src/test/java/com/sngular/kloadgen/randomtool/random/RandomArrayTest.java b/src/test/java/com/sngular/kloadgen/randomtool/random/RandomArrayTest.java index 7f5172b4..a6a713fc 100644 --- a/src/test/java/com/sngular/kloadgen/randomtool/random/RandomArrayTest.java +++ b/src/test/java/com/sngular/kloadgen/randomtool/random/RandomArrayTest.java @@ -25,6 +25,7 @@ private static Stream parametersForGenerateArrayRandomValue() { Arguments.of("long-array", 1, Collections.singletonList("1"), 1L), Arguments.of("short-array", 1, Collections.singletonList("1"), (short) 1), Arguments.of("double-array", 1, Collections.singletonList("1.0"), 1.0), + Arguments.of("map-array", 1, Collections.singletonList("testString:testString,testString:testString"), 2), Arguments.of("float-array", 1, Collections.singletonList("1.0"), 1.0f), Arguments.of("uuid-array", 1, Collections.singletonList("0177f035-e51c-4a46-8b82-5b157371c2a5"), UUID.fromString("0177f035-e51c-4a46-8b82-5b157371c2a5")) diff --git a/src/test/java/com/sngular/kloadgen/randomtool/random/RandomMapTest.java b/src/test/java/com/sngular/kloadgen/randomtool/random/RandomMapTest.java index 86a58dc4..686bb88c 100644 --- a/src/test/java/com/sngular/kloadgen/randomtool/random/RandomMapTest.java +++ b/src/test/java/com/sngular/kloadgen/randomtool/random/RandomMapTest.java @@ -9,6 +9,7 @@ import java.util.Collections; import java.util.List; import java.util.Map; +import java.util.Map.Entry; import java.util.UUID; import java.util.stream.Stream; @@ -27,36 +28,17 @@ private static Stream parametersForGenerateMapRandomValueFromList() { Arguments.of("long-map", 1, Collections.singletonList("testString:1"), Maps.of("testString", 1L), 1), Arguments.of("short-map", 1, Collections.singletonList("testString:1"), Maps.of("testString", (short) 1), 1), Arguments.of("double-map", 1, Collections.singletonList("testString:1.0"), Maps.of("testString", 1.0), 1), - Arguments.of( - "uuid-map", 1, Collections.singletonList("testString:0177f035-e51c-4a46-8b82-5b157371c2a5"), + Arguments.of("uuid-map", 1, Collections.singletonList("testString:0177f035-e51c-4a46-8b82-5b157371c2a5"), Maps.of("testString", UUID.fromString("0177f035-e51c-4a46-8b82-5b157371c2a5")), 1 ) ); } private static Stream parametersForGenerateMapArrayRandomValueFromList() { - return Stream.of( - Arguments.of( - "string-map-array", 1, Collections.singletonList("testString:testString"), - Collections.singletonList(Maps.of("testString", "testString")), 1 - ), - Arguments.of( - "int-map-array", 1, Collections.singletonList("testString:1"), Collections.singletonList(Maps.of("testString", 1)), 1 - ), - Arguments.of( - "long-map-array", 1, Collections.singletonList("testString:1"), Collections.singletonList(Maps.of("testString", 1L)), 1 - ), - Arguments.of( - "short-map-array", 1, Collections.singletonList("testString:1"), Collections.singletonList(Maps.of("testString", (short) 1)), 1 - ), - Arguments.of( - "double-map-array", 1, Collections.singletonList("testString:1.0"), - Collections.singletonList(Maps.of("testString", 1.0)), 1 - ), - Arguments.of( - "uuid-map-array", 1, Collections.singletonList("testString:0177f035-e51c-4a46-8b82-5b157371c2a5"), - Collections.singletonList(Maps.of("testString", UUID.fromString("0177f035-e51c-4a46-8b82-5b157371c2a5"))), 1 - ) + + //The name of this type is due to: SchemaProcessorUtils.getOneDimensionValueType (this remove the last -map) + return Stream.of(Arguments.of("string-array", 5, List.of("key1:[value1,value2,value3]", "key2:[valueB1,valueB2]"), + Map.of("key1", List.of("value1", "value2", "value3"), "key2", List.of("valueB1", "valueB2")), 1) ); } @@ -75,7 +57,7 @@ private static Stream parametersForGenerateMapFixedKeyRandomValue() { @MethodSource("parametersForGenerateMapRandomValueFromList") void generateMapRandomValueFromList( final String fieldType, final Integer valueLength, final List fieldValuesList, final Map expected, final Integer size) { - final Map.Entry[] expectedMap = expected.entrySet().toArray(new Map.Entry[1]); + final Entry[] expectedMap = expected.entrySet().toArray(new Entry[1]); final Map result = (Map) new RandomMap().generateMap(fieldType, valueLength, fieldValuesList, size, Collections.emptyMap()); Assertions.assertThat(result).containsExactly(expectedMap); @@ -83,20 +65,20 @@ void generateMapRandomValueFromList( @ParameterizedTest @MethodSource("parametersForGenerateMapArrayRandomValueFromList") - void generateMapArrayRandomValueFromList(final String fieldType, final Integer valueLength, final List fieldValuesList, final List> expected, + void generateMapArrayRandomValueFromList( + final String fieldType, final Integer valueLength, final List fieldValuesList, final Map> expected, final Integer size) { - final List> result = - (List>) new RandomMap().generateMap(fieldType, valueLength, fieldValuesList, size, Collections.emptyMap()); - Assertions.assertThat(result).containsExactly(expected.get(0)); + final Map> result = (Map>) new RandomMap().generateMap(fieldType, valueLength, fieldValuesList, size, Collections.emptyMap()); + + Assertions.assertThat(result).hasSize(valueLength).containsAllEntriesOf(expected); } @ParameterizedTest @MethodSource("parametersForGenerateMapFixedKeyRandomValue") void generateMapFixedKeyRandomValue(final String fieldType, final Integer valueLength, final List fieldValuesList, final Integer size) { final String[] expectedKeys = fieldValuesList.toArray(new String[1]); - final Map result = - (Map) new RandomMap().generateMap(fieldType, valueLength, fieldValuesList, size, Collections.emptyMap()); + final Map result = (Map) new RandomMap().generateMap(fieldType, valueLength, fieldValuesList, size, Collections.emptyMap()); Assertions.assertThat(result).containsKeys(expectedKeys).doesNotContainValue(null); } } \ No newline at end of file