From b21e311d9a81843503ea1a2b09e22ccd7070ea7c Mon Sep 17 00:00:00 2001 From: Adrian Lagartera Date: Wed, 22 Feb 2023 13:45:10 +0100 Subject: [PATCH 01/30] #332 Add Files Modified To Solve Exception Added a trim statement to avoid problem when cast the schemas from classpath --- .../processor/util/SchemaProcessorUtils.java | 10 +- .../proto-files/messaging_intra_order.proto | 218 ++++++++++++++++++ .../proto-files/messaging_unified_order.proto | 66 ++++++ 3 files changed, 289 insertions(+), 5 deletions(-) create mode 100644 src/test/resources/proto-files/messaging_intra_order.proto create mode 100644 src/test/resources/proto-files/messaging_unified_order.proto diff --git a/src/main/java/com/sngular/kloadgen/processor/util/SchemaProcessorUtils.java b/src/main/java/com/sngular/kloadgen/processor/util/SchemaProcessorUtils.java index ec0db236..e274cf8d 100644 --- a/src/main/java/com/sngular/kloadgen/processor/util/SchemaProcessorUtils.java +++ b/src/main/java/com/sngular/kloadgen/processor/util/SchemaProcessorUtils.java @@ -146,7 +146,7 @@ public static Descriptors.Descriptor buildProtoDescriptor(final ProtoFileElement final DynamicSchema.Builder schemaBuilder = DynamicSchema.newBuilder(); final List imports = schema.getImports(); - for (String importedClass : imports) { + for (final String importedClass : imports) { try (final InputStream resourceStream = SchemaProcessorUtils.class.getClassLoader().getResourceAsStream(importedClass)) { if (null != resourceStream) { final String schemaToString = new String(resourceStream.readAllBytes()); @@ -181,7 +181,7 @@ private static DynamicSchema processImported(final List importedLines) t String packageName = ""; final var linesIterator = importedLines.listIterator(); while (linesIterator.hasNext()) { - final var fileLine = linesIterator.next(); + final var fileLine = linesIterator.next().trim(); if (fileLine.startsWith("package")) { packageName = StringUtils.chop(fileLine.substring(7).trim()); @@ -238,7 +238,7 @@ private static MessageDefinition buildProtoMessageDefinition( final MessageDefinition.Builder msgDef = MessageDefinition.newBuilder(fieldName); final var element = (MessageElement) messageElement; extracted(globalNestedTypesByLevelAndMessage, msgDef, element.getFields(), nextDeepLevel, fieldName); - for (var optionalField : element.getOneOfs()) { + for (final var optionalField : element.getOneOfs()) { extracted(globalNestedTypesByLevelAndMessage, msgDef, optionalField.getFields(), nextDeepLevel, fieldName); } return msgDef.build(); @@ -251,7 +251,7 @@ private static void extracted( final HashMap nestedTypes = processLevelTypes(globalNestedTypesByLevelAndMessage, msgDef, fieldElementList, deepLevel, messageName); - for (var elementField : fieldElementList) { + for (final var elementField : fieldElementList) { final var elementFieldType = elementField.getType(); final var dotType = checkDotType(elementFieldType); if (nestedTypes.containsKey(elementFieldType)) { @@ -343,7 +343,7 @@ private static void addDefinition( if (typeElement instanceof EnumElement) { final var enumElement = (EnumElement) typeElement; final EnumDefinition.Builder builder = EnumDefinition.newBuilder(enumElement.getName()); - for (var constant : enumElement.getConstants()) { + for (final var constant : enumElement.getConstants()) { builder.addValue(constant.getName(), constant.getTag()); } msgDef.addEnumDefinition(builder.build()); diff --git a/src/test/resources/proto-files/messaging_intra_order.proto b/src/test/resources/proto-files/messaging_intra_order.proto new file mode 100644 index 00000000..fc680958 --- /dev/null +++ b/src/test/resources/proto-files/messaging_intra_order.proto @@ -0,0 +1,218 @@ +syntax = "proto3"; + +package demo; + +option java_package = "com.demo.unmanaged.order.proto"; +option java_outer_classname = "IntraOrder"; +option java_multiple_files = true; + +message Order { + + message Address { + int32 address_id = 1; + string address_line_one = 2; + } + + message Contact { + int32 contact_id = 1; + string first_name = 2; + string last_name = 3; + } + + message Dimension { + string unit_of_measure = 1; + int32 value = 2; + } + + message CustomerAccount { + demo.Order.CustomerAccount.BillingParty billing_party = 1; + string name = 2; + + message BillingParty { + string p_name = 1; + repeated demo.Order.Contact p_contact = 2; + } + } + + message LogisticsProviderDetails { + demo.Order.LogisticsProviderDetails.ManagedBy managed_by = 1; + + message ManagedBy { + string organization_name = 1; + string organization_id = 2; + } + } + + message MarketingAreaDetails { + string origin_marketing_area = 1; + string origin_area_name = 2; + } + + message ServiceOfferingDetails { + string business_unit_code = 1; + string service_offering = 2; + } + + message Currency { + string currency_code = 1; + double value = 2; + } + + message OperationalRequirements { + string code = 1; + string type = 2; + demo.Order.OperationalRequirements.IntendedCarrier carrier = 3; + + message IntendedCarrier { + string id = 4; + } + } + + message RatingDetails { + string terms = 1; + string status = 2; + demo.Order.RatingDetails.RateSheets sheets = 3; + + message RateSheets { + repeated demo.Order.RatingDetails.RateSheets.RateSheet in_sheet = 1; + repeated demo.Order.RatingDetails.RateSheets.RateSheet es_sheet = 2; + + message RateSheet { + int32 id = 4; + } + } + } + + message OrderDetails { + repeated demo.Order.OrderDetails.OrderRequirements order_requirements = 1; + demo.Order.OrderDetails.InternationalDetails international_details = 2; + + message OrderRequirements { + string requirement = 1; + } + message InternationalDetails { + demo.Order.OrderDetails.InternationalDetails.CustomsDetails customs_details = 1; + repeated demo.Order.OrderDetails.InternationalDetails.InternationalServices international_services = 2; + demo.Order.OrderDetails.InternationalDetails.InbondDetails inbond_details = 3; + demo.Order.OrderDetails.InternationalDetails.PortDetails port_of_exit_location = 4; + demo.Order.OrderDetails.InternationalDetails.PortDetails port_of_entry_location = 5; + + message InternationalServices { + string service = 1; + } + + message PortDetails { + string location_id = 1; + string location_name = 2; + demo.Order.Address location_address = 3; + } + + message CustomsDetails { + demo.Order.OrderDetails.InternationalDetails.CustomsDetails.CustomsBroker customs_broker = 1; + string clearing_customs_country = 2; + + message CustomsBroker { + string party_name = 1; + demo.Order.Address party_address = 2; + repeated demo.Order.Contact party_contact = 3; + } + } + + message InbondDetails { + string bond_holder_role = 1; + demo.Order.OrderDetails.InternationalDetails.InbondDetails.BondHolderParty bond_holder_party = 2; + + message BondHolderParty { + string party_name = 3; + demo.Order.Address party_address = 4; + repeated demo.Order.Contact party_contact = 5; + } + } + } + } + + message RequestedOperationalServices { + string service_type = 1; + } + + message ReferenceNumbers { + int32 reference_number_id = 1; + string reference_number = 2; + } + + message Appointments { + int32 appointment_id = 1; + string appointment_type_code = 2; + repeated demo.Order.Appointments.AppointmentRequirements appointment_requirements = 3; + repeated demo.Order.Appointments.AppointmentNumbers appointment_numbers = 4; + + message AppointmentRequirements { + string requirement = 5; + } + + message AppointmentNumbers { + string appointment_number = 6; + } + } + + message AdditionalInstructions { + string instruction_type = 1; + string instruction_text = 2; + } + + message Comments { + string comment_type = 1; + string comment = 2; + } + + message Stops { + demo.Order.Stops.StopLocation stop_location = 1; + repeated demo.Order.Stops.ItemHandlingDetails item_handling_details = 2; + + message StopLocation { + string location_id = 1; + demo.Order.Address address = 2; + } + message ItemHandlingDetails { + int32 item_handling_detail_id = 3; + demo.Order.ItemUnit item_handling_unit = 4; + demo.Order.Dimension item_handling_unit_height = 5; + repeated demo.Order.Stops.ItemHandlingDetails.ItemDetails item_details = 6; + + message ItemDetails { + int32 stop_item_id = 7; + demo.Order.ItemUnit packaging_unit = 8; + demo.Order.Dimension item_density = 9; + demo.Order.Dimension item_height = 10; + repeated demo.Order.ReferenceNumbers reference_numbers = 11; + } + } + } + + message EquipmentRequirements { + demo.Order.EquipmentRequirements.EquipmentCharacteristics characteristics = 1; + + message EquipmentCharacteristics { + repeated demo.Order.EquipmentRequirements.EquipmentCharacteristics.EquipmentOptions e_options = 2; + + message EquipmentOptions { + string option = 3; + } + } + } + + message ItemUnit { + int32 quantity = 1; + string type = 2; + } + + message UserDetails { + string person_id = 1; + string first_name = 2; + } + + message SystemDetails { + string order_system = 1; + demo.Order.UserDetails created_by = 2; + } +} diff --git a/src/test/resources/proto-files/messaging_unified_order.proto b/src/test/resources/proto-files/messaging_unified_order.proto new file mode 100644 index 00000000..e56ede6d --- /dev/null +++ b/src/test/resources/proto-files/messaging_unified_order.proto @@ -0,0 +1,66 @@ +syntax = "proto3"; +package demo; + +import "messaging_intra_order.proto"; + +option java_package = "com.demo.unified.aggregator.proto"; +option java_outer_classname = "UnifiedAggregator"; +option java_multiple_files = true; + +message UnifiedOrder { + int32 order_id = 1; + string order_number = 2; + Order.CustomerAccount customer_account = 3; + Order.RatingDetails rating_details = 4; + Order.LogisticsProviderDetails logistics_provider_details = 5; + Order.MarketingAreaDetails marketing_area_details = 6; + Order.ServiceOfferingDetails service_offering_details = 7; + string order_type = 8; + string order_status = 9; + demo.UnifiedOrder.OrderDetails order_details = 10; + Order.OperationalRequirements operational_requirements = 11; + Order.EquipmentRequirements equipment_requirements = 12; + repeated demo.UnifiedOrder.Stops stops = 13; + repeated Order.ReferenceNumbers reference_numbers = 14; + Order.Contact solicitor_contact = 15; + Order.SystemDetails system_details = 16; + repeated Order.AdditionalInstructions additional_instructions = 17; + repeated Order.Comments comments = 18; + + message OrderDetails { + Order.Dimension total_weight = 1; + Order.Currency order_value = 2; + Order.Dimension estimated_distance = 3; + string shipping_option_date = 4; + bool comingle_flag = 5; + bool consolidated_flag = 6; + repeated Order.OrderDetails.OrderRequirements order_requirements = 7; + Order.Currency customer_tendered_rate = 8; + Order.OrderDetails.InternationalDetails international_details = 9; + Order.UserDetails order_owner = 10; + string shipment_identification_number = 11; + string order_tracking_number = 12; + string order_tracking_status = 13; + } + message Stops { + string stop_id = 1; + string stop_reason_code = 2; + string stop_type = 3; + string stop_status = 4; + string customer_provided_date_time = 5; + Order.Stops.StopLocation stop_location = 6; + Order.Contact stop_contact = 7; + repeated Order.Stops.ItemHandlingDetails item_handling_details = 8; + repeated Order.ReferenceNumbers reference_numbers = 9; + repeated Order.Appointments appointments = 10; + Order.Dimension stop_weight = 11; + int32 stop_quantity = 12; + int32 route_stop_id = 13; + Order.Dimension distance_to_next_stop = 14; + string rail_service_level = 15; + string rail_transit_mode = 16; + repeated Order.RequestedOperationalServices requested_operational_services = 17; + repeated Order.AdditionalInstructions additional_instructions = 18; + repeated Order.Comments comments = 19; + } +} From 9a197c728ad7f9f6e379491fe9f11d582ab51a23 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jose=20Enrique=20Garc=C3=ADa=20Maci=C3=B1eiras?= Date: Thu, 23 Feb 2023 18:51:58 +0100 Subject: [PATCH 02/30] #332 Add get dependencies and build proto. --- pom-maven-central.xml | 2 +- pom.xml | 2 +- .../processor/util/SchemaProcessorUtils.java | 24 ++++++++++++++++--- 3 files changed, 23 insertions(+), 5 deletions(-) diff --git a/pom-maven-central.xml b/pom-maven-central.xml index c8c7adf9..ef83cc89 100644 --- a/pom-maven-central.xml +++ b/pom-maven-central.xml @@ -7,7 +7,7 @@ kloadgen - 5.3.0 + 5.3.1 KLoadGen Load Generation Jmeter plugin for Kafka Cluster. Supporting AVRO, JSON Schema and Protobuf schema types. Generate Artificial data based on Data specification diff --git a/pom.xml b/pom.xml index ab04d55b..0caff5b5 100644 --- a/pom.xml +++ b/pom.xml @@ -7,7 +7,7 @@ kloadgen - 5.3.0 + 5.3.1 KLoadGen Load Generation Jmeter plugin for Kafka Cluster. Supporting AVRO, JSON Schema and Protobuf schema types. Generate Artificial data base on Data specification diff --git a/src/main/java/com/sngular/kloadgen/processor/util/SchemaProcessorUtils.java b/src/main/java/com/sngular/kloadgen/processor/util/SchemaProcessorUtils.java index e274cf8d..3fb2bb2f 100644 --- a/src/main/java/com/sngular/kloadgen/processor/util/SchemaProcessorUtils.java +++ b/src/main/java/com/sngular/kloadgen/processor/util/SchemaProcessorUtils.java @@ -25,16 +25,20 @@ import com.google.protobuf.Descriptors; import com.google.protobuf.Descriptors.DescriptorValidationException; import com.sngular.kloadgen.model.FieldValueMapping; +import com.sngular.kloadgen.util.JMeterHelper; import com.sngular.kloadgen.util.ProtobufHelper; import com.squareup.wire.schema.internal.parser.EnumElement; import com.squareup.wire.schema.internal.parser.FieldElement; import com.squareup.wire.schema.internal.parser.MessageElement; import com.squareup.wire.schema.internal.parser.ProtoFileElement; import com.squareup.wire.schema.internal.parser.TypeElement; +import io.confluent.kafka.schemaregistry.client.rest.exceptions.RestClientException; +import io.confluent.kafka.schemaregistry.protobuf.ProtobufSchema; import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.collections4.Predicate; import org.apache.commons.lang3.RandomUtils; import org.apache.commons.lang3.StringUtils; +import org.apache.jmeter.threads.JMeterContextService; public class SchemaProcessorUtils { @@ -156,7 +160,15 @@ public static Descriptors.Descriptor buildProtoDescriptor(final ProtoFileElement schemaBuilder.addDependency(importedSchema.getFileDescriptorSet().getFile(0).getName()); schemaBuilder.addSchema(importedSchema); } + } else { + final var importSchema = JMeterHelper.getParsedSchema(importedClass.substring(0, importedClass.length() - 6), JMeterContextService.getContext().getProperties()); + if (!ProtobufHelper.NOT_ACCEPTED_IMPORTS.contains(importedClass)) { + schemaBuilder.addDependency(((ProtobufSchema) importSchema).toDescriptor().getFullName()); + schemaBuilder.addSchema(convertDynamicSchema((ProtobufSchema) importSchema)); + } } + } catch (RestClientException e) { + throw new RuntimeException(e); } } final MessageElement messageElement = (MessageElement) schema.getTypes().get(0); @@ -170,6 +182,10 @@ public static Descriptors.Descriptor buildProtoDescriptor(final ProtoFileElement return schemaBuilder.build().getMessageDescriptor(messageElement.getName()); } + private static DynamicSchema convertDynamicSchema(final ProtobufSchema importSchema) throws DescriptorValidationException { + return processImported(Arrays.asList(importSchema.rawSchema().toSchema().split("\\n"))); + } + private static Predicate isValid() { return line -> !line.contains("//") && !line.isEmpty(); } @@ -208,9 +224,11 @@ private static MessageDefinition buildMessage(final String messageName, final Li while (messageLines.hasNext() && !exit) { final var field = messageLines.next().trim().split("\\s"); if (ProtobufHelper.isValidType(field[0])) { - messageDefinition.addField(OPTIONAL, field[0], field[1], Integer.parseInt(checkIfChoppable(field[3]))); + messageDefinition.addField(OPTIONAL, field[0], field[1], Integer.parseInt(checkIfGreppable(field[3]))); } else if (ProtobufHelper.LABEL.contains(field[0])) { - messageDefinition.addField(field[0], field[1], field[2], Integer.parseInt(checkIfChoppable(field[4]))); + messageDefinition.addField(field[0], field[1], field[2], Integer.parseInt(checkIfGreppable(field[4]))); + } else if ("message".equalsIgnoreCase(field[0])) { + messageDefinition.addMessageDefinition(buildMessage(field[1], messageLines)); } else if ("}".equalsIgnoreCase(field[0])) { exit = true; } @@ -219,7 +237,7 @@ private static MessageDefinition buildMessage(final String messageName, final Li return messageDefinition.build(); } - private static String checkIfChoppable(final String field) { + private static String checkIfGreppable(final String field) { String choppedField = field; if (field.endsWith(";")) { choppedField = StringUtils.chop(field); From af0dfe3e880a197e933b3ccc38114ebe3db400c2 Mon Sep 17 00:00:00 2001 From: Adrian Lagartera Date: Wed, 1 Mar 2023 17:20:10 +0100 Subject: [PATCH 03/30] #332 Add Test And Documentation Removed some unused test files --- docs/schemas.md | 20 ++ pom-maven-central.xml | 2 +- pom.xml | 2 +- .../impl/ProtobufSRLoadGeneratorTest.java | 85 +++++++ .../mappings/schema_registry_stub.json | 106 ++++++++- .../proto-files/messaging_intra_order.proto | 218 ------------------ .../proto-files/messaging_unified_order.proto | 66 ------ 7 files changed, 212 insertions(+), 287 deletions(-) create mode 100644 src/test/java/com/sngular/kloadgen/loadgen/impl/ProtobufSRLoadGeneratorTest.java delete mode 100644 src/test/resources/proto-files/messaging_intra_order.proto delete mode 100644 src/test/resources/proto-files/messaging_unified_order.proto diff --git a/docs/schemas.md b/docs/schemas.md index 63e2c4bb..57ee747b 100644 --- a/docs/schemas.md +++ b/docs/schemas.md @@ -176,3 +176,23 @@ The same applies when there is more than one level. The last child in the last l The other exception applies to JSON Schema: in order to support `null`values, the collections within objects (type : object) cannot be null. Therefore, they will be **required by default** and they will be **initialized as an empty collection** if the object than contains them is not null. > Within JSON Schema, a maximum of **2 nesting levels** is allowed. + + +### Protobuf Schema - Schema Registry + +If you need use the Protobuf Schema with the Schema Registry, you must put the subject with the same name as the protobuf file, as the following example: + +``` +{ + "schema": "...", + "schemaType": "PROTOBUF", + "references": [ + { + "name": "[the_name_you_want].proto", + "subject": "[the_name_you_want]", + "version": ... + }] +} + +``` +> This example is based on a petition from Schema Registry diff --git a/pom-maven-central.xml b/pom-maven-central.xml index ef83cc89..81b1da3f 100644 --- a/pom-maven-central.xml +++ b/pom-maven-central.xml @@ -7,7 +7,7 @@ kloadgen - 5.3.1 + 5.3.2 KLoadGen Load Generation Jmeter plugin for Kafka Cluster. Supporting AVRO, JSON Schema and Protobuf schema types. Generate Artificial data based on Data specification diff --git a/pom.xml b/pom.xml index 0caff5b5..6086900e 100644 --- a/pom.xml +++ b/pom.xml @@ -7,7 +7,7 @@ kloadgen - 5.3.1 + 5.3.2 KLoadGen Load Generation Jmeter plugin for Kafka Cluster. Supporting AVRO, JSON Schema and Protobuf schema types. Generate Artificial data base on Data specification diff --git a/src/test/java/com/sngular/kloadgen/loadgen/impl/ProtobufSRLoadGeneratorTest.java b/src/test/java/com/sngular/kloadgen/loadgen/impl/ProtobufSRLoadGeneratorTest.java new file mode 100644 index 00000000..fbf19acc --- /dev/null +++ b/src/test/java/com/sngular/kloadgen/loadgen/impl/ProtobufSRLoadGeneratorTest.java @@ -0,0 +1,85 @@ +/* + * This Source Code Form is subject to the terms of the Mozilla Public + * * License, v. 2.0. If a copy of the MPL was not distributed with this + * * file, You can obtain one at https://mozilla.org/MPL/2.0/. + */ + +package com.sngular.kloadgen.loadgen.impl; + +import java.io.File; +import java.io.IOException; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Locale; +import java.util.Map; + +import com.github.tomakehurst.wiremock.junit5.WireMockRuntimeInfo; +import com.github.tomakehurst.wiremock.junit5.WireMockTest; +import com.sngular.kloadgen.exception.KLoadGenException; +import com.sngular.kloadgen.model.FieldValueMapping; +import com.sngular.kloadgen.serializer.EnrichedRecord; +import com.sngular.kloadgen.util.ProducerKeysHelper; +import com.sngular.kloadgen.util.SchemaRegistryKeyHelper; +import io.confluent.kafka.schemaregistry.client.SchemaRegistryClientConfig; +import org.apache.jmeter.threads.JMeterContext; +import org.apache.jmeter.threads.JMeterContextService; +import org.apache.jmeter.threads.JMeterVariables; +import org.apache.jmeter.util.JMeterUtils; +import org.assertj.core.api.Assertions; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +@WireMockTest +class ProtobufSRLoadGeneratorTest { + + @BeforeEach + public void setUp(final WireMockRuntimeInfo wmRuntimeInfo) throws IOException { + final File file = new File("src/test/resources"); + final String absolutePath = file.getAbsolutePath(); + final String kloadPath = absolutePath + "/kloadgen.properties"; + final Map properties = new HashMap<>(); + properties.put(SchemaRegistryKeyHelper.SCHEMA_REGISTRY_URL, wmRuntimeInfo.getHttpBaseUrl()); + properties.put(SchemaRegistryKeyHelper.SCHEMA_REGISTRY_AUTH_KEY, SchemaRegistryKeyHelper.SCHEMA_REGISTRY_AUTH_BASIC_TYPE); + properties.put(SchemaRegistryKeyHelper.SCHEMA_REGISTRY_AUTH_FLAG, ProducerKeysHelper.FLAG_YES); + properties.put(SchemaRegistryClientConfig.BASIC_AUTH_CREDENTIALS_SOURCE, "foo"); + properties.put(SchemaRegistryClientConfig.USER_INFO_CONFIG,"foo"); + JMeterUtils.loadJMeterProperties(kloadPath); + final JMeterContext jmcx = JMeterContextService.getContext(); + jmcx.getProperties().putAll(properties); + jmcx.setVariables(new JMeterVariables()); + JMeterUtils.setLocale(Locale.ENGLISH); + } + + @Test + void testAvroLoadGenerator(final WireMockRuntimeInfo wmRuntimeInfo) throws KLoadGenException { + + final List fieldValueMappingList = Arrays.asList( + FieldValueMapping.builder().fieldName("propertyTest1.importedProperty.nestedProperty").fieldType("string").valueLength(0).fieldValueList("").required(true) + .isAncestorRequired(true).build(), + FieldValueMapping.builder().fieldName("propertyTest1.entityNumberTwo").fieldType("string").valueLength(0).fieldValueList("").required(true) + .isAncestorRequired(true).build(), + FieldValueMapping.builder().fieldName("propertyTest2.propertyNumberOne").fieldType("int").valueLength(0).fieldValueList("").required(true) + .isAncestorRequired(true).build(), + FieldValueMapping.builder().fieldName("propertyTest2.propertyNumberTwo").fieldType("string").valueLength(0).fieldValueList("").required(true) + .isAncestorRequired(true).build() + ); + + final Map originals = new HashMap<>(); + originals.put(SchemaRegistryKeyHelper.SCHEMA_REGISTRY_URL, wmRuntimeInfo.getHttpBaseUrl()); + originals.put(SchemaRegistryKeyHelper.SCHEMA_REGISTRY_USERNAME_KEY, "foo"); + originals.put(SchemaRegistryKeyHelper.SCHEMA_REGISTRY_PASSWORD_KEY, "foo"); + + final ProtobufLoadGenerator protobufLoadGenerator = new ProtobufLoadGenerator(); + protobufLoadGenerator.setUpGenerator(originals, "protobufSubjectWithImport", fieldValueMappingList); + final Object message = protobufLoadGenerator.nextMessage(); + Assertions.assertThat(message).isNotNull().isInstanceOf(EnrichedRecord.class); + + final EnrichedRecord enrichedRecord = (EnrichedRecord) message; + Assertions.assertThat(enrichedRecord.getGenericRecord()).isNotNull(); + Assertions.assertThat(enrichedRecord.getGenericRecord().toString()).contains("propertyTest1"); + Assertions.assertThat(enrichedRecord.getGenericRecord().toString()).contains("entityNumberTwo"); + Assertions.assertThat(enrichedRecord.getGenericRecord().toString()).contains("propertyNumberOne"); + Assertions.assertThat(enrichedRecord.getGenericRecord().toString()).contains("propertyNumberTwo"); + } +} diff --git a/src/test/resources/mappings/schema_registry_stub.json b/src/test/resources/mappings/schema_registry_stub.json index 212444c9..3de8dc1d 100644 --- a/src/test/resources/mappings/schema_registry_stub.json +++ b/src/test/resources/mappings/schema_registry_stub.json @@ -108,6 +108,86 @@ } } }, + { + "priority": 10, + "request": { + "method": "GET", + "urlPattern": "/subjects/protobuf_subject/versions/latest" + }, + "response": { + "status": 200, + "jsonBody": { + "subject": "ProtobufSubject", + "version": 1, + "id": 74, + "schemaType": "PROTOBUF", + "schema": "syntax = \"proto3\";\r\npackage sngular;\r\n\r\noption java_package = \"com.sngular.external.proto\";\r\noption java_outer_classname = \"ExternalMessage\";\r\noption java_multiple_files = true;\r\n\r\nmessage ExternalEntity{\r\n string parentPropertyOne = 1;\r\n int32 parentPropertyTwo = 2;\r\n \r\n message EntityOne {\r\n int32 entityNumberOne = 1;\r\n string entityNumberTwo = 2;\r\n \r\n message NestingEntity {\r\n string nestedProperty = 1;\r\n \r\n }\r\n \r\n }\r\n \r\n message EntityTwo {\r\n int32 propertyNumberOne = 1;\r\n string propertyNumberTwo = 2; \r\n }\r\n\r\n}" + } + } + }, + { + "priority": 10, + "request": { + "method": "GET", + "urlPattern": "/schemas/ids/74\\?fetchMaxId=false&subject=protobuf_subject" + }, + "response": { + "status": 200, + "jsonBody": { + "schemaType": "PROTOBUF", + "schema": "syntax = \"proto3\";\r\npackage sngular;\r\n\r\noption java_package = \"com.sngular.external.proto\";\r\noption java_outer_classname = \"ExternalMessage\";\r\noption java_multiple_files = true;\r\n\r\nmessage ExternalEntity{\r\n string parentPropertyOne = 1;\r\n int32 parentPropertyTwo = 2;\r\n \r\n message EntityOne {\r\n int32 entityNumberOne = 1;\r\n string entityNumberTwo = 2;\r\n \r\n message NestingEntity {\r\n string nestedProperty = 1;\r\n \r\n }\r\n \r\n }\r\n \r\n message EntityTwo {\r\n int32 propertyNumberOne = 1;\r\n string propertyNumberTwo = 2; \r\n }\r\n\r\n}" + }, + "headers": { + "Content-Type": "application/vnd.schemaregistry.v1+json" + } + } + }, + { + "priority": 10, + "request": { + "method": "GET", + "urlPattern": "/subjects/protobufSubjectWithImport/versions/latest" + }, + "response": { + "status": 200, + "jsonBody": { + "subject": "ProtobufSubjectWithImport", + "version": 1, + "id": 75, + "schemaType": "PROTOBUF", + "schema": "syntax = \"proto3\";\r\npackage sngular;\r\n\r\nimport \"protobuf_subject.proto\";\r\n\r\noption java_package = \"com.sngular.internal.proto\";\r\noption java_outer_classname = \"InternalMessage\";\r\noption java_multiple_files = true;\r\n\r\nmessage InternalMessage{\r\n\r\n InternalEntityOne propertyTest1 = 1;\r\n EntityTwo propertyTest2 = 2;\r\n \r\n message InternalEntityOne {\r\n ExternalEntity.EntityOne.NestingEntity importedProperty = 1;\r\n string entityNumberTwo = 2;\r\n }\r\n \r\n message EntityTwo {\r\n int32 propertyNumberOne = 1;\r\n string propertyNumberTwo = 2; \r\n }\r\n\r\n}", + "references" : [{ + "name": "protobuf_subject.proto", + "subject": "protobuf_subject", + "version": 1 + } + ] + } + } + }, + { + "priority": 10, + "request": { + "method": "GET", + "urlPattern": "/schemas/ids/75\\?fetchMaxId=false&subject=protobufSubjectWithImport" + }, + "response": { + "status": 200, + "jsonBody": { + "schemaType": "PROTOBUF", + "schema": "syntax = \"proto3\";\r\npackage sngular;\r\n\r\nimport \"protobuf_subject.proto\";\r\n\r\noption java_package = \"com.sngular.internal.proto\";\r\noption java_outer_classname = \"InternalMessage\";\r\noption java_multiple_files = true;\r\n\r\nmessage InternalMessage{\r\n\r\n InternalEntityOne propertyTest1 = 1;\r\n EntityTwo propertyTest2 = 2;\r\n \r\n message InternalEntityOne {\r\n ExternalEntity.EntityOne.NestingEntity importedProperty = 1;\r\n string entityNumberTwo = 2;\r\n }\r\n \r\n message EntityTwo {\r\n int32 propertyNumberOne = 1;\r\n string propertyNumberTwo = 2; \r\n }\r\n\r\n}", + "references" : [{ + "name": "protobuf_subject.proto", + "subject": "protobuf_subject", + "version": 1 + } + ] + }, + "headers": { + "Content-Type": "application/vnd.schemaregistry.v1+json" + } + } + }, { "priority": 10, "request": { @@ -119,12 +199,36 @@ "jsonBody": [ "avrosubject", "users", - "arrayMap" + "arrayMap", + "protobuf_subject", + "protobufSubjectWithImport" ], "headers": { "Content-Type": "application/vnd.schemaregistry.v1+json" } } + }, + { + "priority": 10, + "request": { + "method": "GET", + "urlPattern" : "/subjects/protobuf_subject/versions/1\\?deleted=true" + }, + "response": { + "status": 200, + "jsonBody": { + "subject": "protobuf_subject", + "version": 1, + "id": 74, + "schemaType": "PROTOBUF", + "schema": "syntax = \"proto3\";\npackage sngular;\n\noption java_package = \"com.sngular.external.proto\";\noption java_outer_classname = \"ExternalMessage\";\noption java_multiple_files = true;\n\nmessage ExternalEntity{\n string parentPropertyOne = 1;\n int32 parentPropertyTwo = 2;\n \n message EntityOne {\n int32 entityNumberOne = 1;\n string entityNumberTwo = 2;\n \n message NestingEntity {\n string nestedProperty = 1;\n \n }\n \n }\n \n message EntityTwo {\n int32 propertyNumberOne = 1;\n string propertyNumberTwo = 2; \n }\n\n}" + }, + "headers": { + "Content-Type": "application/vnd.schemaregistry.v1+json" + } + } + + } ] } diff --git a/src/test/resources/proto-files/messaging_intra_order.proto b/src/test/resources/proto-files/messaging_intra_order.proto deleted file mode 100644 index fc680958..00000000 --- a/src/test/resources/proto-files/messaging_intra_order.proto +++ /dev/null @@ -1,218 +0,0 @@ -syntax = "proto3"; - -package demo; - -option java_package = "com.demo.unmanaged.order.proto"; -option java_outer_classname = "IntraOrder"; -option java_multiple_files = true; - -message Order { - - message Address { - int32 address_id = 1; - string address_line_one = 2; - } - - message Contact { - int32 contact_id = 1; - string first_name = 2; - string last_name = 3; - } - - message Dimension { - string unit_of_measure = 1; - int32 value = 2; - } - - message CustomerAccount { - demo.Order.CustomerAccount.BillingParty billing_party = 1; - string name = 2; - - message BillingParty { - string p_name = 1; - repeated demo.Order.Contact p_contact = 2; - } - } - - message LogisticsProviderDetails { - demo.Order.LogisticsProviderDetails.ManagedBy managed_by = 1; - - message ManagedBy { - string organization_name = 1; - string organization_id = 2; - } - } - - message MarketingAreaDetails { - string origin_marketing_area = 1; - string origin_area_name = 2; - } - - message ServiceOfferingDetails { - string business_unit_code = 1; - string service_offering = 2; - } - - message Currency { - string currency_code = 1; - double value = 2; - } - - message OperationalRequirements { - string code = 1; - string type = 2; - demo.Order.OperationalRequirements.IntendedCarrier carrier = 3; - - message IntendedCarrier { - string id = 4; - } - } - - message RatingDetails { - string terms = 1; - string status = 2; - demo.Order.RatingDetails.RateSheets sheets = 3; - - message RateSheets { - repeated demo.Order.RatingDetails.RateSheets.RateSheet in_sheet = 1; - repeated demo.Order.RatingDetails.RateSheets.RateSheet es_sheet = 2; - - message RateSheet { - int32 id = 4; - } - } - } - - message OrderDetails { - repeated demo.Order.OrderDetails.OrderRequirements order_requirements = 1; - demo.Order.OrderDetails.InternationalDetails international_details = 2; - - message OrderRequirements { - string requirement = 1; - } - message InternationalDetails { - demo.Order.OrderDetails.InternationalDetails.CustomsDetails customs_details = 1; - repeated demo.Order.OrderDetails.InternationalDetails.InternationalServices international_services = 2; - demo.Order.OrderDetails.InternationalDetails.InbondDetails inbond_details = 3; - demo.Order.OrderDetails.InternationalDetails.PortDetails port_of_exit_location = 4; - demo.Order.OrderDetails.InternationalDetails.PortDetails port_of_entry_location = 5; - - message InternationalServices { - string service = 1; - } - - message PortDetails { - string location_id = 1; - string location_name = 2; - demo.Order.Address location_address = 3; - } - - message CustomsDetails { - demo.Order.OrderDetails.InternationalDetails.CustomsDetails.CustomsBroker customs_broker = 1; - string clearing_customs_country = 2; - - message CustomsBroker { - string party_name = 1; - demo.Order.Address party_address = 2; - repeated demo.Order.Contact party_contact = 3; - } - } - - message InbondDetails { - string bond_holder_role = 1; - demo.Order.OrderDetails.InternationalDetails.InbondDetails.BondHolderParty bond_holder_party = 2; - - message BondHolderParty { - string party_name = 3; - demo.Order.Address party_address = 4; - repeated demo.Order.Contact party_contact = 5; - } - } - } - } - - message RequestedOperationalServices { - string service_type = 1; - } - - message ReferenceNumbers { - int32 reference_number_id = 1; - string reference_number = 2; - } - - message Appointments { - int32 appointment_id = 1; - string appointment_type_code = 2; - repeated demo.Order.Appointments.AppointmentRequirements appointment_requirements = 3; - repeated demo.Order.Appointments.AppointmentNumbers appointment_numbers = 4; - - message AppointmentRequirements { - string requirement = 5; - } - - message AppointmentNumbers { - string appointment_number = 6; - } - } - - message AdditionalInstructions { - string instruction_type = 1; - string instruction_text = 2; - } - - message Comments { - string comment_type = 1; - string comment = 2; - } - - message Stops { - demo.Order.Stops.StopLocation stop_location = 1; - repeated demo.Order.Stops.ItemHandlingDetails item_handling_details = 2; - - message StopLocation { - string location_id = 1; - demo.Order.Address address = 2; - } - message ItemHandlingDetails { - int32 item_handling_detail_id = 3; - demo.Order.ItemUnit item_handling_unit = 4; - demo.Order.Dimension item_handling_unit_height = 5; - repeated demo.Order.Stops.ItemHandlingDetails.ItemDetails item_details = 6; - - message ItemDetails { - int32 stop_item_id = 7; - demo.Order.ItemUnit packaging_unit = 8; - demo.Order.Dimension item_density = 9; - demo.Order.Dimension item_height = 10; - repeated demo.Order.ReferenceNumbers reference_numbers = 11; - } - } - } - - message EquipmentRequirements { - demo.Order.EquipmentRequirements.EquipmentCharacteristics characteristics = 1; - - message EquipmentCharacteristics { - repeated demo.Order.EquipmentRequirements.EquipmentCharacteristics.EquipmentOptions e_options = 2; - - message EquipmentOptions { - string option = 3; - } - } - } - - message ItemUnit { - int32 quantity = 1; - string type = 2; - } - - message UserDetails { - string person_id = 1; - string first_name = 2; - } - - message SystemDetails { - string order_system = 1; - demo.Order.UserDetails created_by = 2; - } -} diff --git a/src/test/resources/proto-files/messaging_unified_order.proto b/src/test/resources/proto-files/messaging_unified_order.proto deleted file mode 100644 index e56ede6d..00000000 --- a/src/test/resources/proto-files/messaging_unified_order.proto +++ /dev/null @@ -1,66 +0,0 @@ -syntax = "proto3"; -package demo; - -import "messaging_intra_order.proto"; - -option java_package = "com.demo.unified.aggregator.proto"; -option java_outer_classname = "UnifiedAggregator"; -option java_multiple_files = true; - -message UnifiedOrder { - int32 order_id = 1; - string order_number = 2; - Order.CustomerAccount customer_account = 3; - Order.RatingDetails rating_details = 4; - Order.LogisticsProviderDetails logistics_provider_details = 5; - Order.MarketingAreaDetails marketing_area_details = 6; - Order.ServiceOfferingDetails service_offering_details = 7; - string order_type = 8; - string order_status = 9; - demo.UnifiedOrder.OrderDetails order_details = 10; - Order.OperationalRequirements operational_requirements = 11; - Order.EquipmentRequirements equipment_requirements = 12; - repeated demo.UnifiedOrder.Stops stops = 13; - repeated Order.ReferenceNumbers reference_numbers = 14; - Order.Contact solicitor_contact = 15; - Order.SystemDetails system_details = 16; - repeated Order.AdditionalInstructions additional_instructions = 17; - repeated Order.Comments comments = 18; - - message OrderDetails { - Order.Dimension total_weight = 1; - Order.Currency order_value = 2; - Order.Dimension estimated_distance = 3; - string shipping_option_date = 4; - bool comingle_flag = 5; - bool consolidated_flag = 6; - repeated Order.OrderDetails.OrderRequirements order_requirements = 7; - Order.Currency customer_tendered_rate = 8; - Order.OrderDetails.InternationalDetails international_details = 9; - Order.UserDetails order_owner = 10; - string shipment_identification_number = 11; - string order_tracking_number = 12; - string order_tracking_status = 13; - } - message Stops { - string stop_id = 1; - string stop_reason_code = 2; - string stop_type = 3; - string stop_status = 4; - string customer_provided_date_time = 5; - Order.Stops.StopLocation stop_location = 6; - Order.Contact stop_contact = 7; - repeated Order.Stops.ItemHandlingDetails item_handling_details = 8; - repeated Order.ReferenceNumbers reference_numbers = 9; - repeated Order.Appointments appointments = 10; - Order.Dimension stop_weight = 11; - int32 stop_quantity = 12; - int32 route_stop_id = 13; - Order.Dimension distance_to_next_stop = 14; - string rail_service_level = 15; - string rail_transit_mode = 16; - repeated Order.RequestedOperationalServices requested_operational_services = 17; - repeated Order.AdditionalInstructions additional_instructions = 18; - repeated Order.Comments comments = 19; - } -} From a129cd7846b46cdb91eea5c6cde0bb4cecc48a1c Mon Sep 17 00:00:00 2001 From: Adrian Lagartera Date: Wed, 8 Mar 2023 16:44:35 +0100 Subject: [PATCH 04/30] #332 Add Topic Name Strategy --- .../sngular/kloadgen/sampler/SamplerUtil.java | 28 +++++++++++++------ .../util/NamingStrategyKeyHelper.java | 10 +++++++ 2 files changed, 29 insertions(+), 9 deletions(-) create mode 100644 src/main/java/com/sngular/kloadgen/util/NamingStrategyKeyHelper.java diff --git a/src/main/java/com/sngular/kloadgen/sampler/SamplerUtil.java b/src/main/java/com/sngular/kloadgen/sampler/SamplerUtil.java index 9b1fb59f..1b91c3af 100644 --- a/src/main/java/com/sngular/kloadgen/sampler/SamplerUtil.java +++ b/src/main/java/com/sngular/kloadgen/sampler/SamplerUtil.java @@ -27,6 +27,7 @@ import com.sngular.kloadgen.model.FieldValueMapping; import com.sngular.kloadgen.model.HeaderMapping; import com.sngular.kloadgen.randomtool.generator.StatelessGeneratorTool; +import com.sngular.kloadgen.util.NamingStrategyKeyHelper; import com.sngular.kloadgen.util.ProducerKeysHelper; import com.sngular.kloadgen.util.PropsKeysHelper; import com.sngular.kloadgen.util.SchemaRegistryKeyHelper; @@ -123,6 +124,7 @@ public static Properties setupCommonProperties(final JavaSamplerContext context) props.put(ProducerConfig.COMPRESSION_TYPE_CONFIG, context.getParameter(ProducerConfig.COMPRESSION_TYPE_CONFIG)); props.put(CommonClientConfigs.SECURITY_PROTOCOL_CONFIG, context.getParameter(CommonClientConfigs.SECURITY_PROTOCOL_CONFIG)); props.put(ProducerKeysHelper.SASL_MECHANISM, context.getParameter(ProducerKeysHelper.SASL_MECHANISM)); + props.put(ProducerKeysHelper.KAFKA_TOPIC_CONFIG, context.getParameter(ProducerKeysHelper.KAFKA_TOPIC_CONFIG)); if (Objects.nonNull(context.getParameter(SchemaRegistryKeyHelper.ENABLE_AUTO_SCHEMA_REGISTRATION_CONFIG))) { props.put(SchemaRegistryKeyHelper.ENABLE_AUTO_SCHEMA_REGISTRATION_CONFIG, @@ -135,6 +137,8 @@ public static Properties setupCommonProperties(final JavaSamplerContext context) } }); + // Todo: Add kafk topic name + verifySecurity(context, props); return props; @@ -343,10 +347,16 @@ public static BaseLoadGenerator configureValueGenerator(final Properties props) props.putAll(originals); try { - generator.setUpGenerator( - originals, - jMeterVariables.get(PropsKeysHelper.VALUE_SUBJECT_NAME), - (List) jMeterVariables.getObject(PropsKeysHelper.VALUE_SCHEMA_PROPERTIES)); + if (props.getProperty(ProducerKeysHelper.VALUE_NAME_STRATEGY).equals(NamingStrategyKeyHelper.TOPIC_NAME_STRATEGY)) { + generator.setUpGenerator(originals, props.getProperty(ProducerKeysHelper.KAFKA_TOPIC_CONFIG), + (List) jMeterVariables.getObject(PropsKeysHelper.VALUE_SCHEMA_PROPERTIES)); + } else { + generator.setUpGenerator( + originals, + jMeterVariables.get(PropsKeysHelper.VALUE_SUBJECT_NAME), + (List) jMeterVariables.getObject(PropsKeysHelper.VALUE_SCHEMA_PROPERTIES)); + } + } catch (final KLoadGenException exc) { if (Objects.nonNull(props.get(SchemaRegistryKeyHelper.ENABLE_AUTO_SCHEMA_REGISTRATION_CONFIG))) { generator.setUpGenerator( @@ -358,8 +368,8 @@ public static BaseLoadGenerator configureValueGenerator(final Properties props) } } else { generator.setUpGenerator( - jMeterVariables.get(PropsKeysHelper.VALUE_SCHEMA), - (List) jMeterVariables.getObject(PropsKeysHelper.VALUE_SCHEMA_PROPERTIES)); + jMeterVariables.get(PropsKeysHelper.VALUE_SCHEMA), + (List) jMeterVariables.getObject(PropsKeysHelper.VALUE_SCHEMA_PROPERTIES)); } return generator; @@ -427,10 +437,10 @@ public static BaseLoadGenerator configureKeyGenerator(final Properties props) { public static List populateHeaders(final List kafkaHeaders, final ProducerRecord producerRecord) { final List headersSB = new ArrayList<>(); - for (HeaderMapping kafkaHeader : kafkaHeaders) { + for (final HeaderMapping kafkaHeader : kafkaHeaders) { final String headerValue = STATELESS_GENERATOR_TOOL.generateObject(kafkaHeader.getHeaderName(), kafkaHeader.getHeaderValue(), - 10, - Collections.emptyList()).toString(); + 10, + Collections.emptyList()).toString(); headersSB.add(kafkaHeader.getHeaderName().concat(":").concat(headerValue)); producerRecord.headers().add(kafkaHeader.getHeaderName(), headerValue.getBytes(StandardCharsets.UTF_8)); } diff --git a/src/main/java/com/sngular/kloadgen/util/NamingStrategyKeyHelper.java b/src/main/java/com/sngular/kloadgen/util/NamingStrategyKeyHelper.java new file mode 100644 index 00000000..534c09a5 --- /dev/null +++ b/src/main/java/com/sngular/kloadgen/util/NamingStrategyKeyHelper.java @@ -0,0 +1,10 @@ +package com.sngular.kloadgen.util; + +public class NamingStrategyKeyHelper { + + public static final String TOPIC_NAME_STRATEGY = "io.confluent.kafka.serializers.subject.TopicNameStrategy"; + + public static final String TOPIC_RECORD_NAME_STRATEGY = "io.confluent.kafka.serializers.subject.TopicRecordNameStrategy"; + + public static final String RECORD_NAME_STRATEGY = "io.confluent.kafka.serializers.subject.RecordNameStrategy"; +} From b8426a94622633aa062c43dcc1cf9540cec1495d Mon Sep 17 00:00:00 2001 From: Adrian Lagartera Date: Wed, 8 Mar 2023 16:46:56 +0100 Subject: [PATCH 05/30] #332 Change image from kafka manager to newest version Apparently the original project was renamed into cmak to avoid break the TOS of kafka having the kafka in the name of unofficial tool --- schema_registry_docker/docker-compose-noauth.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/schema_registry_docker/docker-compose-noauth.yml b/schema_registry_docker/docker-compose-noauth.yml index 69d886da..96f64a51 100644 --- a/schema_registry_docker/docker-compose-noauth.yml +++ b/schema_registry_docker/docker-compose-noauth.yml @@ -31,11 +31,11 @@ services: SCHEMA_REGISTRY_LISTENERS: http://0.0.0.0:8081 SCHEMA_REGISTRY_KAFKASTORE_CONNECTION_URL: zookeeper:2181 kafka-manager: - image: kafkamanager/kafka-manager + image: iunera/cmak depends_on: - kafka - zookeeper ports: - 9000:9000 environment: - ZK_HOSTS: http://zookeper:2181 + ZK_HOSTS: zookeeper:2181 From 4858372d5f3dd77bcbd2a1cf0c76ffae24fad77f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alfredo=20Gonz=C3=A1lez=20Fern=C3=A1ndez?= Date: Tue, 21 Mar 2023 13:53:58 +0100 Subject: [PATCH 06/30] Get the schema name from the schema references of imported classes --- pom.xml | 11 +++++++++++ .../impl/ProtobufObjectCreatorFactory.java | 4 ++-- .../processor/util/SchemaProcessorUtils.java | 16 ++++++++++++++-- 3 files changed, 27 insertions(+), 4 deletions(-) diff --git a/pom.xml b/pom.xml index 6086900e..95d59077 100644 --- a/pom.xml +++ b/pom.xml @@ -211,6 +211,17 @@ Europe/Madrid + + alfredo9f + Alfredo González + alfredo.gonzalez@sngular.com + Sngular + https://www.sngular.com + + Senior Backend Developer + + Europe/Madrid + diff --git a/src/main/java/com/sngular/kloadgen/processor/objectcreatorfactory/impl/ProtobufObjectCreatorFactory.java b/src/main/java/com/sngular/kloadgen/processor/objectcreatorfactory/impl/ProtobufObjectCreatorFactory.java index ff87f173..4a00cee6 100644 --- a/src/main/java/com/sngular/kloadgen/processor/objectcreatorfactory/impl/ProtobufObjectCreatorFactory.java +++ b/src/main/java/com/sngular/kloadgen/processor/objectcreatorfactory/impl/ProtobufObjectCreatorFactory.java @@ -42,9 +42,9 @@ public class ProtobufObjectCreatorFactory implements ObjectCreatorFactory { public ProtobufObjectCreatorFactory(final Object schema, final Object metadata) throws DescriptorValidationException, IOException { if (schema instanceof ParsedSchema) { - this.schema = SchemaProcessorUtils.buildProtoDescriptor((ProtoFileElement) ((ParsedSchema) schema).rawSchema()); + this.schema = SchemaProcessorUtils.buildProtoDescriptor((ProtoFileElement) ((ParsedSchema) schema).rawSchema(), metadata); } else if (schema instanceof ProtoFileElement) { - this.schema = SchemaProcessorUtils.buildProtoDescriptor((ProtoFileElement) schema); + this.schema = SchemaProcessorUtils.buildProtoDescriptor((ProtoFileElement) schema, metadata); } else { throw new KLoadGenException("Unsupported schema type"); } diff --git a/src/main/java/com/sngular/kloadgen/processor/util/SchemaProcessorUtils.java b/src/main/java/com/sngular/kloadgen/processor/util/SchemaProcessorUtils.java index 3fb2bb2f..3512fa50 100644 --- a/src/main/java/com/sngular/kloadgen/processor/util/SchemaProcessorUtils.java +++ b/src/main/java/com/sngular/kloadgen/processor/util/SchemaProcessorUtils.java @@ -32,6 +32,8 @@ import com.squareup.wire.schema.internal.parser.MessageElement; import com.squareup.wire.schema.internal.parser.ProtoFileElement; import com.squareup.wire.schema.internal.parser.TypeElement; +import io.confluent.kafka.schemaregistry.client.SchemaMetadata; +import io.confluent.kafka.schemaregistry.client.rest.entities.SchemaReference; import io.confluent.kafka.schemaregistry.client.rest.exceptions.RestClientException; import io.confluent.kafka.schemaregistry.protobuf.ProtobufSchema; import org.apache.commons.collections4.CollectionUtils; @@ -146,7 +148,7 @@ public static String[] splitAndNormalizeFullFieldName(final String fullFieldName return Arrays.stream(fields).map(field -> field.replaceAll("\\[.*]", "")).toArray(String[]::new); } - public static Descriptors.Descriptor buildProtoDescriptor(final ProtoFileElement schema) throws Descriptors.DescriptorValidationException, IOException { + public static Descriptors.Descriptor buildProtoDescriptor(final ProtoFileElement schema, Object metadata) throws Descriptors.DescriptorValidationException, IOException { final DynamicSchema.Builder schemaBuilder = DynamicSchema.newBuilder(); final List imports = schema.getImports(); @@ -161,7 +163,7 @@ public static Descriptors.Descriptor buildProtoDescriptor(final ProtoFileElement schemaBuilder.addSchema(importedSchema); } } else { - final var importSchema = JMeterHelper.getParsedSchema(importedClass.substring(0, importedClass.length() - 6), JMeterContextService.getContext().getProperties()); + final var importSchema = JMeterHelper.getParsedSchema(getSubjectName(importedClass, metadata), JMeterContextService.getContext().getProperties()); if (!ProtobufHelper.NOT_ACCEPTED_IMPORTS.contains(importedClass)) { schemaBuilder.addDependency(((ProtobufSchema) importSchema).toDescriptor().getFullName()); schemaBuilder.addSchema(convertDynamicSchema((ProtobufSchema) importSchema)); @@ -181,7 +183,17 @@ public static Descriptors.Descriptor buildProtoDescriptor(final ProtoFileElement return schemaBuilder.build().getMessageDescriptor(messageElement.getName()); } + private static String getSubjectName(String importedClass, Object metadata) { + List references = ((SchemaMetadata) metadata).getReferences(); + for(final SchemaReference schemaReference: references){ + if(schemaReference.getName().equals(importedClass)){ + return schemaReference.getSubject(); + } + } + + return importedClass; + } private static DynamicSchema convertDynamicSchema(final ProtobufSchema importSchema) throws DescriptorValidationException { return processImported(Arrays.asList(importSchema.rawSchema().toSchema().split("\\n"))); } From 0fd3920f47ad5062a5ebd0f32213528aa9906b78 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?David=20Hern=C3=A1ndez=20Gonz=C3=A1lez?= Date: Mon, 27 Mar 2023 11:41:49 +0200 Subject: [PATCH 07/30] #332 Removed generator set up by TopicNameStrategy --- .../processor/util/SchemaProcessorUtils.java | 42 +++++++++------- .../sngular/kloadgen/sampler/SamplerUtil.java | 50 ++++++++----------- 2 files changed, 43 insertions(+), 49 deletions(-) diff --git a/src/main/java/com/sngular/kloadgen/processor/util/SchemaProcessorUtils.java b/src/main/java/com/sngular/kloadgen/processor/util/SchemaProcessorUtils.java index 3512fa50..bab66e1d 100644 --- a/src/main/java/com/sngular/kloadgen/processor/util/SchemaProcessorUtils.java +++ b/src/main/java/com/sngular/kloadgen/processor/util/SchemaProcessorUtils.java @@ -183,17 +183,21 @@ public static Descriptors.Descriptor buildProtoDescriptor(final ProtoFileElement return schemaBuilder.build().getMessageDescriptor(messageElement.getName()); } - private static String getSubjectName(String importedClass, Object metadata) { - List references = ((SchemaMetadata) metadata).getReferences(); - for(final SchemaReference schemaReference: references){ - if(schemaReference.getName().equals(importedClass)){ - return schemaReference.getSubject(); + private static String getSubjectName(final String importedClass, final Object metadata) { + final List references = ((SchemaMetadata) metadata).getReferences(); + String subjectName = null; + + for (final SchemaReference schemaReference : references) { + if (schemaReference.getName().equals(importedClass)) { + subjectName = schemaReference.getSubject(); + break; } } - return importedClass; + return Objects.requireNonNullElse(subjectName, importedClass); } + private static DynamicSchema convertDynamicSchema(final ProtobufSchema importSchema) throws DescriptorValidationException { return processImported(Arrays.asList(importSchema.rawSchema().toSchema().split("\\n"))); } @@ -258,8 +262,8 @@ private static String checkIfGreppable(final String field) { } private static MessageDefinition buildProtoMessageDefinition( - final String fieldName, final TypeElement messageElement, final HashMap>> globalNestedTypesByLevelAndMessage, - final int previousDeepLevel) { + final String fieldName, final TypeElement messageElement, final HashMap>> globalNestedTypesByLevelAndMessage, + final int previousDeepLevel) { final int nextDeepLevel = previousDeepLevel + 1; @@ -275,11 +279,11 @@ private static MessageDefinition buildProtoMessageDefinition( } private static void extracted( - final HashMap>> globalNestedTypesByLevelAndMessage, final Builder msgDef, final List fieldElementList, - final int deepLevel, final String messageName) { + final HashMap>> globalNestedTypesByLevelAndMessage, final Builder msgDef, final List fieldElementList, + final int deepLevel, final String messageName) { final HashMap nestedTypes = processLevelTypes(globalNestedTypesByLevelAndMessage, msgDef, fieldElementList, deepLevel, - messageName); + messageName); for (final var elementField : fieldElementList) { final var elementFieldType = elementField.getType(); @@ -322,7 +326,7 @@ private static void extracted( msgDef.addField("repeated", "typemapnumber" + elementField.getName(), elementField.getName(), elementField.getTag()); msgDef.addMessageDefinition( - MessageDefinition.newBuilder("typemapnumber" + elementField.getName()).addField(OPTIONAL, "string", "key", 1).addField(OPTIONAL, realType, "value", 2).build()); + MessageDefinition.newBuilder("typemapnumber" + elementField.getName()).addField(OPTIONAL, "string", "key", 1).addField(OPTIONAL, realType, "value", 2).build()); } else if (Objects.nonNull(elementField.getLabel())) { msgDef.addField(elementField.getLabel().toString().toLowerCase(), elementField.getType(), elementField.getName(), elementField.getTag()); } else { @@ -332,9 +336,9 @@ private static void extracted( } private static HashMap processLevelTypes( - final HashMap>> globalNestedTypesByLevelAndMessage, final Builder msgDef, final List fieldElementList, - final int deepLevel, - final String messageName) { + final HashMap>> globalNestedTypesByLevelAndMessage, final Builder msgDef, final List fieldElementList, + final int deepLevel, + final String messageName) { final List allTypesInstantiatedByAttributesWithSimpleNames = new ArrayList<>(); for (final FieldElement fieldElement : fieldElementList) { @@ -367,8 +371,8 @@ private static HashMap processLevelTypes( } private static void addDefinition( - final MessageDefinition.Builder msgDef, final String typeName, final TypeElement typeElement, - final HashMap>> globalNestedTypesByLevelAndMessage, final int deepLevel) { + final MessageDefinition.Builder msgDef, final String typeName, final TypeElement typeElement, + final HashMap>> globalNestedTypesByLevelAndMessage, final int deepLevel) { if (typeElement instanceof EnumElement) { final var enumElement = (EnumElement) typeElement; @@ -385,8 +389,8 @@ private static void addDefinition( } private static void fillNestedTypes( - final TypeElement messageElement, final HashMap>> globalNestedTypesByLevelAndMessage, - final int deepLevel) { + final TypeElement messageElement, final HashMap>> globalNestedTypesByLevelAndMessage, + final int deepLevel) { HashMap> messageNestedTypes = globalNestedTypesByLevelAndMessage.get(deepLevel); if (messageNestedTypes == null) { diff --git a/src/main/java/com/sngular/kloadgen/sampler/SamplerUtil.java b/src/main/java/com/sngular/kloadgen/sampler/SamplerUtil.java index 1b91c3af..533b4099 100644 --- a/src/main/java/com/sngular/kloadgen/sampler/SamplerUtil.java +++ b/src/main/java/com/sngular/kloadgen/sampler/SamplerUtil.java @@ -128,7 +128,7 @@ public static Properties setupCommonProperties(final JavaSamplerContext context) if (Objects.nonNull(context.getParameter(SchemaRegistryKeyHelper.ENABLE_AUTO_SCHEMA_REGISTRATION_CONFIG))) { props.put(SchemaRegistryKeyHelper.ENABLE_AUTO_SCHEMA_REGISTRATION_CONFIG, - context.getParameter(SchemaRegistryKeyHelper.ENABLE_AUTO_SCHEMA_REGISTRATION_CONFIG)); + context.getParameter(SchemaRegistryKeyHelper.ENABLE_AUTO_SCHEMA_REGISTRATION_CONFIG)); } final Iterator parameters = context.getParameterNamesIterator(); parameters.forEachRemaining(parameter -> { @@ -178,7 +178,7 @@ public static Arguments getCommonConsumerDefaultParameters() { defaultParameters.addArgument(ConsumerConfig.SECURITY_PROVIDERS_CONFIG, ""); defaultParameters.addArgument(SslConfigs.SSL_ENABLED_PROTOCOLS_CONFIG, SslConfigs.DEFAULT_SSL_ENABLED_PROTOCOLS); defaultParameters.addArgument(SslConfigs.SSL_ENDPOINT_IDENTIFICATION_ALGORITHM_CONFIG, - SslConfigs.DEFAULT_SSL_ENDPOINT_IDENTIFICATION_ALGORITHM); + SslConfigs.DEFAULT_SSL_ENDPOINT_IDENTIFICATION_ALGORITHM); defaultParameters.addArgument(SslConfigs.SSL_KEYMANAGER_ALGORITHM_CONFIG, SslConfigs.DEFAULT_SSL_KEYMANGER_ALGORITHM); defaultParameters.addArgument(SslConfigs.SSL_KEYSTORE_TYPE_CONFIG, SslConfigs.DEFAULT_SSL_KEYSTORE_TYPE); defaultParameters.addArgument(SslConfigs.SSL_PROVIDER_CONFIG, ""); @@ -300,9 +300,9 @@ private static void verifySecurity(final JavaSamplerContext context, final Prope props.put(SslConfigs.SSL_ENABLED_PROTOCOLS_CONFIG, context.getParameter(SslConfigs.SSL_ENABLED_PROTOCOLS_CONFIG)); props.put(SslConfigs.SSL_ENDPOINT_IDENTIFICATION_ALGORITHM_CONFIG, - propertyOrDefault(context.getParameter(SslConfigs.SSL_ENDPOINT_IDENTIFICATION_ALGORITHM_CONFIG), - ProducerKeysHelper.DEFAULT_SSL_ENDPOINT_IDENTIFICATION_ALGORITHM, - "")); + propertyOrDefault(context.getParameter(SslConfigs.SSL_ENDPOINT_IDENTIFICATION_ALGORITHM_CONFIG), + ProducerKeysHelper.DEFAULT_SSL_ENDPOINT_IDENTIFICATION_ALGORITHM, + "")); props.put(SslConfigs.SSL_KEYMANAGER_ALGORITHM_CONFIG, context.getParameter(SslConfigs.SSL_KEYMANAGER_ALGORITHM_CONFIG)); props.put(SslConfigs.SSL_KEYSTORE_TYPE_CONFIG, context.getParameter(SslConfigs.SSL_KEYSTORE_TYPE_CONFIG)); @@ -338,7 +338,7 @@ public static BaseLoadGenerator configureValueGenerator(final Properties props) } props.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, - Objects.requireNonNullElse(jMeterVariables.get(PropsKeysHelper.VALUE_SERIALIZER_CLASS_PROPERTY), ProducerKeysHelper.VALUE_SERIALIZER_CLASS_CONFIG_DEFAULT)); + Objects.requireNonNullElse(jMeterVariables.get(PropsKeysHelper.VALUE_SERIALIZER_CLASS_PROPERTY), ProducerKeysHelper.VALUE_SERIALIZER_CLASS_CONFIG_DEFAULT)); if (Objects.nonNull(jMeterVariables.get(SchemaRegistryKeyHelper.SCHEMA_REGISTRY_URL))) { final Map originals = new HashMap<>(); @@ -347,29 +347,19 @@ public static BaseLoadGenerator configureValueGenerator(final Properties props) props.putAll(originals); try { - if (props.getProperty(ProducerKeysHelper.VALUE_NAME_STRATEGY).equals(NamingStrategyKeyHelper.TOPIC_NAME_STRATEGY)) { - generator.setUpGenerator(originals, props.getProperty(ProducerKeysHelper.KAFKA_TOPIC_CONFIG), - (List) jMeterVariables.getObject(PropsKeysHelper.VALUE_SCHEMA_PROPERTIES)); - } else { - generator.setUpGenerator( - originals, - jMeterVariables.get(PropsKeysHelper.VALUE_SUBJECT_NAME), - (List) jMeterVariables.getObject(PropsKeysHelper.VALUE_SCHEMA_PROPERTIES)); - } - + generator.setUpGenerator(originals, jMeterVariables.get(PropsKeysHelper.VALUE_SUBJECT_NAME), + (List) jMeterVariables.getObject(PropsKeysHelper.VALUE_SCHEMA_PROPERTIES)); } catch (final KLoadGenException exc) { if (Objects.nonNull(props.get(SchemaRegistryKeyHelper.ENABLE_AUTO_SCHEMA_REGISTRATION_CONFIG))) { - generator.setUpGenerator( - jMeterVariables.get(PropsKeysHelper.VALUE_SCHEMA), - (List) jMeterVariables.getObject(PropsKeysHelper.VALUE_SCHEMA_PROPERTIES)); + generator.setUpGenerator(jMeterVariables.get(PropsKeysHelper.VALUE_SCHEMA), + (List) jMeterVariables.getObject(PropsKeysHelper.VALUE_SCHEMA_PROPERTIES)); } else { throw exc; } } } else { - generator.setUpGenerator( - jMeterVariables.get(PropsKeysHelper.VALUE_SCHEMA), - (List) jMeterVariables.getObject(PropsKeysHelper.VALUE_SCHEMA_PROPERTIES)); + generator.setUpGenerator(jMeterVariables.get(PropsKeysHelper.VALUE_SCHEMA), + (List) jMeterVariables.getObject(PropsKeysHelper.VALUE_SCHEMA_PROPERTIES)); } return generator; @@ -404,7 +394,7 @@ public static BaseLoadGenerator configureKeyGenerator(final Properties props) { } props.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, - Objects.requireNonNullElse(jMeterVariables.get(PropsKeysHelper.VALUE_SERIALIZER_CLASS_PROPERTY), ProducerKeysHelper.KEY_SERIALIZER_CLASS_CONFIG_DEFAULT)); + Objects.requireNonNullElse(jMeterVariables.get(PropsKeysHelper.VALUE_SERIALIZER_CLASS_PROPERTY), ProducerKeysHelper.KEY_SERIALIZER_CLASS_CONFIG_DEFAULT)); if (Objects.nonNull(jMeterVariables.get(SchemaRegistryKeyHelper.SCHEMA_REGISTRY_URL))) { final Map originals = new HashMap<>(); @@ -423,13 +413,13 @@ public static BaseLoadGenerator configureKeyGenerator(final Properties props) { props.putAll(originals); generator.setUpGenerator( - originals, - jMeterVariables.get(PropsKeysHelper.KEY_SUBJECT_NAME), - (List) jMeterVariables.getObject(PropsKeysHelper.KEY_SCHEMA_PROPERTIES)); + originals, + jMeterVariables.get(PropsKeysHelper.KEY_SUBJECT_NAME), + (List) jMeterVariables.getObject(PropsKeysHelper.KEY_SCHEMA_PROPERTIES)); } else { generator.setUpGenerator( - jMeterVariables.get(PropsKeysHelper.KEY_SCHEMA), - (List) jMeterVariables.getObject(PropsKeysHelper.KEY_SCHEMA_PROPERTIES)); + jMeterVariables.get(PropsKeysHelper.KEY_SCHEMA), + (List) jMeterVariables.getObject(PropsKeysHelper.KEY_SCHEMA_PROPERTIES)); } return generator; @@ -439,8 +429,8 @@ public static List populateHeaders(final List kafkaHeader final List headersSB = new ArrayList<>(); for (final HeaderMapping kafkaHeader : kafkaHeaders) { final String headerValue = STATELESS_GENERATOR_TOOL.generateObject(kafkaHeader.getHeaderName(), kafkaHeader.getHeaderValue(), - 10, - Collections.emptyList()).toString(); + 10, + Collections.emptyList()).toString(); headersSB.add(kafkaHeader.getHeaderName().concat(":").concat(headerValue)); producerRecord.headers().add(kafkaHeader.getHeaderName(), headerValue.getBytes(StandardCharsets.UTF_8)); } From dd531d82d781ca8c568578de76d524c849f105fd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?David=20Hern=C3=A1ndez=20Gonz=C3=A1lez?= Date: Tue, 28 Mar 2023 10:03:17 +0200 Subject: [PATCH 08/30] #332 refactor --- .../java/com/sngular/kloadgen/sampler/SamplerUtil.java | 4 ---- .../sngular/kloadgen/util/NamingStrategyKeyHelper.java | 10 ---------- 2 files changed, 14 deletions(-) delete mode 100644 src/main/java/com/sngular/kloadgen/util/NamingStrategyKeyHelper.java diff --git a/src/main/java/com/sngular/kloadgen/sampler/SamplerUtil.java b/src/main/java/com/sngular/kloadgen/sampler/SamplerUtil.java index 533b4099..802b46c9 100644 --- a/src/main/java/com/sngular/kloadgen/sampler/SamplerUtil.java +++ b/src/main/java/com/sngular/kloadgen/sampler/SamplerUtil.java @@ -27,7 +27,6 @@ import com.sngular.kloadgen.model.FieldValueMapping; import com.sngular.kloadgen.model.HeaderMapping; import com.sngular.kloadgen.randomtool.generator.StatelessGeneratorTool; -import com.sngular.kloadgen.util.NamingStrategyKeyHelper; import com.sngular.kloadgen.util.ProducerKeysHelper; import com.sngular.kloadgen.util.PropsKeysHelper; import com.sngular.kloadgen.util.SchemaRegistryKeyHelper; @@ -124,7 +123,6 @@ public static Properties setupCommonProperties(final JavaSamplerContext context) props.put(ProducerConfig.COMPRESSION_TYPE_CONFIG, context.getParameter(ProducerConfig.COMPRESSION_TYPE_CONFIG)); props.put(CommonClientConfigs.SECURITY_PROTOCOL_CONFIG, context.getParameter(CommonClientConfigs.SECURITY_PROTOCOL_CONFIG)); props.put(ProducerKeysHelper.SASL_MECHANISM, context.getParameter(ProducerKeysHelper.SASL_MECHANISM)); - props.put(ProducerKeysHelper.KAFKA_TOPIC_CONFIG, context.getParameter(ProducerKeysHelper.KAFKA_TOPIC_CONFIG)); if (Objects.nonNull(context.getParameter(SchemaRegistryKeyHelper.ENABLE_AUTO_SCHEMA_REGISTRATION_CONFIG))) { props.put(SchemaRegistryKeyHelper.ENABLE_AUTO_SCHEMA_REGISTRATION_CONFIG, @@ -137,8 +135,6 @@ public static Properties setupCommonProperties(final JavaSamplerContext context) } }); - // Todo: Add kafk topic name - verifySecurity(context, props); return props; diff --git a/src/main/java/com/sngular/kloadgen/util/NamingStrategyKeyHelper.java b/src/main/java/com/sngular/kloadgen/util/NamingStrategyKeyHelper.java deleted file mode 100644 index 534c09a5..00000000 --- a/src/main/java/com/sngular/kloadgen/util/NamingStrategyKeyHelper.java +++ /dev/null @@ -1,10 +0,0 @@ -package com.sngular.kloadgen.util; - -public class NamingStrategyKeyHelper { - - public static final String TOPIC_NAME_STRATEGY = "io.confluent.kafka.serializers.subject.TopicNameStrategy"; - - public static final String TOPIC_RECORD_NAME_STRATEGY = "io.confluent.kafka.serializers.subject.TopicRecordNameStrategy"; - - public static final String RECORD_NAME_STRATEGY = "io.confluent.kafka.serializers.subject.RecordNameStrategy"; -} From b037bdfbb8d17c9b053ff70ead70d79eb76c7c62 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?David=20Hern=C3=A1ndez=20Gonz=C3=A1lez?= Date: Tue, 28 Mar 2023 12:04:10 +0200 Subject: [PATCH 09/30] #332 refactor --- .../processor/util/SchemaProcessorUtils.java | 5 +-- .../impl/ProtobufSRLoadGeneratorTest.java | 41 +++++++++++-------- 2 files changed, 24 insertions(+), 22 deletions(-) diff --git a/src/main/java/com/sngular/kloadgen/processor/util/SchemaProcessorUtils.java b/src/main/java/com/sngular/kloadgen/processor/util/SchemaProcessorUtils.java index d51dcd5f..da8aab94 100644 --- a/src/main/java/com/sngular/kloadgen/processor/util/SchemaProcessorUtils.java +++ b/src/main/java/com/sngular/kloadgen/processor/util/SchemaProcessorUtils.java @@ -34,7 +34,6 @@ import com.squareup.wire.schema.internal.parser.TypeElement; import io.confluent.kafka.schemaregistry.client.SchemaMetadata; import io.confluent.kafka.schemaregistry.client.rest.entities.SchemaReference; -import io.confluent.kafka.schemaregistry.client.rest.exceptions.RestClientException; import io.confluent.kafka.schemaregistry.protobuf.ProtobufSchema; import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.collections4.Predicate; @@ -148,7 +147,7 @@ public static String[] splitAndNormalizeFullFieldName(final String fullFieldName return Arrays.stream(fields).map(field -> field.replaceAll("\\[.*]", "")).toArray(String[]::new); } - public static Descriptors.Descriptor buildProtoDescriptor(final ProtoFileElement schema, Object metadata) throws Descriptors.DescriptorValidationException, IOException { + public static Descriptors.Descriptor buildProtoDescriptor(final ProtoFileElement schema, final Object metadata) throws Descriptors.DescriptorValidationException, IOException { final DynamicSchema.Builder schemaBuilder = DynamicSchema.newBuilder(); final List imports = schema.getImports(); @@ -169,8 +168,6 @@ public static Descriptors.Descriptor buildProtoDescriptor(final ProtoFileElement schemaBuilder.addSchema(convertDynamicSchema((ProtobufSchema) importSchema)); } } - } catch (RestClientException e) { - throw new RuntimeException(e); } } final MessageElement messageElement = (MessageElement) schema.getTypes().get(0); diff --git a/src/test/java/com/sngular/kloadgen/loadgen/impl/ProtobufSRLoadGeneratorTest.java b/src/test/java/com/sngular/kloadgen/loadgen/impl/ProtobufSRLoadGeneratorTest.java index fbf19acc..f8167216 100644 --- a/src/test/java/com/sngular/kloadgen/loadgen/impl/ProtobufSRLoadGeneratorTest.java +++ b/src/test/java/com/sngular/kloadgen/loadgen/impl/ProtobufSRLoadGeneratorTest.java @@ -18,6 +18,8 @@ import com.github.tomakehurst.wiremock.junit5.WireMockTest; import com.sngular.kloadgen.exception.KLoadGenException; import com.sngular.kloadgen.model.FieldValueMapping; +import com.sngular.kloadgen.sampler.schemaregistry.SchemaRegistryManager; +import com.sngular.kloadgen.sampler.schemaregistry.SchemaRegistryManagerFactory; import com.sngular.kloadgen.serializer.EnrichedRecord; import com.sngular.kloadgen.util.ProducerKeysHelper; import com.sngular.kloadgen.util.SchemaRegistryKeyHelper; @@ -40,10 +42,11 @@ public void setUp(final WireMockRuntimeInfo wmRuntimeInfo) throws IOException { final String kloadPath = absolutePath + "/kloadgen.properties"; final Map properties = new HashMap<>(); properties.put(SchemaRegistryKeyHelper.SCHEMA_REGISTRY_URL, wmRuntimeInfo.getHttpBaseUrl()); + properties.put(SchemaRegistryKeyHelper.SCHEMA_REGISTRY_NAME, "confluent"); properties.put(SchemaRegistryKeyHelper.SCHEMA_REGISTRY_AUTH_KEY, SchemaRegistryKeyHelper.SCHEMA_REGISTRY_AUTH_BASIC_TYPE); properties.put(SchemaRegistryKeyHelper.SCHEMA_REGISTRY_AUTH_FLAG, ProducerKeysHelper.FLAG_YES); properties.put(SchemaRegistryClientConfig.BASIC_AUTH_CREDENTIALS_SOURCE, "foo"); - properties.put(SchemaRegistryClientConfig.USER_INFO_CONFIG,"foo"); + properties.put(SchemaRegistryClientConfig.USER_INFO_CONFIG, "foo"); JMeterUtils.loadJMeterProperties(kloadPath); final JMeterContext jmcx = JMeterContextService.getContext(); jmcx.getProperties().putAll(properties); @@ -52,34 +55,36 @@ public void setUp(final WireMockRuntimeInfo wmRuntimeInfo) throws IOException { } @Test - void testAvroLoadGenerator(final WireMockRuntimeInfo wmRuntimeInfo) throws KLoadGenException { + void testProtobufLoadGenerator(final WireMockRuntimeInfo wmRuntimeInfo) throws KLoadGenException { final List fieldValueMappingList = Arrays.asList( - FieldValueMapping.builder().fieldName("propertyTest1.importedProperty.nestedProperty").fieldType("string").valueLength(0).fieldValueList("").required(true) - .isAncestorRequired(true).build(), - FieldValueMapping.builder().fieldName("propertyTest1.entityNumberTwo").fieldType("string").valueLength(0).fieldValueList("").required(true) - .isAncestorRequired(true).build(), - FieldValueMapping.builder().fieldName("propertyTest2.propertyNumberOne").fieldType("int").valueLength(0).fieldValueList("").required(true) - .isAncestorRequired(true).build(), - FieldValueMapping.builder().fieldName("propertyTest2.propertyNumberTwo").fieldType("string").valueLength(0).fieldValueList("").required(true) - .isAncestorRequired(true).build() + FieldValueMapping.builder().fieldName("propertyTest1.importedProperty.nestedProperty").fieldType("string").valueLength(0).fieldValueList("").required(true) + .isAncestorRequired(true).build(), + FieldValueMapping.builder().fieldName("propertyTest1.entityNumberTwo").fieldType("string").valueLength(0).fieldValueList("").required(true) + .isAncestorRequired(true).build(), + FieldValueMapping.builder().fieldName("propertyTest2.propertyNumberOne").fieldType("int").valueLength(0).fieldValueList("").required(true) + .isAncestorRequired(true).build(), + FieldValueMapping.builder().fieldName("propertyTest2.propertyNumberTwo").fieldType("string").valueLength(0).fieldValueList("").required(true) + .isAncestorRequired(true).build() ); + final SchemaRegistryManager schemaRegistryManager = SchemaRegistryManagerFactory.getSchemaRegistry("confluent"); final Map originals = new HashMap<>(); - originals.put(SchemaRegistryKeyHelper.SCHEMA_REGISTRY_URL, wmRuntimeInfo.getHttpBaseUrl()); + + originals.put(schemaRegistryManager.getSchemaRegistryUrlKey(), wmRuntimeInfo.getHttpBaseUrl()); + originals.put(SchemaRegistryKeyHelper.SCHEMA_REGISTRY_NAME, "confluent"); originals.put(SchemaRegistryKeyHelper.SCHEMA_REGISTRY_USERNAME_KEY, "foo"); originals.put(SchemaRegistryKeyHelper.SCHEMA_REGISTRY_PASSWORD_KEY, "foo"); final ProtobufLoadGenerator protobufLoadGenerator = new ProtobufLoadGenerator(); protobufLoadGenerator.setUpGenerator(originals, "protobufSubjectWithImport", fieldValueMappingList); - final Object message = protobufLoadGenerator.nextMessage(); + final EnrichedRecord message = protobufLoadGenerator.nextMessage(); Assertions.assertThat(message).isNotNull().isInstanceOf(EnrichedRecord.class); - final EnrichedRecord enrichedRecord = (EnrichedRecord) message; - Assertions.assertThat(enrichedRecord.getGenericRecord()).isNotNull(); - Assertions.assertThat(enrichedRecord.getGenericRecord().toString()).contains("propertyTest1"); - Assertions.assertThat(enrichedRecord.getGenericRecord().toString()).contains("entityNumberTwo"); - Assertions.assertThat(enrichedRecord.getGenericRecord().toString()).contains("propertyNumberOne"); - Assertions.assertThat(enrichedRecord.getGenericRecord().toString()).contains("propertyNumberTwo"); + Assertions.assertThat(message.getGenericRecord()).isNotNull(); + Assertions.assertThat(message.getGenericRecord().toString()).contains("propertyTest1"); + Assertions.assertThat(message.getGenericRecord().toString()).contains("entityNumberTwo"); + Assertions.assertThat(message.getGenericRecord().toString()).contains("propertyNumberOne"); + Assertions.assertThat(message.getGenericRecord().toString()).contains("propertyNumberTwo"); } } From 82a445c513639e7879d1f66b67f4a089a344ba84 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?David=20Hern=C3=A1ndez=20Gonz=C3=A1lez?= Date: Tue, 28 Mar 2023 12:11:35 +0200 Subject: [PATCH 10/30] #332 refactor --- .../kloadgen/loadgen/impl/ProtobufSRLoadGeneratorTest.java | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/test/java/com/sngular/kloadgen/loadgen/impl/ProtobufSRLoadGeneratorTest.java b/src/test/java/com/sngular/kloadgen/loadgen/impl/ProtobufSRLoadGeneratorTest.java index f8167216..97ae8687 100644 --- a/src/test/java/com/sngular/kloadgen/loadgen/impl/ProtobufSRLoadGeneratorTest.java +++ b/src/test/java/com/sngular/kloadgen/loadgen/impl/ProtobufSRLoadGeneratorTest.java @@ -18,7 +18,6 @@ import com.github.tomakehurst.wiremock.junit5.WireMockTest; import com.sngular.kloadgen.exception.KLoadGenException; import com.sngular.kloadgen.model.FieldValueMapping; -import com.sngular.kloadgen.sampler.schemaregistry.SchemaRegistryManager; import com.sngular.kloadgen.sampler.schemaregistry.SchemaRegistryManagerFactory; import com.sngular.kloadgen.serializer.EnrichedRecord; import com.sngular.kloadgen.util.ProducerKeysHelper; @@ -68,7 +67,7 @@ void testProtobufLoadGenerator(final WireMockRuntimeInfo wmRuntimeInfo) throws K .isAncestorRequired(true).build() ); - final SchemaRegistryManager schemaRegistryManager = SchemaRegistryManagerFactory.getSchemaRegistry("confluent"); + final var schemaRegistryManager = SchemaRegistryManagerFactory.getSchemaRegistry("confluent"); final Map originals = new HashMap<>(); originals.put(schemaRegistryManager.getSchemaRegistryUrlKey(), wmRuntimeInfo.getHttpBaseUrl()); @@ -76,9 +75,9 @@ void testProtobufLoadGenerator(final WireMockRuntimeInfo wmRuntimeInfo) throws K originals.put(SchemaRegistryKeyHelper.SCHEMA_REGISTRY_USERNAME_KEY, "foo"); originals.put(SchemaRegistryKeyHelper.SCHEMA_REGISTRY_PASSWORD_KEY, "foo"); - final ProtobufLoadGenerator protobufLoadGenerator = new ProtobufLoadGenerator(); + final var protobufLoadGenerator = new ProtobufLoadGenerator(); protobufLoadGenerator.setUpGenerator(originals, "protobufSubjectWithImport", fieldValueMappingList); - final EnrichedRecord message = protobufLoadGenerator.nextMessage(); + final var message = protobufLoadGenerator.nextMessage(); Assertions.assertThat(message).isNotNull().isInstanceOf(EnrichedRecord.class); Assertions.assertThat(message.getGenericRecord()).isNotNull(); From 622398a4752b122f660ab84aaf5cd8fa2fce7a5e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?David=20Hern=C3=A1ndez=20Gonz=C3=A1lez?= Date: Tue, 28 Mar 2023 12:31:26 +0200 Subject: [PATCH 11/30] #332 updated pom version --- pom-maven-central.xml | 2 +- pom.xml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pom-maven-central.xml b/pom-maven-central.xml index d2522eda..865f9313 100644 --- a/pom-maven-central.xml +++ b/pom-maven-central.xml @@ -7,7 +7,7 @@ kloadgen - 5.4.1 + 5.4.2 KLoadGen Load Generation Jmeter plugin for Kafka Cluster. Supporting AVRO, JSON Schema and Protobuf schema types. Generate Artificial diff --git a/pom.xml b/pom.xml index ca371df1..5f54018f 100644 --- a/pom.xml +++ b/pom.xml @@ -7,7 +7,7 @@ kloadgen - 5.4.1 + 5.4.2 KLoadGen Load Generation Jmeter plugin for Kafka Cluster. Supporting AVRO, JSON Schema and Protobuf schema types. Generate Artificial From eb5f8b9780def501c06ba386b51c2ec68cd8b766 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?David=20Hern=C3=A1ndez=20Gonz=C3=A1lez?= Date: Tue, 28 Mar 2023 13:31:39 +0200 Subject: [PATCH 12/30] #332 fixes PR --- .../processor/util/SchemaProcessorUtils.java | 30 ++++++++-------- .../sngular/kloadgen/sampler/SamplerUtil.java | 34 +++++++------------ .../impl/ProtobufSRLoadGeneratorTest.java | 16 ++++----- 3 files changed, 36 insertions(+), 44 deletions(-) diff --git a/src/main/java/com/sngular/kloadgen/processor/util/SchemaProcessorUtils.java b/src/main/java/com/sngular/kloadgen/processor/util/SchemaProcessorUtils.java index da8aab94..93de1738 100644 --- a/src/main/java/com/sngular/kloadgen/processor/util/SchemaProcessorUtils.java +++ b/src/main/java/com/sngular/kloadgen/processor/util/SchemaProcessorUtils.java @@ -45,6 +45,8 @@ public class SchemaProcessorUtils { private static final String OPTIONAL = "optional"; + private static final String TYPE_MAP_NUMBER = "typemapnumber"; + private SchemaProcessorUtils() { } @@ -262,8 +264,8 @@ private static String checkIfGreppable(final String field) { } private static MessageDefinition buildProtoMessageDefinition( - final String fieldName, final TypeElement messageElement, final HashMap>> globalNestedTypesByLevelAndMessage, - final int previousDeepLevel) { + final String fieldName, final TypeElement messageElement, final HashMap>> globalNestedTypesByLevelAndMessage, + final int previousDeepLevel) { final int nextDeepLevel = previousDeepLevel + 1; @@ -279,11 +281,11 @@ private static MessageDefinition buildProtoMessageDefinition( } private static void extracted( - final HashMap>> globalNestedTypesByLevelAndMessage, final Builder msgDef, final List fieldElementList, - final int deepLevel, final String messageName) { + final HashMap>> globalNestedTypesByLevelAndMessage, final Builder msgDef, final List fieldElementList, + final int deepLevel, final String messageName) { final HashMap nestedTypes = processLevelTypes(globalNestedTypesByLevelAndMessage, msgDef, fieldElementList, deepLevel, - messageName); + messageName); for (final var elementField : fieldElementList) { final var elementFieldType = elementField.getType(); @@ -323,10 +325,10 @@ private static void extracted( addDefinition(msgDef, mapDotType, removed, globalNestedTypesByLevelAndMessage, deepLevel); } - msgDef.addField("repeated", "typemapnumber" + elementField.getName(), elementField.getName(), elementField.getTag()); + msgDef.addField("repeated", TYPE_MAP_NUMBER + elementField.getName(), elementField.getName(), elementField.getTag()); msgDef.addMessageDefinition( - MessageDefinition.newBuilder("typemapnumber" + elementField.getName()).addField(OPTIONAL, "string", "key", 1).addField(OPTIONAL, realType, "value", 2).build()); + MessageDefinition.newBuilder(TYPE_MAP_NUMBER + elementField.getName()).addField(OPTIONAL, "string", "key", 1).addField(OPTIONAL, realType, "value", 2).build()); } else if (Objects.nonNull(elementField.getLabel())) { msgDef.addField(elementField.getLabel().toString().toLowerCase(), elementField.getType(), elementField.getName(), elementField.getTag()); } else { @@ -336,9 +338,9 @@ private static void extracted( } private static HashMap processLevelTypes( - final HashMap>> globalNestedTypesByLevelAndMessage, final Builder msgDef, final List fieldElementList, - final int deepLevel, - final String messageName) { + final HashMap>> globalNestedTypesByLevelAndMessage, final Builder msgDef, final List fieldElementList, + final int deepLevel, + final String messageName) { final List allTypesInstantiatedByAttributesWithSimpleNames = new ArrayList<>(); for (final FieldElement fieldElement : fieldElementList) { @@ -371,8 +373,8 @@ private static HashMap processLevelTypes( } private static void addDefinition( - final MessageDefinition.Builder msgDef, final String typeName, final TypeElement typeElement, - final HashMap>> globalNestedTypesByLevelAndMessage, final int deepLevel) { + final MessageDefinition.Builder msgDef, final String typeName, final TypeElement typeElement, + final HashMap>> globalNestedTypesByLevelAndMessage, final int deepLevel) { if (typeElement instanceof EnumElement) { final var enumElement = (EnumElement) typeElement; @@ -389,8 +391,8 @@ private static void addDefinition( } private static void fillNestedTypes( - final TypeElement messageElement, final HashMap>> globalNestedTypesByLevelAndMessage, - final int deepLevel) { + final TypeElement messageElement, final HashMap>> globalNestedTypesByLevelAndMessage, + final int deepLevel) { HashMap> messageNestedTypes = globalNestedTypesByLevelAndMessage.get(deepLevel); if (messageNestedTypes == null) { diff --git a/src/main/java/com/sngular/kloadgen/sampler/SamplerUtil.java b/src/main/java/com/sngular/kloadgen/sampler/SamplerUtil.java index 526ca091..f0a9a9b3 100644 --- a/src/main/java/com/sngular/kloadgen/sampler/SamplerUtil.java +++ b/src/main/java/com/sngular/kloadgen/sampler/SamplerUtil.java @@ -177,8 +177,7 @@ public static Arguments getCommonConsumerDefaultParameters() { defaultParameters.addArgument(ConsumerConfig.CLIENT_ID_CONFIG, ""); defaultParameters.addArgument(ConsumerConfig.SECURITY_PROVIDERS_CONFIG, ""); defaultParameters.addArgument(SslConfigs.SSL_ENABLED_PROTOCOLS_CONFIG, SslConfigs.DEFAULT_SSL_ENABLED_PROTOCOLS); - defaultParameters.addArgument(SslConfigs.SSL_ENDPOINT_IDENTIFICATION_ALGORITHM_CONFIG, - SslConfigs.DEFAULT_SSL_ENDPOINT_IDENTIFICATION_ALGORITHM); + defaultParameters.addArgument(SslConfigs.SSL_ENDPOINT_IDENTIFICATION_ALGORITHM_CONFIG, SslConfigs.DEFAULT_SSL_ENDPOINT_IDENTIFICATION_ALGORITHM); defaultParameters.addArgument(SslConfigs.SSL_KEYMANAGER_ALGORITHM_CONFIG, SslConfigs.DEFAULT_SSL_KEYMANGER_ALGORITHM); defaultParameters.addArgument(SslConfigs.SSL_KEYSTORE_TYPE_CONFIG, SslConfigs.DEFAULT_SSL_KEYSTORE_TYPE); defaultParameters.addArgument(SslConfigs.SSL_PROVIDER_CONFIG, ""); @@ -303,9 +302,8 @@ private static void verifySecurity(final JavaSamplerContext context, final Prope props.put(SslConfigs.SSL_ENABLED_PROTOCOLS_CONFIG, context.getParameter(SslConfigs.SSL_ENABLED_PROTOCOLS_CONFIG)); props.put(SslConfigs.SSL_ENDPOINT_IDENTIFICATION_ALGORITHM_CONFIG, - propertyOrDefault(context.getParameter(SslConfigs.SSL_ENDPOINT_IDENTIFICATION_ALGORITHM_CONFIG), - ProducerKeysHelper.DEFAULT_SSL_ENDPOINT_IDENTIFICATION_ALGORITHM, - "")); + propertyOrDefault(context.getParameter(SslConfigs.SSL_ENDPOINT_IDENTIFICATION_ALGORITHM_CONFIG), ProducerKeysHelper.DEFAULT_SSL_ENDPOINT_IDENTIFICATION_ALGORITHM, + "")); props.put(SslConfigs.SSL_KEYMANAGER_ALGORITHM_CONFIG, context.getParameter(SslConfigs.SSL_KEYMANAGER_ALGORITHM_CONFIG)); props.put(SslConfigs.SSL_KEYSTORE_TYPE_CONFIG, context.getParameter(SslConfigs.SSL_KEYSTORE_TYPE_CONFIG)); @@ -344,7 +342,7 @@ public static BaseLoadGenerator configureValueGenerator(final Properties props) } props.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, - Objects.requireNonNullElse(jMeterVariables.get(PropsKeysHelper.VALUE_SERIALIZER_CLASS_PROPERTY), ProducerKeysHelper.VALUE_SERIALIZER_CLASS_CONFIG_DEFAULT)); + Objects.requireNonNullElse(jMeterVariables.get(PropsKeysHelper.VALUE_SERIALIZER_CLASS_PROPERTY), ProducerKeysHelper.VALUE_SERIALIZER_CLASS_CONFIG_DEFAULT)); if (Objects.nonNull(jMeterVariables.get(SchemaRegistryKeyHelper.SCHEMA_REGISTRY_NAME))) { final Map originals = new HashMap<>(); @@ -354,18 +352,16 @@ public static BaseLoadGenerator configureValueGenerator(final Properties props) try { generator.setUpGenerator(originals, jMeterVariables.get(PropsKeysHelper.VALUE_SUBJECT_NAME), - (List) jMeterVariables.getObject(PropsKeysHelper.VALUE_SCHEMA_PROPERTIES)); + (List) jMeterVariables.getObject(PropsKeysHelper.VALUE_SCHEMA_PROPERTIES)); } catch (final KLoadGenException exc) { if (Objects.nonNull(props.get(SchemaRegistryKeyHelper.ENABLE_AUTO_SCHEMA_REGISTRATION_CONFIG))) { - generator.setUpGenerator(jMeterVariables.get(PropsKeysHelper.VALUE_SCHEMA), - (List) jMeterVariables.getObject(PropsKeysHelper.VALUE_SCHEMA_PROPERTIES)); + generator.setUpGenerator(jMeterVariables.get(PropsKeysHelper.VALUE_SCHEMA), (List) jMeterVariables.getObject(PropsKeysHelper.VALUE_SCHEMA_PROPERTIES)); } else { throw exc; } } } else { - generator.setUpGenerator(jMeterVariables.get(PropsKeysHelper.VALUE_SCHEMA), - (List) jMeterVariables.getObject(PropsKeysHelper.VALUE_SCHEMA_PROPERTIES)); + generator.setUpGenerator(jMeterVariables.get(PropsKeysHelper.VALUE_SCHEMA), (List) jMeterVariables.getObject(PropsKeysHelper.VALUE_SCHEMA_PROPERTIES)); } return generator; @@ -405,7 +401,7 @@ public static BaseLoadGenerator configureKeyGenerator(final Properties props) { } props.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, - Objects.requireNonNullElse(jMeterVariables.get(PropsKeysHelper.VALUE_SERIALIZER_CLASS_PROPERTY), ProducerKeysHelper.KEY_SERIALIZER_CLASS_CONFIG_DEFAULT)); + Objects.requireNonNullElse(jMeterVariables.get(PropsKeysHelper.VALUE_SERIALIZER_CLASS_PROPERTY), ProducerKeysHelper.KEY_SERIALIZER_CLASS_CONFIG_DEFAULT)); if (Objects.nonNull(jMeterVariables.get(SchemaRegistryKeyHelper.SCHEMA_REGISTRY_NAME))) { final Map originals = new HashMap<>(); @@ -424,14 +420,10 @@ public static BaseLoadGenerator configureKeyGenerator(final Properties props) { props.putAll(originals); - generator.setUpGenerator( - originals, - jMeterVariables.get(PropsKeysHelper.KEY_SUBJECT_NAME), - (List) jMeterVariables.getObject(PropsKeysHelper.KEY_SCHEMA_PROPERTIES)); + generator.setUpGenerator(originals, jMeterVariables.get(PropsKeysHelper.KEY_SUBJECT_NAME), + (List) jMeterVariables.getObject(PropsKeysHelper.KEY_SCHEMA_PROPERTIES)); } else { - generator.setUpGenerator( - jMeterVariables.get(PropsKeysHelper.KEY_SCHEMA), - (List) jMeterVariables.getObject(PropsKeysHelper.KEY_SCHEMA_PROPERTIES)); + generator.setUpGenerator(jMeterVariables.get(PropsKeysHelper.KEY_SCHEMA), (List) jMeterVariables.getObject(PropsKeysHelper.KEY_SCHEMA_PROPERTIES)); } return generator; @@ -440,9 +432,7 @@ public static BaseLoadGenerator configureKeyGenerator(final Properties props) { public static List populateHeaders(final List kafkaHeaders, final ProducerRecord producerRecord) { final List headersSB = new ArrayList<>(); for (final HeaderMapping kafkaHeader : kafkaHeaders) { - final String headerValue = STATELESS_GENERATOR_TOOL.generateObject(kafkaHeader.getHeaderName(), kafkaHeader.getHeaderValue(), - 10, - Collections.emptyList()).toString(); + final String headerValue = STATELESS_GENERATOR_TOOL.generateObject(kafkaHeader.getHeaderName(), kafkaHeader.getHeaderValue(), 10, Collections.emptyList()).toString(); headersSB.add(kafkaHeader.getHeaderName().concat(":").concat(headerValue)); producerRecord.headers().add(kafkaHeader.getHeaderName(), headerValue.getBytes(StandardCharsets.UTF_8)); } diff --git a/src/test/java/com/sngular/kloadgen/loadgen/impl/ProtobufSRLoadGeneratorTest.java b/src/test/java/com/sngular/kloadgen/loadgen/impl/ProtobufSRLoadGeneratorTest.java index 97ae8687..d10474d9 100644 --- a/src/test/java/com/sngular/kloadgen/loadgen/impl/ProtobufSRLoadGeneratorTest.java +++ b/src/test/java/com/sngular/kloadgen/loadgen/impl/ProtobufSRLoadGeneratorTest.java @@ -57,14 +57,14 @@ public void setUp(final WireMockRuntimeInfo wmRuntimeInfo) throws IOException { void testProtobufLoadGenerator(final WireMockRuntimeInfo wmRuntimeInfo) throws KLoadGenException { final List fieldValueMappingList = Arrays.asList( - FieldValueMapping.builder().fieldName("propertyTest1.importedProperty.nestedProperty").fieldType("string").valueLength(0).fieldValueList("").required(true) - .isAncestorRequired(true).build(), - FieldValueMapping.builder().fieldName("propertyTest1.entityNumberTwo").fieldType("string").valueLength(0).fieldValueList("").required(true) - .isAncestorRequired(true).build(), - FieldValueMapping.builder().fieldName("propertyTest2.propertyNumberOne").fieldType("int").valueLength(0).fieldValueList("").required(true) - .isAncestorRequired(true).build(), - FieldValueMapping.builder().fieldName("propertyTest2.propertyNumberTwo").fieldType("string").valueLength(0).fieldValueList("").required(true) - .isAncestorRequired(true).build() + FieldValueMapping.builder().fieldName("propertyTest1.importedProperty.nestedProperty").fieldType("string").valueLength(0).fieldValueList("").required(true) + .isAncestorRequired(true).build(), + FieldValueMapping.builder().fieldName("propertyTest1.entityNumberTwo").fieldType("string").valueLength(0).fieldValueList("").required(true) + .isAncestorRequired(true).build(), + FieldValueMapping.builder().fieldName("propertyTest2.propertyNumberOne").fieldType("int").valueLength(0).fieldValueList("").required(true) + .isAncestorRequired(true).build(), + FieldValueMapping.builder().fieldName("propertyTest2.propertyNumberTwo").fieldType("string").valueLength(0).fieldValueList("").required(true) + .isAncestorRequired(true).build() ); final var schemaRegistryManager = SchemaRegistryManagerFactory.getSchemaRegistry("confluent"); From e86011b0fed8b7ed0753d3c6dfac44cfa55de09f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?David=20Hern=C3=A1ndez=20Gonz=C3=A1lez?= Date: Tue, 28 Mar 2023 13:35:49 +0200 Subject: [PATCH 13/30] #332 fixes PR --- .../kloadgen/processor/util/SchemaProcessorUtils.java | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/main/java/com/sngular/kloadgen/processor/util/SchemaProcessorUtils.java b/src/main/java/com/sngular/kloadgen/processor/util/SchemaProcessorUtils.java index 93de1738..cc183f86 100644 --- a/src/main/java/com/sngular/kloadgen/processor/util/SchemaProcessorUtils.java +++ b/src/main/java/com/sngular/kloadgen/processor/util/SchemaProcessorUtils.java @@ -164,10 +164,11 @@ public static Descriptors.Descriptor buildProtoDescriptor(final ProtoFileElement schemaBuilder.addSchema(importedSchema); } } else { - final var importSchema = JMeterHelper.getParsedSchema(getSubjectName(importedClass, metadata), JMeterContextService.getContext().getProperties()); + final var importedProtobufSchema = (ProtobufSchema) JMeterHelper.getParsedSchema(getSubjectName(importedClass, metadata), + JMeterContextService.getContext().getProperties()); if (!ProtobufHelper.NOT_ACCEPTED_IMPORTS.contains(importedClass)) { - schemaBuilder.addDependency(((ProtobufSchema) importSchema).toDescriptor().getFullName()); - schemaBuilder.addSchema(convertDynamicSchema((ProtobufSchema) importSchema)); + schemaBuilder.addDependency(importedProtobufSchema.toDescriptor().getFullName()); + schemaBuilder.addSchema(convertDynamicSchema(importedProtobufSchema)); } } } From 5fa06494f23d1cdf634679e60c9dbaa954b71b8b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alfredo=20Gonz=C3=A1lez=20Fern=C3=A1ndez?= Date: Tue, 28 Mar 2023 13:52:16 +0200 Subject: [PATCH 14/30] update pom --- pom-maven-central.xml | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/pom-maven-central.xml b/pom-maven-central.xml index 865f9313..f919e8ec 100644 --- a/pom-maven-central.xml +++ b/pom-maven-central.xml @@ -235,6 +235,17 @@ Europe/Madrid + + alfredo9f + Alfredo González + alfredo.gonzalez@sngular.com + Sngular + https://www.sngular.com + + Senior Backend Developer + + Europe/Madrid + From 1994dff3118eda4cdcd719f2ae46b4bb66037416 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jose=20Enrique=20Garc=C3=ADa=20Maci=C3=B1eiras?= Date: Fri, 15 Sep 2023 11:58:52 +0200 Subject: [PATCH 15/30] Merge with Master --- pom-maven-central.xml | 89 +++++++++++++++---- pom.xml | 38 ++++++-- ...KafkaHeadersConfigElementBeanInfoTest.java | 2 +- ...leSerializedConfigElementBeanInfoTest.java | 2 +- ...eySerializedConfigElementBeanInfoTest.java | 2 +- ...leSerializedConfigElementBeanInfoTest.java | 14 +-- ...ueSerializedConfigElementBeanInfoTest.java | 12 +-- .../extractor/SchemaExtractorTest.java | 38 ++++---- .../ProtobufApicurioExtractorTest.java | 3 +- 9 files changed, 141 insertions(+), 59 deletions(-) diff --git a/pom-maven-central.xml b/pom-maven-central.xml index 94443301..5bf8a08f 100644 --- a/pom-maven-central.xml +++ b/pom-maven-central.xml @@ -25,17 +25,6 @@ - - AnxoGomezSngular - Anxo Gómez Sánchez - anxo.gomez@sngular.com - Sngular - https://sngular.github.io/ - - Developer - - Europe/Madrid - jegarcia Jose Enrique Garcia Maciñeiras @@ -195,13 +184,24 @@ davidgarciago David García Gondell david.garciag@sngular.com - Corunet - https://corunet.github.io/ + Sngular + https://sngular.github.io/ Junior Developer Europe/Madrid + + AnxoGomezSngular + Anxo Gómez Sánchez + anxo.gomez@sngular.com + Sngular + https://sngular.github.io/ + + Developer + + Europe/Madrid + pedrogarcia Pedro Garcia @@ -235,6 +235,17 @@ Europe/Madrid + + rabad-sng + Raúl Abad + raul.abad@sngular.com + Sngular + https://www.sngular.com + + Senior Backend Developer + + Europe/Madrid + dhergon David Hernández González @@ -307,7 +318,7 @@ 4.4 3.12.0 2.8.6 - 31.0.1-jre + 32.0.0-jre 2.4.0-b180830.0359 11 provided @@ -324,7 +335,7 @@ 4.5.0 2.0.0-alpha1 1.3.1 - 2.32.0 + 2.35.0 @@ -625,7 +636,43 @@ + + + + org.apache.maven.plugins + maven-checkstyle-plugin + 3.1.2 + + + com.puppycrawl.tools + checkstyle + 10.2 + + + + + + + org.apache.maven.plugins + maven-checkstyle-plugin + + checkstyle.xml + UTF-8 + true + true + false + + + + validate + validate + + check + + + + com.github.ekryd.sortpom sortpom-maven-plugin @@ -934,4 +981,16 @@ + + + + org.apache.maven.plugins + maven-checkstyle-plugin + + checkstyle.xml + + + + + diff --git a/pom.xml b/pom.xml index f55bdb1e..ec0f061a 100644 --- a/pom.xml +++ b/pom.xml @@ -11,7 +11,7 @@ KLoadGen Load Generation Jmeter plugin for Kafka Cluster. Supporting AVRO, JSON Schema and Protobuf schema types. Generate Artificial - data base on Data specification + data based on Data specification https://sngular.github.io/kloadgen/ @@ -191,6 +191,17 @@ Europe/Madrid + + AnxoGomezSngular + Anxo Gómez Sánchez + anxo.gomez@sngular.com + Sngular + https://sngular.github.io/ + + Developer + + Europe/Madrid + pedrogarcia Pedro Garcia @@ -213,6 +224,17 @@ Europe/Madrid + + nuriaricorf + Nuria Rico Freire + nuria.rico@sngular.com + Sngular + https://sngular.github.io/ + + Junior Developer + + Europe/Madrid + rabad-sng Raúl Abad @@ -288,7 +310,7 @@ - 3.22.0 + 3.24.2 1.2.0 1.5.1 1.11.0 @@ -300,9 +322,9 @@ 2.4.0-b180830.0359 11 provided - 5.5 - 2.6.0 - 5.8.2 + 5.6.2 + 2.8.0 + 5.10.0 7.1.1 2.4.1.Final 3.1.0 @@ -667,7 +689,7 @@ org.apache.maven.plugins maven-compiler-plugin - 3.6.0 + 3.8.1 true true @@ -706,7 +728,7 @@ org.apache.maven.plugins maven-surefire-plugin - 2.22.2 + 3.1.2 @@ -761,7 +783,7 @@ org.apache.maven.plugins maven-shade-plugin - 2.4.2 + 3.2.4 diff --git a/src/test/java/com/sngular/kloadgen/config/kafkaheaders/KafkaHeadersConfigElementBeanInfoTest.java b/src/test/java/com/sngular/kloadgen/config/kafkaheaders/KafkaHeadersConfigElementBeanInfoTest.java index f2e58370..cc7579d2 100644 --- a/src/test/java/com/sngular/kloadgen/config/kafkaheaders/KafkaHeadersConfigElementBeanInfoTest.java +++ b/src/test/java/com/sngular/kloadgen/config/kafkaheaders/KafkaHeadersConfigElementBeanInfoTest.java @@ -29,7 +29,7 @@ public void setUp() { @Test final void shouldGenerateElements() { final PropertyDescriptor[] propertyDescriptors = kafkaHeadersConfigElementBeanInfo.getPropertyDescriptors(); - Assertions.assertThat(propertyDescriptors).hasSize(1); + Assertions.assertThat(propertyDescriptors).hasSize(3); Assertions.assertThat(propertyDescriptors[0].getName()).isEqualTo(KAFKA_HEADERS); } } \ No newline at end of file diff --git a/src/test/java/com/sngular/kloadgen/config/keyfileserialized/KeyFileSerializedConfigElementBeanInfoTest.java b/src/test/java/com/sngular/kloadgen/config/keyfileserialized/KeyFileSerializedConfigElementBeanInfoTest.java index 4aec184b..e64b7005 100644 --- a/src/test/java/com/sngular/kloadgen/config/keyfileserialized/KeyFileSerializedConfigElementBeanInfoTest.java +++ b/src/test/java/com/sngular/kloadgen/config/keyfileserialized/KeyFileSerializedConfigElementBeanInfoTest.java @@ -39,7 +39,7 @@ public void setUp() { @Test void shouldGenerateElements() { final PropertyDescriptor[] propertyDescriptors = keyFileSerializedConfigElementBeanInfo.getPropertyDescriptors(); - Assertions.assertThat(propertyDescriptors).hasSize(6); + Assertions.assertThat(propertyDescriptors).hasSize(8); Assertions.assertThat(propertyDescriptors[0].getName()).isEqualTo(KEY_NAME_STRATEGY); Assertions.assertThat(propertyDescriptors[1].getName()).isEqualTo(KEY_SCHEMA_DEFINITION); Assertions.assertThat(propertyDescriptors[2].getName()).isEqualTo(KEY_SCHEMA_PROPERTIES); diff --git a/src/test/java/com/sngular/kloadgen/config/keyserialized/KeySerializedConfigElementBeanInfoTest.java b/src/test/java/com/sngular/kloadgen/config/keyserialized/KeySerializedConfigElementBeanInfoTest.java index c57ad23f..f5433c59 100644 --- a/src/test/java/com/sngular/kloadgen/config/keyserialized/KeySerializedConfigElementBeanInfoTest.java +++ b/src/test/java/com/sngular/kloadgen/config/keyserialized/KeySerializedConfigElementBeanInfoTest.java @@ -37,7 +37,7 @@ public void setUp() { @Test void shouldGenerateElements() { final PropertyDescriptor[] propertyDescriptors = keySerializedConfigElementBeanInfo.getPropertyDescriptors(); - Assertions.assertThat(propertyDescriptors).hasSize(5); + Assertions.assertThat(propertyDescriptors).hasSize(7); Assertions.assertThat(propertyDescriptors[0].getName()).isEqualTo(KEY_NAME_STRATEGY); Assertions.assertThat(propertyDescriptors[1].getName()).isEqualTo(KEY_SCHEMA_PROPERTIES); Assertions.assertThat(propertyDescriptors[2].getName()).isEqualTo(KEY_SCHEMA_TYPE); diff --git a/src/test/java/com/sngular/kloadgen/config/valuefileserialized/ValueFileSerializedConfigElementBeanInfoTest.java b/src/test/java/com/sngular/kloadgen/config/valuefileserialized/ValueFileSerializedConfigElementBeanInfoTest.java index 1346ec90..73e52752 100644 --- a/src/test/java/com/sngular/kloadgen/config/valuefileserialized/ValueFileSerializedConfigElementBeanInfoTest.java +++ b/src/test/java/com/sngular/kloadgen/config/valuefileserialized/ValueFileSerializedConfigElementBeanInfoTest.java @@ -39,12 +39,12 @@ public void setUp() { @Test void shouldGenerateElements() { final PropertyDescriptor[] propertyDescriptors = valueFileSerializedConfigElementBeanInfo.getPropertyDescriptors(); - Assertions.assertThat(propertyDescriptors).hasSize(6); - Assertions.assertThat(propertyDescriptors[0].getName()).isEqualTo(VALUE_NAME_STRATEGY); - Assertions.assertThat(propertyDescriptors[1].getName()).isEqualTo(VALUE_SCHEMA_DEFINITION); - Assertions.assertThat(propertyDescriptors[2].getName()).isEqualTo(VALUE_SCHEMA_PROPERTIES); - Assertions.assertThat(propertyDescriptors[3].getName()).isEqualTo(VALUE_SCHEMA_TYPE); - Assertions.assertThat(propertyDescriptors[4].getName()).isEqualTo(SERIALIZER_PROPERTY); - Assertions.assertThat(propertyDescriptors[5].getName()).isEqualTo(VALUE_SUBJECT_NAME); + Assertions.assertThat(propertyDescriptors).hasSize(8); + Assertions.assertThat(propertyDescriptors[2].getName()).isEqualTo(VALUE_NAME_STRATEGY); + Assertions.assertThat(propertyDescriptors[3].getName()).isEqualTo(VALUE_SCHEMA_DEFINITION); + Assertions.assertThat(propertyDescriptors[4].getName()).isEqualTo(VALUE_SCHEMA_PROPERTIES); + Assertions.assertThat(propertyDescriptors[5].getName()).isEqualTo(VALUE_SCHEMA_TYPE); + Assertions.assertThat(propertyDescriptors[6].getName()).isEqualTo(SERIALIZER_PROPERTY); + Assertions.assertThat(propertyDescriptors[7].getName()).isEqualTo(VALUE_SUBJECT_NAME); } } \ No newline at end of file diff --git a/src/test/java/com/sngular/kloadgen/config/valueserialized/ValueSerializedConfigElementBeanInfoTest.java b/src/test/java/com/sngular/kloadgen/config/valueserialized/ValueSerializedConfigElementBeanInfoTest.java index 6bd40d5d..e4477b9e 100644 --- a/src/test/java/com/sngular/kloadgen/config/valueserialized/ValueSerializedConfigElementBeanInfoTest.java +++ b/src/test/java/com/sngular/kloadgen/config/valueserialized/ValueSerializedConfigElementBeanInfoTest.java @@ -36,11 +36,11 @@ public void setUp() { @Test void shouldGenerateElements() { final var propertyDescriptors = valueSerializedConfigElementBeanInfo.getPropertyDescriptors(); - Assertions.assertThat(propertyDescriptors).hasSize(5); - Assertions.assertThat(propertyDescriptors[0].getName()).isEqualTo(VALUE_NAME_STRATEGY); - Assertions.assertThat(propertyDescriptors[1].getName()).isEqualTo(VALUE_SCHEMA_PROPERTIES); - Assertions.assertThat(propertyDescriptors[2].getName()).isEqualTo(VALUE_SCHEMA_TYPE); - Assertions.assertThat(propertyDescriptors[3].getName()).isEqualTo(VALUE_SERIALIZER_CONFIGURATION); - Assertions.assertThat(propertyDescriptors[4].getName()).isEqualTo(VALUE_SUBJECT_NAME); + Assertions.assertThat(propertyDescriptors).hasSize(7); + Assertions.assertThat(propertyDescriptors[2].getName()).isEqualTo(VALUE_NAME_STRATEGY); + Assertions.assertThat(propertyDescriptors[3].getName()).isEqualTo(VALUE_SCHEMA_PROPERTIES); + Assertions.assertThat(propertyDescriptors[4].getName()).isEqualTo(VALUE_SCHEMA_TYPE); + Assertions.assertThat(propertyDescriptors[5].getName()).isEqualTo(VALUE_SERIALIZER_CONFIGURATION); + Assertions.assertThat(propertyDescriptors[6].getName()).isEqualTo(VALUE_SUBJECT_NAME); } } \ No newline at end of file diff --git a/src/test/java/com/sngular/kloadgen/extractor/SchemaExtractorTest.java b/src/test/java/com/sngular/kloadgen/extractor/SchemaExtractorTest.java index eca31c37..1baabf55 100644 --- a/src/test/java/com/sngular/kloadgen/extractor/SchemaExtractorTest.java +++ b/src/test/java/com/sngular/kloadgen/extractor/SchemaExtractorTest.java @@ -10,6 +10,7 @@ import java.io.IOException; import java.util.Locale; import java.util.Properties; + import com.sngular.kloadgen.common.SchemaRegistryEnum; import com.sngular.kloadgen.exception.KLoadGenException; import com.sngular.kloadgen.schemaregistry.adapter.impl.ApicurioParsedSchemaMetadata; @@ -21,7 +22,6 @@ import com.sngular.kloadgen.util.JMeterHelper; import com.sngular.kloadgen.util.SchemaRegistryKeyHelper; import io.confluent.kafka.schemaregistry.ParsedSchema; -import io.confluent.kafka.schemaregistry.client.rest.exceptions.RestClientException; import org.apache.jmeter.threads.JMeterContext; import org.apache.jmeter.threads.JMeterContextService; import org.apache.jmeter.threads.JMeterVariables; @@ -44,9 +44,9 @@ class SchemaExtractorTest { private final Properties properties = new Properties(); - private MockedStatic jMeterHelperMockedStatic; + private MockedStatic jmeterHelperMockedStatic; - private MockedStatic jMeterContextServiceMockedStatic; + private MockedStatic jmeterContextServiceMockedStatic; @BeforeEach void setUp() { @@ -56,28 +56,28 @@ void setUp() { final JMeterContext jmcx = JMeterContextService.getContext(); jmcx.setVariables(new JMeterVariables()); JMeterUtils.setLocale(Locale.ENGLISH); - jMeterHelperMockedStatic = Mockito.mockStatic(JMeterHelper.class); - jMeterContextServiceMockedStatic = Mockito.mockStatic(JMeterContextService.class, Answers.RETURNS_DEEP_STUBS); + jmeterHelperMockedStatic = Mockito.mockStatic(JMeterHelper.class); + jmeterContextServiceMockedStatic = Mockito.mockStatic(JMeterContextService.class, Answers.RETURNS_DEEP_STUBS); } @AfterEach void tearDown() { - jMeterHelperMockedStatic.close(); - jMeterContextServiceMockedStatic.close(); + jmeterHelperMockedStatic.close(); + jmeterContextServiceMockedStatic.close(); properties.clear(); } @Test @DisplayName("Test flatPropertiesList with AVRO") - void testFlatPropertiesListWithAVRO() throws IOException, RestClientException { + void testFlatPropertiesListWithAVRO() throws IOException { final File testFile = fileHelper.getFile("/avro-files/embedded-avros-example-test.avsc"); properties.setProperty(SchemaRegistryKeyHelper.SCHEMA_REGISTRY_NAME, SchemaRegistryEnum.APICURIO.toString()); final ParsedSchema parsedSchema = SchemaParseUtil.getParsedSchema(testFile, "AVRO"); final var baseParsedSchema = new BaseParsedSchema<>(ApicurioParsedSchemaMetadata.parse(parsedSchema)); - jMeterHelperMockedStatic.when(() -> JMeterHelper.getParsedSchema(Mockito.anyString(), Mockito.any(Properties.class))).thenReturn(baseParsedSchema); - jMeterContextServiceMockedStatic.when(() -> JMeterContextService.getContext().getProperties()).thenReturn(properties); + jmeterHelperMockedStatic.when(() -> JMeterHelper.getParsedSchema(Mockito.anyString(), Mockito.any(Properties.class))).thenReturn(baseParsedSchema); + jmeterContextServiceMockedStatic.when(() -> JMeterContextService.getContext().getProperties()).thenReturn(properties); final var result = SchemaExtractor.flatPropertiesList("avroSubject"); Assertions.assertThat(result).isNotNull(); @@ -85,15 +85,15 @@ void testFlatPropertiesListWithAVRO() throws IOException, RestClientException { @Test @DisplayName("Test flatPropertiesList with Json") - void testFlatPropertiesListWithJson() throws IOException, RestClientException { + void testFlatPropertiesListWithJson() throws IOException { final File testFile = fileHelper.getFile("/jsonschema/basic.jcs"); properties.setProperty(SchemaRegistryKeyHelper.SCHEMA_REGISTRY_NAME, SchemaRegistryEnum.CONFLUENT.toString()); final var parsedSchema = SchemaParseUtil.getParsedSchema(testFile, "JSON"); final var baseParsedSchema = new BaseParsedSchema<>(ConfluentParsedSchemaMetadata.parse(parsedSchema)); - jMeterContextServiceMockedStatic.when(() -> JMeterContextService.getContext().getProperties()).thenReturn(properties); - jMeterHelperMockedStatic.when(() -> JMeterHelper.getParsedSchema(Mockito.anyString(), Mockito.any(Properties.class))).thenReturn(baseParsedSchema); + jmeterContextServiceMockedStatic.when(() -> JMeterContextService.getContext().getProperties()).thenReturn(properties); + jmeterHelperMockedStatic.when(() -> JMeterHelper.getParsedSchema(Mockito.anyString(), Mockito.any(Properties.class))).thenReturn(baseParsedSchema); final var result = SchemaExtractor.flatPropertiesList("jsonSubject"); Assertions.assertThat(result).isNotNull(); @@ -101,30 +101,30 @@ void testFlatPropertiesListWithJson() throws IOException, RestClientException { @Test @DisplayName("Test flatPropertiesList with Protobuf") - void testFlatPropertiesListWithProtobuf() throws RestClientException, IOException { + void testFlatPropertiesListWithProtobuf() throws IOException { final File testFile = fileHelper.getFile("/proto-files/easyTest.proto"); properties.setProperty(SchemaRegistryKeyHelper.SCHEMA_REGISTRY_NAME, SchemaRegistryEnum.CONFLUENT.toString()); final var parsedSchema = SchemaParseUtil.getParsedSchema(testFile, "PROTOBUF"); final var baseParsedSchema = new BaseParsedSchema<>(ConfluentParsedSchemaMetadata.parse(parsedSchema)); - jMeterContextServiceMockedStatic.when(() -> JMeterContextService.getContext().getProperties()).thenReturn(properties); - jMeterHelperMockedStatic.when(() -> JMeterHelper.getParsedSchema(Mockito.anyString(), Mockito.any(Properties.class))).thenReturn(baseParsedSchema); + jmeterContextServiceMockedStatic.when(() -> JMeterContextService.getContext().getProperties()).thenReturn(properties); + jmeterHelperMockedStatic.when(() -> JMeterHelper.getParsedSchema(Mockito.anyString(), Mockito.any(Properties.class))).thenReturn(baseParsedSchema); final var result = SchemaExtractor.flatPropertiesList("protobufSubject"); Assertions.assertThat(result).isNotNull(); } @Test @DisplayName("Test flatPropertiesList throws exception schema type not supported") - void testFlatPropertiesListWithException() throws IOException, RestClientException { + void testFlatPropertiesListWithException() { final var parsedSchema = new ParsedSchemaUtil(); final var baseParsedSchema = new BaseParsedSchema<>(ConfluentParsedSchemaMetadata.parse(parsedSchema)); properties.setProperty(SchemaRegistryKeyHelper.SCHEMA_REGISTRY_NAME, SchemaRegistryEnum.CONFLUENT.toString()); - jMeterContextServiceMockedStatic.when(() -> JMeterContextService.getContext().getProperties()).thenReturn(properties); + jmeterContextServiceMockedStatic.when(() -> JMeterContextService.getContext().getProperties()).thenReturn(properties); - jMeterHelperMockedStatic.when(() -> JMeterHelper.getParsedSchema(Mockito.anyString(), Mockito.any(Properties.class))).thenReturn(baseParsedSchema); + jmeterHelperMockedStatic.when(() -> JMeterHelper.getParsedSchema(Mockito.anyString(), Mockito.any(Properties.class))).thenReturn(baseParsedSchema); Assertions.assertThatExceptionOfType(KLoadGenException.class) .isThrownBy(() -> SchemaExtractor.flatPropertiesList("exceptionSubject") ).withMessage(String.format("Schema type not supported %s", parsedSchema.schemaType())); diff --git a/src/test/java/com/sngular/kloadgen/extractor/extractors/protobuff/ProtobufApicurioExtractorTest.java b/src/test/java/com/sngular/kloadgen/extractor/extractors/protobuff/ProtobufApicurioExtractorTest.java index f71c04a2..8f35d767 100644 --- a/src/test/java/com/sngular/kloadgen/extractor/extractors/protobuff/ProtobufApicurioExtractorTest.java +++ b/src/test/java/com/sngular/kloadgen/extractor/extractors/protobuff/ProtobufApicurioExtractorTest.java @@ -3,6 +3,7 @@ import java.io.File; import java.util.List; import java.util.Locale; + import com.sngular.kloadgen.extractor.extractors.Extractor; import com.sngular.kloadgen.model.FieldValueMapping; import com.sngular.kloadgen.testutil.FileHelper; @@ -24,7 +25,7 @@ class ProtobufApicurioExtractorTest { private final Extractor protoBufApicurioExtractor = new ProtoBufApicurioExtractor(); - private Location location = Location.get("",""); + private final Location location = Location.get("", ""); @BeforeEach public void setUp() { From 88978ce12e4052ee15f676411e3b27ae12c71004 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jose=20Enrique=20Garc=C3=ADa=20Maci=C3=B1eiras?= Date: Thu, 21 Sep 2023 08:45:26 +0200 Subject: [PATCH 16/30] #340 update dependencies --- pom.xml | 62 +++++++-------- .../kloadgen/extractor/SchemaExtractor.java | 21 ++--- .../extractors/ExtractorFactory.java | 51 ++++++------ .../avro/AbstractAvroFileExtractor.java | 17 ++-- .../avro/AvroApicurioExtractor.java | 1 + .../kloadgen/processor/SchemaProcessor.java | 3 +- .../ObjectCreatorFactoryHelper.java | 20 ++--- .../impl/AvroObjectCreatorFactory.java | 31 ++++---- .../impl/ProtobufObjectCreatorFactory.java | 7 +- .../processor/util/SchemaProcessorUtils.java | 10 +-- .../SchemaRegistryManagerFactory.java | 15 ++-- .../impl/AbstractParsedSchemaAdapter.java | 8 ++ ...ApicurioAbstractParsedSchemaMetadata.java} | 9 ++- .../adapter/impl/BaseParsedSchema.java | 4 +- ...onfluentAbstractParsedSchemaMetadata.java} | 25 +++--- .../adapter/impl/GenericSchemaRegistry.java | 3 +- .../impl/GenericSchemaRegistryAdapter.java | 9 ++- .../adapter/impl/ParsedSchemaAdapter.java | 7 -- .../impl/ApicurioSchemaRegistry.java | 30 +++---- .../impl/ConfluentSchemaRegistry.java | 34 ++++---- .../sngular/kloadgen/util/JMeterHelper.java | 3 + .../extractor/SchemaExtractorTest.java | 12 +-- .../ProtobufSchemaProcessorTest.java | 50 +++++++----- .../serializer/ProtobufSerializerTest.java | 78 +++++++++---------- .../resources/proto-files/complexTest.proto | 1 - 25 files changed, 249 insertions(+), 262 deletions(-) create mode 100644 src/main/java/com/sngular/kloadgen/schemaregistry/adapter/impl/AbstractParsedSchemaAdapter.java rename src/main/java/com/sngular/kloadgen/schemaregistry/adapter/impl/{ApicurioParsedSchemaMetadata.java => ApicurioAbstractParsedSchemaMetadata.java} (58%) rename src/main/java/com/sngular/kloadgen/schemaregistry/adapter/impl/{ConfluentParsedSchemaMetadata.java => ConfluentAbstractParsedSchemaMetadata.java} (53%) delete mode 100644 src/main/java/com/sngular/kloadgen/schemaregistry/adapter/impl/ParsedSchemaAdapter.java diff --git a/pom.xml b/pom.xml index ec0f061a..e1586d1d 100644 --- a/pom.xml +++ b/pom.xml @@ -7,7 +7,7 @@ kloadgen - 5.6.7 + 5.6.8 KLoadGen Load Generation Jmeter plugin for Kafka Cluster. Supporting AVRO, JSON Schema and Protobuf schema types. Generate Artificial @@ -290,6 +290,17 @@ Europe/Madrid + + enrique.gonzalo + Enrique Gonzalo Legarra + enrique.gonzalo@sngular.com + Sngular + https://sngular.github.io/ + + Senior FullStack Developer + + Europe/Madrid + @@ -313,21 +324,21 @@ 3.24.2 1.2.0 1.5.1 - 1.11.0 + 1.11.2 1.9.4 4.4 3.12.0 2.8.6 32.0.0-jre 2.4.0-b180830.0359 - 11 + 17 provided 5.6.2 - 2.8.0 - 5.10.0 + 2.6.0 + 5.8.2 7.1.1 - 2.4.1.Final - 3.1.0 + 2.4.3.Final + 3.5.1 1.18.22 0.9.5 4.2.0 @@ -335,21 +346,10 @@ 4.5.0 2.0.0-alpha1 1.3.1 - 2.35.0 + 2.35.1 - - org.apache.jmeter - jorphan - ${jmeter.version} - - - org.slf4j - slf4j-api - - - org.apache.kafka kafka-clients @@ -365,11 +365,6 @@ - - com.google.guava - guava - ${guava.version} - com.eclipsesource.minimal-json minimal-json @@ -389,7 +384,12 @@ com.fasterxml.jackson.core jackson-core - 2.13.1 + 2.15.2 + + + com.fasterxml.jackson.core + jackson-annotations + 2.15.2 org.projectlombok @@ -461,7 +461,7 @@ org.apache.avro avro-protobuf - 1.11.0 + 1.11.2 com.github.os72 @@ -502,7 +502,7 @@ com.github.everit-org.json-schema org.everit.json.schema - 1.14.0 + 1.14.2 @@ -748,10 +748,6 @@ ${jmeter.version} provided - - org.apache.jmeter - jorphan - org.apache.logging.log4j log4j-slf4j-impl @@ -818,10 +814,6 @@ ApacheJMeter_java ${jmeter.version} - - org.apache.jmeter - jorphan - org.apache.logging.log4j log4j-slf4j-impl diff --git a/src/main/java/com/sngular/kloadgen/extractor/SchemaExtractor.java b/src/main/java/com/sngular/kloadgen/extractor/SchemaExtractor.java index bab1aee1..a7a7b760 100644 --- a/src/main/java/com/sngular/kloadgen/extractor/SchemaExtractor.java +++ b/src/main/java/com/sngular/kloadgen/extractor/SchemaExtractor.java @@ -7,24 +7,23 @@ import java.nio.file.Paths; import java.util.List; import java.util.stream.Stream; + import com.sngular.kloadgen.common.SchemaRegistryEnum; import com.sngular.kloadgen.extractor.extractors.ExtractorFactory; import com.sngular.kloadgen.model.FieldValueMapping; import io.confluent.kafka.schemaregistry.ParsedSchema; +import lombok.Getter; import org.apache.commons.lang3.tuple.Pair; -public class SchemaExtractor { +@Getter +public final class SchemaExtractor { private SchemaRegistryEnum type; public SchemaExtractor() { } - public SchemaRegistryEnum getType(){ - return type; - } - - public void setType(SchemaRegistryEnum type){ + public void setType(final SchemaRegistryEnum type) { this.type = type; } @@ -33,11 +32,11 @@ public static Pair> flatPropertiesList(final Str } public static List flatPropertiesList(final ParsedSchema parserSchema) { - return ExtractorFactory.getExtractor(parserSchema.schemaType(), "CONFLUENT").processSchema(parserSchema, SchemaRegistryEnum.CONFLUENT); + return ExtractorFactory.getExtractor(parserSchema.schemaType()).processSchema(parserSchema, SchemaRegistryEnum.CONFLUENT); } - public static List schemaTypesList(final File schemaFile, final String schemaType, String registry) throws IOException { - return ExtractorFactory.getExtractor(schemaType, registry).getSchemaNameList(readLineByLine(schemaFile.getPath()), + public static List schemaTypesList(final File schemaFile, final String schemaType, final String registry) throws IOException { + return ExtractorFactory.getExtractor(schemaType).getSchemaNameList(readLineByLine(schemaFile.getPath()), ExtractorFactory.getSchemaRegistry(registry)); } @@ -51,8 +50,4 @@ private static String readLineByLine(final String filePath) throws IOException { return contentBuilder.toString(); } - private static List processSchema(final ParsedSchema schema) { - return ExtractorFactory.getExtractor(schema.schemaType(),SchemaRegistryEnum.CONFLUENT.name()).processSchema(schema.rawSchema().toString(), SchemaRegistryEnum.CONFLUENT); - } - } diff --git a/src/main/java/com/sngular/kloadgen/extractor/extractors/ExtractorFactory.java b/src/main/java/com/sngular/kloadgen/extractor/extractors/ExtractorFactory.java index 452b9b9a..9ff5903f 100644 --- a/src/main/java/com/sngular/kloadgen/extractor/extractors/ExtractorFactory.java +++ b/src/main/java/com/sngular/kloadgen/extractor/extractors/ExtractorFactory.java @@ -4,6 +4,7 @@ import java.util.List; import java.util.Objects; import java.util.Properties; + import com.sngular.kloadgen.common.SchemaRegistryEnum; import com.sngular.kloadgen.common.SchemaTypeEnum; import com.sngular.kloadgen.exception.KLoadGenException; @@ -11,8 +12,8 @@ import com.sngular.kloadgen.extractor.extractors.json.JsonExtractor; import com.sngular.kloadgen.extractor.extractors.protobuff.ProtobuffExtractor; import com.sngular.kloadgen.model.FieldValueMapping; -import com.sngular.kloadgen.schemaregistry.adapter.impl.ApicurioParsedSchemaMetadata; -import com.sngular.kloadgen.schemaregistry.adapter.impl.ParsedSchemaAdapter; +import com.sngular.kloadgen.schemaregistry.adapter.impl.AbstractParsedSchemaAdapter; +import com.sngular.kloadgen.schemaregistry.adapter.impl.ApicurioAbstractParsedSchemaMetadata; import com.sngular.kloadgen.util.JMeterHelper; import com.sngular.kloadgen.util.SchemaRegistryKeyHelper; import org.apache.commons.lang3.EnumUtils; @@ -20,30 +21,28 @@ import org.apache.jmeter.threads.JMeterContextService; public final class ExtractorFactory { - private static AvroExtractor avroExtractor = new AvroExtractor(); + private static final AvroExtractor AVRO_EXTRACTOR = new AvroExtractor(); - private static JsonExtractor jsonExtractor = new JsonExtractor(); + private static final JsonExtractor JSON_EXTRACTOR = new JsonExtractor(); - private static ProtobuffExtractor protobuffExtractor = new ProtobuffExtractor(); + private static final ProtobuffExtractor PROTOBUFF_EXTRACTOR = new ProtobuffExtractor(); private ExtractorFactory() { } - public static ExtractorRegistry getExtractor(final String schemaType, final String schemaRegistryEnum) { - - SchemaRegistryEnum registryEnum = getSchemaRegistry(schemaRegistryEnum); + public static ExtractorRegistry getExtractor(final String schemaType) { if (schemaType != null && EnumUtils.isValidEnum(SchemaTypeEnum.class, schemaType.toUpperCase())) { final ExtractorRegistry response; switch (SchemaTypeEnum.valueOf(schemaType.toUpperCase())) { case JSON: - response = jsonExtractor; + response = JSON_EXTRACTOR; break; case AVRO: - response = avroExtractor; + response = AVRO_EXTRACTOR; break; case PROTOBUF: - response = protobuffExtractor; + response = PROTOBUFF_EXTRACTOR; break; default: throw new KLoadGenException(String.format("Schema type not supported %s", schemaType)); @@ -54,7 +53,7 @@ public static ExtractorRegistry getExtractor(final String schemaType, final Stri } } - public static SchemaRegistryEnum getSchemaRegistry(String schemaRegistryEnum) { + public static SchemaRegistryEnum getSchemaRegistry(final String schemaRegistryEnum) { if (schemaRegistryEnum != null && EnumUtils.isValidEnum(SchemaRegistryEnum.class, schemaRegistryEnum.toUpperCase())) { return SchemaRegistryEnum.valueOf(schemaRegistryEnum.toUpperCase()); } else { @@ -66,27 +65,21 @@ public static Pair> flatPropertiesList(final Str final Properties properties = JMeterContextService.getContext().getProperties(); final var schemaParsed = JMeterHelper.getParsedSchema(subjectName, properties); final String registryName = properties.getProperty(SchemaRegistryKeyHelper.SCHEMA_REGISTRY_NAME); - String schemaType = null; - final ParsedSchemaAdapter parsedSchemaAdapter = schemaParsed.getParsedSchemaAdapter(); - schemaType = parsedSchemaAdapter.getType(); + final AbstractParsedSchemaAdapter abstractParsedSchemaAdapter = schemaParsed.getParsedSchemaAdapter(); + final String schemaType = abstractParsedSchemaAdapter.getType(); - List attributeList = new ArrayList<>(); - SchemaRegistryEnum schemaRegistryEnum = SchemaRegistryEnum.valueOf(registryName.toUpperCase()); + final List attributeList = new ArrayList<>(); + final SchemaRegistryEnum schemaRegistryEnum = SchemaRegistryEnum.valueOf(registryName.toUpperCase()); - Object schema = null; + final Object schema; if (Objects.nonNull(registryName)) { //TODO change parser - switch (schemaRegistryEnum) { - case APICURIO: - schema = ((ApicurioParsedSchemaMetadata) parsedSchemaAdapter).getSchema(); - break; - case CONFLUENT: - schema = parsedSchemaAdapter.getRawSchema(); - break; - default: - throw new KLoadGenException("Schema Registry Type nos supported " + registryName.toUpperCase()); - } - attributeList = getExtractor(schemaType,registryName.toUpperCase()).processSchema(schema, schemaRegistryEnum); + schema = switch (schemaRegistryEnum) { + case APICURIO -> ((ApicurioAbstractParsedSchemaMetadata) abstractParsedSchemaAdapter).getSchema(); + case CONFLUENT -> abstractParsedSchemaAdapter.getRawSchema(); + default -> throw new KLoadGenException("Schema Registry Type nos supported " + registryName.toUpperCase()); + }; + attributeList.addAll(getExtractor(schemaType).processSchema(schema, schemaRegistryEnum)); } return Pair.of(schemaType, attributeList); } diff --git a/src/main/java/com/sngular/kloadgen/extractor/extractors/avro/AbstractAvroFileExtractor.java b/src/main/java/com/sngular/kloadgen/extractor/extractors/avro/AbstractAvroFileExtractor.java index 2346533d..5c48fff4 100644 --- a/src/main/java/com/sngular/kloadgen/extractor/extractors/avro/AbstractAvroFileExtractor.java +++ b/src/main/java/com/sngular/kloadgen/extractor/extractors/avro/AbstractAvroFileExtractor.java @@ -6,6 +6,7 @@ import java.util.List; import java.util.Objects; import java.util.Set; + import com.sngular.kloadgen.extractor.extractors.SchemaExtractorUtil; import com.sngular.kloadgen.model.FieldValueMapping; import com.sngular.kloadgen.randomtool.random.RandomObject; @@ -22,7 +23,7 @@ public abstract class AbstractAvroFileExtractor { protected AbstractAvroFileExtractor() { } - public List processSchemaDefault(final Schema schemaReceived) { + public final List processSchemaDefault(final Schema schemaReceived) { final var attributeList = new ArrayList(); Schema aux = null; if (checkIfUnion(schemaReceived)) { @@ -30,19 +31,17 @@ public List processSchemaDefault(final Schema schemaReceived) } else if (checkIfRecord(schemaReceived)) { aux = schemaReceived; } - if(aux != null && (checkIfUnion(schemaReceived) || checkIfRecord(schemaReceived))){ + if (aux != null && (checkIfUnion(schemaReceived) || checkIfRecord(schemaReceived))) { aux.getFields().forEach(field -> processField(field, attributeList, true, true)); } return attributeList; } - public List getSchemaNameList(Schema schema) { - List result = new ArrayList<>(); - result.addAll(extractSchemaNames(schema)); - return result; + public final List getSchemaNameList(final Schema schema) { + return new ArrayList<>(extractSchemaNames(schema)); } - public void processField( + public final void processField( final Schema.Field innerField, final List completeFieldList, final boolean isAncestorRequired, final boolean isAncestor) { if (checkIfRecord(innerField.schema())) { processRecordFieldList(innerField.name(), ".", processFieldList(innerField.schema().getFields(), isAncestorRequired), completeFieldList); @@ -353,8 +352,8 @@ private boolean checkIfRequiredField(final Schema innerSchema) { return result; } - private Set extractSchemaNames(Schema schema) { - Set schemaNames = new HashSet<>(); + private Set extractSchemaNames(final Schema schema) { + final Set schemaNames = new HashSet<>(); if (checkIfRecord(schema)) { schemaNames.add(schema.getName()); schema.getFields().forEach(field -> schemaNames.addAll(extractSchemaNames(field.schema()))); diff --git a/src/main/java/com/sngular/kloadgen/extractor/extractors/avro/AvroApicurioExtractor.java b/src/main/java/com/sngular/kloadgen/extractor/extractors/avro/AvroApicurioExtractor.java index 411b7392..af440bf2 100644 --- a/src/main/java/com/sngular/kloadgen/extractor/extractors/avro/AvroApicurioExtractor.java +++ b/src/main/java/com/sngular/kloadgen/extractor/extractors/avro/AvroApicurioExtractor.java @@ -1,6 +1,7 @@ package com.sngular.kloadgen.extractor.extractors.avro; import java.util.List; + import com.sngular.kloadgen.extractor.extractors.Extractor; import com.sngular.kloadgen.model.FieldValueMapping; import io.apicurio.registry.utils.serde.avro.AvroSchemaUtils; diff --git a/src/main/java/com/sngular/kloadgen/processor/SchemaProcessor.java b/src/main/java/com/sngular/kloadgen/processor/SchemaProcessor.java index 31848647..b5f44f6a 100644 --- a/src/main/java/com/sngular/kloadgen/processor/SchemaProcessor.java +++ b/src/main/java/com/sngular/kloadgen/processor/SchemaProcessor.java @@ -11,6 +11,7 @@ import java.util.List; import java.util.Objects; import java.util.stream.Collectors; + import com.sngular.kloadgen.common.SchemaTypeEnum; import com.sngular.kloadgen.model.FieldValueMapping; import com.sngular.kloadgen.processor.model.SchemaProcessorPOJO; @@ -223,7 +224,7 @@ private boolean searchFieldWithSharedPathAndMakeItProcessable( if (otherFieldIsRequired) { shouldProcess = false; } else { - if (listFieldsSharingPath.size() > 0) { + if (!listFieldsSharingPath.isEmpty()) { makeFieldValueMappingRequiredAndNotNullable(fetchFieldSharingPathToMakeItRequired(listFieldsSharingPath)); shouldProcess = fieldValueMapping.getRequired(); } else { diff --git a/src/main/java/com/sngular/kloadgen/processor/objectcreatorfactory/ObjectCreatorFactoryHelper.java b/src/main/java/com/sngular/kloadgen/processor/objectcreatorfactory/ObjectCreatorFactoryHelper.java index fa5722b6..380fba6c 100644 --- a/src/main/java/com/sngular/kloadgen/processor/objectcreatorfactory/ObjectCreatorFactoryHelper.java +++ b/src/main/java/com/sngular/kloadgen/processor/objectcreatorfactory/ObjectCreatorFactoryHelper.java @@ -1,6 +1,7 @@ package com.sngular.kloadgen.processor.objectcreatorfactory; import java.io.IOException; + import com.google.protobuf.Descriptors.DescriptorValidationException; import com.sngular.kloadgen.common.SchemaTypeEnum; import com.sngular.kloadgen.exception.KLoadGenException; @@ -19,19 +20,12 @@ private ObjectCreatorFactoryHelper() {} public static ObjectCreatorFactory getInstance(final SchemaTypeEnum schemaType, final Object schema, final BaseSchemaMetadata metadata) { final ObjectCreatorFactory objectCreatorFactory; try { - switch (schemaType) { - case JSON: - objectCreatorFactory = new JsonObjectCreatorFactory(); - break; - case AVRO: - objectCreatorFactory = new AvroObjectCreatorFactory(schema, metadata); - break; - case PROTOBUF: - objectCreatorFactory = new ProtobufObjectCreatorFactory(schema, metadata); - break; - default: - throw new KLoadGenException("Unsupported schema type"); - } + objectCreatorFactory = switch (schemaType) { + case JSON -> new JsonObjectCreatorFactory(); + case AVRO -> new AvroObjectCreatorFactory(schema, metadata); + case PROTOBUF -> new ProtobufObjectCreatorFactory(schema, metadata); + default -> throw new KLoadGenException("Unsupported schema type"); + }; } catch (KLoadGenException | DescriptorValidationException | IOException e) { final String logMsg = "Please, make sure that the schema sources fed are correct"; log.error(logMsg, e); diff --git a/src/main/java/com/sngular/kloadgen/processor/objectcreatorfactory/impl/AvroObjectCreatorFactory.java b/src/main/java/com/sngular/kloadgen/processor/objectcreatorfactory/impl/AvroObjectCreatorFactory.java index 303d51c0..c92f8344 100644 --- a/src/main/java/com/sngular/kloadgen/processor/objectcreatorfactory/impl/AvroObjectCreatorFactory.java +++ b/src/main/java/com/sngular/kloadgen/processor/objectcreatorfactory/impl/AvroObjectCreatorFactory.java @@ -11,16 +11,17 @@ import java.util.Set; import java.util.concurrent.atomic.AtomicBoolean; import java.util.function.Function; + import com.sngular.kloadgen.exception.KLoadGenException; import com.sngular.kloadgen.model.ConstraintTypeEnum; import com.sngular.kloadgen.processor.model.SchemaProcessorPOJO; import com.sngular.kloadgen.processor.objectcreatorfactory.ObjectCreatorFactory; import com.sngular.kloadgen.processor.util.SchemaProcessorUtils; import com.sngular.kloadgen.randomtool.generator.AvroGeneratorTool; -import com.sngular.kloadgen.schemaregistry.adapter.impl.ApicurioParsedSchemaMetadata; +import com.sngular.kloadgen.schemaregistry.adapter.impl.AbstractParsedSchemaAdapter; +import com.sngular.kloadgen.schemaregistry.adapter.impl.ApicurioAbstractParsedSchemaMetadata; import com.sngular.kloadgen.schemaregistry.adapter.impl.BaseParsedSchema; import com.sngular.kloadgen.schemaregistry.adapter.impl.BaseSchemaMetadata; -import com.sngular.kloadgen.schemaregistry.adapter.impl.ParsedSchemaAdapter; import com.sngular.kloadgen.schemaregistry.adapter.impl.SchemaMetadataAdapter; import com.sngular.kloadgen.serializer.EnrichedRecord; import io.confluent.kafka.schemaregistry.ParsedSchema; @@ -30,7 +31,7 @@ import org.apache.avro.generic.GenericRecord; import org.apache.commons.collections4.IteratorUtils; -public class AvroObjectCreatorFactory implements ObjectCreatorFactory { +public final class AvroObjectCreatorFactory implements ObjectCreatorFactory { private static final AvroGeneratorTool AVRO_GENERATOR_TOOL = new AvroGeneratorTool(); @@ -49,9 +50,9 @@ public AvroObjectCreatorFactory(final Object schema, final BaseSchemaMetadata generateFunction, final boolean isInnerMap) { + public Object createMap(final SchemaProcessorPOJO pojo, final Function generateFunction, final boolean isInnerMap) { Map map = new HashMap<>(); if (pojo.isLastFilterTypeOfLastElement()) { @@ -92,7 +93,7 @@ private String generateString(final Integer valueLength) { } @Override - public final Object createArray(final SchemaProcessorPOJO pojo, final Function generateFunction, final boolean isInnerArray) { + public Object createArray(final SchemaProcessorPOJO pojo, final Function generateFunction, final boolean isInnerArray) { List list = new ArrayList<>(); if (pojo.isLastFilterTypeOfLastElement()) { @@ -111,7 +112,7 @@ public final Object createArray(final SchemaProcessorPOJO pojo, final Function createFinalArray(final SchemaProcessorPOJO pojo) { pojo.getFieldValuesList(), pojo.getFieldSize(), Collections.emptyMap()); } - public final Object assignObject(final String targetObjectName, final String fieldName, final Object objectToAssign) { + public Object assignObject(final String targetObjectName, final String fieldName, final Object objectToAssign) { final GenericRecord entityObject = entity.get(targetObjectName); entityObject.put(fieldName, objectToAssign); return entityObject; diff --git a/src/main/java/com/sngular/kloadgen/processor/objectcreatorfactory/impl/ProtobufObjectCreatorFactory.java b/src/main/java/com/sngular/kloadgen/processor/objectcreatorfactory/impl/ProtobufObjectCreatorFactory.java index 892c4673..abab03c3 100644 --- a/src/main/java/com/sngular/kloadgen/processor/objectcreatorfactory/impl/ProtobufObjectCreatorFactory.java +++ b/src/main/java/com/sngular/kloadgen/processor/objectcreatorfactory/impl/ProtobufObjectCreatorFactory.java @@ -10,6 +10,7 @@ import java.util.Objects; import java.util.concurrent.atomic.AtomicBoolean; import java.util.function.Function; + import com.google.protobuf.Descriptors; import com.google.protobuf.Descriptors.Descriptor; import com.google.protobuf.Descriptors.DescriptorValidationException; @@ -23,9 +24,9 @@ import com.sngular.kloadgen.processor.objectcreatorfactory.ObjectCreatorFactory; import com.sngular.kloadgen.processor.util.SchemaProcessorUtils; import com.sngular.kloadgen.randomtool.generator.ProtoBufGeneratorTool; +import com.sngular.kloadgen.schemaregistry.adapter.impl.AbstractParsedSchemaAdapter; import com.sngular.kloadgen.schemaregistry.adapter.impl.BaseParsedSchema; import com.sngular.kloadgen.schemaregistry.adapter.impl.BaseSchemaMetadata; -import com.sngular.kloadgen.schemaregistry.adapter.impl.ParsedSchemaAdapter; import com.sngular.kloadgen.schemaregistry.adapter.impl.SchemaMetadataAdapter; import com.sngular.kloadgen.serializer.EnrichedRecord; import com.squareup.wire.schema.internal.parser.ProtoFileElement; @@ -49,7 +50,7 @@ public ProtobufObjectCreatorFactory(final Object schema, final BaseSchemaMetadat this.schema = SchemaProcessorUtils.buildProtoDescriptor((ProtoFileElement) schema, metadata); } else if (schema instanceof BaseParsedSchema) { final BaseParsedSchema schemaParse = (BaseParsedSchema) schema; - final ParsedSchemaAdapter adapterParse = schemaParse.getParsedSchemaAdapter(); + final AbstractParsedSchemaAdapter adapterParse = schemaParse.getParsedSchemaAdapter(); final Object schemaParsed = adapterParse.getRawSchema(); this.schema = SchemaProcessorUtils.buildProtoDescriptor((ProtoFileElement) schemaParsed, metadata); } else { @@ -170,7 +171,7 @@ public final boolean isOptionalFieldAccordingToSchema(final String completeField } @Override - public Object getRootNode(final String rootNode) { + public final Object getRootNode(final String rootNode) { return entity.get(rootNode); } diff --git a/src/main/java/com/sngular/kloadgen/processor/util/SchemaProcessorUtils.java b/src/main/java/com/sngular/kloadgen/processor/util/SchemaProcessorUtils.java index af608b25..2a5e3d22 100644 --- a/src/main/java/com/sngular/kloadgen/processor/util/SchemaProcessorUtils.java +++ b/src/main/java/com/sngular/kloadgen/processor/util/SchemaProcessorUtils.java @@ -17,6 +17,7 @@ import java.util.Objects; import java.util.regex.Matcher; import java.util.regex.Pattern; + import com.github.os72.protobuf.dynamic.DynamicSchema; import com.github.os72.protobuf.dynamic.EnumDefinition; import com.github.os72.protobuf.dynamic.MessageDefinition; @@ -24,8 +25,8 @@ import com.google.protobuf.Descriptors; import com.google.protobuf.Descriptors.DescriptorValidationException; import com.sngular.kloadgen.model.FieldValueMapping; +import com.sngular.kloadgen.schemaregistry.adapter.impl.AbstractParsedSchemaAdapter; import com.sngular.kloadgen.schemaregistry.adapter.impl.BaseSchemaMetadata; -import com.sngular.kloadgen.schemaregistry.adapter.impl.ParsedSchemaAdapter; import com.sngular.kloadgen.schemaregistry.adapter.impl.SchemaMetadataAdapter; import com.sngular.kloadgen.util.JMeterHelper; import com.sngular.kloadgen.util.ProtobufHelper; @@ -167,8 +168,8 @@ public static Descriptors.Descriptor buildProtoDescriptor(final ProtoFileElement schemaBuilder.addSchema(importedSchema); } } else { - final ParsedSchemaAdapter protoFileElement = JMeterHelper.getParsedSchema(getSubjectName(importedClass, metadata), - JMeterContextService.getContext().getProperties()).getParsedSchemaAdapter(); + final AbstractParsedSchemaAdapter protoFileElement = JMeterHelper.getParsedSchema(getSubjectName(importedClass, metadata), + JMeterContextService.getContext().getProperties()).getParsedSchemaAdapter(); final var importedProtobufSchema = new ProtobufSchema(protoFileElement.getRawSchema(), metadata.getSchemaMetadataAdapter().getReferences(), new HashMap<>()); if (!ProtobufHelper.NOT_ACCEPTED_IMPORTS.contains(importedClass)) { schemaBuilder.addDependency(importedProtobufSchema.toDescriptor().getFullName()); @@ -381,8 +382,7 @@ private static void addDefinition( final MessageDefinition.Builder msgDef, final String typeName, final TypeElement typeElement, final HashMap>> globalNestedTypesByLevelAndMessage, final int deepLevel) { - if (typeElement instanceof EnumElement) { - final var enumElement = (EnumElement) typeElement; + if (typeElement instanceof final EnumElement enumElement) { final EnumDefinition.Builder builder = EnumDefinition.newBuilder(enumElement.getName()); for (final var constant : enumElement.getConstants()) { builder.addValue(constant.getName(), constant.getTag()); diff --git a/src/main/java/com/sngular/kloadgen/schemaregistry/SchemaRegistryManagerFactory.java b/src/main/java/com/sngular/kloadgen/schemaregistry/SchemaRegistryManagerFactory.java index cbdd0206..ddd75acf 100644 --- a/src/main/java/com/sngular/kloadgen/schemaregistry/SchemaRegistryManagerFactory.java +++ b/src/main/java/com/sngular/kloadgen/schemaregistry/SchemaRegistryManagerFactory.java @@ -2,9 +2,6 @@ import java.util.Map; -import static com.sngular.kloadgen.common.SchemaRegistryEnum.APICURIO; -import static com.sngular.kloadgen.common.SchemaRegistryEnum.CONFLUENT; - import com.sngular.kloadgen.common.SchemaRegistryEnum; import com.sngular.kloadgen.exception.KLoadGenException; import com.sngular.kloadgen.schemaregistry.impl.ApicurioSchemaRegistry; @@ -14,12 +11,16 @@ @Slf4j public class SchemaRegistryManagerFactory { - static Map schemaRegistryMap = Map.of(CONFLUENT, new ConfluentSchemaRegistry(), APICURIO, new ApicurioSchemaRegistry()); + private static final Map SCHEMA_REGISTRY_MAP = + Map.of(SchemaRegistryEnum.CONFLUENT, new ConfluentSchemaRegistry(), SchemaRegistryEnum.APICURIO, new ApicurioSchemaRegistry()); + + protected SchemaRegistryManagerFactory() { + } - public static SchemaRegistryAdapter getSchemaRegistry(String registry) { + public static SchemaRegistryAdapter getSchemaRegistry(final String registry) { try { - SchemaRegistryEnum schemaRegistryEnum = SchemaRegistryEnum.valueOf(registry.toUpperCase()); - return schemaRegistryMap.get(schemaRegistryEnum); + final SchemaRegistryEnum schemaRegistryEnum = SchemaRegistryEnum.valueOf(registry.toUpperCase()); + return SCHEMA_REGISTRY_MAP.get(schemaRegistryEnum); } catch (final IllegalArgumentException e) { final String logMsg = "Can not parse the registry " + registry; log.error(logMsg, e); diff --git a/src/main/java/com/sngular/kloadgen/schemaregistry/adapter/impl/AbstractParsedSchemaAdapter.java b/src/main/java/com/sngular/kloadgen/schemaregistry/adapter/impl/AbstractParsedSchemaAdapter.java new file mode 100644 index 00000000..527f6ee4 --- /dev/null +++ b/src/main/java/com/sngular/kloadgen/schemaregistry/adapter/impl/AbstractParsedSchemaAdapter.java @@ -0,0 +1,8 @@ +package com.sngular.kloadgen.schemaregistry.adapter.impl; + +public abstract class AbstractParsedSchemaAdapter { + + public abstract T getType(); + + public abstract T getRawSchema(); +} diff --git a/src/main/java/com/sngular/kloadgen/schemaregistry/adapter/impl/ApicurioParsedSchemaMetadata.java b/src/main/java/com/sngular/kloadgen/schemaregistry/adapter/impl/ApicurioAbstractParsedSchemaMetadata.java similarity index 58% rename from src/main/java/com/sngular/kloadgen/schemaregistry/adapter/impl/ApicurioParsedSchemaMetadata.java rename to src/main/java/com/sngular/kloadgen/schemaregistry/adapter/impl/ApicurioAbstractParsedSchemaMetadata.java index 72ef8f66..f18cdb15 100644 --- a/src/main/java/com/sngular/kloadgen/schemaregistry/adapter/impl/ApicurioParsedSchemaMetadata.java +++ b/src/main/java/com/sngular/kloadgen/schemaregistry/adapter/impl/ApicurioAbstractParsedSchemaMetadata.java @@ -8,7 +8,7 @@ @Getter @Setter @NoArgsConstructor -public class ApicurioParsedSchemaMetadata extends ParsedSchemaAdapter { +public class ApicurioAbstractParsedSchemaMetadata extends AbstractParsedSchemaAdapter { private Object schema; @@ -16,12 +16,13 @@ public class ApicurioParsedSchemaMetadata extends ParsedSchemaAdapter { private String type; - public ApicurioParsedSchemaMetadata(ParsedSchema parsedSchema){ + public ApicurioAbstractParsedSchemaMetadata(final ParsedSchema parsedSchema) { this.rawSchema = parsedSchema.canonicalString(); this.type = parsedSchema.schemaType(); this.schema = parsedSchema.rawSchema(); } - public static ParsedSchemaAdapter parse(final ParsedSchema parsedSchema) { - return new ApicurioParsedSchemaMetadata(parsedSchema); + + public static AbstractParsedSchemaAdapter parse(final ParsedSchema parsedSchema) { + return new ApicurioAbstractParsedSchemaMetadata(parsedSchema); } } diff --git a/src/main/java/com/sngular/kloadgen/schemaregistry/adapter/impl/BaseParsedSchema.java b/src/main/java/com/sngular/kloadgen/schemaregistry/adapter/impl/BaseParsedSchema.java index 55df89e6..407c1339 100644 --- a/src/main/java/com/sngular/kloadgen/schemaregistry/adapter/impl/BaseParsedSchema.java +++ b/src/main/java/com/sngular/kloadgen/schemaregistry/adapter/impl/BaseParsedSchema.java @@ -2,7 +2,7 @@ import org.apache.avro.Schema.Parser; -public class BaseParsedSchema extends Parser { +public class BaseParsedSchema extends Parser { private final T parsedSchemaAdapter; @@ -10,7 +10,7 @@ public BaseParsedSchema(final T parsedSchemaAdapter) { this.parsedSchemaAdapter = parsedSchemaAdapter; } - public T getParsedSchemaAdapter(){ + public final T getParsedSchemaAdapter() { return parsedSchemaAdapter; } } diff --git a/src/main/java/com/sngular/kloadgen/schemaregistry/adapter/impl/ConfluentParsedSchemaMetadata.java b/src/main/java/com/sngular/kloadgen/schemaregistry/adapter/impl/ConfluentAbstractParsedSchemaMetadata.java similarity index 53% rename from src/main/java/com/sngular/kloadgen/schemaregistry/adapter/impl/ConfluentParsedSchemaMetadata.java rename to src/main/java/com/sngular/kloadgen/schemaregistry/adapter/impl/ConfluentAbstractParsedSchemaMetadata.java index a24ea866..42e1f5e3 100644 --- a/src/main/java/com/sngular/kloadgen/schemaregistry/adapter/impl/ConfluentParsedSchemaMetadata.java +++ b/src/main/java/com/sngular/kloadgen/schemaregistry/adapter/impl/ConfluentAbstractParsedSchemaMetadata.java @@ -6,7 +6,7 @@ import org.apache.avro.Schema; @Getter -public class ConfluentParsedSchemaMetadata extends ParsedSchemaAdapter { +public final class ConfluentAbstractParsedSchemaMetadata extends AbstractParsedSchemaAdapter { private String schemaType; @@ -14,40 +14,41 @@ public class ConfluentParsedSchemaMetadata extends ParsedSchemaAdapter { private String canonicalString; - private Object rawSchema; + private Object rawSchema; - private ConfluentParsedSchemaMetadata(ParsedSchema parsedSchema){ + private ConfluentAbstractParsedSchemaMetadata(final ParsedSchema parsedSchema) { this.schemaType = parsedSchema.schemaType(); this.name = parsedSchema.name(); this.canonicalString = parsedSchema.canonicalString(); this.rawSchema = parsedSchema.rawSchema(); } - private ConfluentParsedSchemaMetadata(Schema schema){ + private ConfluentAbstractParsedSchemaMetadata(final Schema schema) { this.schemaType = schema.getType().getName(); this.name = schema.getName(); } - public ConfluentParsedSchemaMetadata(ProtobufSchema schema){ + public ConfluentAbstractParsedSchemaMetadata(final ProtobufSchema schema) { this.schemaType = schema.schemaType(); this.name = schema.name(); this.rawSchema = schema.rawSchema(); this.canonicalString = schema.canonicalString(); } - public ConfluentParsedSchemaMetadata() { + public ConfluentAbstractParsedSchemaMetadata() { } - public static ParsedSchemaAdapter parse(final ParsedSchema parsedSchema) { - return new ConfluentParsedSchemaMetadata(parsedSchema); + public static AbstractParsedSchemaAdapter parse(final ParsedSchema parsedSchema) { + return new ConfluentAbstractParsedSchemaMetadata(parsedSchema); } - public static ParsedSchemaAdapter parse(final Schema schema) { - return new ConfluentParsedSchemaMetadata(schema); + public static AbstractParsedSchemaAdapter parse(final Schema schema) { + return new ConfluentAbstractParsedSchemaMetadata(schema); } - public static ParsedSchemaAdapter parse (final ProtobufSchema schema){ - return new ConfluentParsedSchemaMetadata(schema); + + public static AbstractParsedSchemaAdapter parse(final ProtobufSchema schema) { + return new ConfluentAbstractParsedSchemaMetadata(schema); } @Override diff --git a/src/main/java/com/sngular/kloadgen/schemaregistry/adapter/impl/GenericSchemaRegistry.java b/src/main/java/com/sngular/kloadgen/schemaregistry/adapter/impl/GenericSchemaRegistry.java index 2ad7c68b..b8bbaca5 100644 --- a/src/main/java/com/sngular/kloadgen/schemaregistry/adapter/impl/GenericSchemaRegistry.java +++ b/src/main/java/com/sngular/kloadgen/schemaregistry/adapter/impl/GenericSchemaRegistry.java @@ -11,9 +11,10 @@ @Setter @AllArgsConstructor @NoArgsConstructor -public class GenericSchemaRegistry{ +public class GenericSchemaRegistry { private GenericSchemaRegistryAdapter genericSchemaRegistryAdapter; + private T id; private T version; diff --git a/src/main/java/com/sngular/kloadgen/schemaregistry/adapter/impl/GenericSchemaRegistryAdapter.java b/src/main/java/com/sngular/kloadgen/schemaregistry/adapter/impl/GenericSchemaRegistryAdapter.java index b1ef073a..c04b8c6d 100644 --- a/src/main/java/com/sngular/kloadgen/schemaregistry/adapter/impl/GenericSchemaRegistryAdapter.java +++ b/src/main/java/com/sngular/kloadgen/schemaregistry/adapter/impl/GenericSchemaRegistryAdapter.java @@ -1,11 +1,12 @@ package com.sngular.kloadgen.schemaregistry.adapter.impl; -public interface GenericSchemaRegistryAdapter { - T getId(); +public interface GenericSchemaRegistryAdapter { - T getVersion(); + T getId(); - U getSchemaType(); + T getVersion(); + + U getSchemaType(); } diff --git a/src/main/java/com/sngular/kloadgen/schemaregistry/adapter/impl/ParsedSchemaAdapter.java b/src/main/java/com/sngular/kloadgen/schemaregistry/adapter/impl/ParsedSchemaAdapter.java deleted file mode 100644 index 2d1848b9..00000000 --- a/src/main/java/com/sngular/kloadgen/schemaregistry/adapter/impl/ParsedSchemaAdapter.java +++ /dev/null @@ -1,7 +0,0 @@ -package com.sngular.kloadgen.schemaregistry.adapter.impl; - -public abstract class ParsedSchemaAdapter { - public abstract T getType(); - - public abstract T getRawSchema(); -} diff --git a/src/main/java/com/sngular/kloadgen/schemaregistry/impl/ApicurioSchemaRegistry.java b/src/main/java/com/sngular/kloadgen/schemaregistry/impl/ApicurioSchemaRegistry.java index b0797090..ad990e80 100644 --- a/src/main/java/com/sngular/kloadgen/schemaregistry/impl/ApicurioSchemaRegistry.java +++ b/src/main/java/com/sngular/kloadgen/schemaregistry/impl/ApicurioSchemaRegistry.java @@ -8,17 +8,16 @@ import java.util.List; import java.util.Map; import java.util.Objects; + import com.google.protobuf.Message; import com.sngular.kloadgen.common.SchemaTypeEnum; import com.sngular.kloadgen.exception.KLoadGenException; import com.sngular.kloadgen.schemaregistry.SchemaRegistryAdapter; -import com.sngular.kloadgen.schemaregistry.SchemaRegistryConstants; -import com.sngular.kloadgen.schemaregistry.adapter.impl.ApicurioParsedSchemaMetadata; +import com.sngular.kloadgen.schemaregistry.adapter.impl.ApicurioAbstractParsedSchemaMetadata; import com.sngular.kloadgen.schemaregistry.adapter.impl.ApicurioSchemaMetadata; import com.sngular.kloadgen.schemaregistry.adapter.impl.BaseParsedSchema; import com.sngular.kloadgen.schemaregistry.adapter.impl.BaseSchemaMetadata; import com.sngular.kloadgen.schemaregistry.adapter.impl.SchemaMetadataAdapter; -import com.sngular.kloadgen.util.SchemaRegistryKeyHelper; import io.apicurio.registry.resolver.SchemaParser; import io.apicurio.registry.rest.client.RegistryClient; import io.apicurio.registry.rest.client.RegistryClientFactory; @@ -36,30 +35,25 @@ public final class ApicurioSchemaRegistry implements SchemaRegistryAdapter { - private static final Map propertiesMap = Map.of(SchemaRegistryKeyHelper.SCHEMA_REGISTRY_NAME, SchemaRegistryConstants.SCHEMA_REGISTRY_APICURIO, - SchemaRegistryKeyHelper.SCHEMA_REGISTRY_URL_KEY, SerdeConfig.REGISTRY_URL); - private RegistryClient schemaRegistryClient; - private ApicurioParsedSchemaMetadata apicurioParsedSchemaMetadata; - - public final String getSchemaRegistryUrlKey() { + public String getSchemaRegistryUrlKey() { return SerdeConfig.REGISTRY_URL; } @Override - public final void setSchemaRegistryClient(final String url, final Map properties) { + public void setSchemaRegistryClient(final String url, final Map properties) { this.schemaRegistryClient = RegistryClientFactory.create(url); } @Override - public final void setSchemaRegistryClient(final Map properties) { + public void setSchemaRegistryClient(final Map properties) { final String url = Objects.toString(properties.get(this.getSchemaRegistryUrlKey()), ""); this.schemaRegistryClient = RegistryClientFactory.create(url); } @Override - public final Collection getAllSubjects() throws KLoadGenException { + public Collection getAllSubjects() throws KLoadGenException { final Collection subjects = new ArrayList<>(); try { final List artifacts = this.schemaRegistryClient.searchArtifacts(null, null, null, @@ -75,7 +69,7 @@ public final Collection getAllSubjects() throws KLoadGenException { } @Override - public final BaseSchemaMetadata getLatestSchemaMetadata(final String artifactId) throws KLoadGenException { + public BaseSchemaMetadata getLatestSchemaMetadata(final String artifactId) throws KLoadGenException { try { final SearchedArtifact searchedArtifact = getLastestSearchedArtifact(artifactId); final ArtifactMetaData artifactMetaData = this.schemaRegistryClient.getArtifactMetaData(searchedArtifact.getGroupId(), searchedArtifact.getId()); @@ -86,8 +80,8 @@ public final BaseSchemaMetadata getLatestSchemaMetadata( } @Override - public final BaseParsedSchema getSchemaBySubject(final String artifactId) { - final ApicurioParsedSchemaMetadata schema = new ApicurioParsedSchemaMetadata(); + public BaseParsedSchema getSchemaBySubject(final String artifactId) { + final ApicurioAbstractParsedSchemaMetadata schema = new ApicurioAbstractParsedSchemaMetadata(); try { final SearchedArtifact searchedArtifact = getLastestSearchedArtifact(artifactId); final InputStream inputStream = this.schemaRegistryClient.getLatestArtifact(searchedArtifact.getGroupId(), searchedArtifact.getId()); @@ -100,7 +94,7 @@ public final BaseParsedSchema getSchemaBySubject(f } } - private static void setSchemaBySchemaType(final ApicurioParsedSchemaMetadata schema, final byte[] result, final String searchedArtifactType) { + private static void setSchemaBySchemaType(final ApicurioAbstractParsedSchemaMetadata schema, final byte[] result, final String searchedArtifactType) { switch (SchemaTypeEnum.valueOf(searchedArtifactType)) { case AVRO: @@ -122,9 +116,9 @@ private static void setSchemaBySchemaType(final ApicurioParsedSchemaMetadata sch } @Override - public final BaseParsedSchema getSchemaBySubjectAndId( + public final BaseParsedSchema getSchemaBySubjectAndId( final String subjectName, final BaseSchemaMetadata metadata) { - final ApicurioParsedSchemaMetadata schema = new ApicurioParsedSchemaMetadata(); + final ApicurioAbstractParsedSchemaMetadata schema = new ApicurioAbstractParsedSchemaMetadata(); final SchemaMetadataAdapter schemaMetadataAdapter = metadata.getSchemaMetadataAdapter(); try { diff --git a/src/main/java/com/sngular/kloadgen/schemaregistry/impl/ConfluentSchemaRegistry.java b/src/main/java/com/sngular/kloadgen/schemaregistry/impl/ConfluentSchemaRegistry.java index c516ec84..1f112843 100644 --- a/src/main/java/com/sngular/kloadgen/schemaregistry/impl/ConfluentSchemaRegistry.java +++ b/src/main/java/com/sngular/kloadgen/schemaregistry/impl/ConfluentSchemaRegistry.java @@ -5,16 +5,15 @@ import java.util.Collection; import java.util.List; import java.util.Map; + import com.sngular.kloadgen.exception.KLoadGenException; import com.sngular.kloadgen.schemaregistry.SchemaRegistryAdapter; -import com.sngular.kloadgen.schemaregistry.SchemaRegistryConstants; +import com.sngular.kloadgen.schemaregistry.adapter.impl.AbstractParsedSchemaAdapter; import com.sngular.kloadgen.schemaregistry.adapter.impl.BaseParsedSchema; import com.sngular.kloadgen.schemaregistry.adapter.impl.BaseSchemaMetadata; -import com.sngular.kloadgen.schemaregistry.adapter.impl.ConfluentParsedSchemaMetadata; +import com.sngular.kloadgen.schemaregistry.adapter.impl.ConfluentAbstractParsedSchemaMetadata; import com.sngular.kloadgen.schemaregistry.adapter.impl.ConfluentSchemaMetadata; -import com.sngular.kloadgen.schemaregistry.adapter.impl.ParsedSchemaAdapter; import com.sngular.kloadgen.schemaregistry.adapter.impl.SchemaMetadataAdapter; -import com.sngular.kloadgen.util.SchemaRegistryKeyHelper; import io.confluent.kafka.schemaregistry.ParsedSchema; import io.confluent.kafka.schemaregistry.avro.AvroSchemaProvider; import io.confluent.kafka.schemaregistry.client.CachedSchemaRegistryClient; @@ -23,21 +22,15 @@ import io.confluent.kafka.schemaregistry.json.JsonSchemaProvider; import io.confluent.kafka.schemaregistry.protobuf.ProtobufSchemaProvider; import io.confluent.kafka.serializers.AbstractKafkaSchemaSerDeConfig; +import lombok.extern.slf4j.Slf4j; import org.apache.jmeter.threads.JMeterContextService; -public class ConfluentSchemaRegistry implements SchemaRegistryAdapter { +@Slf4j +public final class ConfluentSchemaRegistry implements SchemaRegistryAdapter { private SchemaRegistryClient schemaRegistryClient; - private Map propertiesMap; - public ConfluentSchemaRegistry() { - this.propertiesMap = Map.of(SchemaRegistryKeyHelper.SCHEMA_REGISTRY_NAME, SchemaRegistryConstants.SCHEMA_REGISTRY_CONFLUENT, - SchemaRegistryKeyHelper.SCHEMA_REGISTRY_URL_KEY, AbstractKafkaSchemaSerDeConfig.SCHEMA_REGISTRY_URL_CONFIG, - SchemaRegistryConstants.BASIC_AUTH_CREDENTIALS, AbstractKafkaSchemaSerDeConfig.BASIC_AUTH_CREDENTIALS_SOURCE, - SchemaRegistryConstants.USER_INFO_CONFIG, AbstractKafkaSchemaSerDeConfig.USER_INFO_CONFIG, - SchemaRegistryConstants.BEARER_AUTH_CREDENTIALS, AbstractKafkaSchemaSerDeConfig.BEARER_AUTH_CREDENTIALS_SOURCE, - SchemaRegistryConstants.BEARER_AUTH_TOKEN_CONFIG, AbstractKafkaSchemaSerDeConfig.BEARER_AUTH_TOKEN_CONFIG); } @Override @@ -47,12 +40,14 @@ public String getSchemaRegistryUrlKey() { @Override public void setSchemaRegistryClient(final String url, final Map properties) { + log.debug("CREATEION"); this.schemaRegistryClient = new CachedSchemaRegistryClient(List.of(checkPropertyOrVariable(url)), 1000, List.of(new AvroSchemaProvider(), new JsonSchemaProvider(), new ProtobufSchemaProvider()), properties); } @Override public void setSchemaRegistryClient(final Map properties) { + log.debug("CREATITIN"); final String url = properties.get(this.getSchemaRegistryUrlKey()).toString(); this.schemaRegistryClient = new CachedSchemaRegistryClient(List.of(checkPropertyOrVariable(url)), 1000, List.of(new AvroSchemaProvider(), new JsonSchemaProvider(), new ProtobufSchemaProvider()), properties); @@ -78,22 +73,23 @@ public BaseSchemaMetadata getLatestSchemaMetadata(final } } - public BaseParsedSchema getSchemaBySubject(final String subjectName) { + public BaseParsedSchema getSchemaBySubject(final String subjectName) { try { final ConfluentSchemaMetadata schemaMetadata = ConfluentSchemaMetadata.parse(this.schemaRegistryClient.getLatestSchemaMetadata(subjectName)); final ParsedSchema parsedSchema = this.schemaRegistryClient.getSchemaBySubjectAndId(subjectName, schemaMetadata.getId()); - final ParsedSchemaAdapter parsedSchemaAdapter = ConfluentParsedSchemaMetadata.parse(parsedSchema); - return new BaseParsedSchema(parsedSchemaAdapter); + final AbstractParsedSchemaAdapter abstractParsedSchemaAdapter = ConfluentAbstractParsedSchemaMetadata.parse(parsedSchema); + return new BaseParsedSchema(abstractParsedSchemaAdapter); } catch (RestClientException | IOException e) { throw new KLoadGenException(e.getMessage()); } } - public BaseParsedSchema getSchemaBySubjectAndId(final String subjectName, BaseSchemaMetadata metadata) { + public BaseParsedSchema getSchemaBySubjectAndId(final String subjectName, + final BaseSchemaMetadata metadata) { try { final ParsedSchema parsedSchema = this.schemaRegistryClient.getSchemaBySubjectAndId(subjectName, metadata.getSchemaMetadataAdapter().getId()); - final ParsedSchemaAdapter parsedSchemaAdapter = ConfluentParsedSchemaMetadata.parse(parsedSchema); - return new BaseParsedSchema(parsedSchemaAdapter); + final AbstractParsedSchemaAdapter abstractParsedSchemaAdapter = ConfluentAbstractParsedSchemaMetadata.parse(parsedSchema); + return new BaseParsedSchema(abstractParsedSchemaAdapter); } catch (RestClientException | IOException e) { throw new KLoadGenException(e.getMessage()); } diff --git a/src/main/java/com/sngular/kloadgen/util/JMeterHelper.java b/src/main/java/com/sngular/kloadgen/util/JMeterHelper.java index 444f3186..ba029b23 100644 --- a/src/main/java/com/sngular/kloadgen/util/JMeterHelper.java +++ b/src/main/java/com/sngular/kloadgen/util/JMeterHelper.java @@ -7,14 +7,17 @@ package com.sngular.kloadgen.util; import java.util.Properties; + import com.sngular.kloadgen.exception.KLoadGenException; import com.sngular.kloadgen.schemaregistry.SchemaRegistryAdapter; import com.sngular.kloadgen.schemaregistry.SchemaRegistryManagerFactory; import com.sngular.kloadgen.schemaregistry.adapter.impl.BaseParsedSchema; +import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; import org.apache.jmeter.threads.JMeterContextService; @SuppressWarnings("checkstyle:AbbreviationAsWordInName") +@Slf4j public final class JMeterHelper { private JMeterHelper() { diff --git a/src/test/java/com/sngular/kloadgen/extractor/SchemaExtractorTest.java b/src/test/java/com/sngular/kloadgen/extractor/SchemaExtractorTest.java index 1baabf55..9f62af25 100644 --- a/src/test/java/com/sngular/kloadgen/extractor/SchemaExtractorTest.java +++ b/src/test/java/com/sngular/kloadgen/extractor/SchemaExtractorTest.java @@ -13,9 +13,9 @@ import com.sngular.kloadgen.common.SchemaRegistryEnum; import com.sngular.kloadgen.exception.KLoadGenException; -import com.sngular.kloadgen.schemaregistry.adapter.impl.ApicurioParsedSchemaMetadata; +import com.sngular.kloadgen.schemaregistry.adapter.impl.ApicurioAbstractParsedSchemaMetadata; import com.sngular.kloadgen.schemaregistry.adapter.impl.BaseParsedSchema; -import com.sngular.kloadgen.schemaregistry.adapter.impl.ConfluentParsedSchemaMetadata; +import com.sngular.kloadgen.schemaregistry.adapter.impl.ConfluentAbstractParsedSchemaMetadata; import com.sngular.kloadgen.testutil.FileHelper; import com.sngular.kloadgen.testutil.ParsedSchemaUtil; import com.sngular.kloadgen.testutil.SchemaParseUtil; @@ -75,7 +75,7 @@ void testFlatPropertiesListWithAVRO() throws IOException { properties.setProperty(SchemaRegistryKeyHelper.SCHEMA_REGISTRY_NAME, SchemaRegistryEnum.APICURIO.toString()); final ParsedSchema parsedSchema = SchemaParseUtil.getParsedSchema(testFile, "AVRO"); - final var baseParsedSchema = new BaseParsedSchema<>(ApicurioParsedSchemaMetadata.parse(parsedSchema)); + final var baseParsedSchema = new BaseParsedSchema<>(ApicurioAbstractParsedSchemaMetadata.parse(parsedSchema)); jmeterHelperMockedStatic.when(() -> JMeterHelper.getParsedSchema(Mockito.anyString(), Mockito.any(Properties.class))).thenReturn(baseParsedSchema); jmeterContextServiceMockedStatic.when(() -> JMeterContextService.getContext().getProperties()).thenReturn(properties); final var result = SchemaExtractor.flatPropertiesList("avroSubject"); @@ -91,7 +91,7 @@ void testFlatPropertiesListWithJson() throws IOException { properties.setProperty(SchemaRegistryKeyHelper.SCHEMA_REGISTRY_NAME, SchemaRegistryEnum.CONFLUENT.toString()); final var parsedSchema = SchemaParseUtil.getParsedSchema(testFile, "JSON"); - final var baseParsedSchema = new BaseParsedSchema<>(ConfluentParsedSchemaMetadata.parse(parsedSchema)); + final var baseParsedSchema = new BaseParsedSchema<>(ConfluentAbstractParsedSchemaMetadata.parse(parsedSchema)); jmeterContextServiceMockedStatic.when(() -> JMeterContextService.getContext().getProperties()).thenReturn(properties); jmeterHelperMockedStatic.when(() -> JMeterHelper.getParsedSchema(Mockito.anyString(), Mockito.any(Properties.class))).thenReturn(baseParsedSchema); final var result = SchemaExtractor.flatPropertiesList("jsonSubject"); @@ -107,7 +107,7 @@ void testFlatPropertiesListWithProtobuf() throws IOException { properties.setProperty(SchemaRegistryKeyHelper.SCHEMA_REGISTRY_NAME, SchemaRegistryEnum.CONFLUENT.toString()); final var parsedSchema = SchemaParseUtil.getParsedSchema(testFile, "PROTOBUF"); - final var baseParsedSchema = new BaseParsedSchema<>(ConfluentParsedSchemaMetadata.parse(parsedSchema)); + final var baseParsedSchema = new BaseParsedSchema<>(ConfluentAbstractParsedSchemaMetadata.parse(parsedSchema)); jmeterContextServiceMockedStatic.when(() -> JMeterContextService.getContext().getProperties()).thenReturn(properties); jmeterHelperMockedStatic.when(() -> JMeterHelper.getParsedSchema(Mockito.anyString(), Mockito.any(Properties.class))).thenReturn(baseParsedSchema); final var result = SchemaExtractor.flatPropertiesList("protobufSubject"); @@ -119,7 +119,7 @@ void testFlatPropertiesListWithProtobuf() throws IOException { void testFlatPropertiesListWithException() { final var parsedSchema = new ParsedSchemaUtil(); - final var baseParsedSchema = new BaseParsedSchema<>(ConfluentParsedSchemaMetadata.parse(parsedSchema)); + final var baseParsedSchema = new BaseParsedSchema<>(ConfluentAbstractParsedSchemaMetadata.parse(parsedSchema)); properties.setProperty(SchemaRegistryKeyHelper.SCHEMA_REGISTRY_NAME, SchemaRegistryEnum.CONFLUENT.toString()); jmeterContextServiceMockedStatic.when(() -> JMeterContextService.getContext().getProperties()).thenReturn(properties); diff --git a/src/test/java/com/sngular/kloadgen/processor/ProtobufSchemaProcessorTest.java b/src/test/java/com/sngular/kloadgen/processor/ProtobufSchemaProcessorTest.java index 4c6f3a37..40e1ba83 100644 --- a/src/test/java/com/sngular/kloadgen/processor/ProtobufSchemaProcessorTest.java +++ b/src/test/java/com/sngular/kloadgen/processor/ProtobufSchemaProcessorTest.java @@ -7,18 +7,22 @@ import java.util.List; import java.util.Locale; import java.util.Map; + import com.google.protobuf.Descriptors; -import com.google.protobuf.Descriptors.DescriptorValidationException; import com.google.protobuf.DynamicMessage; import com.sngular.kloadgen.common.SchemaTypeEnum; import com.sngular.kloadgen.exception.KLoadGenException; import com.sngular.kloadgen.extractor.SchemaExtractor; import com.sngular.kloadgen.model.FieldValueMapping; +import com.sngular.kloadgen.schemaregistry.SchemaRegistryAdapter; +import com.sngular.kloadgen.schemaregistry.SchemaRegistryManagerFactory; import com.sngular.kloadgen.schemaregistry.adapter.impl.BaseSchemaMetadata; import com.sngular.kloadgen.schemaregistry.adapter.impl.ConfluentSchemaMetadata; import com.sngular.kloadgen.serializer.EnrichedRecord; import com.sngular.kloadgen.testutil.FileHelper; import com.sngular.kloadgen.testutil.SchemaParseUtil; +import com.sngular.kloadgen.util.SchemaRegistryKeyHelper; +import org.apache.commons.collections.MapUtils; import org.apache.jmeter.threads.JMeterContext; import org.apache.jmeter.threads.JMeterContextService; import org.apache.jmeter.threads.JMeterVariables; @@ -45,18 +49,20 @@ public void setUp() { final JMeterContext jmcx = JMeterContextService.getContext(); jmcx.setVariables(new JMeterVariables()); JMeterUtils.setLocale(Locale.ENGLISH); + JMeterContextService.getContext().getProperties().put(SchemaRegistryKeyHelper.SCHEMA_REGISTRY_NAME, "CONFLUENT"); + } @Test @DisplayName("Be able to process embedded schema") - void textEmbeddedTypeTestSchemaProcessor() throws KLoadGenException, IOException, DescriptorValidationException { + void textEmbeddedTypeTestSchemaProcessor() throws KLoadGenException, IOException { final File testFile = fileHelper.getFile("/proto-files/embeddedTypeTest.proto"); final List fieldValueMappingList = List.of( FieldValueMapping.builder().fieldName("phones.addressesPhone[1:].id[1]").fieldType("string-array").fieldValueList("Pablo").required(true).isAncestorRequired(true).build(), FieldValueMapping.builder().fieldName("phones.phoneType").fieldType("enum").fieldValueList("[MOBILE, HOME, WORK]").required(true).isAncestorRequired(true).build()); final SchemaProcessor protobufSchemaProcessor = new SchemaProcessor(); - protobufSchemaProcessor.processSchema(SchemaTypeEnum.PROTOBUF, SchemaParseUtil.getParsedSchema(testFile,SchemaTypeEnum.PROTOBUF.name()), + protobufSchemaProcessor.processSchema(SchemaTypeEnum.PROTOBUF, SchemaParseUtil.getParsedSchema(testFile, SchemaTypeEnum.PROTOBUF.name()), confluentBaseSchemaMetadata, fieldValueMappingList); final EnrichedRecord message = (EnrichedRecord) protobufSchemaProcessor.next(); final DynamicMessage genericRecord = (DynamicMessage) message.getGenericRecord(); @@ -83,14 +89,14 @@ private String getIdFieldForEmbeddedTypeTest(final List assertValues) { @Test @DisplayName("Be able to process complex types like StringValue or Int32Value and get values by default") - void testProtobufGoogleTypes() throws IOException, DescriptorValidationException { + void testProtobufGoogleTypes() throws IOException { final File testFile = fileHelper.getFile("/proto-files/googleTypesTest.proto"); final List fieldValueMappingList = List.of( FieldValueMapping.builder().fieldName("id").fieldType("Int32Value").required(true).isAncestorRequired(true).build(), FieldValueMapping.builder().fieldName("occurrence_id").fieldType("StringValue").fieldValueList("Isabel").required(true).isAncestorRequired(true).build(), FieldValueMapping.builder().fieldName("load_number").fieldType("Int32Value").required(true).isAncestorRequired(true).build()); final SchemaProcessor protobufSchemaProcessor = new SchemaProcessor(); - protobufSchemaProcessor.processSchema(SchemaTypeEnum.PROTOBUF, SchemaParseUtil.getParsedSchema(testFile,SchemaTypeEnum.PROTOBUF.name()), + protobufSchemaProcessor.processSchema(SchemaTypeEnum.PROTOBUF, SchemaParseUtil.getParsedSchema(testFile, SchemaTypeEnum.PROTOBUF.name()), confluentBaseSchemaMetadata, fieldValueMappingList); final EnrichedRecord message = (EnrichedRecord) protobufSchemaProcessor.next(); final DynamicMessage genericRecord = (DynamicMessage) message.getGenericRecord(); @@ -113,12 +119,12 @@ void testProtobufGoogleTypes() throws IOException, DescriptorValidationException @Test @DisplayName("Be able to process enum in the schema") - void testProtoBufEnumSchemaProcessor() throws IOException, DescriptorValidationException { + void testProtoBufEnumSchemaProcessor() throws IOException { final File testFile = fileHelper.getFile("/proto-files/enumTest.proto"); final List fieldValueMappingList = schemaExtractor.flatPropertiesList(SchemaParseUtil.getParsedSchema(testFile, "PROTOBUF")); fieldValueMappingList.get(0).setFieldValuesList("HOME, WORK"); final SchemaProcessor protobufSchemaProcessor = new SchemaProcessor(); - protobufSchemaProcessor.processSchema(SchemaTypeEnum.PROTOBUF, SchemaParseUtil.getParsedSchema(testFile,SchemaTypeEnum.PROTOBUF.name()), + protobufSchemaProcessor.processSchema(SchemaTypeEnum.PROTOBUF, SchemaParseUtil.getParsedSchema(testFile, SchemaTypeEnum.PROTOBUF.name()), confluentBaseSchemaMetadata, fieldValueMappingList); final EnrichedRecord message = (EnrichedRecord) protobufSchemaProcessor.next(); final DynamicMessage genericRecord = (DynamicMessage) message.getGenericRecord(); @@ -152,11 +158,11 @@ void testProtoBufEnumSchemaProcessor() throws IOException, DescriptorValidationE @Test @DisplayName("Be able to process easy schema") - void testProtoBufEasyTestProcessor() throws IOException, DescriptorValidationException { + void testProtoBufEasyTestProcessor() throws IOException { final File testFile = fileHelper.getFile("/proto-files/easyTest.proto"); final List fieldValueMappingList = schemaExtractor.flatPropertiesList(SchemaParseUtil.getParsedSchema(testFile, "PROTOBUF")); final SchemaProcessor protobufSchemaProcessor = new SchemaProcessor(); - protobufSchemaProcessor.processSchema(SchemaTypeEnum.PROTOBUF, SchemaParseUtil.getParsedSchema(testFile,SchemaTypeEnum.PROTOBUF.name()), + protobufSchemaProcessor.processSchema(SchemaTypeEnum.PROTOBUF, SchemaParseUtil.getParsedSchema(testFile, SchemaTypeEnum.PROTOBUF.name()), confluentBaseSchemaMetadata, fieldValueMappingList); final EnrichedRecord message = (EnrichedRecord) protobufSchemaProcessor.next(); final DynamicMessage genericRecord = (DynamicMessage) message.getGenericRecord(); @@ -182,12 +188,12 @@ void testProtoBufEasyTestProcessor() throws IOException, DescriptorValidationExc @Test @DisplayName("Be able to process oneOf fields") - void testProtoBufOneOfProcessor() throws IOException, DescriptorValidationException { + void testProtoBufOneOfProcessor() throws IOException { final File testFile = fileHelper.getFile("/proto-files/oneOfTest.proto"); final List fieldValueMappingList = schemaExtractor.flatPropertiesList(SchemaParseUtil.getParsedSchema(testFile, "PROTOBUF")); final SchemaProcessor protobufSchemaProcessor = new SchemaProcessor(); - protobufSchemaProcessor.processSchema(SchemaTypeEnum.PROTOBUF, SchemaParseUtil.getParsedSchema(testFile,SchemaTypeEnum.PROTOBUF.name()), + protobufSchemaProcessor.processSchema(SchemaTypeEnum.PROTOBUF, SchemaParseUtil.getParsedSchema(testFile, SchemaTypeEnum.PROTOBUF.name()), confluentBaseSchemaMetadata, fieldValueMappingList); final EnrichedRecord message = (EnrichedRecord) protobufSchemaProcessor.next(); final DynamicMessage genericRecord = (DynamicMessage) message.getGenericRecord(); @@ -213,7 +219,7 @@ void testProtoBufOneOfProcessor() throws IOException, DescriptorValidationExcept @Test @DisplayName("Be able to process map in schema") - void testProtoBufMapTestProcessor() throws IOException, DescriptorValidationException { + void testProtoBufMapTestProcessor() throws IOException { final File testFile = fileHelper.getFile("/proto-files/mapTest.proto"); final List fieldValueMappingList = Arrays.asList( FieldValueMapping.builder().fieldName("name[:]").fieldType("string-map").fieldValueList("Pablo").required(true).isAncestorRequired(true).build(), @@ -269,8 +275,11 @@ private Object getSubFieldForMapTestProcessor(final DynamicMessage dynamicMessag @Test @DisplayName("Be able to process complex schema") - void testProtoBufComplexTestProcessor() throws IOException, DescriptorValidationException { + void testProtoBufComplexTestProcessor() throws IOException { final File testFile = fileHelper.getFile("/proto-files/complexTest.proto"); + final SchemaRegistryAdapter schemaRegistryManager = SchemaRegistryManagerFactory.getSchemaRegistry("CONFLUENT"); + schemaRegistryManager.setSchemaRegistryClient("http://localhost:8080", MapUtils.EMPTY_MAP); + final List fieldValueMappingList = Arrays.asList( FieldValueMapping.builder().fieldName("phone_types[].phone").fieldType("long").required(true).isAncestorRequired(true).build(), FieldValueMapping.builder().fieldName("phone_types[].principal").fieldType("boolean").required(true).isAncestorRequired(true).build(), @@ -319,7 +328,7 @@ void testProtoBufComplexTestProcessor() throws IOException, DescriptorValidation @Test @DisplayName("Be able to process provided complex schema") - void testProtoBufProvidedComplexTestProcessor() throws IOException, DescriptorValidationException { + void testProtoBufProvidedComplexTestProcessor() throws IOException { final File testFile = fileHelper.getFile("/proto-files/providedTest.proto"); final List fieldValueMappingList = Arrays.asList( FieldValueMapping.builder().fieldName("id").fieldType("int").required(true).isAncestorRequired(true).build(), @@ -400,7 +409,7 @@ void testProtoBufProvidedComplexTestProcessor() throws IOException, DescriptorVa } @Test - void testFailing() throws IOException, DescriptorValidationException { + void testFailing() throws IOException { final File testFile = fileHelper.getFile("/proto-files/deveTest.proto"); final List fieldValueMappingList = Arrays.asList( FieldValueMapping.builder().fieldName("load_type").fieldType("string").required(true).isAncestorRequired(true).build(), @@ -462,7 +471,8 @@ void testFailing() throws IOException, DescriptorValidationException { ); final SchemaProcessor protobufSchemaProcessor = new SchemaProcessor(); - protobufSchemaProcessor.processSchema(SchemaTypeEnum.PROTOBUF, SchemaParseUtil.getParsedSchema(testFile,SchemaTypeEnum.PROTOBUF.name()), confluentBaseSchemaMetadata, fieldValueMappingList); + protobufSchemaProcessor.processSchema(SchemaTypeEnum.PROTOBUF, + SchemaParseUtil.getParsedSchema(testFile, SchemaTypeEnum.PROTOBUF.name()), confluentBaseSchemaMetadata, fieldValueMappingList); final EnrichedRecord message = (EnrichedRecord) protobufSchemaProcessor.next(); final DynamicMessage genericRecord = (DynamicMessage) message.getGenericRecord(); final List assertKeys = new ArrayList<>(); @@ -493,7 +503,8 @@ void testProtoBufProvidedWithNotNestedTypeProcessor() throws IOException { FieldValueMapping.builder().fieldName("customer_account.billing_party.address.address_line_one").fieldType("string").required(true).isAncestorRequired(true).build(), FieldValueMapping.builder().fieldName("customer_account.billing_party.address.address_line_two").fieldType("string").required(true).isAncestorRequired(true).build()); final SchemaProcessor protobufSchemaProcessor = new SchemaProcessor(); - protobufSchemaProcessor.processSchema(SchemaTypeEnum.PROTOBUF, SchemaParseUtil.getParsedSchema(testFile,SchemaTypeEnum.PROTOBUF.name()), confluentBaseSchemaMetadata, fieldValueMappingList); + protobufSchemaProcessor.processSchema(SchemaTypeEnum.PROTOBUF, + SchemaParseUtil.getParsedSchema(testFile, SchemaTypeEnum.PROTOBUF.name()), confluentBaseSchemaMetadata, fieldValueMappingList); final EnrichedRecord message = (EnrichedRecord) protobufSchemaProcessor.next(); final DynamicMessage genericRecord = (DynamicMessage) message.getGenericRecord(); final List assertKeys = new ArrayList<>(); @@ -519,14 +530,15 @@ void testProtoBufProvidedWithNotNestedTypeProcessor() throws IOException { @Test @DisplayName("Be able to process Date and TimeOfDay types") - void testDateTimeTypes() throws IOException, DescriptorValidationException { + void testDateTimeTypes() throws IOException { final File testFile = fileHelper.getFile("/proto-files/dateTimeTest.proto"); final List fieldValueMappingList = List.of( FieldValueMapping.builder().fieldName("incident_date").fieldType(".google.type.Date").fieldValueList("2022-05-30").required(true).isAncestorRequired(true).build(), FieldValueMapping.builder().fieldName("incident_time").fieldType(".google.type.TimeOfDay").fieldValueList("14:20:30-05:00").required(true).isAncestorRequired(true) .build()); final SchemaProcessor protobufSchemaProcessor = new SchemaProcessor(); - protobufSchemaProcessor.processSchema(SchemaTypeEnum.PROTOBUF, SchemaParseUtil.getParsedSchema(testFile,SchemaTypeEnum.PROTOBUF.name()), confluentBaseSchemaMetadata, fieldValueMappingList); + protobufSchemaProcessor.processSchema(SchemaTypeEnum.PROTOBUF, + SchemaParseUtil.getParsedSchema(testFile, SchemaTypeEnum.PROTOBUF.name()), confluentBaseSchemaMetadata, fieldValueMappingList); final EnrichedRecord message = (EnrichedRecord) protobufSchemaProcessor.next(); final DynamicMessage genericRecord = (DynamicMessage) message.getGenericRecord(); final Map map = genericRecord.getAllFields(); diff --git a/src/test/java/com/sngular/kloadgen/serializer/ProtobufSerializerTest.java b/src/test/java/com/sngular/kloadgen/serializer/ProtobufSerializerTest.java index 2a55a409..79e16c1d 100644 --- a/src/test/java/com/sngular/kloadgen/serializer/ProtobufSerializerTest.java +++ b/src/test/java/com/sngular/kloadgen/serializer/ProtobufSerializerTest.java @@ -3,30 +3,23 @@ import java.io.File; import java.io.IOException; import java.util.List; +import java.util.Locale; import java.util.stream.Stream; import java.util.stream.Stream.Builder; -import static com.sngular.kloadgen.serializer.ProtobuffSerializerTestFixture.TEST_COMPLETE_PROTO; -import static com.sngular.kloadgen.serializer.ProtobuffSerializerTestFixture.TEST_COMPLEX; -import static com.sngular.kloadgen.serializer.ProtobuffSerializerTestFixture.TEST_DATE_TIME; -import static com.sngular.kloadgen.serializer.ProtobuffSerializerTestFixture.TEST_DEVE; -import static com.sngular.kloadgen.serializer.ProtobuffSerializerTestFixture.TEST_EASY; -import static com.sngular.kloadgen.serializer.ProtobuffSerializerTestFixture.TEST_EMBEDDED_TYPE; -import static com.sngular.kloadgen.serializer.ProtobuffSerializerTestFixture.TEST_ENUM; -import static com.sngular.kloadgen.serializer.ProtobuffSerializerTestFixture.TEST_GOOGLE_TYPES; -import static com.sngular.kloadgen.serializer.ProtobuffSerializerTestFixture.TEST_ISSUE_311; -import static com.sngular.kloadgen.serializer.ProtobuffSerializerTestFixture.TEST_MAP; -import static com.sngular.kloadgen.serializer.ProtobuffSerializerTestFixture.TEST_ONE_OF; -import static com.sngular.kloadgen.serializer.ProtobuffSerializerTestFixture.TEST_PROVIDED; - import com.sngular.kloadgen.common.SchemaTypeEnum; import com.sngular.kloadgen.model.FieldValueMapping; import com.sngular.kloadgen.processor.SchemaProcessor; import com.sngular.kloadgen.schemaregistry.adapter.impl.BaseSchemaMetadata; import com.sngular.kloadgen.schemaregistry.adapter.impl.ConfluentSchemaMetadata; import com.sngular.kloadgen.testutil.SchemaParseUtil; +import com.sngular.kloadgen.util.SchemaRegistryKeyHelper; import io.confluent.kafka.schemaregistry.ParsedSchema; import lombok.extern.slf4j.Slf4j; +import org.apache.jmeter.threads.JMeterContext; +import org.apache.jmeter.threads.JMeterContextService; +import org.apache.jmeter.threads.JMeterVariables; +import org.apache.jmeter.util.JMeterUtils; import org.assertj.core.api.Assertions; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Named; @@ -42,42 +35,49 @@ class ProtobufSerializerTest { @BeforeEach void setUp() { protobufSerializer = new ProtobufSerializer(); + final File file = new File("src/test/resources"); + final String absolutePath = file.getAbsolutePath(); + JMeterUtils.loadJMeterProperties(absolutePath + "/kloadgen.properties"); + final JMeterContext jmcx = JMeterContextService.getContext(); + jmcx.setVariables(new JMeterVariables()); + JMeterUtils.setLocale(Locale.ENGLISH); + JMeterContextService.getContext().getProperties().put(SchemaRegistryKeyHelper.SCHEMA_REGISTRY_NAME, "CONFLUENT"); } private static Stream getSchemaToTest() { - Builder builder = Stream.builder(); + final Builder builder = Stream.builder(); - File testCompleteProtoFile = TEST_COMPLETE_PROTO.getFirst(); - builder.add(Arguments.arguments(Named.of(testCompleteProtoFile.getName(), testCompleteProtoFile), TEST_COMPLETE_PROTO.getSecond())); - File testComplexFile = TEST_COMPLEX.getFirst(); - builder.add(Arguments.arguments(Named.of(testComplexFile.getName(), testComplexFile), TEST_COMPLEX.getSecond())); - File testDateTimeFile = TEST_DATE_TIME.getFirst(); - builder.add(Arguments.arguments(Named.of(testDateTimeFile.getName(), testDateTimeFile), TEST_DATE_TIME.getSecond())); - File testDeveFile = TEST_DEVE.getFirst(); - builder.add(Arguments.arguments(Named.of(testDeveFile.getName(), testDeveFile), TEST_DEVE.getSecond())); - File testEasyFile = TEST_EASY.getFirst(); - builder.add(Arguments.arguments(Named.of(testEasyFile.getName(), testEasyFile), TEST_EASY.getSecond())); - File testEmbeddedTypeFile = TEST_EMBEDDED_TYPE.getFirst(); - builder.add(Arguments.arguments(Named.of(testEmbeddedTypeFile.getName(), testEmbeddedTypeFile), TEST_EMBEDDED_TYPE.getSecond())); - File testEnumFile = TEST_ENUM.getFirst(); - builder.add(Arguments.arguments(Named.of(testEnumFile.getName(), testEnumFile), TEST_ENUM.getSecond())); - File testGoogleTypesFile = TEST_GOOGLE_TYPES.getFirst(); - builder.add(Arguments.arguments(Named.of(testGoogleTypesFile.getName(), testGoogleTypesFile), TEST_GOOGLE_TYPES.getSecond())); - File testIssue311File = TEST_ISSUE_311.getFirst(); - builder.add(Arguments.arguments(Named.of(testIssue311File.getName(), testIssue311File), TEST_ISSUE_311.getSecond())); - File testMapFile = TEST_MAP.getFirst(); - builder.add(Arguments.arguments(Named.of(testMapFile.getName(), testMapFile), TEST_MAP.getSecond())); - File testOneOfFile = TEST_ONE_OF.getFirst(); - builder.add(Arguments.arguments(Named.of(testOneOfFile.getName(), testOneOfFile), TEST_ONE_OF.getSecond())); - File testProvidedFile = TEST_PROVIDED.getFirst(); - builder.add(Arguments.arguments(Named.of(testProvidedFile.getName(), testProvidedFile), TEST_PROVIDED.getSecond())); + final File testCompleteProtoFile = ProtobuffSerializerTestFixture.TEST_COMPLETE_PROTO.getFirst(); + builder.add(Arguments.arguments(Named.of(testCompleteProtoFile.getName(), testCompleteProtoFile), ProtobuffSerializerTestFixture.TEST_COMPLETE_PROTO.getSecond())); + final File testComplexFile = ProtobuffSerializerTestFixture.TEST_COMPLEX.getFirst(); + builder.add(Arguments.arguments(Named.of(testComplexFile.getName(), testComplexFile), ProtobuffSerializerTestFixture.TEST_COMPLEX.getSecond())); + final File testDateTimeFile = ProtobuffSerializerTestFixture.TEST_DATE_TIME.getFirst(); + builder.add(Arguments.arguments(Named.of(testDateTimeFile.getName(), testDateTimeFile), ProtobuffSerializerTestFixture.TEST_DATE_TIME.getSecond())); + final File testDeveFile = ProtobuffSerializerTestFixture.TEST_DEVE.getFirst(); + builder.add(Arguments.arguments(Named.of(testDeveFile.getName(), testDeveFile), ProtobuffSerializerTestFixture.TEST_DEVE.getSecond())); + final File testEasyFile = ProtobuffSerializerTestFixture.TEST_EASY.getFirst(); + builder.add(Arguments.arguments(Named.of(testEasyFile.getName(), testEasyFile), ProtobuffSerializerTestFixture.TEST_EASY.getSecond())); + final File testEmbeddedTypeFile = ProtobuffSerializerTestFixture.TEST_EMBEDDED_TYPE.getFirst(); + builder.add(Arguments.arguments(Named.of(testEmbeddedTypeFile.getName(), testEmbeddedTypeFile), ProtobuffSerializerTestFixture.TEST_EMBEDDED_TYPE.getSecond())); + final File testEnumFile = ProtobuffSerializerTestFixture.TEST_ENUM.getFirst(); + builder.add(Arguments.arguments(Named.of(testEnumFile.getName(), testEnumFile), ProtobuffSerializerTestFixture.TEST_ENUM.getSecond())); + final File testGoogleTypesFile = ProtobuffSerializerTestFixture.TEST_GOOGLE_TYPES.getFirst(); + builder.add(Arguments.arguments(Named.of(testGoogleTypesFile.getName(), testGoogleTypesFile), ProtobuffSerializerTestFixture.TEST_GOOGLE_TYPES.getSecond())); + final File testIssue311File = ProtobuffSerializerTestFixture.TEST_ISSUE_311.getFirst(); + builder.add(Arguments.arguments(Named.of(testIssue311File.getName(), testIssue311File), ProtobuffSerializerTestFixture.TEST_ISSUE_311.getSecond())); + final File testMapFile = ProtobuffSerializerTestFixture.TEST_MAP.getFirst(); + builder.add(Arguments.arguments(Named.of(testMapFile.getName(), testMapFile), ProtobuffSerializerTestFixture.TEST_MAP.getSecond())); + final File testOneOfFile = ProtobuffSerializerTestFixture.TEST_ONE_OF.getFirst(); + builder.add(Arguments.arguments(Named.of(testOneOfFile.getName(), testOneOfFile), ProtobuffSerializerTestFixture.TEST_ONE_OF.getSecond())); + final File testProvidedFile = ProtobuffSerializerTestFixture.TEST_PROVIDED.getFirst(); + builder.add(Arguments.arguments(Named.of(testProvidedFile.getName(), testProvidedFile), ProtobuffSerializerTestFixture.TEST_PROVIDED.getSecond())); return builder.build(); } @ParameterizedTest @MethodSource("getSchemaToTest") - void serialize(File schemaFile, List fieldValueMappings) throws IOException { + void serialize(final File schemaFile, final List fieldValueMappings) throws IOException { final ParsedSchema parsedSchema = SchemaParseUtil.getParsedSchema(schemaFile, "Protobuf"); final SchemaProcessor protobufSchemaProcessor = new SchemaProcessor(); final BaseSchemaMetadata confluentBaseSchemaMetadata = diff --git a/src/test/resources/proto-files/complexTest.proto b/src/test/resources/proto-files/complexTest.proto index dea0058f..4c10d313 100644 --- a/src/test/resources/proto-files/complexTest.proto +++ b/src/test/resources/proto-files/complexTest.proto @@ -8,7 +8,6 @@ option java_outer_classname = "Protos"; import "google/protobuf/timestamp.proto"; import "google/protobuf/descriptor.proto"; -import "google/protobuf/compiler/plugin.proto"; message Test { repeated PhoneTypes phone_types = 1; From 46e047bc2cf39efe65e18c4ee52b14cf95b05597 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jose=20Enrique=20Garc=C3=ADa=20Maci=C3=B1eiras?= Date: Thu, 21 Sep 2023 21:37:55 +0200 Subject: [PATCH 17/30] #332 Adding google proto types --- pom-maven-central.xml | 5 +++++ pom.xml | 5 +++++ 2 files changed, 10 insertions(+) diff --git a/pom-maven-central.xml b/pom-maven-central.xml index 5bf8a08f..4bade7db 100644 --- a/pom-maven-central.xml +++ b/pom-maven-central.xml @@ -601,6 +601,11 @@ ${jaxb-api.version} compile + + com.google.api.grpc + proto-google-common-protos + 2.24.0 + org.junit.jupiter junit-jupiter-params diff --git a/pom.xml b/pom.xml index e1586d1d..ce0a62ab 100644 --- a/pom.xml +++ b/pom.xml @@ -474,6 +474,11 @@ 4.0.1 runtime + + com.google.api.grpc + proto-google-common-protos + 2.24.0 + org.slf4j slf4j-api From 977edaf5ca8d4017d1ed3885899f2ceba26e3d13 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 29 Sep 2023 22:08:11 +0000 Subject: [PATCH 18/30] Bump org.apache.avro:avro from 1.11.0 to 1.11.3 Bumps org.apache.avro:avro from 1.11.0 to 1.11.3. --- updated-dependencies: - dependency-name: org.apache.avro:avro dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index b9fad739..b6111591 100644 --- a/pom.xml +++ b/pom.xml @@ -302,7 +302,7 @@ 3.22.0 1.2.0 1.5.1 - 1.11.0 + 1.11.3 1.9.4 4.4 3.12.0 From 0324595e1f8c3294e05512f96c75985f04c7a1b2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jose=20Enrique=20Garc=C3=ADa=20Maci=C3=B1eiras?= Date: Sat, 30 Sep 2023 11:07:30 +0200 Subject: [PATCH 19/30] #332 Fixing size serializer. --- .../extractors/avro/AvroConfluentExtractor.java | 1 + .../extractor/extractors/avro/AvroExtractor.java | 11 ++++------- .../kloadgen/serializer/ProtobufSerializer.java | 3 ++- 3 files changed, 7 insertions(+), 8 deletions(-) diff --git a/src/main/java/com/sngular/kloadgen/extractor/extractors/avro/AvroConfluentExtractor.java b/src/main/java/com/sngular/kloadgen/extractor/extractors/avro/AvroConfluentExtractor.java index b572dce2..7405c3ab 100644 --- a/src/main/java/com/sngular/kloadgen/extractor/extractors/avro/AvroConfluentExtractor.java +++ b/src/main/java/com/sngular/kloadgen/extractor/extractors/avro/AvroConfluentExtractor.java @@ -1,6 +1,7 @@ package com.sngular.kloadgen.extractor.extractors.avro; import java.util.List; + import com.sngular.kloadgen.extractor.extractors.Extractor; import com.sngular.kloadgen.model.FieldValueMapping; import io.confluent.kafka.schemaregistry.avro.AvroSchema; diff --git a/src/main/java/com/sngular/kloadgen/extractor/extractors/avro/AvroExtractor.java b/src/main/java/com/sngular/kloadgen/extractor/extractors/avro/AvroExtractor.java index 7ac96c7d..88e48b8d 100644 --- a/src/main/java/com/sngular/kloadgen/extractor/extractors/avro/AvroExtractor.java +++ b/src/main/java/com/sngular/kloadgen/extractor/extractors/avro/AvroExtractor.java @@ -3,9 +3,6 @@ import java.util.List; import java.util.Map; -import static com.sngular.kloadgen.common.SchemaRegistryEnum.APICURIO; -import static com.sngular.kloadgen.common.SchemaRegistryEnum.CONFLUENT; - import com.sngular.kloadgen.common.SchemaRegistryEnum; import com.sngular.kloadgen.extractor.extractors.Extractor; import com.sngular.kloadgen.extractor.extractors.ExtractorRegistry; @@ -13,14 +10,14 @@ public class AvroExtractor implements ExtractorRegistry { - static Map schemaRegistryMap = Map.of(CONFLUENT, new AvroConfluentExtractor(), APICURIO, - new AvroApicurioExtractor()); + private static Map schemaRegistryMap = Map.of(SchemaRegistryEnum.CONFLUENT, new AvroConfluentExtractor(), SchemaRegistryEnum.APICURIO, + new AvroApicurioExtractor()); - public final List processSchema(final Object schema, SchemaRegistryEnum registryEnum) { + public final List processSchema(final Object schema, final SchemaRegistryEnum registryEnum) { return schemaRegistryMap.get(registryEnum).processSchema(schema); } - public final List getSchemaNameList(final String schema, SchemaRegistryEnum registryEnum) { + public final List getSchemaNameList(final String schema, final SchemaRegistryEnum registryEnum) { return schemaRegistryMap.get(registryEnum).getSchemaNameList(schema); } diff --git a/src/main/java/com/sngular/kloadgen/serializer/ProtobufSerializer.java b/src/main/java/com/sngular/kloadgen/serializer/ProtobufSerializer.java index 63d64fb2..18a264cd 100644 --- a/src/main/java/com/sngular/kloadgen/serializer/ProtobufSerializer.java +++ b/src/main/java/com/sngular/kloadgen/serializer/ProtobufSerializer.java @@ -9,6 +9,7 @@ import java.io.ByteArrayOutputStream; import java.io.IOException; import java.nio.ByteBuffer; +import java.util.Arrays; import javax.xml.bind.DatatypeConverter; @@ -38,7 +39,7 @@ public final byte[] serialize(final String topic, final T data) { final ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream(); byteArrayOutputStream.write(MAGIC_BYTE); - byteArrayOutputStream.write(ByteBuffer.allocate(ID_SIZE).put(data.getSchemaMetadata().getId().toString().getBytes()).array()); + byteArrayOutputStream.write(ByteBuffer.allocate(ID_SIZE).put(Arrays.copyOfRange(data.getSchemaMetadata().getId().toString().getBytes(), 0, ID_SIZE)).array()); final Descriptor descriptor = ((DynamicMessage) data.getGenericRecord()).getDescriptorForType(); final ProtobufSchema schema = new ProtobufSchema(descriptor); final MessageIndexes indexes = schema.toMessageIndexes(descriptor.getFullName()); From d15d7b26ff9429dc5bd20007ebffcaab6be0a8a7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jose=20Enrique=20Garc=C3=ADa=20Maci=C3=B1eiras?= Date: Sat, 30 Sep 2023 11:38:58 +0200 Subject: [PATCH 20/30] #332 Include proto files in package --- pom-maven-central.xml | 3 +++ pom.xml | 3 +++ 2 files changed, 6 insertions(+) diff --git a/pom-maven-central.xml b/pom-maven-central.xml index 4bade7db..6a0c02a8 100644 --- a/pom-maven-central.xml +++ b/pom-maven-central.xml @@ -852,6 +852,9 @@ org/apache/jmeter/jmeter.properties + + *.proto + ${project.artifactId}-${project.version} diff --git a/pom.xml b/pom.xml index ce0a62ab..e4f4bb80 100644 --- a/pom.xml +++ b/pom.xml @@ -791,6 +791,9 @@ org/apache/jmeter/jmeter.properties + + *.proto + ${project.artifactId}-${project.version} From a3cad52b71826a020ca7546e24731baf4d2e1897 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jose=20Enrique=20Garc=C3=ADa=20Maci=C3=B1eiras?= Date: Tue, 3 Oct 2023 10:54:19 +0200 Subject: [PATCH 21/30] #332 Fix styles and pom version --- pom-maven-central.xml | 2 +- .../kloadgen/extractor/extractors/ExtractorRegistry.java | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/pom-maven-central.xml b/pom-maven-central.xml index 6a0c02a8..71eefe2e 100644 --- a/pom-maven-central.xml +++ b/pom-maven-central.xml @@ -7,7 +7,7 @@ kloadgen - 5.6.7 + 5.6.8 KLoadGen Load Generation Jmeter plugin for Kafka Cluster. Supporting AVRO, JSON Schema and Protobuf schema types. Generate Artificial diff --git a/src/main/java/com/sngular/kloadgen/extractor/extractors/ExtractorRegistry.java b/src/main/java/com/sngular/kloadgen/extractor/extractors/ExtractorRegistry.java index a7ca2b84..99f0a803 100644 --- a/src/main/java/com/sngular/kloadgen/extractor/extractors/ExtractorRegistry.java +++ b/src/main/java/com/sngular/kloadgen/extractor/extractors/ExtractorRegistry.java @@ -1,6 +1,7 @@ package com.sngular.kloadgen.extractor.extractors; import java.util.List; + import com.sngular.kloadgen.common.SchemaRegistryEnum; import com.sngular.kloadgen.model.FieldValueMapping; From a442f7a3759b09dc1a976c8086773a1d654d291f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jose=20Enrique=20Garc=C3=ADa=20Maci=C3=B1eiras?= Date: Tue, 3 Oct 2023 12:22:05 +0200 Subject: [PATCH 22/30] #332 Fix styles and pom version --- pom-maven-central.xml | 2 +- .../extractor/extractors/Extractor.java | 1 + .../json/AbstractJsonExtractor.java | 398 +++++++++--------- .../extractors/json/JsonDefaultExtractor.java | 37 +- .../extractors/json/JsonExtractor.java | 15 +- .../protobuff/AbstractProtoFileExtractor.java | 29 +- .../protobuff/ProtoBufApicurioExtractor.java | 1 + .../protobuff/ProtoBufConfluentExtractor.java | 3 +- .../protobuff/ProtobuffExtractor.java | 18 +- .../loadgen/impl/AvroSRLoadGenerator.java | 1 + .../loadgen/impl/JsonSRLoadGenerator.java | 1 + .../loadgen/impl/ProtobufLoadGenerator.java | 5 +- .../impl/JsonObjectCreatorFactory.java | 2 +- .../editor/KeyDeserializerPropertyEditor.java | 3 +- .../editor/NameStrategyPropertyEditor.java | 7 +- .../generator/ProtoBufGeneratorTool.java | 34 +- .../sampler/KafkaConsumerSampler.java | 1 + .../sampler/KafkaProducerSampler.java | 1 + .../adapter/impl/ApicurioSchemaMetadata.java | 7 +- .../adapter/impl/BaseSchemaMetadata.java | 2 +- .../adapter/impl/ConfluentSchemaMetadata.java | 19 +- .../schema/KLoadSchemaMetadata.java | 12 +- .../kloadgen/serializer/AvroDeserializer.java | 3 +- .../avro/AvroApicurioExtractorTest.java | 7 +- .../avro/AvroConfluentExtractorTest.java | 1 + .../extractors/json/JsonDefaultTest.java | 16 +- .../ProtobufConfluentExtractorTest.java | 4 +- .../loadgen/impl/AvroSRLoadGeneratorTest.java | 1 + .../impl/ProtobufSRLoadGeneratorTest.java | 1 + .../processor/AvroSchemaProcessorTest.java | 5 +- .../editor/FileSubjectPropertyEditorTest.java | 1 + .../serializer/AvroDeserializerTest.java | 67 ++- .../AvroSerializersTestFixture.java | 151 +++---- .../GenericRecordAvroSerializersTest.java | 70 ++- .../serializer/JsonSerializerTest.java | 67 ++- .../serializer/JsonSerializerTestFixture.java | 304 ++++++------- .../sngular/kloadgen/testutil/FileHelper.java | 1 + 37 files changed, 634 insertions(+), 664 deletions(-) diff --git a/pom-maven-central.xml b/pom-maven-central.xml index 1c6393f2..c710f436 100644 --- a/pom-maven-central.xml +++ b/pom-maven-central.xml @@ -331,7 +331,7 @@ 2.8.6 32.0.0-jre 2.4.0-b180830.0359 - 11 + 17 provided 5.5 2.6.0 diff --git a/src/main/java/com/sngular/kloadgen/extractor/extractors/Extractor.java b/src/main/java/com/sngular/kloadgen/extractor/extractors/Extractor.java index 91c243af..b67f7eaa 100644 --- a/src/main/java/com/sngular/kloadgen/extractor/extractors/Extractor.java +++ b/src/main/java/com/sngular/kloadgen/extractor/extractors/Extractor.java @@ -1,6 +1,7 @@ package com.sngular.kloadgen.extractor.extractors; import java.util.List; + import com.sngular.kloadgen.model.FieldValueMapping; public interface Extractor { diff --git a/src/main/java/com/sngular/kloadgen/extractor/extractors/json/AbstractJsonExtractor.java b/src/main/java/com/sngular/kloadgen/extractor/extractors/json/AbstractJsonExtractor.java index 11fe2683..0e8f865c 100644 --- a/src/main/java/com/sngular/kloadgen/extractor/extractors/json/AbstractJsonExtractor.java +++ b/src/main/java/com/sngular/kloadgen/extractor/extractors/json/AbstractJsonExtractor.java @@ -3,6 +3,7 @@ import java.util.ArrayList; import java.util.List; import java.util.Objects; + import com.sngular.kloadgen.exception.KLoadGenException; import com.sngular.kloadgen.extractor.extractors.SchemaExtractorUtil; import com.sngular.kloadgen.extractor.parser.impl.JSONSchemaParser; @@ -19,218 +20,229 @@ import org.apache.commons.collections4.Transformer; import org.apache.commons.lang3.StringUtils; -public class AbstractJsonExtractor { - - public final JSONSchemaParser jsonSchemaParser = new JSONSchemaParser(); +public abstract class AbstractJsonExtractor { - private List extractInternalFields(final ObjectField field, final Boolean isAncestorRequired) { - final List completeFieldList = new ArrayList<>(); - for (Field innerField : field.getProperties()) { - completeFieldList.addAll(processField(innerField, false, isAncestorRequired)); - } - return completeFieldList; - } + private static final JSONSchemaParser JSON_SCHEMA_PARSER = new JSONSchemaParser(); - private Transformer fixName(final String fieldName, final String splitter) { - return fieldValue -> { - fieldValue.setFieldName(extractFieldName(fieldName) + splitter + fieldValue.getFieldName()); - return fieldValue; - }; - } + public final JSONSchemaParser getSchemaParser() { + return JSON_SCHEMA_PARSER; + } - private static String extractFieldName(final String fieldName) { - String fieldNameClean = fieldName; - if (fieldName.endsWith("[][]") || fieldName.endsWith("[:][]")) { - fieldNameClean = fieldName.substring(0, fieldName.length() - 2); - } else if (fieldName.endsWith("[][:]") || fieldName.endsWith("[:][:]")) { - fieldNameClean = fieldName.substring(0, fieldName.length() - 3); - } - return fieldNameClean; + private static String extractFieldName(final String fieldName) { + String fieldNameClean = fieldName; + if (fieldName.endsWith("[][]") || fieldName.endsWith("[:][]")) { + fieldNameClean = fieldName.substring(0, fieldName.length() - 2); + } else if (fieldName.endsWith("[][:]") || fieldName.endsWith("[:][:]")) { + fieldNameClean = fieldName.substring(0, fieldName.length() - 3); } + return fieldNameClean; + } - public List processField(final Field innerField, final Boolean isRootElement, final Boolean isAncestorRequired) { - final List completeFieldList = new ArrayList<>(); - if (innerField instanceof ObjectField) { - processRecordFieldList(innerField.getName(), ".", - extractInternalFields((ObjectField) innerField, isAncestorRequired != null ? isAncestorRequired : ((ObjectField) innerField).isFieldRequired()), - completeFieldList, checkRequiredElement(isRootElement, isAncestorRequired, ((ObjectField) innerField).isFieldRequired())); - } else if (innerField instanceof ArrayField) { - completeFieldList.addAll(extractArrayInternalFields((ArrayField) innerField, isRootElement, - checkRequiredElement(isRootElement, isAncestorRequired, ((ArrayField) innerField).isFieldRequired()), "")); - } else if (innerField instanceof EnumField) { - completeFieldList.add(FieldValueMapping - .builder() - .fieldName(innerField.getName()) - .fieldType(innerField.getType()) - .valueLength(0) - .fieldValueList(String.join(",", ((EnumField) innerField).getEnumValues())) - .build()); - } else if (innerField instanceof MapField) { - completeFieldList.addAll( - extractMapInternalFields((MapField) innerField, isRootElement, - checkRequiredElement(isRootElement, isAncestorRequired, ((MapField) innerField).isFieldRequired()), "")); - } else if (innerField instanceof NumberField) { - final FieldValueMapping.FieldValueMappingBuilder builder = FieldValueMapping - .builder() - .fieldName(innerField.getName()) - .fieldType(innerField.getType()); - - addConstraint(builder, ConstraintTypeEnum.EXCLUDED_MAXIMUM_VALUE, getSafeNumberAsString(((NumberField) innerField).getExclusiveMaximum())); - addConstraint(builder, ConstraintTypeEnum.EXCLUDED_MINIMUM_VALUE, getSafeNumberAsString(((NumberField) innerField).getExclusiveMinimum())); - addConstraint(builder, ConstraintTypeEnum.MAXIMUM_VALUE, getSafeNumberAsString(((NumberField) innerField).getMaximum())); - addConstraint(builder, ConstraintTypeEnum.MINIMUM_VALUE, getSafeNumberAsString(((NumberField) innerField).getMinimum())); - addConstraint(builder, ConstraintTypeEnum.MULTIPLE_OF, getSafeNumberAsString(((NumberField) innerField).getMultipleOf())); - - completeFieldList.add(builder.build()); - } else if (innerField instanceof StringField) { - final FieldValueMapping.FieldValueMappingBuilder builder = FieldValueMapping - .builder() - .fieldName(innerField.getName()) - .fieldType(innerField.getType()); - - addConstraint(builder, ConstraintTypeEnum.REGEX, ((StringField) innerField).getRegex()); - addConstraint(builder, ConstraintTypeEnum.MAXIMUM_VALUE, getSafeNumberAsString(((StringField) innerField).getMaxlength())); - addConstraint(builder, ConstraintTypeEnum.MINIMUM_VALUE, getSafeNumberAsString(((StringField) innerField).getMinLength())); - addConstraint(builder, ConstraintTypeEnum.FORMAT, ((StringField) innerField).getFormat()); - - completeFieldList.add(builder.build()); - } else { - completeFieldList.add(FieldValueMapping.builder().fieldName(innerField.getName()).fieldType(innerField.getType()).build()); - } - return completeFieldList; + private List extractInternalFields(final ObjectField field, final Boolean isAncestorRequired) { + final List completeFieldList = new ArrayList<>(); + for (Field innerField : field.getProperties()) { + completeFieldList.addAll(processField(innerField, false, isAncestorRequired)); } - - private void addConstraint(final FieldValueMapping.FieldValueMappingBuilder builder, final ConstraintTypeEnum constraint, final String constrainValue) { - if (StringUtils.isNotBlank(constrainValue)) { - builder.constraint(constraint, constrainValue); - } - } - - private String getSafeNumberAsString(final Number exclusiveMaximum) { - String result = null; - if (Objects.nonNull(exclusiveMaximum)) { - result = exclusiveMaximum.toString(); - } - return result; + return completeFieldList; + } + + private Transformer fixName(final String fieldName, final String splitter) { + return fieldValue -> { + fieldValue.setFieldName(extractFieldName(fieldName) + splitter + fieldValue.getFieldName()); + return fieldValue; + }; + } + + public final List processField(final Field innerField, final Boolean isRootElement, final Boolean isAncestorRequired) { + final List completeFieldList = new ArrayList<>(); + if (innerField instanceof ObjectField) { + processRecordFieldList(innerField.getName(), ".", + extractInternalFields((ObjectField) innerField, isAncestorRequired != null ? isAncestorRequired : ((ObjectField) innerField).isFieldRequired()), + completeFieldList, checkRequiredElement(isRootElement, isAncestorRequired, ((ObjectField) innerField).isFieldRequired())); + } else if (innerField instanceof ArrayField) { + completeFieldList.addAll(extractArrayInternalFields((ArrayField) innerField, isRootElement, + checkRequiredElement(isRootElement, isAncestorRequired, ((ArrayField) innerField).isFieldRequired()), "")); + } else if (innerField instanceof EnumField) { + completeFieldList.add(FieldValueMapping + .builder() + .fieldName(innerField.getName()) + .fieldType(innerField.getType()) + .valueLength(0) + .fieldValueList(String.join(",", ((EnumField) innerField).getEnumValues())) + .build()); + } else if (innerField instanceof MapField) { + completeFieldList.addAll( + extractMapInternalFields((MapField) innerField, isRootElement, + checkRequiredElement(isRootElement, isAncestorRequired, ((MapField) innerField).isFieldRequired()), "")); + } else if (innerField instanceof NumberField) { + final FieldValueMapping.FieldValueMappingBuilder builder = FieldValueMapping + .builder() + .fieldName(innerField.getName()) + .fieldType(innerField.getType()); + + addConstraint(builder, ConstraintTypeEnum.EXCLUDED_MAXIMUM_VALUE, getSafeNumberAsString(((NumberField) innerField).getExclusiveMaximum())); + addConstraint(builder, ConstraintTypeEnum.EXCLUDED_MINIMUM_VALUE, getSafeNumberAsString(((NumberField) innerField).getExclusiveMinimum())); + addConstraint(builder, ConstraintTypeEnum.MAXIMUM_VALUE, getSafeNumberAsString(((NumberField) innerField).getMaximum())); + addConstraint(builder, ConstraintTypeEnum.MINIMUM_VALUE, getSafeNumberAsString(((NumberField) innerField).getMinimum())); + addConstraint(builder, ConstraintTypeEnum.MULTIPLE_OF, getSafeNumberAsString(((NumberField) innerField).getMultipleOf())); + + completeFieldList.add(builder.build()); + } else if (innerField instanceof StringField) { + final FieldValueMapping.FieldValueMappingBuilder builder = FieldValueMapping + .builder() + .fieldName(innerField.getName()) + .fieldType(innerField.getType()); + + addConstraint(builder, ConstraintTypeEnum.REGEX, ((StringField) innerField).getRegex()); + addConstraint(builder, ConstraintTypeEnum.MAXIMUM_VALUE, getSafeNumberAsString(((StringField) innerField).getMaxlength())); + addConstraint(builder, ConstraintTypeEnum.MINIMUM_VALUE, getSafeNumberAsString(((StringField) innerField).getMinLength())); + addConstraint(builder, ConstraintTypeEnum.FORMAT, ((StringField) innerField).getFormat()); + + completeFieldList.add(builder.build()); + } else { + completeFieldList.add(FieldValueMapping.builder().fieldName(innerField.getName()).fieldType(innerField.getType()).build()); } + return completeFieldList; + } - private List extractObjectInternalFields(final ObjectField value, final String innerFieldName, final Boolean isAncestorRequired, final String breadCrumb, - final String endValue) { - final List completeFieldList = new ArrayList<>(); - final List requiredInternalFields = value.getRequired(); - for (Field propertiesField : value.getProperties()) { - final List processedField = processField(propertiesField, false, isAncestorRequired); - processedField.get(0).setAncestorRequired(isAncestorRequired != null && isAncestorRequired); - processedField.get(0).setRequired(checkRequiredByType(propertiesField, requiredInternalFields, processedField.get(0))); - CollectionUtils.collect( - processedField, - fixName(StringUtils.isNotEmpty(breadCrumb) ? breadCrumb + endValue : innerFieldName, endValue + "."), - completeFieldList); - } - - return completeFieldList; + private void addConstraint(final FieldValueMapping.FieldValueMappingBuilder builder, final ConstraintTypeEnum constraint, final String constrainValue) { + if (StringUtils.isNotBlank(constrainValue)) { + builder.constraint(constraint, constrainValue); } + } - private List extractArrayInternalFields( - final ArrayField innerField, final Boolean isRootElement, - final Boolean isAncestorRequired, final String breadCrumb) { - - final List completeFieldList = new ArrayList<>(); - - for (Field value : innerField.getValues()) { - if (value instanceof ObjectField) { - completeFieldList.addAll(extractObjectInternalFields((ObjectField) value, innerField.getName(), isAncestorRequired, breadCrumb, "[]")); - } else if (value instanceof ArrayField) { - completeFieldList.addAll(extractArrayInternalFields( - (ArrayField) value, false, isAncestorRequired, generateBreadCrumb(breadCrumb, innerField.getName(), "[]"))); - } else if (value instanceof MapField) { - completeFieldList.addAll(extractMapInternalFields( - (MapField) value, false, isAncestorRequired, generateBreadCrumb(breadCrumb, innerField.getName(), "[]"))); - } else { - final String name = (StringUtils.isNotEmpty(breadCrumb) ? breadCrumb : innerField.getName() + "[]") + (StringUtils.isNotEmpty(breadCrumb) ? "[]" : breadCrumb); - completeFieldList.add( - FieldValueMapping.builder() - .fieldName(name) - .fieldType(value.getType() + SchemaExtractorUtil.ARRAY_TYPE_POSTFIX + ((StringUtils.isNotEmpty(breadCrumb) && breadCrumb.endsWith("[]")) ? - SchemaExtractorUtil.ARRAY_TYPE_POSTFIX : - (StringUtils.isNotEmpty(breadCrumb) && breadCrumb.endsWith("[:]")) ? "-map" : breadCrumb)) - .required(!name.endsWith("][]") && !name.endsWith("][:]") && innerField.isFieldRequired()) - .isAncestorRequired(!isRootElement && isAncestorRequired != null && isAncestorRequired) - .build()); - } - } - return completeFieldList; + private String getSafeNumberAsString(final Number exclusiveMaximum) { + String result = null; + if (Objects.nonNull(exclusiveMaximum)) { + result = exclusiveMaximum.toString(); } - - private List extractMapInternalFields( - final MapField innerField, final Boolean isRootElement, - final Boolean isAncestorRequired, final String breadCrumb) { - final List completeFieldList = new ArrayList<>(); - final Field value = innerField.getMapType(); - - if (value instanceof ObjectField) { - completeFieldList.addAll(extractObjectInternalFields((ObjectField) value, innerField.getName(), isAncestorRequired, breadCrumb, "[:]")); - } else if (value instanceof ArrayField) { - completeFieldList.addAll(extractArrayInternalFields( - (ArrayField) value, false, isAncestorRequired, generateBreadCrumb(breadCrumb, innerField.getName(), "[:]"))); - } else if (value instanceof MapField) { - completeFieldList.addAll(extractMapInternalFields( - (MapField) value, false, isAncestorRequired, generateBreadCrumb(breadCrumb, innerField.getName(), "[:]"))); - } else { - final String name = (StringUtils.isNotEmpty(breadCrumb) ? breadCrumb : innerField.getName() + "[:]") + (StringUtils.isNotEmpty(breadCrumb) ? "[:]" : breadCrumb); - completeFieldList.add( - FieldValueMapping.builder() - .fieldName(name) - .fieldType(value.getType() + "-map" + ((StringUtils.isNotEmpty(breadCrumb) && breadCrumb.endsWith("[:]")) ? "-map" : - (StringUtils.isNotEmpty(breadCrumb) && breadCrumb.endsWith("[]")) ? SchemaExtractorUtil.ARRAY_TYPE_POSTFIX : breadCrumb)) - .required(!name.endsWith("][]") && !name.endsWith("][:]") && innerField.isFieldRequired()) - .isAncestorRequired(!isRootElement && isAncestorRequired != null && isAncestorRequired) - .build()); - } - return completeFieldList; + return result; + } + + private List extractObjectInternalFields( + final ObjectField value, final String innerFieldName, final Boolean isAncestorRequired, final String breadCrumb, + final String endValue) { + final List completeFieldList = new ArrayList<>(); + final List requiredInternalFields = value.getRequired(); + for (Field propertiesField : value.getProperties()) { + final List processedField = processField(propertiesField, false, isAncestorRequired); + processedField.get(0).setAncestorRequired(isAncestorRequired != null && isAncestorRequired); + processedField.get(0).setRequired(checkRequiredByType(propertiesField, requiredInternalFields, processedField.get(0))); + CollectionUtils.collect( + processedField, + fixName(StringUtils.isNotEmpty(breadCrumb) ? breadCrumb + endValue : innerFieldName, endValue + "."), + completeFieldList); } - private void processRecordFieldList( - final String fieldName, final String splitter, final List internalFields, - final List completeFieldList, final boolean isAncestorRequired) { - internalFields.forEach(internalField -> { - if (Objects.nonNull(internalField.getFieldName())) { - internalField.setFieldName(fieldName + splitter + internalField.getFieldName()); - } else { - internalField.setFieldName(fieldName); - } - final String[] splittedName = internalField.getFieldName().split("\\."); - String parentName = splittedName[splittedName.length - 2]; - parentName = parentName.replace("[]", ""); - if (fieldName.equals(parentName)) { - internalField.setAncestorRequired(isAncestorRequired); - } - completeFieldList.add(internalField); - }); + return completeFieldList; + } + + private List extractArrayInternalFields( + final ArrayField innerField, final Boolean isRootElement, + final Boolean isAncestorRequired, final String breadCrumb) { + + final List completeFieldList = new ArrayList<>(); + + for (Field value : innerField.getValues()) { + if (value instanceof ObjectField) { + completeFieldList.addAll(extractObjectInternalFields((ObjectField) value, innerField.getName(), isAncestorRequired, breadCrumb, "[]")); + } else if (value instanceof ArrayField) { + completeFieldList.addAll(extractArrayInternalFields( + (ArrayField) value, false, isAncestorRequired, generateBreadCrumb(breadCrumb, innerField.getName(), "[]"))); + } else if (value instanceof MapField) { + completeFieldList.addAll(extractMapInternalFields( + (MapField) value, false, isAncestorRequired, generateBreadCrumb(breadCrumb, innerField.getName(), "[]"))); + } else { + final String name = (StringUtils.isNotEmpty(breadCrumb) ? breadCrumb : innerField.getName() + "[]") + (StringUtils.isNotEmpty(breadCrumb) ? "[]" : breadCrumb); + completeFieldList.add( + FieldValueMapping.builder() + .fieldName(name) + .fieldType(value.getType() + + SchemaExtractorUtil.ARRAY_TYPE_POSTFIX + + ((StringUtils.isNotEmpty(breadCrumb) + && breadCrumb.endsWith("[]")) + ? SchemaExtractorUtil.ARRAY_TYPE_POSTFIX : (StringUtils.isNotEmpty(breadCrumb) && breadCrumb.endsWith("[:]")) ? "-map" : breadCrumb)) + .required(!name.endsWith("][]") && !name.endsWith("][:]") && innerField.isFieldRequired()) + .isAncestorRequired(!isRootElement && isAncestorRequired != null && isAncestorRequired) + .build()); + } } - - private Boolean checkRequiredByType( - final Field innerField, final List requiredInternalFields, - final FieldValueMapping fieldValueMapping) { - boolean result = fieldValueMapping.getRequired(); - if (!(innerField instanceof ArrayField) && !(innerField instanceof MapField)) { - result = requiredInternalFields.contains(fieldValueMapping.getFieldName()); - } - return result; + return completeFieldList; + } + + private List extractMapInternalFields( + final MapField innerField, final Boolean isRootElement, + final Boolean isAncestorRequired, final String breadCrumb) { + final List completeFieldList = new ArrayList<>(); + final Field value = innerField.getMapType(); + + if (value instanceof ObjectField) { + completeFieldList.addAll(extractObjectInternalFields((ObjectField) value, innerField.getName(), isAncestorRequired, breadCrumb, "[:]")); + } else if (value instanceof ArrayField) { + completeFieldList.addAll(extractArrayInternalFields( + (ArrayField) value, false, isAncestorRequired, generateBreadCrumb(breadCrumb, innerField.getName(), "[:]"))); + } else if (value instanceof MapField) { + completeFieldList.addAll(extractMapInternalFields( + (MapField) value, false, isAncestorRequired, generateBreadCrumb(breadCrumb, innerField.getName(), "[:]"))); + } else { + final String name = (StringUtils.isNotEmpty(breadCrumb) ? breadCrumb : innerField.getName() + "[:]") + (StringUtils.isNotEmpty(breadCrumb) ? "[:]" : breadCrumb); + completeFieldList.add( + FieldValueMapping.builder() + .fieldName(name) + .fieldType(value.getType() + "-map" + ((StringUtils.isNotEmpty(breadCrumb) && breadCrumb.endsWith("[:]")) ? "-map" : + (StringUtils.isNotEmpty( + breadCrumb) && + breadCrumb.endsWith("[]")) + ? + SchemaExtractorUtil.ARRAY_TYPE_POSTFIX + : breadCrumb)) + .required(!name.endsWith("][]") && !name.endsWith("][:]") && innerField.isFieldRequired()) + .isAncestorRequired(!isRootElement && isAncestorRequired != null && isAncestorRequired) + .build()); } - - private Boolean checkRequiredElement(final Boolean isRootElement, final Boolean isAncestorRequired, final Boolean isFieldRequired) { - final boolean isRequired = isAncestorRequired != null && isAncestorRequired; - return isRootElement != null && isRootElement ? isFieldRequired : isRequired; + return completeFieldList; + } + + private void processRecordFieldList(final String fieldName, final String splitter, final List internalFields, + final List completeFieldList, final boolean isAncestorRequired) { + internalFields.forEach(internalField -> { + if (Objects.nonNull(internalField.getFieldName())) { + internalField.setFieldName(fieldName + splitter + internalField.getFieldName()); + } else { + internalField.setFieldName(fieldName); + } + final String[] splittedName = internalField.getFieldName().split("\\."); + String parentName = splittedName[splittedName.length - 2]; + parentName = parentName.replace("[]", ""); + if (fieldName.equals(parentName)) { + internalField.setAncestorRequired(isAncestorRequired); + } + completeFieldList.add(internalField); + }); + } + + private Boolean checkRequiredByType( + final Field innerField, final List requiredInternalFields, + final FieldValueMapping fieldValueMapping) { + boolean result = fieldValueMapping.getRequired(); + if (!(innerField instanceof ArrayField) && !(innerField instanceof MapField)) { + result = requiredInternalFields.contains(fieldValueMapping.getFieldName()); } + return result; + } - private String generateBreadCrumb(final String breadCrumb, final String fieldName, final String endValue) { + private Boolean checkRequiredElement(final Boolean isRootElement, final Boolean isAncestorRequired, final Boolean isFieldRequired) { + final boolean isRequired = isAncestorRequired != null && isAncestorRequired; + return isRootElement != null && isRootElement ? isFieldRequired : isRequired; + } - if (breadCrumb.endsWith("[]") || breadCrumb.endsWith("[:]")) { - throw new KLoadGenException("Wrong Json Schema, 3+ consecutive nested collections are not allowed"); - } + private String generateBreadCrumb(final String breadCrumb, final String fieldName, final String endValue) { - return fieldName + endValue; + if (breadCrumb.endsWith("[]") || breadCrumb.endsWith("[:]")) { + throw new KLoadGenException("Wrong Json Schema, 3+ consecutive nested collections are not allowed"); } + return fieldName + endValue; + } + } diff --git a/src/main/java/com/sngular/kloadgen/extractor/extractors/json/JsonDefaultExtractor.java b/src/main/java/com/sngular/kloadgen/extractor/extractors/json/JsonDefaultExtractor.java index 8f2a0f57..528e836c 100644 --- a/src/main/java/com/sngular/kloadgen/extractor/extractors/json/JsonDefaultExtractor.java +++ b/src/main/java/com/sngular/kloadgen/extractor/extractors/json/JsonDefaultExtractor.java @@ -5,6 +5,7 @@ import java.util.List; import java.util.Set; import java.util.stream.Collectors; + import com.sngular.kloadgen.extractor.extractors.Extractor; import com.sngular.kloadgen.model.FieldValueMapping; import com.sngular.kloadgen.model.json.Field; @@ -12,27 +13,27 @@ public class JsonDefaultExtractor extends AbstractJsonExtractor implements Extractor { - @Override - public List processSchema(org.everit.json.schema.Schema schema) { - - Schema parsed = jsonSchemaParser.parse(schema.toString()); + @Override + public final List processSchema(final org.everit.json.schema.Schema schema) { - final List attributeList = new ArrayList<>(); - parsed.getProperties().forEach(field -> attributeList.addAll(processField(field, true, null))); + final Schema parsed = getSchemaParser().parse(schema.toString()); - final Set requiredFields = new HashSet<>(parsed.getRequiredFields()); + final List attributeList = new ArrayList<>(); + parsed.getProperties().forEach(field -> attributeList.addAll(processField(field, true, null))); - for (FieldValueMapping field : attributeList) { - if (!field.getFieldName().contains("[]") && !field.getFieldName().contains("[:]")) { - field.setRequired(requiredFields.contains(field.getFieldName())); - } - } - return attributeList; - } + final Set requiredFields = new HashSet<>(parsed.getRequiredFields()); - @Override - public List getSchemaNameList(String schema) { - Schema parsed = jsonSchemaParser.parse(schema); - return parsed.getProperties().stream().map(Field::getName).collect(Collectors.toList()); + for (FieldValueMapping field : attributeList) { + if (!field.getFieldName().contains("[]") && !field.getFieldName().contains("[:]")) { + field.setRequired(requiredFields.contains(field.getFieldName())); + } } + return attributeList; + } + + @Override + public final List getSchemaNameList(final String schema) { + final Schema parsed = getSchemaParser().parse(schema); + return parsed.getProperties().stream().map(Field::getName).collect(Collectors.toList()); + } } diff --git a/src/main/java/com/sngular/kloadgen/extractor/extractors/json/JsonExtractor.java b/src/main/java/com/sngular/kloadgen/extractor/extractors/json/JsonExtractor.java index 24d47aba..1f754f2b 100644 --- a/src/main/java/com/sngular/kloadgen/extractor/extractors/json/JsonExtractor.java +++ b/src/main/java/com/sngular/kloadgen/extractor/extractors/json/JsonExtractor.java @@ -3,9 +3,6 @@ import java.util.List; import java.util.Map; -import static com.sngular.kloadgen.common.SchemaRegistryEnum.APICURIO; -import static com.sngular.kloadgen.common.SchemaRegistryEnum.CONFLUENT; - import com.sngular.kloadgen.common.SchemaRegistryEnum; import com.sngular.kloadgen.extractor.extractors.Extractor; import com.sngular.kloadgen.extractor.extractors.ExtractorRegistry; @@ -14,15 +11,15 @@ public class JsonExtractor implements ExtractorRegistry { - static Map schemaRegistryMap = Map.of(CONFLUENT, new JsonDefaultExtractor(), - APICURIO, new JsonDefaultExtractor()); + private static final Map SCHEMA_REGISTRY_MAP = Map.of(SchemaRegistryEnum.CONFLUENT, new JsonDefaultExtractor(), + SchemaRegistryEnum.APICURIO, new JsonDefaultExtractor()); - public final List processSchema(final Object schemaReceived, SchemaRegistryEnum registryEnum) { - return schemaRegistryMap.get(registryEnum).processSchema(schemaReceived); + public final List processSchema(final Object schemaReceived, final SchemaRegistryEnum registryEnum) { + return SCHEMA_REGISTRY_MAP.get(registryEnum).processSchema(schemaReceived); } - public final List getSchemaNameList(final String schema, SchemaRegistryEnum registryEnum) { - return schemaRegistryMap.get(registryEnum).getSchemaNameList(schema); + public final List getSchemaNameList(final String schema, final SchemaRegistryEnum registryEnum) { + return SCHEMA_REGISTRY_MAP.get(registryEnum).getSchemaNameList(schema); } } diff --git a/src/main/java/com/sngular/kloadgen/extractor/extractors/protobuff/AbstractProtoFileExtractor.java b/src/main/java/com/sngular/kloadgen/extractor/extractors/protobuff/AbstractProtoFileExtractor.java index f155e547..32282b7d 100644 --- a/src/main/java/com/sngular/kloadgen/extractor/extractors/protobuff/AbstractProtoFileExtractor.java +++ b/src/main/java/com/sngular/kloadgen/extractor/extractors/protobuff/AbstractProtoFileExtractor.java @@ -11,6 +11,7 @@ import java.util.Objects; import java.util.regex.Matcher; import java.util.regex.Pattern; + import com.github.os72.protobuf.dynamic.DynamicSchema; import com.google.protobuf.Descriptors; import com.sngular.kloadgen.exception.KLoadGenException; @@ -39,14 +40,14 @@ public abstract class AbstractProtoFileExtractor { protected AbstractProtoFileExtractor() { } - public List processSchemaDefault(ProtoFileElement schema){ + public final List processSchemaDefault(final ProtoFileElement schema) { final List attributeList = new ArrayList<>(); final HashMap nestedTypes = new HashMap<>(); schema.getTypes().forEach(field -> processField(field, attributeList, schema.getImports(), true, nestedTypes)); return attributeList; } - public List getSchemaNameListDefault(String schema){ + public final List getSchemaNameListDefault(final String schema) { final DynamicSchema dynamicSchema; try { dynamicSchema = DynamicSchema.parseFrom(schema.getBytes(StandardCharsets.UTF_8)); @@ -61,8 +62,7 @@ public static void processField( final List imports, final boolean isAncestorRequired, final Map nestedTypes) { fillNestedTypes(field, nestedTypes); - if (field instanceof MessageElement) { - final var messageField = (MessageElement) field; + if (field instanceof final MessageElement messageField) { if (!messageField.getOneOfs().isEmpty()) { extractOneOfs((MessageElement) field, completeFieldList, nestedTypes, isAncestorRequired); } @@ -79,9 +79,8 @@ public static void processField( } } - private static void processField( - final FieldElement field, - final List completeFieldList, final List imports, final boolean isAncestorRequired, final Map nestedTypes) { + private static void processField(final FieldElement field, final List completeFieldList, final List imports, final boolean isAncestorRequired, + final Map nestedTypes) { final Field.Label label = checkNullLabel(field); final boolean isArray = "repeated".equalsIgnoreCase(Objects.requireNonNull(label.toString())); final boolean isOptional = Objects.equals(Objects.requireNonNull(label.toString()), "optional"); @@ -108,8 +107,8 @@ private static List processFieldList(final TypeElement fieldL return completeFieldList; } - private static void extractOneOfs( - final MessageElement field, final List completeFieldList, final Map nestedTypes, final boolean isAncestorRequired) { + private static void extractOneOfs(final MessageElement field, final List completeFieldList, final Map nestedTypes, + final boolean isAncestorRequired) { final List oneOfs = new ArrayList<>(field.getOneOfs()); for (OneOfElement oneOfElement : oneOfs) { if (!oneOfElement.getFields().isEmpty()) { @@ -130,8 +129,7 @@ private static void extractOneOfs( } } - private static FieldValueMapping extractDotTypeWhenNotNestedType( - final FieldElement subfield, final boolean isArray, final String dotType, final boolean isRequired, + private static FieldValueMapping extractDotTypeWhenNotNestedType(final FieldElement subfield, final boolean isArray, final String dotType, final boolean isRequired, final boolean isAncestorRequired) { final FieldValueMapping completeFieldList; if (isArray) { @@ -143,9 +141,8 @@ private static FieldValueMapping extractDotTypeWhenNotNestedType( return completeFieldList; } - private static void extractMapType( - final List completeFieldList, final Map nestedTypes, final FieldElement subfield, final List imports, - final boolean isRequired, final boolean isAncestorRequired) { + private static void extractMapType(final List completeFieldList, final Map nestedTypes, final FieldElement subfield, + final List imports, final boolean isRequired, final boolean isAncestorRequired) { final String subFieldType = extractInternalMapFields(subfield); final String dotTypeMap = checkDotType(subFieldType, imports); if (ProtobufHelper.isValidType(subFieldType)) { @@ -169,8 +166,8 @@ private static String extractInternalMapFields(final FieldElement subfield) { return mapSplit[1].replace(">", "").trim(); } - private static void extractPrimitiveTypes( - final List completeFieldList, final FieldElement subfield, final boolean isArray, final boolean isRequired, final boolean isAncestorRequired) { + private static void extractPrimitiveTypes(final List completeFieldList, final FieldElement subfield, final boolean isArray, final boolean isRequired, + final boolean isAncestorRequired) { if (isArray) { completeFieldList.add(FieldValueMapping.builder().fieldName(subfield.getName() + "[]") .fieldType(subfield.getType().replace(subfield.getType(), ProtobufHelper.translateType(subfield.getType())) + ARRAY_POSTFIX) diff --git a/src/main/java/com/sngular/kloadgen/extractor/extractors/protobuff/ProtoBufApicurioExtractor.java b/src/main/java/com/sngular/kloadgen/extractor/extractors/protobuff/ProtoBufApicurioExtractor.java index 893acaeb..509fe09c 100644 --- a/src/main/java/com/sngular/kloadgen/extractor/extractors/protobuff/ProtoBufApicurioExtractor.java +++ b/src/main/java/com/sngular/kloadgen/extractor/extractors/protobuff/ProtoBufApicurioExtractor.java @@ -1,6 +1,7 @@ package com.sngular.kloadgen.extractor.extractors.protobuff; import java.util.List; + import com.sngular.kloadgen.extractor.extractors.Extractor; import com.sngular.kloadgen.model.FieldValueMapping; import com.squareup.wire.schema.internal.parser.ProtoFileElement; diff --git a/src/main/java/com/sngular/kloadgen/extractor/extractors/protobuff/ProtoBufConfluentExtractor.java b/src/main/java/com/sngular/kloadgen/extractor/extractors/protobuff/ProtoBufConfluentExtractor.java index 06a39183..3b24537a 100644 --- a/src/main/java/com/sngular/kloadgen/extractor/extractors/protobuff/ProtoBufConfluentExtractor.java +++ b/src/main/java/com/sngular/kloadgen/extractor/extractors/protobuff/ProtoBufConfluentExtractor.java @@ -1,6 +1,7 @@ package com.sngular.kloadgen.extractor.extractors.protobuff; import java.util.List; + import com.sngular.kloadgen.extractor.extractors.Extractor; import com.sngular.kloadgen.model.FieldValueMapping; import io.confluent.kafka.schemaregistry.protobuf.ProtobufSchema; @@ -8,7 +9,7 @@ public class ProtoBufConfluentExtractor extends AbstractProtoFileExtractor implements Extractor { public final List processSchema(final ProtobufSchema schemaReceived) { - return processSchemaDefault(schemaReceived.rawSchema()); + return processSchemaDefault(schemaReceived.rawSchema()); } public final List getSchemaNameList(final String schema) { diff --git a/src/main/java/com/sngular/kloadgen/extractor/extractors/protobuff/ProtobuffExtractor.java b/src/main/java/com/sngular/kloadgen/extractor/extractors/protobuff/ProtobuffExtractor.java index de451c8a..9989903a 100644 --- a/src/main/java/com/sngular/kloadgen/extractor/extractors/protobuff/ProtobuffExtractor.java +++ b/src/main/java/com/sngular/kloadgen/extractor/extractors/protobuff/ProtobuffExtractor.java @@ -1,11 +1,9 @@ package com.sngular.kloadgen.extractor.extractors.protobuff; +import java.util.ArrayList; import java.util.List; import java.util.Map; -import static com.sngular.kloadgen.common.SchemaRegistryEnum.APICURIO; -import static com.sngular.kloadgen.common.SchemaRegistryEnum.CONFLUENT; - import com.sngular.kloadgen.common.SchemaRegistryEnum; import com.sngular.kloadgen.extractor.extractors.Extractor; import com.sngular.kloadgen.extractor.extractors.ExtractorRegistry; @@ -14,17 +12,19 @@ public class ProtobuffExtractor implements ExtractorRegistry { - static Map schemaRegistryMap = Map.of(CONFLUENT, new ProtoBufConfluentExtractor(), - APICURIO, new ProtoBufApicurioExtractor()); + static Map schemaRegistryMap = Map.of(SchemaRegistryEnum.CONFLUENT, new ProtoBufConfluentExtractor(), + SchemaRegistryEnum.APICURIO, new ProtoBufApicurioExtractor()); - public final List processSchema(final Object schemaReceived, SchemaRegistryEnum registryEnum) { + public final List processSchema(final Object schemaReceived, final SchemaRegistryEnum registryEnum) { + final var resultSchema = new ArrayList(); if (schemaReceived instanceof ProtoFileElement) { - return schemaRegistryMap.get(APICURIO).processSchema(schemaReceived); + resultSchema.addAll(schemaRegistryMap.get(SchemaRegistryEnum.APICURIO).processSchema(schemaReceived)); } - return schemaRegistryMap.get(registryEnum).processSchema(schemaReceived); + resultSchema.addAll(schemaRegistryMap.get(registryEnum).processSchema(schemaReceived)); + return resultSchema; } - public final List getSchemaNameList(final String schema, SchemaRegistryEnum registryEnum) { + public final List getSchemaNameList(final String schema, final SchemaRegistryEnum registryEnum) { return schemaRegistryMap.get(registryEnum).getSchemaNameList(schema); } diff --git a/src/main/java/com/sngular/kloadgen/loadgen/impl/AvroSRLoadGenerator.java b/src/main/java/com/sngular/kloadgen/loadgen/impl/AvroSRLoadGenerator.java index 321431cf..3576c24b 100644 --- a/src/main/java/com/sngular/kloadgen/loadgen/impl/AvroSRLoadGenerator.java +++ b/src/main/java/com/sngular/kloadgen/loadgen/impl/AvroSRLoadGenerator.java @@ -9,6 +9,7 @@ import java.io.IOException; import java.util.List; import java.util.Map; + import com.sngular.kloadgen.common.SchemaTypeEnum; import com.sngular.kloadgen.exception.KLoadGenException; import com.sngular.kloadgen.loadgen.BaseLoadGenerator; diff --git a/src/main/java/com/sngular/kloadgen/loadgen/impl/JsonSRLoadGenerator.java b/src/main/java/com/sngular/kloadgen/loadgen/impl/JsonSRLoadGenerator.java index c8632afb..55faf553 100644 --- a/src/main/java/com/sngular/kloadgen/loadgen/impl/JsonSRLoadGenerator.java +++ b/src/main/java/com/sngular/kloadgen/loadgen/impl/JsonSRLoadGenerator.java @@ -10,6 +10,7 @@ import java.util.Collections; import java.util.List; import java.util.Map; + import com.sngular.kloadgen.common.SchemaTypeEnum; import com.sngular.kloadgen.exception.KLoadGenException; import com.sngular.kloadgen.loadgen.BaseLoadGenerator; diff --git a/src/main/java/com/sngular/kloadgen/loadgen/impl/ProtobufLoadGenerator.java b/src/main/java/com/sngular/kloadgen/loadgen/impl/ProtobufLoadGenerator.java index 2b63c099..9e1f747e 100644 --- a/src/main/java/com/sngular/kloadgen/loadgen/impl/ProtobufLoadGenerator.java +++ b/src/main/java/com/sngular/kloadgen/loadgen/impl/ProtobufLoadGenerator.java @@ -4,6 +4,7 @@ import java.util.Collections; import java.util.List; import java.util.Map; + import com.sngular.kloadgen.common.SchemaTypeEnum; import com.sngular.kloadgen.exception.KLoadGenException; import com.sngular.kloadgen.loadgen.BaseLoadGenerator; @@ -45,8 +46,8 @@ public final void setUpGenerator(final String schema, final List( - ConfluentSchemaMetadata.parse(new SchemaMetadata(1, 1, SchemaTypeEnum.PROTOBUF.name(), Collections.emptyList(), schema))) - , fieldExprMappings); + ConfluentSchemaMetadata.parse(new SchemaMetadata(1, 1, SchemaTypeEnum.PROTOBUF.name(), Collections.emptyList(), schema))), + fieldExprMappings); } @Override diff --git a/src/main/java/com/sngular/kloadgen/processor/objectcreatorfactory/impl/JsonObjectCreatorFactory.java b/src/main/java/com/sngular/kloadgen/processor/objectcreatorfactory/impl/JsonObjectCreatorFactory.java index d9b97298..909e2d40 100644 --- a/src/main/java/com/sngular/kloadgen/processor/objectcreatorfactory/impl/JsonObjectCreatorFactory.java +++ b/src/main/java/com/sngular/kloadgen/processor/objectcreatorfactory/impl/JsonObjectCreatorFactory.java @@ -123,7 +123,7 @@ public final boolean isOptionalFieldAccordingToSchema(final String completeField } @Override - public Object getRootNode(final String rootNode) { + public final Object getRootNode(final String rootNode) { return entity.get(rootNode); } diff --git a/src/main/java/com/sngular/kloadgen/property/editor/KeyDeserializerPropertyEditor.java b/src/main/java/com/sngular/kloadgen/property/editor/KeyDeserializerPropertyEditor.java index 4a3a7464..f8369ea6 100644 --- a/src/main/java/com/sngular/kloadgen/property/editor/KeyDeserializerPropertyEditor.java +++ b/src/main/java/com/sngular/kloadgen/property/editor/KeyDeserializerPropertyEditor.java @@ -59,7 +59,8 @@ private void init() { private void fillDeserializer(final JComboBox objectJComboBox) { deserializerComboBox = objectJComboBox; final Reflections reflections = new Reflections(new ConfigurationBuilder().addUrls(ClasspathHelper.forClass(Deserializer.class)).filterInputsBy( - new FilterBuilder().includePackage(SerDesPackageValue.COM_SNGULAR_KLOADGEN_SERIALIZER).includePackage(SerDesPackageValue.IO_CONFLUENT_KAFKA_SERIALIZERS)) + new FilterBuilder().includePackage(SerDesPackageValue.COM_SNGULAR_KLOADGEN_SERIALIZER) + .includePackage(SerDesPackageValue.IO_CONFLUENT_KAFKA_SERIALIZERS)) .setScanners(Scanners.SubTypes)); ReflectionUtils.extractDeserializers(deserializerComboBox, reflections, Deserializer.class); } diff --git a/src/main/java/com/sngular/kloadgen/property/editor/NameStrategyPropertyEditor.java b/src/main/java/com/sngular/kloadgen/property/editor/NameStrategyPropertyEditor.java index 96da679a..dbd337a0 100644 --- a/src/main/java/com/sngular/kloadgen/property/editor/NameStrategyPropertyEditor.java +++ b/src/main/java/com/sngular/kloadgen/property/editor/NameStrategyPropertyEditor.java @@ -14,11 +14,10 @@ import java.beans.PropertyEditorSupport; import java.util.Objects; -import javax.swing.JComboBox; -import javax.swing.JPanel; - import io.apicurio.registry.resolver.strategy.ArtifactReferenceResolverStrategy; import io.confluent.kafka.serializers.subject.strategy.SubjectNameStrategy; +import javax.swing.JComboBox; +import javax.swing.JPanel; import lombok.extern.slf4j.Slf4j; import org.apache.jmeter.gui.ClearGui; import org.apache.jmeter.testbeans.gui.TestBeanPropertyEditor; @@ -62,7 +61,7 @@ private void fillSerializer(final JComboBox objectJComboBox) { getComboItemValues(ArtifactReferenceResolverStrategy.class); } - private void getComboItemValues(Class targetClass) { + private void getComboItemValues(final Class targetClass) { final Reflections reflections = new Reflections( new ConfigurationBuilder() .addUrls(ClasspathHelper.forClass(targetClass)) diff --git a/src/main/java/com/sngular/kloadgen/randomtool/generator/ProtoBufGeneratorTool.java b/src/main/java/com/sngular/kloadgen/randomtool/generator/ProtoBufGeneratorTool.java index b7b94f31..f3fe5164 100644 --- a/src/main/java/com/sngular/kloadgen/randomtool/generator/ProtoBufGeneratorTool.java +++ b/src/main/java/com/sngular/kloadgen/randomtool/generator/ProtoBufGeneratorTool.java @@ -29,13 +29,13 @@ public class ProtoBufGeneratorTool { private final Map context = new HashMap<>(); - public final List generateArray(final String fieldName, final String fieldType, final int arraySize, final Integer valueLength, final List fieldValuesList) { + public final List generateArray(final String fieldName, final String fieldType, final int arraySize, final Integer valueLength, final List fieldValuesList) { final List parameterList = new ArrayList<>(fieldValuesList); parameterList.replaceAll( fieldValue -> fieldValue.matches("\\$\\{\\w*}") ? JMeterContextService.getContext().getVariables().get(fieldValue.substring(2, fieldValue.length() - 1)) : fieldValue); - final List value = new ArrayList<>(arraySize); + final List value = new ArrayList<>(arraySize); if ("seq".equals(fieldType)) { if (!fieldValuesList.isEmpty() && (fieldValuesList.size() > 1 || RandomSequence.isTypeNotSupported(fieldType))) { value.add(RandomSequence.generateSeq(fieldName, fieldType, parameterList, context)); @@ -45,7 +45,7 @@ public final List generateArray(final String fieldName, final String fieldType, } } } else { - value.addAll((ArrayList) RANDOM_ARRAY.generateArray(fieldType, valueLength, parameterList, arraySize, Collections.emptyMap())); + value.addAll((List) RANDOM_ARRAY.generateArray(fieldType, valueLength, parameterList, arraySize, Collections.emptyMap())); } return value; @@ -62,6 +62,20 @@ public final Object generateObject(final Descriptors.EnumDescriptor descriptor, return value; } + public final Object generateObject( + final FieldDescriptor descriptor, final String fieldType, final Integer valueLength, final List fieldValuesList, final Map constraints) { + Object result = null; + if (Objects.nonNull(descriptor.getJavaType())) { + result = generateRawObject(fieldType, valueLength, fieldValuesList, constraints); + } + return result; + } + + public final Object generateRawObject( + final String fieldType, final Integer valueLength, final List fieldValuesList, final Map constraints) { + return RANDOM_OBJECT.generateRandom(fieldType, valueLength, fieldValuesList, constraints); + } + private Object getEnumOrGenerate(final Descriptors.EnumDescriptor descriptor, final String fieldType, final List parameterList) { Object value = null; @@ -79,20 +93,6 @@ private Object getEnumOrGenerate(final Descriptors.EnumDescriptor descriptor, fi return value; } - public final Object generateObject( - final FieldDescriptor descriptor, final String fieldType, final Integer valueLength, final List fieldValuesList, final Map constraints) { - Object result = null; - if (Objects.nonNull(descriptor.getJavaType())) { - result = generateRawObject(fieldType, valueLength, fieldValuesList, constraints); - } - return result; - } - - public final Object generateRawObject( - final String fieldType, final Integer valueLength, final List fieldValuesList, final Map constraints) { - return RANDOM_OBJECT.generateRandom(fieldType, valueLength, fieldValuesList, constraints); - } - private Object getArrayEnumOrGenerate(final Descriptors.EnumDescriptor descriptor, final String fieldType, final int arraySize, final List parameterList) { final List value = new ArrayList<>(arraySize); for (int i = 0; i < arraySize; i++) { diff --git a/src/main/java/com/sngular/kloadgen/sampler/KafkaConsumerSampler.java b/src/main/java/com/sngular/kloadgen/sampler/KafkaConsumerSampler.java index 693883ba..7e2cbe7a 100644 --- a/src/main/java/com/sngular/kloadgen/sampler/KafkaConsumerSampler.java +++ b/src/main/java/com/sngular/kloadgen/sampler/KafkaConsumerSampler.java @@ -14,6 +14,7 @@ import java.util.Collections; import java.util.Objects; import java.util.Properties; + import com.sngular.kloadgen.util.ProducerKeysHelper; import lombok.AllArgsConstructor; import lombok.NoArgsConstructor; diff --git a/src/main/java/com/sngular/kloadgen/sampler/KafkaProducerSampler.java b/src/main/java/com/sngular/kloadgen/sampler/KafkaProducerSampler.java index 036694cc..16e78dfa 100644 --- a/src/main/java/com/sngular/kloadgen/sampler/KafkaProducerSampler.java +++ b/src/main/java/com/sngular/kloadgen/sampler/KafkaProducerSampler.java @@ -15,6 +15,7 @@ import java.util.Properties; import java.util.Set; import java.util.concurrent.ExecutionException; + import com.sngular.kloadgen.exception.KLoadGenException; import com.sngular.kloadgen.loadgen.BaseLoadGenerator; import com.sngular.kloadgen.model.HeaderMapping; diff --git a/src/main/java/com/sngular/kloadgen/schemaregistry/adapter/impl/ApicurioSchemaMetadata.java b/src/main/java/com/sngular/kloadgen/schemaregistry/adapter/impl/ApicurioSchemaMetadata.java index 1e0e22ba..3107cf79 100644 --- a/src/main/java/com/sngular/kloadgen/schemaregistry/adapter/impl/ApicurioSchemaMetadata.java +++ b/src/main/java/com/sngular/kloadgen/schemaregistry/adapter/impl/ApicurioSchemaMetadata.java @@ -4,6 +4,7 @@ import java.util.Date; import java.util.List; import java.util.Map; + import io.apicurio.registry.rest.v2.beans.ArtifactMetaData; import io.apicurio.registry.types.ArtifactState; import lombok.Getter; @@ -60,13 +61,13 @@ public ApicurioSchemaMetadata(final ArtifactMetaData artifactMetaData) { } @Override - public String getType() { + public final String getType() { return this.schemaType; } @Override - public List getReferences() { - return Collections.emptyList(); + public final List getReferences() { + return Collections.emptyList(); } } diff --git a/src/main/java/com/sngular/kloadgen/schemaregistry/adapter/impl/BaseSchemaMetadata.java b/src/main/java/com/sngular/kloadgen/schemaregistry/adapter/impl/BaseSchemaMetadata.java index 04e84d1d..9eee1d8a 100644 --- a/src/main/java/com/sngular/kloadgen/schemaregistry/adapter/impl/BaseSchemaMetadata.java +++ b/src/main/java/com/sngular/kloadgen/schemaregistry/adapter/impl/BaseSchemaMetadata.java @@ -3,7 +3,7 @@ import lombok.Builder; @Builder -public class BaseSchemaMetadata { +public final class BaseSchemaMetadata { private final T schemaMetadataAdapter; diff --git a/src/main/java/com/sngular/kloadgen/schemaregistry/adapter/impl/ConfluentSchemaMetadata.java b/src/main/java/com/sngular/kloadgen/schemaregistry/adapter/impl/ConfluentSchemaMetadata.java index c7a3aaa0..db16a217 100644 --- a/src/main/java/com/sngular/kloadgen/schemaregistry/adapter/impl/ConfluentSchemaMetadata.java +++ b/src/main/java/com/sngular/kloadgen/schemaregistry/adapter/impl/ConfluentSchemaMetadata.java @@ -1,6 +1,7 @@ package com.sngular.kloadgen.schemaregistry.adapter.impl; import java.util.List; + import io.confluent.kafka.schemaregistry.client.SchemaMetadata; import io.confluent.kafka.schemaregistry.client.rest.entities.SchemaReference; import lombok.Getter; @@ -8,17 +9,17 @@ @Getter public class ConfluentSchemaMetadata implements SchemaMetadataAdapter { - private Integer id; + private final Integer id; - private Integer version; + private final Integer version; - private String schemaType; + private final String schemaType; - private String schema; + private final String schema; - private List references; + private final List references; - private ConfluentSchemaMetadata(SchemaMetadata schemaMetadata) { + private ConfluentSchemaMetadata(final SchemaMetadata schemaMetadata) { this.id = schemaMetadata.getId(); this.version = schemaMetadata.getVersion(); this.schemaType = schemaMetadata.getSchemaType(); @@ -26,17 +27,17 @@ private ConfluentSchemaMetadata(SchemaMetadata schemaMetadata) { this.references = schemaMetadata.getReferences(); } - public static ConfluentSchemaMetadata parse(SchemaMetadata schemaMetadata) { + public static ConfluentSchemaMetadata parse(final SchemaMetadata schemaMetadata) { return new ConfluentSchemaMetadata(schemaMetadata); } @Override - public Integer getGlobalId() { + public final Integer getGlobalId() { return this.getId(); } @Override - public String getType() { + public final String getType() { return this.schemaType; } } diff --git a/src/main/java/com/sngular/kloadgen/schemaregistry/schema/KLoadSchemaMetadata.java b/src/main/java/com/sngular/kloadgen/schemaregistry/schema/KLoadSchemaMetadata.java index 37e1ed35..fe1342e0 100644 --- a/src/main/java/com/sngular/kloadgen/schemaregistry/schema/KLoadSchemaMetadata.java +++ b/src/main/java/com/sngular/kloadgen/schemaregistry/schema/KLoadSchemaMetadata.java @@ -1,14 +1,4 @@ package com.sngular.kloadgen.schemaregistry.schema; -import lombok.Value; - -@Value -public class KLoadSchemaMetadata { - - String id; - - String version; - - String schemaType; - +public record KLoadSchemaMetadata(String id, String version, String schemaType) { } diff --git a/src/main/java/com/sngular/kloadgen/serializer/AvroDeserializer.java b/src/main/java/com/sngular/kloadgen/serializer/AvroDeserializer.java index 97717b61..1bc7c78b 100644 --- a/src/main/java/com/sngular/kloadgen/serializer/AvroDeserializer.java +++ b/src/main/java/com/sngular/kloadgen/serializer/AvroDeserializer.java @@ -10,6 +10,7 @@ import java.nio.ByteBuffer; import java.util.HashMap; import java.util.Map; + import com.sngular.kloadgen.util.PropsKeysHelper; import lombok.extern.slf4j.Slf4j; import org.apache.avro.Schema; @@ -53,7 +54,7 @@ public final Object deserialize(final String topic, final byte[] data) { final var buffer = getByteBuffer(data); final var reader = new GenericDatumReader<>(avroSchema); - final int length = buffer.limit() - 1-4; + final int length = buffer.limit() - 5; final int start = buffer.position() + buffer.arrayOffset(); try { diff --git a/src/test/java/com/sngular/kloadgen/extractor/extractors/avro/AvroApicurioExtractorTest.java b/src/test/java/com/sngular/kloadgen/extractor/extractors/avro/AvroApicurioExtractorTest.java index f9ba863b..387197f7 100644 --- a/src/test/java/com/sngular/kloadgen/extractor/extractors/avro/AvroApicurioExtractorTest.java +++ b/src/test/java/com/sngular/kloadgen/extractor/extractors/avro/AvroApicurioExtractorTest.java @@ -1,6 +1,7 @@ package com.sngular.kloadgen.extractor.extractors.avro; import java.util.List; + import com.sngular.kloadgen.extractor.extractors.Extractor; import com.sngular.kloadgen.model.FieldValueMapping; import com.sngular.kloadgen.testutil.FileHelper; @@ -66,7 +67,6 @@ void testOptionalEnum() throws Exception { void testFlatPropertiesOptionalMapArray() throws Exception { final String testFile = fileHelper.getContent("/avro-files/testOptionalMap.avsc"); - final List parsedSchema = avroApicurioExtractor.getSchemaNameList(testFile); final Schema schema = new Schema.Parser().parse(testFile); final List fieldValueMappingList = avroApicurioExtractor.processSchema(schema); @@ -96,7 +96,6 @@ void testFlatPropertiesOptionalMapArray() throws Exception { @DisplayName("Should extract Map of Record") void testFlatPropertiesMap() throws Exception { final String testFile = fileHelper.getContent("/avro-files/testMap.avsc"); - final List parsedSchema = avroApicurioExtractor.getSchemaNameList(testFile); final Schema schema = new Schema.Parser().parse(testFile); final List fieldValueMappingList = avroApicurioExtractor.processSchema(schema); Assertions.assertThat(fieldValueMappingList) @@ -128,7 +127,6 @@ void testFlatPropertiesMap() throws Exception { void testFlatPropertiesLogicalTypes() throws Exception { final String testFile = fileHelper.getContent("/avro-files/testLogicalTypes.avsc"); - final List parsedSchema = avroApicurioExtractor.getSchemaNameList(testFile); final Schema schema = new Schema.Parser().parse(testFile); final List fieldValueMappingList = avroApicurioExtractor.processSchema(schema); Assertions.assertThat(fieldValueMappingList) @@ -158,7 +156,6 @@ void testFlatPropertiesLogicalTypes() throws Exception { void testFlatPropertiesOptionalArray() throws Exception { final String testFile = fileHelper.getContent("/avro-files/issue.avsc"); - final List parsedSchema = avroApicurioExtractor.getSchemaNameList(testFile); final Schema schema = new Schema.Parser().parse(testFile); final List fieldValueMappingList = avroApicurioExtractor.processSchema(schema); Assertions.assertThat(fieldValueMappingList) @@ -178,7 +175,6 @@ void testFlatPropertiesOptionalArray() throws Exception { void testFlatPropertiesUnionRecordAvros() throws Exception { final String testFile = fileHelper.getContent("/avro-files/testUnionRecord.avsc"); - final List parsedSchema = avroApicurioExtractor.getSchemaNameList(testFile); final Schema schema = new Schema.Parser().parse(testFile); final List fieldValueMappingList = avroApicurioExtractor.processSchema(schema); @@ -209,7 +205,6 @@ void testFlatPropertiesUnionRecordAvros() throws Exception { void testFlatPropertiesRecordUnionReverseOrder() throws Exception { final String testFile = fileHelper.getContent("/avro-files/testUnionReverseOrder.avsc"); - final List parsedSchema = avroApicurioExtractor.getSchemaNameList(testFile); final Schema schema = new Schema.Parser().parse(testFile); final List fieldValueMappingList = avroApicurioExtractor.processSchema(schema); diff --git a/src/test/java/com/sngular/kloadgen/extractor/extractors/avro/AvroConfluentExtractorTest.java b/src/test/java/com/sngular/kloadgen/extractor/extractors/avro/AvroConfluentExtractorTest.java index 31330329..16c6d67c 100644 --- a/src/test/java/com/sngular/kloadgen/extractor/extractors/avro/AvroConfluentExtractorTest.java +++ b/src/test/java/com/sngular/kloadgen/extractor/extractors/avro/AvroConfluentExtractorTest.java @@ -1,6 +1,7 @@ package com.sngular.kloadgen.extractor.extractors.avro; import java.util.List; + import com.sngular.kloadgen.extractor.extractors.Extractor; import com.sngular.kloadgen.model.FieldValueMapping; import com.sngular.kloadgen.testutil.FileHelper; diff --git a/src/test/java/com/sngular/kloadgen/extractor/extractors/json/JsonDefaultTest.java b/src/test/java/com/sngular/kloadgen/extractor/extractors/json/JsonDefaultTest.java index f417e56e..db6c0818 100644 --- a/src/test/java/com/sngular/kloadgen/extractor/extractors/json/JsonDefaultTest.java +++ b/src/test/java/com/sngular/kloadgen/extractor/extractors/json/JsonDefaultTest.java @@ -3,9 +3,9 @@ import java.util.List; import java.util.Map; import java.util.Set; + import com.sngular.kloadgen.exception.KLoadGenException; import com.sngular.kloadgen.extractor.extractors.Extractor; -import com.sngular.kloadgen.extractor.parser.impl.JSONSchemaParser; import com.sngular.kloadgen.model.ConstraintTypeEnum; import com.sngular.kloadgen.model.FieldValueMapping; import com.sngular.kloadgen.testutil.FileHelper; @@ -22,8 +22,6 @@ class JsonDefaultTest { private final Extractor jsonDefaultExtractor = new JsonDefaultExtractor(); - public final JSONSchemaParser jsonSchemaParser = new JSONSchemaParser(); - @Test @DisplayName("Should extract basic types") void testBasic() throws Exception { @@ -35,12 +33,12 @@ void testBasic() throws Exception { final List fieldValueMappingList = jsonDefaultExtractor.processSchema(schema); Assertions.assertThat(fieldValueMappingList) - .hasSize(3) - .containsExactlyInAnyOrder( - FieldValueMapping.builder().fieldName("firstName").fieldType("string").constraints(constraints).required(false).isAncestorRequired(false).build(), - FieldValueMapping.builder().fieldName("lastName").fieldType("string").constraints(constraints).required(true).isAncestorRequired(false).build(), - FieldValueMapping.builder().fieldName("age").fieldType("number").required(true).isAncestorRequired(false).build() - ); + .hasSize(3) + .containsExactlyInAnyOrder( + FieldValueMapping.builder().fieldName("firstName").fieldType("string").constraints(constraints).required(false).isAncestorRequired(false).build(), + FieldValueMapping.builder().fieldName("lastName").fieldType("string").constraints(constraints).required(true).isAncestorRequired(false).build(), + FieldValueMapping.builder().fieldName("age").fieldType("number").required(true).isAncestorRequired(false).build() + ); } @Test diff --git a/src/test/java/com/sngular/kloadgen/extractor/extractors/protobuff/ProtobufConfluentExtractorTest.java b/src/test/java/com/sngular/kloadgen/extractor/extractors/protobuff/ProtobufConfluentExtractorTest.java index e4bf3f0f..6f46ca64 100644 --- a/src/test/java/com/sngular/kloadgen/extractor/extractors/protobuff/ProtobufConfluentExtractorTest.java +++ b/src/test/java/com/sngular/kloadgen/extractor/extractors/protobuff/ProtobufConfluentExtractorTest.java @@ -3,6 +3,7 @@ import java.io.File; import java.util.List; import java.util.Locale; + import com.sngular.kloadgen.extractor.extractors.Extractor; import com.sngular.kloadgen.model.FieldValueMapping; import com.sngular.kloadgen.testutil.FileHelper; @@ -53,7 +54,8 @@ void testFlatProperties() throws Exception { void testEmbeddedTypes() throws Exception { final String testFile = fileHelper.getContent("/proto-files/embeddedTypeTest.proto"); final ProtobufSchema schema = new ProtobufSchema(testFile); - final List fieldValueMappingList = protoBufConfluentExtractor.processSchema(schema); Assertions.assertThat(fieldValueMappingList) + final List fieldValueMappingList = protoBufConfluentExtractor.processSchema(schema); + Assertions.assertThat(fieldValueMappingList) .hasSize(2) .containsExactlyInAnyOrder( FieldValueMapping.builder().fieldName("phones.addressesPhone[:].id[]").fieldType("string-array").required(true).isAncestorRequired(true).build(), diff --git a/src/test/java/com/sngular/kloadgen/loadgen/impl/AvroSRLoadGeneratorTest.java b/src/test/java/com/sngular/kloadgen/loadgen/impl/AvroSRLoadGeneratorTest.java index f052b75a..cff6bd0d 100644 --- a/src/test/java/com/sngular/kloadgen/loadgen/impl/AvroSRLoadGeneratorTest.java +++ b/src/test/java/com/sngular/kloadgen/loadgen/impl/AvroSRLoadGeneratorTest.java @@ -13,6 +13,7 @@ import java.util.List; import java.util.Locale; import java.util.Map; + import com.github.tomakehurst.wiremock.junit5.WireMockRuntimeInfo; import com.github.tomakehurst.wiremock.junit5.WireMockTest; import com.sngular.kloadgen.exception.KLoadGenException; diff --git a/src/test/java/com/sngular/kloadgen/loadgen/impl/ProtobufSRLoadGeneratorTest.java b/src/test/java/com/sngular/kloadgen/loadgen/impl/ProtobufSRLoadGeneratorTest.java index 008251f5..55a2c7c5 100644 --- a/src/test/java/com/sngular/kloadgen/loadgen/impl/ProtobufSRLoadGeneratorTest.java +++ b/src/test/java/com/sngular/kloadgen/loadgen/impl/ProtobufSRLoadGeneratorTest.java @@ -13,6 +13,7 @@ import java.util.List; import java.util.Locale; import java.util.Map; + import com.github.tomakehurst.wiremock.junit5.WireMockRuntimeInfo; import com.github.tomakehurst.wiremock.junit5.WireMockTest; import com.sngular.kloadgen.exception.KLoadGenException; diff --git a/src/test/java/com/sngular/kloadgen/processor/AvroSchemaProcessorTest.java b/src/test/java/com/sngular/kloadgen/processor/AvroSchemaProcessorTest.java index 0dc3fe04..9608c3d2 100644 --- a/src/test/java/com/sngular/kloadgen/processor/AvroSchemaProcessorTest.java +++ b/src/test/java/com/sngular/kloadgen/processor/AvroSchemaProcessorTest.java @@ -17,6 +17,7 @@ import java.util.stream.Collectors; import java.util.stream.IntStream; import java.util.stream.Stream; + import com.sngular.kloadgen.common.SchemaTypeEnum; import com.sngular.kloadgen.exception.KLoadGenException; import com.sngular.kloadgen.model.FieldValueMapping; @@ -189,9 +190,9 @@ private GenericRecord setUpEntityForEmbeddedAvroTest(final ParsedSchema parsedSc GenericData.Record entity = null; //TODO HERE if (parsedSchema.rawSchema() instanceof Schema) { - Schema schema = (Schema) parsedSchema.rawSchema(); + final Schema schema = (Schema) parsedSchema.rawSchema(); if (Schema.Type.UNION.equals(schema.getType())) { - entity = new GenericData.Record(schema.getTypes().get(schema.getTypes().size()-1)); + entity = new GenericData.Record(schema.getTypes().get(schema.getTypes().size() - 1)); } else { entity = new GenericData.Record(schema); } diff --git a/src/test/java/com/sngular/kloadgen/property/editor/FileSubjectPropertyEditorTest.java b/src/test/java/com/sngular/kloadgen/property/editor/FileSubjectPropertyEditorTest.java index 38ef589e..8edde20b 100644 --- a/src/test/java/com/sngular/kloadgen/property/editor/FileSubjectPropertyEditorTest.java +++ b/src/test/java/com/sngular/kloadgen/property/editor/FileSubjectPropertyEditorTest.java @@ -9,6 +9,7 @@ import java.io.File; import java.io.IOException; import java.util.List; + import com.sngular.kloadgen.model.FieldValueMapping; import com.sngular.kloadgen.testutil.FileHelper; import com.sngular.kloadgen.testutil.SchemaParseUtil; diff --git a/src/test/java/com/sngular/kloadgen/serializer/AvroDeserializerTest.java b/src/test/java/com/sngular/kloadgen/serializer/AvroDeserializerTest.java index 125b1895..057bc851 100644 --- a/src/test/java/com/sngular/kloadgen/serializer/AvroDeserializerTest.java +++ b/src/test/java/com/sngular/kloadgen/serializer/AvroDeserializerTest.java @@ -6,20 +6,6 @@ import java.util.stream.Stream; import java.util.stream.Stream.Builder; -import javax.xml.bind.DatatypeConverter; - -import static com.sngular.kloadgen.serializer.AvroSerializersTestFixture.TEST_FILE_ISSUE; -import static com.sngular.kloadgen.serializer.AvroSerializersTestFixture.TEST_ISSUE; -import static com.sngular.kloadgen.serializer.AvroSerializersTestFixture.TEST_LOGICAL_TYPES; -import static com.sngular.kloadgen.serializer.AvroSerializersTestFixture.TEST_MAP; -import static com.sngular.kloadgen.serializer.AvroSerializersTestFixture.TEST_NULL_ON_OPTIONAL_FIELDS; -import static com.sngular.kloadgen.serializer.AvroSerializersTestFixture.TEST_OPTIONAL_ENUM; -import static com.sngular.kloadgen.serializer.AvroSerializersTestFixture.TEST_OPTIONAL_MAP; -import static com.sngular.kloadgen.serializer.AvroSerializersTestFixture.TEST_SUBENTITY_ARRAY; -import static com.sngular.kloadgen.serializer.AvroSerializersTestFixture.TEST_UNION_RECORD; -import static com.sngular.kloadgen.serializer.AvroSerializersTestFixture.TEST_USER; -import static com.sngular.kloadgen.serializer.SerializerTestFixture.readSchema; - import com.sngular.kloadgen.common.SchemaTypeEnum; import com.sngular.kloadgen.model.FieldValueMapping; import com.sngular.kloadgen.processor.SchemaProcessor; @@ -28,6 +14,7 @@ import com.sngular.kloadgen.testutil.SchemaParseUtil; import com.sngular.kloadgen.util.PropsKeysHelper; import io.confluent.kafka.schemaregistry.ParsedSchema; +import javax.xml.bind.DatatypeConverter; import lombok.extern.slf4j.Slf4j; import org.assertj.core.api.Assertions; import org.junit.jupiter.api.BeforeEach; @@ -53,39 +40,39 @@ void setUp() { private static Stream getSchemaToTest() { - Builder builder = Stream.builder(); - - File testSubentityArrayFile = TEST_SUBENTITY_ARRAY.getFirst(); - builder.add(Arguments.arguments(Named.of(testSubentityArrayFile.getName(), testSubentityArrayFile), TEST_SUBENTITY_ARRAY.getSecond())); - File testIssueFile = TEST_ISSUE.getFirst(); - builder.add(Arguments.arguments(Named.of(testIssueFile.getName(), testIssueFile), TEST_ISSUE.getSecond())); - File testOptionalEnumFile = TEST_OPTIONAL_ENUM.getFirst(); - builder.add(Arguments.arguments(Named.of(testOptionalEnumFile.getName(), testOptionalEnumFile), TEST_OPTIONAL_ENUM.getSecond())); - File testFileIssueFile = TEST_FILE_ISSUE.getFirst(); - builder.add(Arguments.arguments(Named.of(testFileIssueFile.getName(), testFileIssueFile), TEST_FILE_ISSUE.getSecond())); - File testLogicalTypesFile = TEST_LOGICAL_TYPES.getFirst(); - builder.add(Arguments.arguments(Named.of(testLogicalTypesFile.getName(), testLogicalTypesFile), TEST_LOGICAL_TYPES.getSecond())); - File testMapFile = TEST_MAP.getFirst(); - builder.add(Arguments.arguments(Named.of(testMapFile.getName(), testMapFile), TEST_MAP.getSecond())); - File testNullOnOptionalFieldsFile = TEST_NULL_ON_OPTIONAL_FIELDS.getFirst(); + final Builder builder = Stream.builder(); + + final File testSubentityArrayFile = AvroSerializersTestFixture.TEST_SUBENTITY_ARRAY.getFirst(); + builder.add(Arguments.arguments(Named.of(testSubentityArrayFile.getName(), testSubentityArrayFile), AvroSerializersTestFixture.TEST_SUBENTITY_ARRAY.getSecond())); + final File testIssueFile = AvroSerializersTestFixture.TEST_ISSUE.getFirst(); + builder.add(Arguments.arguments(Named.of(testIssueFile.getName(), testIssueFile), AvroSerializersTestFixture.TEST_ISSUE.getSecond())); + final File testOptionalEnumFile = AvroSerializersTestFixture.TEST_OPTIONAL_ENUM.getFirst(); + builder.add(Arguments.arguments(Named.of(testOptionalEnumFile.getName(), testOptionalEnumFile), AvroSerializersTestFixture.TEST_OPTIONAL_ENUM.getSecond())); + final File testFileIssueFile = AvroSerializersTestFixture.TEST_FILE_ISSUE.getFirst(); + builder.add(Arguments.arguments(Named.of(testFileIssueFile.getName(), testFileIssueFile), AvroSerializersTestFixture.TEST_FILE_ISSUE.getSecond())); + final File testLogicalTypesFile = AvroSerializersTestFixture.TEST_LOGICAL_TYPES.getFirst(); + builder.add(Arguments.arguments(Named.of(testLogicalTypesFile.getName(), testLogicalTypesFile), AvroSerializersTestFixture.TEST_LOGICAL_TYPES.getSecond())); + final File testMapFile = AvroSerializersTestFixture.TEST_MAP.getFirst(); + builder.add(Arguments.arguments(Named.of(testMapFile.getName(), testMapFile), AvroSerializersTestFixture.TEST_MAP.getSecond())); + final File testNullOnOptionalFieldsFile = AvroSerializersTestFixture.TEST_NULL_ON_OPTIONAL_FIELDS.getFirst(); builder.add( - Arguments.arguments(Named.of(testNullOnOptionalFieldsFile.getName(), testNullOnOptionalFieldsFile), TEST_NULL_ON_OPTIONAL_FIELDS.getSecond())); - File testOptionalMapFile = TEST_OPTIONAL_MAP.getFirst(); - builder.add(Arguments.arguments(Named.of(testOptionalMapFile.getName(), testOptionalMapFile), TEST_OPTIONAL_MAP.getSecond())); - File testNullOnOptionalFields = TEST_NULL_ON_OPTIONAL_FIELDS.getFirst(); - builder.add(Arguments.arguments(Named.of(testNullOnOptionalFields.getName(), testNullOnOptionalFields), TEST_NULL_ON_OPTIONAL_FIELDS.getSecond())); - File testUnionRecordFile = TEST_UNION_RECORD.getFirst(); - builder.add(Arguments.arguments(Named.of(testUnionRecordFile.getName(), testUnionRecordFile), TEST_UNION_RECORD.getSecond())); - File testUserFile = TEST_USER.getFirst(); - builder.add(Arguments.arguments(Named.of(testUserFile.getName(), testUserFile), TEST_USER.getSecond())); + Arguments.arguments(Named.of(testNullOnOptionalFieldsFile.getName(), testNullOnOptionalFieldsFile), AvroSerializersTestFixture.TEST_NULL_ON_OPTIONAL_FIELDS.getSecond())); + final File testOptionalMapFile = AvroSerializersTestFixture.TEST_OPTIONAL_MAP.getFirst(); + builder.add(Arguments.arguments(Named.of(testOptionalMapFile.getName(), testOptionalMapFile), AvroSerializersTestFixture.TEST_OPTIONAL_MAP.getSecond())); + final File testNullOnOptionalFields = AvroSerializersTestFixture.TEST_NULL_ON_OPTIONAL_FIELDS.getFirst(); + builder.add(Arguments.arguments(Named.of(testNullOnOptionalFields.getName(), testNullOnOptionalFields), AvroSerializersTestFixture.TEST_NULL_ON_OPTIONAL_FIELDS.getSecond())); + final File testUnionRecordFile = AvroSerializersTestFixture.TEST_UNION_RECORD.getFirst(); + builder.add(Arguments.arguments(Named.of(testUnionRecordFile.getName(), testUnionRecordFile), AvroSerializersTestFixture.TEST_UNION_RECORD.getSecond())); + final File testUserFile = AvroSerializersTestFixture.TEST_USER.getFirst(); + builder.add(Arguments.arguments(Named.of(testUserFile.getName(), testUserFile), AvroSerializersTestFixture.TEST_USER.getSecond())); return builder.build(); } @ParameterizedTest @MethodSource("getSchemaToTest") - void deserialize(File schemaFile, List fieldValueMappings) throws Exception { - final var schemaStr = readSchema(schemaFile); + void deserialize(final File schemaFile, final List fieldValueMappings) throws Exception { + final var schemaStr = SerializerTestFixture.readSchema(schemaFile); final BaseSchemaMetadata confluentBaseSchemaMetadata = new BaseSchemaMetadata<>( ConfluentSchemaMetadata.parse(new io.confluent.kafka.schemaregistry.client.SchemaMetadata(1, 1, diff --git a/src/test/java/com/sngular/kloadgen/serializer/AvroSerializersTestFixture.java b/src/test/java/com/sngular/kloadgen/serializer/AvroSerializersTestFixture.java index a9cf10c6..db88d04d 100644 --- a/src/test/java/com/sngular/kloadgen/serializer/AvroSerializersTestFixture.java +++ b/src/test/java/com/sngular/kloadgen/serializer/AvroSerializersTestFixture.java @@ -1,7 +1,5 @@ package com.sngular.kloadgen.serializer; -import static com.sngular.kloadgen.serializer.SerializerTestFixture.createFieldValueMapping; - import java.io.File; import java.util.Arrays; import java.util.List; @@ -14,132 +12,135 @@ public final class AvroSerializersTestFixture { - static final List> serializerList = List.of( - new GenericAvroRecordSerializer<>(), - new GenericAvroRecordBinarySerializer<>() + static final List> SERIALIZER_LIST = List.of( + new GenericAvroRecordSerializer<>(), + new GenericAvroRecordBinarySerializer<>() ); static final Pair> TEST_SUBENTITY_ARRAY = new Pair<>( new FileHelper().getFile("/avro-files/avros-example-with-sub-entity-array-test.avsc"), Arrays.asList( - createFieldValueMapping("subEntity.anotherLevel.subEntityIntArray[2]", "int-array"), - createFieldValueMapping("subEntity.anotherLevel.subEntityRecordArray[2].name", "string"), - createFieldValueMapping("topLevelIntArray[3]", "int-array"), - createFieldValueMapping("topLevelRecordArray[3].name", "string")) + SerializerTestFixture.createFieldValueMapping("subEntity.anotherLevel.subEntityIntArray[2]", "int-array"), + SerializerTestFixture.createFieldValueMapping("subEntity.anotherLevel.subEntityRecordArray[2].name", "string"), + SerializerTestFixture.createFieldValueMapping("topLevelIntArray[3]", "int-array"), + SerializerTestFixture.createFieldValueMapping("topLevelRecordArray[3].name", "string")) ); static final Pair> TEST_EMBEDED_AVROS_EXAMPLE = new Pair<>( new FileHelper().getFile("/avro-files/embedded-avros-example-test.avsc"), Arrays.asList( - createFieldValueMapping("fieldMySchema.testInt_id", "int", "[100]"), - createFieldValueMapping("fieldMySchema.testLong", "long"), - createFieldValueMapping("fieldMySchema.fieldString", "string"), - createFieldValueMapping("timestamp", "long")) + SerializerTestFixture.createFieldValueMapping("fieldMySchema.testInt_id", "int", "[100]"), + SerializerTestFixture.createFieldValueMapping("fieldMySchema.testLong", "long"), + SerializerTestFixture.createFieldValueMapping("fieldMySchema.fieldString", "string"), + SerializerTestFixture.createFieldValueMapping("timestamp", "long")) ); static final Pair> TEST_ISSUE = new Pair<>( new FileHelper().getFile("/avro-files/issue.avsc"), Arrays.asList( - createFieldValueMapping("mainObject.arrayValue[].optional1", "string"), - createFieldValueMapping("mainObject.arrayValue[].optional2", "string"), - createFieldValueMapping("mainObject.arrayValue[].optional3", "string")) + SerializerTestFixture.createFieldValueMapping("mainObject.arrayValue[].optional1", "string"), + SerializerTestFixture.createFieldValueMapping("mainObject.arrayValue[].optional2", "string"), + SerializerTestFixture.createFieldValueMapping("mainObject.arrayValue[].optional3", "string")) ); static final Pair> TEST_OPTIONAL_ENUM = new Pair<>( new FileHelper().getFile("/avro-files/optionalEnum.avsc"), - Arrays.asList(createFieldValueMapping("aggregateAttribute.fruitList.fruits[].fruitType", "enum")) + Arrays.asList(SerializerTestFixture.createFieldValueMapping("aggregateAttribute.fruitList.fruits[].fruitType", "enum")) ); static final Pair> TEST_FILE_ISSUE = new Pair<>( new FileHelper().getFile("/avro-files/testFileIssue.avsc"), Arrays.asList( - createFieldValueMapping("timestamp", "long_timestamp-millis"), - createFieldValueMapping("queueId", "string"), - createFieldValueMapping("name", "string"), - createFieldValueMapping("description", "string"), - createFieldValueMapping("accountDefault", "boolean"), - createFieldValueMapping("accountId", "string")) + SerializerTestFixture.createFieldValueMapping("timestamp", "long_timestamp-millis"), + SerializerTestFixture.createFieldValueMapping("queueId", "string"), + SerializerTestFixture.createFieldValueMapping("name", "string"), + SerializerTestFixture.createFieldValueMapping("description", "string"), + SerializerTestFixture.createFieldValueMapping("accountDefault", "boolean"), + SerializerTestFixture.createFieldValueMapping("accountId", "string")) ); static final Pair> TEST_LOGICAL_TYPES = new Pair<>( new FileHelper().getFile("/avro-files/testLogicalTypes.avsc"), Arrays.asList( - createFieldValueMapping("Date", "int_date"), - createFieldValueMapping("TimeMillis", "int_time-millis"), - createFieldValueMapping("TimeMicros", "long_time-micros"), - createFieldValueMapping("TimestampMillis", "long_timestamp-millis"), - createFieldValueMapping("TimestampMicros", "long_timestamp-micros"), - createFieldValueMapping("LocalTimestampMillis", "long_local-timestamp-millis"), - createFieldValueMapping("LocalTimestampMicros", "long_local-timestamp-micros"), - createFieldValueMapping("UUID", "string_uuid"), - createFieldValueMapping("Decimal", "bytes_decimal"), - createFieldValueMapping("DecimalFixed", "fixed_decimal")) + SerializerTestFixture.createFieldValueMapping("Date", "int_date"), + SerializerTestFixture.createFieldValueMapping("TimeMillis", "int_time-millis"), + SerializerTestFixture.createFieldValueMapping("TimeMicros", "long_time-micros"), + SerializerTestFixture.createFieldValueMapping("TimestampMillis", "long_timestamp-millis"), + SerializerTestFixture.createFieldValueMapping("TimestampMicros", "long_timestamp-micros"), + SerializerTestFixture.createFieldValueMapping("LocalTimestampMillis", "long_local-timestamp-millis"), + SerializerTestFixture.createFieldValueMapping("LocalTimestampMicros", "long_local-timestamp-micros"), + SerializerTestFixture.createFieldValueMapping("UUID", "string_uuid"), + SerializerTestFixture.createFieldValueMapping("Decimal", "bytes_decimal"), + SerializerTestFixture.createFieldValueMapping("DecimalFixed", "fixed_decimal")) ); static final Pair> TEST_MAP = new Pair<>( new FileHelper().getFile("/avro-files/testMap.avsc"), Arrays.asList( - createFieldValueMapping("theMap[:][].otherType.addTypeId", "string"), - createFieldValueMapping("theMap[:][].otherType.name", "string"), - createFieldValueMapping("theMap[:][].otherType.otherField", "string"), - createFieldValueMapping("theMap[:][].addAmount", "bytes_decimal"), - createFieldValueMapping("theMap[:][].addCode", "string"), - createFieldValueMapping("theMap[:][].metadataMap[:]", "string-map"), - createFieldValueMapping("theMap[:][].metadataArray[]", "string-array"), - createFieldValueMapping("theMap[:][].metadataMapMap[:][:]", "string-map-map"), - createFieldValueMapping("theMap[:][].metadataArrayArray[][]", "string-array-array")) + SerializerTestFixture.createFieldValueMapping("theMap[:][].otherType.addTypeId", "string"), + SerializerTestFixture.createFieldValueMapping("theMap[:][].otherType.name", "string"), + SerializerTestFixture.createFieldValueMapping("theMap[:][].otherType.otherField", "string"), + SerializerTestFixture.createFieldValueMapping("theMap[:][].addAmount", "bytes_decimal"), + SerializerTestFixture.createFieldValueMapping("theMap[:][].addCode", "string"), + SerializerTestFixture.createFieldValueMapping("theMap[:][].metadataMap[:]", "string-map"), + SerializerTestFixture.createFieldValueMapping("theMap[:][].metadataArray[]", "string-array"), + SerializerTestFixture.createFieldValueMapping("theMap[:][].metadataMapMap[:][:]", "string-map-map"), + SerializerTestFixture.createFieldValueMapping("theMap[:][].metadataArrayArray[][]", "string-array-array")) ); static final Pair> TEST_NULL_ON_OPTIONAL_FIELDS = new Pair<>( new FileHelper().getFile("/avro-files/testNullOnOptionalFields.avsc"), Arrays.asList( - createFieldValueMapping("name", "string"), - createFieldValueMapping("favorite_number", "int"), - createFieldValueMapping("favorite_color", "string"), - createFieldValueMapping("emails[]", "string-array"), - createFieldValueMapping("phones[]", "int-array"), - createFieldValueMapping("friends[]", "string-array"), - createFieldValueMapping("favorite_cars[].brand", "string"), - createFieldValueMapping("favorite_cars[].power", "int"), - createFieldValueMapping("favorite_cars[].parts[]", "string-array"), - createFieldValueMapping("favorite_cars2[].brand2", "string"), - createFieldValueMapping("favorite_cars2[].power2", "int"), - createFieldValueMapping("favorite_cars2[].parts[]2", "string-array")) + SerializerTestFixture.createFieldValueMapping("name", "string"), + SerializerTestFixture.createFieldValueMapping("favorite_number", "int"), + SerializerTestFixture.createFieldValueMapping("favorite_color", "string"), + SerializerTestFixture.createFieldValueMapping("emails[]", "string-array"), + SerializerTestFixture.createFieldValueMapping("phones[]", "int-array"), + SerializerTestFixture.createFieldValueMapping("friends[]", "string-array"), + SerializerTestFixture.createFieldValueMapping("favorite_cars[].brand", "string"), + SerializerTestFixture.createFieldValueMapping("favorite_cars[].power", "int"), + SerializerTestFixture.createFieldValueMapping("favorite_cars[].parts[]", "string-array"), + SerializerTestFixture.createFieldValueMapping("favorite_cars2[].brand2", "string"), + SerializerTestFixture.createFieldValueMapping("favorite_cars2[].power2", "int"), + SerializerTestFixture.createFieldValueMapping("favorite_cars2[].parts[]2", "string-array")) ); static final Pair> TEST_OPTIONAL_MAP = new Pair<>( new FileHelper().getFile("/avro-files/testOptionalMap.avsc"), Arrays.asList( - createFieldValueMapping("mapOfString[:]", "string-map"), - createFieldValueMapping("arrayOfString[]", "string-array"), - createFieldValueMapping("arrayOfMap[][:]", "string-map-array"), - createFieldValueMapping("mapOfArray[:][]", "int-array-map"), - createFieldValueMapping("mapOfArrayOfRecord[:][].name", "string"), - createFieldValueMapping("mapOfArrayOfRecord[:][].age", "int"), - createFieldValueMapping("arrayOfMapOfRecord[][:].name", "string"), - createFieldValueMapping("arrayOfMapOfRecord[][:].age", "int")) + SerializerTestFixture.createFieldValueMapping("mapOfString[:]", "string-map"), + SerializerTestFixture.createFieldValueMapping("arrayOfString[]", "string-array"), + SerializerTestFixture.createFieldValueMapping("arrayOfMap[][:]", "string-map-array"), + SerializerTestFixture.createFieldValueMapping("mapOfArray[:][]", "int-array-map"), + SerializerTestFixture.createFieldValueMapping("mapOfArrayOfRecord[:][].name", "string"), + SerializerTestFixture.createFieldValueMapping("mapOfArrayOfRecord[:][].age", "int"), + SerializerTestFixture.createFieldValueMapping("arrayOfMapOfRecord[][:].name", "string"), + SerializerTestFixture.createFieldValueMapping("arrayOfMapOfRecord[][:].age", "int")) ); static final Pair> TEST_UNION_RECORD = new Pair<>( new FileHelper().getFile("/avro-files/testUnionRecord.avsc"), Arrays.asList( - createFieldValueMapping("validateInnerObject.attribute1", "string"), - createFieldValueMapping("validateInnerObject.attribute2", "string"), - createFieldValueMapping("products[].Price.price", "string"), - createFieldValueMapping("products[].Price.priceType", "string"), - createFieldValueMapping("products[].Price.currency", "string"), - createFieldValueMapping("products[].Price.discount", "string"), - createFieldValueMapping("products[].Price.validateInnerObject.attribute1", "string"), - createFieldValueMapping("products[].Price.validateInnerObject.attribute2", "string")) + SerializerTestFixture.createFieldValueMapping("validateInnerObject.attribute1", "string"), + SerializerTestFixture.createFieldValueMapping("validateInnerObject.attribute2", "string"), + SerializerTestFixture.createFieldValueMapping("products[].Price.price", "string"), + SerializerTestFixture.createFieldValueMapping("products[].Price.priceType", "string"), + SerializerTestFixture.createFieldValueMapping("products[].Price.currency", "string"), + SerializerTestFixture.createFieldValueMapping("products[].Price.discount", "string"), + SerializerTestFixture.createFieldValueMapping("products[].Price.validateInnerObject.attribute1", "string"), + SerializerTestFixture.createFieldValueMapping("products[].Price.validateInnerObject.attribute2", "string")) ); static final Pair> TEST_USER = new Pair<>( new FileHelper().getFile("/avro-files/userTest.avsc"), Arrays.asList( - createFieldValueMapping("id", "int"), - createFieldValueMapping("twitterAccounts[].status", "enum"), - createFieldValueMapping("twitterAccounts[].status2", "enum"), - createFieldValueMapping("toDoItems[].status3", "enum"), - createFieldValueMapping("toDoItems[].status4", "enum")) + SerializerTestFixture.createFieldValueMapping("id", "int"), + SerializerTestFixture.createFieldValueMapping("twitterAccounts[].status", "enum"), + SerializerTestFixture.createFieldValueMapping("twitterAccounts[].status2", "enum"), + SerializerTestFixture.createFieldValueMapping("toDoItems[].status3", "enum"), + SerializerTestFixture.createFieldValueMapping("toDoItems[].status4", "enum")) ); + + private AvroSerializersTestFixture() { + } } diff --git a/src/test/java/com/sngular/kloadgen/serializer/GenericRecordAvroSerializersTest.java b/src/test/java/com/sngular/kloadgen/serializer/GenericRecordAvroSerializersTest.java index da585b52..290771ff 100644 --- a/src/test/java/com/sngular/kloadgen/serializer/GenericRecordAvroSerializersTest.java +++ b/src/test/java/com/sngular/kloadgen/serializer/GenericRecordAvroSerializersTest.java @@ -5,20 +5,6 @@ import java.util.stream.Stream; import java.util.stream.Stream.Builder; -import static com.sngular.kloadgen.serializer.AvroSerializersTestFixture.TEST_EMBEDED_AVROS_EXAMPLE; -import static com.sngular.kloadgen.serializer.AvroSerializersTestFixture.TEST_FILE_ISSUE; -import static com.sngular.kloadgen.serializer.AvroSerializersTestFixture.TEST_ISSUE; -import static com.sngular.kloadgen.serializer.AvroSerializersTestFixture.TEST_LOGICAL_TYPES; -import static com.sngular.kloadgen.serializer.AvroSerializersTestFixture.TEST_MAP; -import static com.sngular.kloadgen.serializer.AvroSerializersTestFixture.TEST_NULL_ON_OPTIONAL_FIELDS; -import static com.sngular.kloadgen.serializer.AvroSerializersTestFixture.TEST_OPTIONAL_ENUM; -import static com.sngular.kloadgen.serializer.AvroSerializersTestFixture.TEST_OPTIONAL_MAP; -import static com.sngular.kloadgen.serializer.AvroSerializersTestFixture.TEST_SUBENTITY_ARRAY; -import static com.sngular.kloadgen.serializer.AvroSerializersTestFixture.TEST_UNION_RECORD; -import static com.sngular.kloadgen.serializer.AvroSerializersTestFixture.TEST_USER; -import static com.sngular.kloadgen.serializer.AvroSerializersTestFixture.serializerList; -import static com.sngular.kloadgen.serializer.SerializerTestFixture.readSchema; - import com.sngular.kloadgen.common.SchemaTypeEnum; import com.sngular.kloadgen.model.FieldValueMapping; import com.sngular.kloadgen.processor.SchemaProcessor; @@ -40,34 +26,34 @@ public class GenericRecordAvroSerializersTest { private static Stream getSerializerAndSchemaToTest() { - Builder builder = Stream.builder(); - serializerList.forEach(serializer -> { - Named> serializerArgument = Named.of(serializer.getClass().getName(), serializer); - File testSubentityArrayFile = TEST_SUBENTITY_ARRAY.getFirst(); - builder.add(Arguments.arguments(serializerArgument, Named.of(testSubentityArrayFile.getName(), testSubentityArrayFile), TEST_SUBENTITY_ARRAY.getSecond())); - File testEmbededAvrosExampleFile = TEST_EMBEDED_AVROS_EXAMPLE.getFirst(); - builder.add(Arguments.arguments(serializerArgument, Named.of(testEmbededAvrosExampleFile.getName(), testEmbededAvrosExampleFile), TEST_EMBEDED_AVROS_EXAMPLE.getSecond())); - File testIssueFile = TEST_ISSUE.getFirst(); - builder.add(Arguments.arguments(serializerArgument, Named.of(testIssueFile.getName(), testIssueFile), TEST_ISSUE.getSecond())); - File testOptionalEnumFile = TEST_OPTIONAL_ENUM.getFirst(); - builder.add(Arguments.arguments(serializerArgument, Named.of(testOptionalEnumFile.getName(), testOptionalEnumFile), TEST_OPTIONAL_ENUM.getSecond())); - File testFileIssueFile = TEST_FILE_ISSUE.getFirst(); - builder.add(Arguments.arguments(serializerArgument, Named.of(testFileIssueFile.getName(), testFileIssueFile), TEST_FILE_ISSUE.getSecond())); - File testLogicalTypesFile = TEST_LOGICAL_TYPES.getFirst(); - builder.add(Arguments.arguments(serializerArgument, Named.of(testLogicalTypesFile.getName(), testLogicalTypesFile), TEST_LOGICAL_TYPES.getSecond())); - File testMapFile = TEST_MAP.getFirst(); - builder.add(Arguments.arguments(serializerArgument, Named.of(testMapFile.getName(), testMapFile), TEST_MAP.getSecond())); - File testNullOnOptionalFieldsFile = TEST_NULL_ON_OPTIONAL_FIELDS.getFirst(); + final Builder builder = Stream.builder(); + AvroSerializersTestFixture.SERIALIZER_LIST.forEach(serializer -> { + final Named> serializerArgument = Named.of(serializer.getClass().getName(), serializer); + final File testSubentityArrayFile = AvroSerializersTestFixture.TEST_SUBENTITY_ARRAY.getFirst(); + builder.add(Arguments.arguments(serializerArgument, Named.of(testSubentityArrayFile.getName(), testSubentityArrayFile), AvroSerializersTestFixture.TEST_SUBENTITY_ARRAY.getSecond())); + final File testEmbededAvrosExampleFile = AvroSerializersTestFixture.TEST_EMBEDED_AVROS_EXAMPLE.getFirst(); + builder.add(Arguments.arguments(serializerArgument, Named.of(testEmbededAvrosExampleFile.getName(), testEmbededAvrosExampleFile), AvroSerializersTestFixture.TEST_EMBEDED_AVROS_EXAMPLE.getSecond())); + final File testIssueFile = AvroSerializersTestFixture.TEST_ISSUE.getFirst(); + builder.add(Arguments.arguments(serializerArgument, Named.of(testIssueFile.getName(), testIssueFile), AvroSerializersTestFixture.TEST_ISSUE.getSecond())); + final File testOptionalEnumFile = AvroSerializersTestFixture.TEST_OPTIONAL_ENUM.getFirst(); + builder.add(Arguments.arguments(serializerArgument, Named.of(testOptionalEnumFile.getName(), testOptionalEnumFile), AvroSerializersTestFixture.TEST_OPTIONAL_ENUM.getSecond())); + final File testFileIssueFile = AvroSerializersTestFixture.TEST_FILE_ISSUE.getFirst(); + builder.add(Arguments.arguments(serializerArgument, Named.of(testFileIssueFile.getName(), testFileIssueFile), AvroSerializersTestFixture.TEST_FILE_ISSUE.getSecond())); + final File testLogicalTypesFile = AvroSerializersTestFixture.TEST_LOGICAL_TYPES.getFirst(); + builder.add(Arguments.arguments(serializerArgument, Named.of(testLogicalTypesFile.getName(), testLogicalTypesFile), AvroSerializersTestFixture.TEST_LOGICAL_TYPES.getSecond())); + final File testMapFile = AvroSerializersTestFixture.TEST_MAP.getFirst(); + builder.add(Arguments.arguments(serializerArgument, Named.of(testMapFile.getName(), testMapFile), AvroSerializersTestFixture.TEST_MAP.getSecond())); + final File testNullOnOptionalFieldsFile = AvroSerializersTestFixture.TEST_NULL_ON_OPTIONAL_FIELDS.getFirst(); builder.add( - Arguments.arguments(serializerArgument, Named.of(testNullOnOptionalFieldsFile.getName(), testNullOnOptionalFieldsFile), TEST_NULL_ON_OPTIONAL_FIELDS.getSecond())); - File testOptionalMapFile = TEST_OPTIONAL_MAP.getFirst(); - builder.add(Arguments.arguments(serializerArgument, Named.of(testOptionalMapFile.getName(), testOptionalMapFile), TEST_OPTIONAL_MAP.getSecond())); - File testNullOnOptionalFields = TEST_NULL_ON_OPTIONAL_FIELDS.getFirst(); - builder.add(Arguments.arguments(serializerArgument, Named.of(testNullOnOptionalFields.getName(), testNullOnOptionalFields), TEST_NULL_ON_OPTIONAL_FIELDS.getSecond())); - File testUnionRecordFile = TEST_UNION_RECORD.getFirst(); - builder.add(Arguments.arguments(serializerArgument, Named.of(testUnionRecordFile.getName(), testUnionRecordFile), TEST_UNION_RECORD.getSecond())); - File testUserFile = TEST_USER.getFirst(); - builder.add(Arguments.arguments(serializerArgument, Named.of(testUserFile.getName(), testUserFile), TEST_USER.getSecond())); + Arguments.arguments(serializerArgument, Named.of(testNullOnOptionalFieldsFile.getName(), testNullOnOptionalFieldsFile), AvroSerializersTestFixture.TEST_NULL_ON_OPTIONAL_FIELDS.getSecond())); + final File testOptionalMapFile = AvroSerializersTestFixture.TEST_OPTIONAL_MAP.getFirst(); + builder.add(Arguments.arguments(serializerArgument, Named.of(testOptionalMapFile.getName(), testOptionalMapFile), AvroSerializersTestFixture.TEST_OPTIONAL_MAP.getSecond())); + final File testNullOnOptionalFields = AvroSerializersTestFixture.TEST_NULL_ON_OPTIONAL_FIELDS.getFirst(); + builder.add(Arguments.arguments(serializerArgument, Named.of(testNullOnOptionalFields.getName(), testNullOnOptionalFields), AvroSerializersTestFixture.TEST_NULL_ON_OPTIONAL_FIELDS.getSecond())); + final File testUnionRecordFile = AvroSerializersTestFixture.TEST_UNION_RECORD.getFirst(); + builder.add(Arguments.arguments(serializerArgument, Named.of(testUnionRecordFile.getName(), testUnionRecordFile), AvroSerializersTestFixture.TEST_UNION_RECORD.getSecond())); + final File testUserFile = AvroSerializersTestFixture.TEST_USER.getFirst(); + builder.add(Arguments.arguments(serializerArgument, Named.of(testUserFile.getName(), testUserFile), AvroSerializersTestFixture.TEST_USER.getSecond())); }); return builder.build(); } @@ -75,7 +61,7 @@ private static Stream getSerializerAndSchemaToTest() { @ParameterizedTest @MethodSource("getSerializerAndSchemaToTest") void genericAvroRecordSerializerTest(Serializer serializer, File schemaFile, List fieldValueMappings) throws Exception { - final var schemaStr = readSchema(schemaFile); + final var schemaStr = SerializerTestFixture.readSchema(schemaFile); final BaseSchemaMetadata confluentBaseSchemaMetadata = new BaseSchemaMetadata<>( ConfluentSchemaMetadata.parse(new io.confluent.kafka.schemaregistry.client.SchemaMetadata(1, 1, diff --git a/src/test/java/com/sngular/kloadgen/serializer/JsonSerializerTest.java b/src/test/java/com/sngular/kloadgen/serializer/JsonSerializerTest.java index 0ef09474..922f6093 100644 --- a/src/test/java/com/sngular/kloadgen/serializer/JsonSerializerTest.java +++ b/src/test/java/com/sngular/kloadgen/serializer/JsonSerializerTest.java @@ -5,19 +5,6 @@ import java.util.stream.Stream; import java.util.stream.Stream.Builder; -import static com.sngular.kloadgen.serializer.JsonSerializerTestFixture.TEST_BASIC; -import static com.sngular.kloadgen.serializer.JsonSerializerTestFixture.TEST_BASIC_ARRAY; -import static com.sngular.kloadgen.serializer.JsonSerializerTestFixture.TEST_BASIC_NUMBER; -import static com.sngular.kloadgen.serializer.JsonSerializerTestFixture.TEST_COLLECTIONS; -import static com.sngular.kloadgen.serializer.JsonSerializerTestFixture.TEST_COMPLEX_DEFINITIONS; -import static com.sngular.kloadgen.serializer.JsonSerializerTestFixture.TEST_COMPLEX_DOCUMENT; -import static com.sngular.kloadgen.serializer.JsonSerializerTestFixture.TEST_MAP; -import static com.sngular.kloadgen.serializer.JsonSerializerTestFixture.TEST_MEDIUM_DOCUMENT; -import static com.sngular.kloadgen.serializer.JsonSerializerTestFixture.TEST_MULTIPLE_TYPE; -import static com.sngular.kloadgen.serializer.JsonSerializerTestFixture.TEST_MULTIPLE_TYPE_SINGLE; -import static com.sngular.kloadgen.serializer.JsonSerializerTestFixture.TEST_NESTED_COLLECTIONS; -import static com.sngular.kloadgen.serializer.SerializerTestFixture.readSchema; - import com.fasterxml.jackson.databind.node.ObjectNode; import com.sngular.kloadgen.common.SchemaTypeEnum; import com.sngular.kloadgen.model.FieldValueMapping; @@ -36,41 +23,41 @@ public class JsonSerializerTest { private static final SchemaProcessor JSON_SCHEMA_PROCESSOR = new SchemaProcessor(); - private static final GenericJsonRecordSerializer serializer = new GenericJsonRecordSerializer(); + private static final GenericJsonRecordSerializer SERIALIZER = new GenericJsonRecordSerializer(); private static Stream getSchemaToTest() { - Builder builder = Stream.builder(); + final Builder builder = Stream.builder(); - File testBasicFile = TEST_BASIC.getFirst(); - builder.add(Arguments.arguments(Named.of(testBasicFile.getName(), testBasicFile), TEST_BASIC.getSecond())); - File testBasicArraysFile = TEST_BASIC_ARRAY.getFirst(); - builder.add(Arguments.arguments(Named.of(testBasicArraysFile.getName(), testBasicArraysFile), TEST_BASIC_ARRAY.getSecond())); - File testBasicNumberFile = TEST_BASIC_NUMBER.getFirst(); - builder.add(Arguments.arguments(Named.of(testBasicNumberFile.getName(), testBasicNumberFile), TEST_BASIC_NUMBER.getSecond())); - File testCollectionsFile = TEST_COLLECTIONS.getFirst(); - builder.add(Arguments.arguments(Named.of(testCollectionsFile.getName(), testCollectionsFile), TEST_COLLECTIONS.getSecond())); - File testComplexDefinitionsFile = TEST_COMPLEX_DEFINITIONS.getFirst(); - builder.add(Arguments.arguments(Named.of(testComplexDefinitionsFile.getName(), testComplexDefinitionsFile), TEST_COMPLEX_DEFINITIONS.getSecond())); - File testComplexDocumentFile = TEST_COMPLEX_DOCUMENT.getFirst(); - builder.add(Arguments.arguments(Named.of(testComplexDocumentFile.getName(), testComplexDocumentFile), TEST_COMPLEX_DOCUMENT.getSecond())); - File testMediumDocumentFile = TEST_MEDIUM_DOCUMENT.getFirst(); - builder.add(Arguments.arguments(Named.of(testMediumDocumentFile.getName(), testMediumDocumentFile), TEST_MEDIUM_DOCUMENT.getSecond())); - File testMultipleTypeFile = TEST_MULTIPLE_TYPE.getFirst(); - builder.add(Arguments.arguments(Named.of(testMultipleTypeFile.getName(), testMultipleTypeFile), TEST_MULTIPLE_TYPE.getSecond())); - File testMultipleTypeSingleFile = TEST_MULTIPLE_TYPE_SINGLE.getFirst(); - builder.add(Arguments.arguments(Named.of(testMultipleTypeSingleFile.getName(), testMultipleTypeSingleFile), TEST_MULTIPLE_TYPE_SINGLE.getSecond())); - File testNestedCollectionsFile = TEST_NESTED_COLLECTIONS.getFirst(); - builder.add(Arguments.arguments(Named.of(testNestedCollectionsFile.getName(), testNestedCollectionsFile), TEST_NESTED_COLLECTIONS.getSecond())); - File testMapFile = TEST_MAP.getFirst(); - builder.add(Arguments.arguments(Named.of(testMapFile.getName(), testMapFile), TEST_MAP.getSecond())); + final File testBasicFile = JsonSerializerTestFixture.TEST_BASIC.getFirst(); + builder.add(Arguments.arguments(Named.of(testBasicFile.getName(), testBasicFile), JsonSerializerTestFixture.TEST_BASIC.getSecond())); + final File testBasicArraysFile = JsonSerializerTestFixture.TEST_BASIC_ARRAY.getFirst(); + builder.add(Arguments.arguments(Named.of(testBasicArraysFile.getName(), testBasicArraysFile), JsonSerializerTestFixture.TEST_BASIC_ARRAY.getSecond())); + final File testBasicNumberFile = JsonSerializerTestFixture.TEST_BASIC_NUMBER.getFirst(); + builder.add(Arguments.arguments(Named.of(testBasicNumberFile.getName(), testBasicNumberFile), JsonSerializerTestFixture.TEST_BASIC_NUMBER.getSecond())); + final File testCollectionsFile = JsonSerializerTestFixture.TEST_COLLECTIONS.getFirst(); + builder.add(Arguments.arguments(Named.of(testCollectionsFile.getName(), testCollectionsFile), JsonSerializerTestFixture.TEST_COLLECTIONS.getSecond())); + final File testComplexDefinitionsFile = JsonSerializerTestFixture.TEST_COMPLEX_DEFINITIONS.getFirst(); + builder.add(Arguments.arguments(Named.of(testComplexDefinitionsFile.getName(), testComplexDefinitionsFile), JsonSerializerTestFixture.TEST_COMPLEX_DEFINITIONS.getSecond())); + final File testComplexDocumentFile = JsonSerializerTestFixture.TEST_COMPLEX_DOCUMENT.getFirst(); + builder.add(Arguments.arguments(Named.of(testComplexDocumentFile.getName(), testComplexDocumentFile), JsonSerializerTestFixture.TEST_COMPLEX_DOCUMENT.getSecond())); + final File testMediumDocumentFile = JsonSerializerTestFixture.TEST_MEDIUM_DOCUMENT.getFirst(); + builder.add(Arguments.arguments(Named.of(testMediumDocumentFile.getName(), testMediumDocumentFile), JsonSerializerTestFixture.TEST_MEDIUM_DOCUMENT.getSecond())); + final File testMultipleTypeFile = JsonSerializerTestFixture.TEST_MULTIPLE_TYPE.getFirst(); + builder.add(Arguments.arguments(Named.of(testMultipleTypeFile.getName(), testMultipleTypeFile), JsonSerializerTestFixture.TEST_MULTIPLE_TYPE.getSecond())); + final File testMultipleTypeSingleFile = JsonSerializerTestFixture.TEST_MULTIPLE_TYPE_SINGLE.getFirst(); + builder.add(Arguments.arguments(Named.of(testMultipleTypeSingleFile.getName(), testMultipleTypeSingleFile), JsonSerializerTestFixture.TEST_MULTIPLE_TYPE_SINGLE.getSecond())); + final File testNestedCollectionsFile = JsonSerializerTestFixture.TEST_NESTED_COLLECTIONS.getFirst(); + builder.add(Arguments.arguments(Named.of(testNestedCollectionsFile.getName(), testNestedCollectionsFile), JsonSerializerTestFixture.TEST_NESTED_COLLECTIONS.getSecond())); + final File testMapFile = JsonSerializerTestFixture.TEST_MAP.getFirst(); + builder.add(Arguments.arguments(Named.of(testMapFile.getName(), testMapFile), JsonSerializerTestFixture.TEST_MAP.getSecond())); return builder.build(); } @ParameterizedTest @MethodSource("getSchemaToTest") - void recordSerializersTestLogicalTypes(File schemaFile, List fieldValueMappings) throws Exception { - final var schemaStr = readSchema(schemaFile); + final void recordSerializersTestLogicalTypes(final File schemaFile, final List fieldValueMappings) throws Exception { + final var schemaStr = SerializerTestFixture.readSchema(schemaFile); final BaseSchemaMetadata confluentBaseSchemaMetadata = new BaseSchemaMetadata<>( ConfluentSchemaMetadata.parse(new io.confluent.kafka.schemaregistry.client.SchemaMetadata(1, 1, @@ -80,7 +67,7 @@ void recordSerializersTestLogicalTypes(File schemaFile, List JSON_SCHEMA_PROCESSOR.processSchema(SchemaTypeEnum.JSON, parsedSchema, confluentBaseSchemaMetadata, fieldValueMappings); final var objectNode = JSON_SCHEMA_PROCESSOR.next(); - final var message = serializer.serialize("the-topic", (ObjectNode) objectNode); + final var message = SERIALIZER.serialize("the-topic", (ObjectNode) objectNode); Assertions.assertThat(message).isNotNull(); } diff --git a/src/test/java/com/sngular/kloadgen/serializer/JsonSerializerTestFixture.java b/src/test/java/com/sngular/kloadgen/serializer/JsonSerializerTestFixture.java index 91e9baea..9e3da2ac 100644 --- a/src/test/java/com/sngular/kloadgen/serializer/JsonSerializerTestFixture.java +++ b/src/test/java/com/sngular/kloadgen/serializer/JsonSerializerTestFixture.java @@ -1,7 +1,5 @@ package com.sngular.kloadgen.serializer; -import static com.sngular.kloadgen.serializer.SerializerTestFixture.createFieldValueMapping; - import java.io.File; import java.util.Arrays; import java.util.List; @@ -10,102 +8,102 @@ import com.sngular.kloadgen.testutil.FileHelper; import org.apache.commons.math3.util.Pair; -public class JsonSerializerTestFixture { +final class JsonSerializerTestFixture { static final Pair> TEST_BASIC = new Pair<>( new FileHelper().getFile("/jsonschema/basic.jcs"), Arrays.asList( - createFieldValueMapping("firstName", "string"), - createFieldValueMapping("lastName", "string"), - createFieldValueMapping("age", "number")) + SerializerTestFixture.createFieldValueMapping("firstName", "string"), + SerializerTestFixture.createFieldValueMapping("lastName", "string"), + SerializerTestFixture.createFieldValueMapping("age", "number")) ); static final Pair> TEST_BASIC_ARRAY = new Pair<>( new FileHelper().getFile("/jsonschema/basic-array.jcs"), Arrays.asList( - createFieldValueMapping("fruits[]", "string-array"), - createFieldValueMapping("vegetables[].veggieName", "string"), - createFieldValueMapping("vegetables[].veggieLike", "boolean")) + SerializerTestFixture.createFieldValueMapping("fruits[]", "string-array"), + SerializerTestFixture.createFieldValueMapping("vegetables[].veggieName", "string"), + SerializerTestFixture.createFieldValueMapping("vegetables[].veggieLike", "boolean")) ); static final Pair> TEST_BASIC_NUMBER = new Pair<>( new FileHelper().getFile("/jsonschema/basic-number.jcs"), Arrays.asList( - createFieldValueMapping("latitude", "number"), - createFieldValueMapping("longitude", "number")) + SerializerTestFixture.createFieldValueMapping("latitude", "number"), + SerializerTestFixture.createFieldValueMapping("longitude", "number")) ); static final Pair> TEST_COLLECTIONS = new Pair<>( new FileHelper().getFile("/jsonschema/collections.jcs"), Arrays.asList( - createFieldValueMapping("mapOfStrings[:]", "string-map"), - createFieldValueMapping("arrayOfObjectsOfBasicTypes[].stringOfObject", "string"), - createFieldValueMapping("arrayOfObjectsOfBasicTypes[].numberOfObject", "number"), - createFieldValueMapping("objectOfCollectionsOfBasicTypes.arrayOfStrings[]", "string-array"), - createFieldValueMapping("objectOfCollectionsOfBasicTypes.mapOfIntegers[:]", "number-map"), - createFieldValueMapping("objectOfCollectionsOfBasicTypes.stringControl", "string"), - createFieldValueMapping("objectOfCollectionsOfObject.stringControl", "string"), - createFieldValueMapping("objectOfCollectionsOfObject.arrayOfObjectsPerson[].namePerson", "string"), - createFieldValueMapping("objectOfCollectionsOfObject.arrayOfObjectsPerson[].phonePerson", "number"), - createFieldValueMapping("objectOfCollectionsOfObject.mapOfObjectsDog[:].nameDog", "string"), - createFieldValueMapping("objectOfCollectionsOfObject.mapOfObjectsDog[:].vetData.dogId", "number"), - createFieldValueMapping("objectOfCollectionsOfObject.mapOfObjectsDog[:].vetData.breedName", "string")) + SerializerTestFixture.createFieldValueMapping("mapOfStrings[:]", "string-map"), + SerializerTestFixture.createFieldValueMapping("arrayOfObjectsOfBasicTypes[].stringOfObject", "string"), + SerializerTestFixture.createFieldValueMapping("arrayOfObjectsOfBasicTypes[].numberOfObject", "number"), + SerializerTestFixture.createFieldValueMapping("objectOfCollectionsOfBasicTypes.arrayOfStrings[]", "string-array"), + SerializerTestFixture.createFieldValueMapping("objectOfCollectionsOfBasicTypes.mapOfIntegers[:]", "number-map"), + SerializerTestFixture.createFieldValueMapping("objectOfCollectionsOfBasicTypes.stringControl", "string"), + SerializerTestFixture.createFieldValueMapping("objectOfCollectionsOfObject.stringControl", "string"), + SerializerTestFixture.createFieldValueMapping("objectOfCollectionsOfObject.arrayOfObjectsPerson[].namePerson", "string"), + SerializerTestFixture.createFieldValueMapping("objectOfCollectionsOfObject.arrayOfObjectsPerson[].phonePerson", "number"), + SerializerTestFixture.createFieldValueMapping("objectOfCollectionsOfObject.mapOfObjectsDog[:].nameDog", "string"), + SerializerTestFixture.createFieldValueMapping("objectOfCollectionsOfObject.mapOfObjectsDog[:].vetData.dogId", "number"), + SerializerTestFixture.createFieldValueMapping("objectOfCollectionsOfObject.mapOfObjectsDog[:].vetData.breedName", "string")) ); static final Pair> TEST_COMPLEX_DEFINITIONS = new Pair<>( new FileHelper().getFile("/jsonschema/complex-definitions.jcs"), Arrays.asList( - createFieldValueMapping("objectOfDefinitions.stringControl", "string"), - createFieldValueMapping("objectOfDefinitions.arrayOfStrings[]", "string-array"), - createFieldValueMapping("objectOfDefinitions.mapOfStrings[:]", "string-map"), - createFieldValueMapping("arrayOfObjects[].stringOfObject", "string"), - createFieldValueMapping("arrayOfObjects[].numberOfObject", "number"), - createFieldValueMapping("mapOfObjects[:].arrayOfInternalObject[]", "string-array"), - createFieldValueMapping("mapOfMaps[:][:].stringControlObject", "string"), - createFieldValueMapping("mapOfMaps[:][:].arrayOfArraysOfStrings[][]", "string-array-array")) + SerializerTestFixture.createFieldValueMapping("objectOfDefinitions.stringControl", "string"), + SerializerTestFixture.createFieldValueMapping("objectOfDefinitions.arrayOfStrings[]", "string-array"), + SerializerTestFixture.createFieldValueMapping("objectOfDefinitions.mapOfStrings[:]", "string-map"), + SerializerTestFixture.createFieldValueMapping("arrayOfObjects[].stringOfObject", "string"), + SerializerTestFixture.createFieldValueMapping("arrayOfObjects[].numberOfObject", "number"), + SerializerTestFixture.createFieldValueMapping("mapOfObjects[:].arrayOfInternalObject[]", "string-array"), + SerializerTestFixture.createFieldValueMapping("mapOfMaps[:][:].stringControlObject", "string"), + SerializerTestFixture.createFieldValueMapping("mapOfMaps[:][:].arrayOfArraysOfStrings[][]", "string-array-array")) ); static final Pair> TEST_COMPLEX_DOCUMENT = new Pair<>( new FileHelper().getFile("/jsonschema/complex-document.jcs"), Arrays.asList( - createFieldValueMapping("_id", "string"), - createFieldValueMapping("userId", "number"), - createFieldValueMapping("storeId", "number"), - createFieldValueMapping("snapshotId", "string"), - createFieldValueMapping("addressId", "string"), - createFieldValueMapping("addressLine", "string"), - createFieldValueMapping("alias", "string"), - createFieldValueMapping("contactInformation.email", "string"), - createFieldValueMapping("contactInformation.firstName", "string"), - createFieldValueMapping("contactInformation.middleName", "string"), - createFieldValueMapping("contactInformation.lastName", "string"), - createFieldValueMapping("contactInformation.honorific", "string"), - createFieldValueMapping("contactInformation.phones[].prefix", "string"), - createFieldValueMapping("contactInformation.phones[].number", "string"), - createFieldValueMapping("countryCode", "string"), - createFieldValueMapping("location.streetName", "string"), - createFieldValueMapping("location.streetNumber", "string"), - createFieldValueMapping("location.floor", "string"), - createFieldValueMapping("location.door", "string"), - createFieldValueMapping("location.doorCode", "string"), - createFieldValueMapping("location.zipCode", "string"), - createFieldValueMapping("geopoliticalSubdivisions.level1.code", "string"), - createFieldValueMapping("geopoliticalSubdivisions.level1.freeForm", "string"), - createFieldValueMapping("geopoliticalSubdivisions.level2.code", "string"), - createFieldValueMapping("geopoliticalSubdivisions.level2.freeForm", "string"), - createFieldValueMapping("geopoliticalSubdivisions.level3.code", "string"), - createFieldValueMapping("geopoliticalSubdivisions.level3.freeForm", "string"), - createFieldValueMapping("geopoliticalSubdivisions.level4.code", "string"), - createFieldValueMapping("geopoliticalSubdivisions.level4.freeForm", "string"), - createFieldValueMapping("_metadata.createdAt", "timestamp"), - createFieldValueMapping("_metadata.createdBy", "string"), - createFieldValueMapping("_metadata.lastUpdatedAt", "timestamp"), - createFieldValueMapping("_metadata.lastUpdatedBy", "string"), - createFieldValueMapping("_metadata.deletedAt", "timestamp"), - createFieldValueMapping("_metadata.projectVersion", "string"), - createFieldValueMapping("_metadata.projectName", "string"), - createFieldValueMapping("_metadata.deletedBy", "string"), - createFieldValueMapping("_metadata.schema", "number"), + SerializerTestFixture.createFieldValueMapping("_id", "string"), + SerializerTestFixture.createFieldValueMapping("userId", "number"), + SerializerTestFixture.createFieldValueMapping("storeId", "number"), + SerializerTestFixture.createFieldValueMapping("snapshotId", "string"), + SerializerTestFixture.createFieldValueMapping("addressId", "string"), + SerializerTestFixture.createFieldValueMapping("addressLine", "string"), + SerializerTestFixture.createFieldValueMapping("alias", "string"), + SerializerTestFixture.createFieldValueMapping("contactInformation.email", "string"), + SerializerTestFixture.createFieldValueMapping("contactInformation.firstName", "string"), + SerializerTestFixture.createFieldValueMapping("contactInformation.middleName", "string"), + SerializerTestFixture.createFieldValueMapping("contactInformation.lastName", "string"), + SerializerTestFixture.createFieldValueMapping("contactInformation.honorific", "string"), + SerializerTestFixture.createFieldValueMapping("contactInformation.phones[].prefix", "string"), + SerializerTestFixture.createFieldValueMapping("contactInformation.phones[].number", "string"), + SerializerTestFixture.createFieldValueMapping("countryCode", "string"), + SerializerTestFixture.createFieldValueMapping("location.streetName", "string"), + SerializerTestFixture.createFieldValueMapping("location.streetNumber", "string"), + SerializerTestFixture.createFieldValueMapping("location.floor", "string"), + SerializerTestFixture.createFieldValueMapping("location.door", "string"), + SerializerTestFixture.createFieldValueMapping("location.doorCode", "string"), + SerializerTestFixture.createFieldValueMapping("location.zipCode", "string"), + SerializerTestFixture.createFieldValueMapping("geopoliticalSubdivisions.level1.code", "string"), + SerializerTestFixture.createFieldValueMapping("geopoliticalSubdivisions.level1.freeForm", "string"), + SerializerTestFixture.createFieldValueMapping("geopoliticalSubdivisions.level2.code", "string"), + SerializerTestFixture.createFieldValueMapping("geopoliticalSubdivisions.level2.freeForm", "string"), + SerializerTestFixture.createFieldValueMapping("geopoliticalSubdivisions.level3.code", "string"), + SerializerTestFixture.createFieldValueMapping("geopoliticalSubdivisions.level3.freeForm", "string"), + SerializerTestFixture.createFieldValueMapping("geopoliticalSubdivisions.level4.code", "string"), + SerializerTestFixture.createFieldValueMapping("geopoliticalSubdivisions.level4.freeForm", "string"), + SerializerTestFixture.createFieldValueMapping("_metadata.createdAt", "timestamp"), + SerializerTestFixture.createFieldValueMapping("_metadata.createdBy", "string"), + SerializerTestFixture.createFieldValueMapping("_metadata.lastUpdatedAt", "timestamp"), + SerializerTestFixture.createFieldValueMapping("_metadata.lastUpdatedBy", "string"), + SerializerTestFixture.createFieldValueMapping("_metadata.deletedAt", "timestamp"), + SerializerTestFixture.createFieldValueMapping("_metadata.projectVersion", "string"), + SerializerTestFixture.createFieldValueMapping("_metadata.projectName", "string"), + SerializerTestFixture.createFieldValueMapping("_metadata.deletedBy", "string"), + SerializerTestFixture.createFieldValueMapping("_metadata.schema", "number"), FieldValueMapping.builder().fieldName("_entity").fieldType("enum").valueLength(0).fieldValueList("[AddressSnapshot]").required(true) .isAncestorRequired(true).build(), FieldValueMapping.builder().fieldName("_class").fieldType("enum").valueLength(0).fieldValueList("[AddressSnapshot]").required(true) @@ -119,21 +117,21 @@ public class JsonSerializerTestFixture { .isAncestorRequired(true).build(), FieldValueMapping.builder().fieldName("_entity").fieldType("enum").valueLength(0).fieldValueList("[OrderPublicDetailDocument]").required(true) .isAncestorRequired(true).build(), - createFieldValueMapping("_metadata.createdAt", "number"), - createFieldValueMapping("_metadata.createdBy", "string"), - createFieldValueMapping("_metadata.deletedAt", "number"), - createFieldValueMapping("_metadata.deletedBy", "string"), - createFieldValueMapping("_metadata.lastUpdatedAt", "number"), - createFieldValueMapping("_metadata.lastUpdatedBy", "string"), - createFieldValueMapping("_metadata.projectName", "string"), - createFieldValueMapping("_metadata.projectVersion", "string"), - createFieldValueMapping("_metadata.schema", "number"), - createFieldValueMapping("orderId", "string"), - createFieldValueMapping("storeId", "number"), - createFieldValueMapping("paymentId", "string"), - createFieldValueMapping("parentId", "string"), - createFieldValueMapping("externalId", "string"), - createFieldValueMapping("editorId", "number"), + SerializerTestFixture.createFieldValueMapping("_metadata.createdAt", "number"), + SerializerTestFixture.createFieldValueMapping("_metadata.createdBy", "string"), + SerializerTestFixture.createFieldValueMapping("_metadata.deletedAt", "number"), + SerializerTestFixture.createFieldValueMapping("_metadata.deletedBy", "string"), + SerializerTestFixture.createFieldValueMapping("_metadata.lastUpdatedAt", "number"), + SerializerTestFixture.createFieldValueMapping("_metadata.lastUpdatedBy", "string"), + SerializerTestFixture.createFieldValueMapping("_metadata.projectName", "string"), + SerializerTestFixture.createFieldValueMapping("_metadata.projectVersion", "string"), + SerializerTestFixture.createFieldValueMapping("_metadata.schema", "number"), + SerializerTestFixture.createFieldValueMapping("orderId", "string"), + SerializerTestFixture.createFieldValueMapping("storeId", "number"), + SerializerTestFixture.createFieldValueMapping("paymentId", "string"), + SerializerTestFixture.createFieldValueMapping("parentId", "string"), + SerializerTestFixture.createFieldValueMapping("externalId", "string"), + SerializerTestFixture.createFieldValueMapping("editorId", "number"), FieldValueMapping.builder().fieldName("type").fieldType("enum").valueLength(0) .fieldValueList("[REGULAR, REPLACEMENT, EXCHANGE, REPOSITION, ONLINE_EXCHANGE, STORE_EXCHANGE]").required(true) .isAncestorRequired(true).build(), @@ -151,111 +149,113 @@ public class JsonSerializerTestFixture { .isAncestorRequired(true).build(), FieldValueMapping.builder().fieldName("manualReviewStatus").fieldType("enum").valueLength(0).fieldValueList("[NA, PENDING, ACCEPTED, REFUSED]").required(true) .isAncestorRequired(true).build(), - createFieldValueMapping("customerId", "number"), + SerializerTestFixture.createFieldValueMapping("customerId", "number"), FieldValueMapping.builder().fieldName("customerType").fieldType("enum").valueLength(0).fieldValueList("[REGISTERED, GUEST]").required(true) .isAncestorRequired(true).build(), FieldValueMapping.builder().fieldName("guestAction").fieldType("enum").valueLength(0).fieldValueList("[NO_ACTION, CONVERSION, LINK_ACCOUNT]").required(true) .isAncestorRequired(true).build(), - createFieldValueMapping("locale", "string"), - createFieldValueMapping("preorder", "boolean"), - createFieldValueMapping("fastSint", "boolean"), - createFieldValueMapping("pvpTaxesIncluded", "boolean"), - createFieldValueMapping("placedDatetime", "string"), - createFieldValueMapping("cancelledDatetime", "string"), - createFieldValueMapping("lastUpdateDatetime", "string"), - createFieldValueMapping("snapshotDatetime", "string"), - createFieldValueMapping("duty.amount.value", "number"), - createFieldValueMapping("duty.amount.currency", "string"), - createFieldValueMapping("duty.amount.exponent", "number"), - createFieldValueMapping("duty.confirmedAmount.value", "number"), - createFieldValueMapping("duty.confirmedAmount.currency", "string"), - createFieldValueMapping("duty.confirmedAmount.exponent", "number"), + SerializerTestFixture.createFieldValueMapping("locale", "string"), + SerializerTestFixture.createFieldValueMapping("preorder", "boolean"), + SerializerTestFixture.createFieldValueMapping("fastSint", "boolean"), + SerializerTestFixture.createFieldValueMapping("pvpTaxesIncluded", "boolean"), + SerializerTestFixture.createFieldValueMapping("placedDatetime", "string"), + SerializerTestFixture.createFieldValueMapping("cancelledDatetime", "string"), + SerializerTestFixture.createFieldValueMapping("lastUpdateDatetime", "string"), + SerializerTestFixture.createFieldValueMapping("snapshotDatetime", "string"), + SerializerTestFixture.createFieldValueMapping("duty.amount.value", "number"), + SerializerTestFixture.createFieldValueMapping("duty.amount.currency", "string"), + SerializerTestFixture.createFieldValueMapping("duty.amount.exponent", "number"), + SerializerTestFixture.createFieldValueMapping("duty.confirmedAmount.value", "number"), + SerializerTestFixture.createFieldValueMapping("duty.confirmedAmount.currency", "string"), + SerializerTestFixture.createFieldValueMapping("duty.confirmedAmount.exponent", "number"), FieldValueMapping.builder().fieldName("billing.status").fieldType("enum").valueLength(0).fieldValueList("[NOT_APPLIED, PENDING, BILLED]").required(true) .isAncestorRequired(true).build(), - createFieldValueMapping("billing.addressId", "string"), - createFieldValueMapping("origin.systemCode", "string"), + SerializerTestFixture.createFieldValueMapping("billing.addressId", "string"), + SerializerTestFixture.createFieldValueMapping("origin.systemCode", "string"), FieldValueMapping.builder().fieldName("origin.systemType").fieldType("enum").valueLength(0).fieldValueList("[CHECKOUT, MPS, BACKOFFICE, STORE]").required(true) .isAncestorRequired(true).build(), - createFieldValueMapping("origin.systemUser", "string"), - createFieldValueMapping("origin.systemDeviceId", "string"), - createFieldValueMapping("origin.deviceType", "string"), - createFieldValueMapping("origin.deviceUserAgent", "string"), - createFieldValueMapping("orderItems[].id", "string"), - createFieldValueMapping("orderItems[].partNumber", "string"), + SerializerTestFixture.createFieldValueMapping("origin.systemUser", "string"), + SerializerTestFixture.createFieldValueMapping("origin.systemDeviceId", "string"), + SerializerTestFixture.createFieldValueMapping("origin.deviceType", "string"), + SerializerTestFixture.createFieldValueMapping("origin.deviceUserAgent", "string"), + SerializerTestFixture.createFieldValueMapping("orderItems[].id", "string"), + SerializerTestFixture.createFieldValueMapping("orderItems[].partNumber", "string"), FieldValueMapping.builder().fieldName("orderItems[].type").fieldType("enum").valueLength(0) .fieldValueList("[PRODUCT, VIRTUAL_GIFT_CARD, PHYSICAL_GIFT_CARD, SHIPPING_COST, SERVICE]").required(true) .isAncestorRequired(true).build(), FieldValueMapping.builder().fieldName("orderItems[].status").fieldType("enum").valueLength(0).fieldValueList("[RECEIVED, PREPARING, SHIPPED, CLOSED, CANCELLED]") .required(true) .isAncestorRequired(true).build(), - createFieldValueMapping("orderItems[].statusReason", "string"), - createFieldValueMapping("orderItems[].statusDatetime", "string"), - createFieldValueMapping("orderItems[].pvpAmount.value", "number"), - createFieldValueMapping("orderItems[].pvpAmount.currency", "string"), - createFieldValueMapping("orderItems[].pvpAmount.exponent", "number"), - createFieldValueMapping("orderItems[].taxesAmount.value", "number"), - createFieldValueMapping("orderItems[].taxesAmount.currency", "string"), - createFieldValueMapping("orderItems[].taxesAmount.exponent", "number"), - createFieldValueMapping("orderItems[].taxesPercentage", "number"), - createFieldValueMapping("orderItems[].initialMinimumDate", "string"), - createFieldValueMapping("orderItems[].initialMaximumDate", "string"), - createFieldValueMapping("orderItems[].adjustment.value", "number"), - createFieldValueMapping("orderItems[].adjustment.currency", "string"), - createFieldValueMapping("orderItems[].adjustment.exponent", "number"), - createFieldValueMapping("orderItems[].product.catalogEntryId", "number"), + SerializerTestFixture.createFieldValueMapping("orderItems[].statusReason", "string"), + SerializerTestFixture.createFieldValueMapping("orderItems[].statusDatetime", "string"), + SerializerTestFixture.createFieldValueMapping("orderItems[].pvpAmount.value", "number"), + SerializerTestFixture.createFieldValueMapping("orderItems[].pvpAmount.currency", "string"), + SerializerTestFixture.createFieldValueMapping("orderItems[].pvpAmount.exponent", "number"), + SerializerTestFixture.createFieldValueMapping("orderItems[].taxesAmount.value", "number"), + SerializerTestFixture.createFieldValueMapping("orderItems[].taxesAmount.currency", "string"), + SerializerTestFixture.createFieldValueMapping("orderItems[].taxesAmount.exponent", "number"), + SerializerTestFixture.createFieldValueMapping("orderItems[].taxesPercentage", "number"), + SerializerTestFixture.createFieldValueMapping("orderItems[].initialMinimumDate", "string"), + SerializerTestFixture.createFieldValueMapping("orderItems[].initialMaximumDate", "string"), + SerializerTestFixture.createFieldValueMapping("orderItems[].adjustment.value", "number"), + SerializerTestFixture.createFieldValueMapping("orderItems[].adjustment.currency", "string"), + SerializerTestFixture.createFieldValueMapping("orderItems[].adjustment.exponent", "number"), + SerializerTestFixture.createFieldValueMapping("orderItems[].product.catalogEntryId", "number"), FieldValueMapping.builder().fieldName("orderItems[].product.stockMode").fieldType("enum").valueLength(0) .fieldValueList("[UNKNOWN, NORMAL, TRANSIT_PRESALE, VIRTUALSTOCK_PRESALE, SOD_PRESALE, PREORDER]").required(true) .isAncestorRequired(true).build(), - createFieldValueMapping("orderItems[].virtualGiftCard.catalogEntryId", "number"), - createFieldValueMapping("orderItems[].virtualGiftCard.senderName", "string"), - createFieldValueMapping("orderItems[].virtualGiftCard.message", "string"), - createFieldValueMapping("orderItems[].virtualGiftCard.style", "string"), - createFieldValueMapping("orderItems[].virtualGiftCard.receiverName", "string"), - createFieldValueMapping("orderItems[].virtualGiftCard.deliveryDate", "string"), - createFieldValueMapping("orderItems[].physicalGiftCard.catalogEntryId", "number"), - createFieldValueMapping("orderItems[].physicalGiftCard.senderName", "string"), - createFieldValueMapping("orderItems[].physicalGiftCard.message", "string"), - createFieldValueMapping("orderItems[].physicalGiftCard.receiverPhonePrefix", "string"), - createFieldValueMapping("orderItems[].physicalGiftCard.receiverPhoneNumber", "string")) + SerializerTestFixture.createFieldValueMapping("orderItems[].virtualGiftCard.catalogEntryId", "number"), + SerializerTestFixture.createFieldValueMapping("orderItems[].virtualGiftCard.senderName", "string"), + SerializerTestFixture.createFieldValueMapping("orderItems[].virtualGiftCard.message", "string"), + SerializerTestFixture.createFieldValueMapping("orderItems[].virtualGiftCard.style", "string"), + SerializerTestFixture.createFieldValueMapping("orderItems[].virtualGiftCard.receiverName", "string"), + SerializerTestFixture.createFieldValueMapping("orderItems[].virtualGiftCard.deliveryDate", "string"), + SerializerTestFixture.createFieldValueMapping("orderItems[].physicalGiftCard.catalogEntryId", "number"), + SerializerTestFixture.createFieldValueMapping("orderItems[].physicalGiftCard.senderName", "string"), + SerializerTestFixture.createFieldValueMapping("orderItems[].physicalGiftCard.message", "string"), + SerializerTestFixture.createFieldValueMapping("orderItems[].physicalGiftCard.receiverPhonePrefix", "string"), + SerializerTestFixture.createFieldValueMapping("orderItems[].physicalGiftCard.receiverPhoneNumber", "string")) ); static final Pair> TEST_MULTIPLE_TYPE = new Pair<>( new FileHelper().getFile("/jsonschema/multiple-type.jcs"), Arrays.asList( - createFieldValueMapping("id", "number"), - createFieldValueMapping("version", "number"), - createFieldValueMapping("dtype", "string"), - createFieldValueMapping("timestamp", "string"), - createFieldValueMapping("event_type", "string")) + SerializerTestFixture.createFieldValueMapping("id", "number"), + SerializerTestFixture.createFieldValueMapping("version", "number"), + SerializerTestFixture.createFieldValueMapping("dtype", "string"), + SerializerTestFixture.createFieldValueMapping("timestamp", "string"), + SerializerTestFixture.createFieldValueMapping("event_type", "string")) ); static final Pair> TEST_MULTIPLE_TYPE_SINGLE = new Pair<>( new FileHelper().getFile("/jsonschema/multiple-type-single.jcs"), Arrays.asList( - createFieldValueMapping("id", "number")) + SerializerTestFixture.createFieldValueMapping("id", "number")) ); static final Pair> TEST_NESTED_COLLECTIONS = new Pair<>( new FileHelper().getFile("/jsonschema/nested-collections.jcs"), Arrays.asList( - createFieldValueMapping("arrayOfMapsOfObjects[][:].stringObject", "string"), - createFieldValueMapping("arrayOfMapsOfObjects[][:].numberObject", "number"), - createFieldValueMapping("arrayOfArraysOfStrings[][]", "string-array-array"), - createFieldValueMapping("mapOfArraysOfStrings[:][]", "string-array-map"), - createFieldValueMapping("mapOfMapsOfObjects[:][:].name4Object", "string"), - createFieldValueMapping("mapOfMapsOfObjects[:][:].number4Object", "number"), - createFieldValueMapping("mapOfObjectsOfCollections[:].arrayOfMapsOfObject[][:].stringControl", "string"), - createFieldValueMapping("mapOfObjectsOfCollections[:].arrayOfMapsOfObject[][:].numberControl", "number")) + SerializerTestFixture.createFieldValueMapping("arrayOfMapsOfObjects[][:].stringObject", "string"), + SerializerTestFixture.createFieldValueMapping("arrayOfMapsOfObjects[][:].numberObject", "number"), + SerializerTestFixture.createFieldValueMapping("arrayOfArraysOfStrings[][]", "string-array-array"), + SerializerTestFixture.createFieldValueMapping("mapOfArraysOfStrings[:][]", "string-array-map"), + SerializerTestFixture.createFieldValueMapping("mapOfMapsOfObjects[:][:].name4Object", "string"), + SerializerTestFixture.createFieldValueMapping("mapOfMapsOfObjects[:][:].number4Object", "number"), + SerializerTestFixture.createFieldValueMapping("mapOfObjectsOfCollections[:].arrayOfMapsOfObject[][:].stringControl", "string"), + SerializerTestFixture.createFieldValueMapping("mapOfObjectsOfCollections[:].arrayOfMapsOfObject[][:].numberControl", "number")) ); static final Pair> TEST_MAP = new Pair<>( new FileHelper().getFile("/jsonschema/test-map.jcs"), Arrays.asList( - createFieldValueMapping("firstName", "string"), - createFieldValueMapping("lastName", "string"), - createFieldValueMapping("age", "number"), - createFieldValueMapping("testMap.itemType[:]", "number-map"), - createFieldValueMapping("testMap.itemTipo[:]", "string-map")) + SerializerTestFixture.createFieldValueMapping("firstName", "string"), + SerializerTestFixture.createFieldValueMapping("lastName", "string"), + SerializerTestFixture.createFieldValueMapping("age", "number"), + SerializerTestFixture.createFieldValueMapping("testMap.itemType[:]", "number-map"), + SerializerTestFixture.createFieldValueMapping("testMap.itemTipo[:]", "string-map")) ); + private JsonSerializerTestFixture() { + } } diff --git a/src/test/java/com/sngular/kloadgen/testutil/FileHelper.java b/src/test/java/com/sngular/kloadgen/testutil/FileHelper.java index 899ec446..1e8b7f06 100644 --- a/src/test/java/com/sngular/kloadgen/testutil/FileHelper.java +++ b/src/test/java/com/sngular/kloadgen/testutil/FileHelper.java @@ -10,6 +10,7 @@ import java.io.IOException; import java.net.URL; import java.nio.charset.StandardCharsets; + import org.apache.commons.io.IOUtils; public final class FileHelper { From 8b8e936274f12d05592e8e615c1fa224d87b3d23 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jose=20Enrique=20Garc=C3=ADa=20Maci=C3=B1eiras?= Date: Tue, 3 Oct 2023 18:07:35 +0200 Subject: [PATCH 23/30] #332 Fix styles and test --- .../extractors/ExtractorFactory.java | 1 - .../json/AbstractJsonExtractor.java | 9 +- .../protobuff/ProtobuffExtractor.java | 11 +- .../adapter/impl/BaseSchemaMetadata.java | 5 +- .../schema/KLoadSchemaMetadata.java | 4 - .../ProtobuffSerializerTestFixture.java | 239 +++++++++--------- 6 files changed, 131 insertions(+), 138 deletions(-) delete mode 100644 src/main/java/com/sngular/kloadgen/schemaregistry/schema/KLoadSchemaMetadata.java diff --git a/src/main/java/com/sngular/kloadgen/extractor/extractors/ExtractorFactory.java b/src/main/java/com/sngular/kloadgen/extractor/extractors/ExtractorFactory.java index 9ff5903f..371d2de4 100644 --- a/src/main/java/com/sngular/kloadgen/extractor/extractors/ExtractorFactory.java +++ b/src/main/java/com/sngular/kloadgen/extractor/extractors/ExtractorFactory.java @@ -77,7 +77,6 @@ public static Pair> flatPropertiesList(final Str schema = switch (schemaRegistryEnum) { case APICURIO -> ((ApicurioAbstractParsedSchemaMetadata) abstractParsedSchemaAdapter).getSchema(); case CONFLUENT -> abstractParsedSchemaAdapter.getRawSchema(); - default -> throw new KLoadGenException("Schema Registry Type nos supported " + registryName.toUpperCase()); }; attributeList.addAll(getExtractor(schemaType).processSchema(schema, schemaRegistryEnum)); } diff --git a/src/main/java/com/sngular/kloadgen/extractor/extractors/json/AbstractJsonExtractor.java b/src/main/java/com/sngular/kloadgen/extractor/extractors/json/AbstractJsonExtractor.java index 0e8f865c..07a7525b 100644 --- a/src/main/java/com/sngular/kloadgen/extractor/extractors/json/AbstractJsonExtractor.java +++ b/src/main/java/com/sngular/kloadgen/extractor/extractors/json/AbstractJsonExtractor.java @@ -189,12 +189,9 @@ private List extractMapInternalFields( completeFieldList.add( FieldValueMapping.builder() .fieldName(name) - .fieldType(value.getType() + "-map" + ((StringUtils.isNotEmpty(breadCrumb) && breadCrumb.endsWith("[:]")) ? "-map" : - (StringUtils.isNotEmpty( - breadCrumb) && - breadCrumb.endsWith("[]")) - ? - SchemaExtractorUtil.ARRAY_TYPE_POSTFIX + .fieldType(value.getType() + "-map" + ((StringUtils.isNotEmpty(breadCrumb) + && breadCrumb.endsWith("[:]")) ? "-map" + : (StringUtils.isNotEmpty(breadCrumb) && breadCrumb.endsWith("[]")) ? SchemaExtractorUtil.ARRAY_TYPE_POSTFIX : breadCrumb)) .required(!name.endsWith("][]") && !name.endsWith("][:]") && innerField.isFieldRequired()) .isAncestorRequired(!isRootElement && isAncestorRequired != null && isAncestorRequired) diff --git a/src/main/java/com/sngular/kloadgen/extractor/extractors/protobuff/ProtobuffExtractor.java b/src/main/java/com/sngular/kloadgen/extractor/extractors/protobuff/ProtobuffExtractor.java index 9989903a..d95212cc 100644 --- a/src/main/java/com/sngular/kloadgen/extractor/extractors/protobuff/ProtobuffExtractor.java +++ b/src/main/java/com/sngular/kloadgen/extractor/extractors/protobuff/ProtobuffExtractor.java @@ -12,20 +12,21 @@ public class ProtobuffExtractor implements ExtractorRegistry { - static Map schemaRegistryMap = Map.of(SchemaRegistryEnum.CONFLUENT, new ProtoBufConfluentExtractor(), - SchemaRegistryEnum.APICURIO, new ProtoBufApicurioExtractor()); + private static final Map SCHEMA_REGISTRY_MAP = Map.of(SchemaRegistryEnum.CONFLUENT, new ProtoBufConfluentExtractor(), + SchemaRegistryEnum.APICURIO, new ProtoBufApicurioExtractor()); public final List processSchema(final Object schemaReceived, final SchemaRegistryEnum registryEnum) { final var resultSchema = new ArrayList(); if (schemaReceived instanceof ProtoFileElement) { - resultSchema.addAll(schemaRegistryMap.get(SchemaRegistryEnum.APICURIO).processSchema(schemaReceived)); + resultSchema.addAll(SCHEMA_REGISTRY_MAP.get(SchemaRegistryEnum.APICURIO).processSchema(schemaReceived)); + } else { + resultSchema.addAll(SCHEMA_REGISTRY_MAP.get(registryEnum).processSchema(schemaReceived)); } - resultSchema.addAll(schemaRegistryMap.get(registryEnum).processSchema(schemaReceived)); return resultSchema; } public final List getSchemaNameList(final String schema, final SchemaRegistryEnum registryEnum) { - return schemaRegistryMap.get(registryEnum).getSchemaNameList(schema); + return SCHEMA_REGISTRY_MAP.get(registryEnum).getSchemaNameList(schema); } } \ No newline at end of file diff --git a/src/main/java/com/sngular/kloadgen/schemaregistry/adapter/impl/BaseSchemaMetadata.java b/src/main/java/com/sngular/kloadgen/schemaregistry/adapter/impl/BaseSchemaMetadata.java index 9eee1d8a..f4c04009 100644 --- a/src/main/java/com/sngular/kloadgen/schemaregistry/adapter/impl/BaseSchemaMetadata.java +++ b/src/main/java/com/sngular/kloadgen/schemaregistry/adapter/impl/BaseSchemaMetadata.java @@ -1,7 +1,9 @@ package com.sngular.kloadgen.schemaregistry.adapter.impl; import lombok.Builder; +import lombok.Getter; +@Getter @Builder public final class BaseSchemaMetadata { @@ -11,7 +13,4 @@ public BaseSchemaMetadata(final T schemaMetadataAdapter) { this.schemaMetadataAdapter = schemaMetadataAdapter; } - public T getSchemaMetadataAdapter() { - return schemaMetadataAdapter; - } } diff --git a/src/main/java/com/sngular/kloadgen/schemaregistry/schema/KLoadSchemaMetadata.java b/src/main/java/com/sngular/kloadgen/schemaregistry/schema/KLoadSchemaMetadata.java deleted file mode 100644 index fe1342e0..00000000 --- a/src/main/java/com/sngular/kloadgen/schemaregistry/schema/KLoadSchemaMetadata.java +++ /dev/null @@ -1,4 +0,0 @@ -package com.sngular.kloadgen.schemaregistry.schema; - -public record KLoadSchemaMetadata(String id, String version, String schemaType) { -} diff --git a/src/test/java/com/sngular/kloadgen/serializer/ProtobuffSerializerTestFixture.java b/src/test/java/com/sngular/kloadgen/serializer/ProtobuffSerializerTestFixture.java index 67276d8a..136a8ca7 100644 --- a/src/test/java/com/sngular/kloadgen/serializer/ProtobuffSerializerTestFixture.java +++ b/src/test/java/com/sngular/kloadgen/serializer/ProtobuffSerializerTestFixture.java @@ -1,7 +1,5 @@ package com.sngular.kloadgen.serializer; -import static com.sngular.kloadgen.serializer.SerializerTestFixture.createFieldValueMapping; - import java.io.File; import java.util.Arrays; import java.util.List; @@ -15,16 +13,16 @@ public class ProtobuffSerializerTestFixture { static final Pair> TEST_COMPLETE_PROTO = new Pair<>( new FileHelper().getFile("/proto-files/completeProto.proto"), Arrays.asList( - createFieldValueMapping("name", "string"), - createFieldValueMapping("id", "int"), - createFieldValueMapping("addressesArray[].id[]", "string-array"), - createFieldValueMapping("addressesArray[].zipcode", "long"), - createFieldValueMapping("addressesDot[].id[]", "string-array"), - createFieldValueMapping("addressesDot[].zipcode", "long"), - createFieldValueMapping("addressesMap[:].id[]", "string-array"), - createFieldValueMapping("addressesMap[:].zipcode", "long"), - createFieldValueMapping("addressesNoDotMap[:].id[]", "string-array"), - createFieldValueMapping("addressesNoDotMap[:].zipcode", "long"), + SerializerTestFixture.createFieldValueMapping("name", "string"), + SerializerTestFixture.createFieldValueMapping("id", "int"), + SerializerTestFixture.createFieldValueMapping("addressesArray[].id[]", "string-array"), + SerializerTestFixture.createFieldValueMapping("addressesArray[].zipcode", "long"), + SerializerTestFixture.createFieldValueMapping("addressesDot[].id[]", "string-array"), + SerializerTestFixture.createFieldValueMapping("addressesDot[].zipcode", "long"), + SerializerTestFixture.createFieldValueMapping("addressesMap[:].id[]", "string-array"), + SerializerTestFixture.createFieldValueMapping("addressesMap[:].zipcode", "long"), + SerializerTestFixture.createFieldValueMapping("addressesNoDotMap[:].id[]", "string-array"), + SerializerTestFixture.createFieldValueMapping("addressesNoDotMap[:].zipcode", "long"), FieldValueMapping.builder().fieldName("phones[]").fieldType("enum").valueLength(0).fieldValueList("[MOBILE, HOME, WORK]").required(true) .isAncestorRequired(true).build()) ); @@ -32,100 +30,100 @@ public class ProtobuffSerializerTestFixture { static final Pair> TEST_COMPLEX = new Pair<>( new FileHelper().getFile("/proto-files/complexTest.proto"), Arrays.asList( - createFieldValueMapping("phone_types[].phone", "long"), - createFieldValueMapping("phone_types[].principal", "boolean"), - createFieldValueMapping("name", "string"), - createFieldValueMapping("age", "int"), - createFieldValueMapping("address[].street[]", "string-array"), - createFieldValueMapping("address[].number_street", "int"), - createFieldValueMapping("pets[:].pet_name", "string"), - createFieldValueMapping("pets[:].pet_age", "int"), - createFieldValueMapping("pets[:].owner", "string"), - createFieldValueMapping("descriptors[:]", "string-map"), - createFieldValueMapping("dates[]", "string-array"), - createFieldValueMapping("response", "string"), - createFieldValueMapping("presents[:].options[]", "string-array")) + SerializerTestFixture.createFieldValueMapping("phone_types[].phone", "long"), + SerializerTestFixture.createFieldValueMapping("phone_types[].principal", "boolean"), + SerializerTestFixture.createFieldValueMapping("name", "string"), + SerializerTestFixture.createFieldValueMapping("age", "int"), + SerializerTestFixture.createFieldValueMapping("address[].street[]", "string-array"), + SerializerTestFixture.createFieldValueMapping("address[].number_street", "int"), + SerializerTestFixture.createFieldValueMapping("pets[:].pet_name", "string"), + SerializerTestFixture.createFieldValueMapping("pets[:].pet_age", "int"), + SerializerTestFixture.createFieldValueMapping("pets[:].owner", "string"), + SerializerTestFixture.createFieldValueMapping("descriptors[:]", "string-map"), + SerializerTestFixture.createFieldValueMapping("dates[]", "string-array"), + SerializerTestFixture.createFieldValueMapping("response", "string"), + SerializerTestFixture.createFieldValueMapping("presents[:].options[]", "string-array")) ); static final Pair> TEST_DATE_TIME = new Pair<>( new FileHelper().getFile("/proto-files/dateTimeTest.proto"), Arrays.asList( - createFieldValueMapping("incident_date", ".google.type.Date"), - createFieldValueMapping("incident_time", ".google.type.TimeOfDay")) + SerializerTestFixture.createFieldValueMapping("incident_date", ".google.type.Date"), + SerializerTestFixture.createFieldValueMapping("incident_time", ".google.type.TimeOfDay")) ); static final Pair> TEST_DEVE = new Pair<>( new FileHelper().getFile("/proto-files/deveTest.proto"), Arrays.asList( - createFieldValueMapping("load_type", "string"), + SerializerTestFixture.createFieldValueMapping("load_type", "string"), FieldValueMapping.builder().fieldName("shipment.carrier_identifier.type").fieldType("enum").valueLength(0) .fieldValueList("[CARRIER_IDENTIFIER_TYPE_UNSPECIFIED, CARRIER_IDENTIFIER_TYPE_DOT_NUMBER]").required(true) .isAncestorRequired(true).build(), - createFieldValueMapping("shipment.carrier_identifier.value", "string"), + SerializerTestFixture.createFieldValueMapping("shipment.carrier_identifier.value", "string"), FieldValueMapping.builder().fieldName("shipment.shipment_identifiers[].type").fieldType("enum").valueLength(0) .fieldValueList("[SHIPMENT_IDENTIFIER_TYPE_UNSPECIFIED, SHIPMENT_IDENTIFIER_TYPE_BILL_OF_LADING, SHIPMENT_IDENTIFIER_TYPE_ORDER]").required(true) .isAncestorRequired(true).build(), - createFieldValueMapping("shipment.shipment_identifiers[].value", "string"), + SerializerTestFixture.createFieldValueMapping("shipment.shipment_identifiers[].value", "string"), FieldValueMapping.builder().fieldName("shipment.equipment_identifiers[].type").fieldType("enum").valueLength(0) .fieldValueList( - "[EQUIPMENT_IDENTIFIER_TYPE_UNSPECIFIED, EQUIPMENT_IDENTIFIER_TYPE_MOBILE_PHONE_NUMBER, EQUIPMENT_IDENTIFIER_TYPE_VEHICLE_ID, " + - "EQUIPMENT_IDENTIFIER_TYPE_LICENSE_PLATE, EQUIPMENT_IDENTIFIER_TYPE_SENSITECH_DEVICE_ID, EQUIPMENT_IDENTIFIER_TYPE_EMERSON_DEVICE_ID, " + - "EQUIPMENT_IDENTIFIER_TYPE_TIVE_DEVICE_ID, EQUIPMENT_IDENTIFIER_TYPE_CONTAINER_ID]") + "[EQUIPMENT_IDENTIFIER_TYPE_UNSPECIFIED, EQUIPMENT_IDENTIFIER_TYPE_MOBILE_PHONE_NUMBER, EQUIPMENT_IDENTIFIER_TYPE_VEHICLE_ID, " + + "EQUIPMENT_IDENTIFIER_TYPE_LICENSE_PLATE, EQUIPMENT_IDENTIFIER_TYPE_SENSITECH_DEVICE_ID, EQUIPMENT_IDENTIFIER_TYPE_EMERSON_DEVICE_ID, " + + "EQUIPMENT_IDENTIFIER_TYPE_TIVE_DEVICE_ID, EQUIPMENT_IDENTIFIER_TYPE_CONTAINER_ID]") .required(true) .isAncestorRequired(true).build(), - createFieldValueMapping("shipment.equipment_identifiers[].value", "string"), - createFieldValueMapping("shipment.attributes[]", "string-array"), - createFieldValueMapping("latest_status_update.timestamp", ".google.protobuf.Timestamp"), + SerializerTestFixture.createFieldValueMapping("shipment.equipment_identifiers[].value", "string"), + SerializerTestFixture.createFieldValueMapping("shipment.attributes[]", "string-array"), + SerializerTestFixture.createFieldValueMapping("latest_status_update.timestamp", ".google.protobuf.Timestamp"), FieldValueMapping.builder().fieldName("latest_status_update.status_code").fieldType("enum").valueLength(0) .fieldValueList( - "[STATUS_UPDATE_CODE_UNSPECIFIED, STATUS_UPDATE_CODE_DISPATCHED, STATUS_UPDATE_CODE_IN_TRANSIT, STATUS_UPDATE_CODE_AT_STOP, " + - "STATUS_UPDATE_CODE_COMPLETED, STATUS_UPDATE_CODE_TRACKING_FAILED, STATUS_UPDATE_CODE_INFO, STATUS_UPDATE_CODE_DELETED]") + "[STATUS_UPDATE_CODE_UNSPECIFIED, STATUS_UPDATE_CODE_DISPATCHED, STATUS_UPDATE_CODE_IN_TRANSIT, STATUS_UPDATE_CODE_AT_STOP, " + + "STATUS_UPDATE_CODE_COMPLETED, STATUS_UPDATE_CODE_TRACKING_FAILED, STATUS_UPDATE_CODE_INFO, STATUS_UPDATE_CODE_DELETED]") .required(true) .isAncestorRequired(true).build(), FieldValueMapping.builder().fieldName("latest_status_update.status_reason").fieldType("enum").valueLength(0) .fieldValueList( - "[STATUS_UPDATE_REASON_UNSPECIFIED, STATUS_UPDATE_REASON_PENDING_TRACKING_METHOD, STATUS_UPDATE_REASON_SCHEDULED, " + - "STATUS_UPDATE_REASON_PENDING_APPROVAL, STATUS_UPDATE_REASON_ACQUIRING_LOCATION, STATUS_UPDATE_REASON_PENDING_CARRIER, " + - "STATUS_UPDATE_REASON_IN_MOTION, STATUS_UPDATE_REASON_IDLE, STATUS_UPDATE_REASON_APPROVAL_DENIED, STATUS_UPDATE_REASON_TIMED_OUT, " + - "STATUS_UPDATE_REASON_CANCELED, STATUS_UPDATE_REASON_DEPARTED_FINAL_STOP, STATUS_UPDATE_REASON_ARRIVED_FINAL_STOP, " + - "STATUS_UPDATE_REASON_ARRIVED_FAILED_TO_ACQUIRE_LOCATION, STATUS_UPDATE_REASON_INFO]") + "[STATUS_UPDATE_REASON_UNSPECIFIED, STATUS_UPDATE_REASON_PENDING_TRACKING_METHOD, STATUS_UPDATE_REASON_SCHEDULED, " + + "STATUS_UPDATE_REASON_PENDING_APPROVAL, STATUS_UPDATE_REASON_ACQUIRING_LOCATION, STATUS_UPDATE_REASON_PENDING_CARRIER, " + + "STATUS_UPDATE_REASON_IN_MOTION, STATUS_UPDATE_REASON_IDLE, STATUS_UPDATE_REASON_APPROVAL_DENIED, STATUS_UPDATE_REASON_TIMED_OUT, " + + "STATUS_UPDATE_REASON_CANCELED, STATUS_UPDATE_REASON_DEPARTED_FINAL_STOP, STATUS_UPDATE_REASON_ARRIVED_FINAL_STOP, " + + "STATUS_UPDATE_REASON_ARRIVED_FAILED_TO_ACQUIRE_LOCATION, STATUS_UPDATE_REASON_INFO]") .required(true) .isAncestorRequired(true).build(), - createFieldValueMapping("latest_status_update.geo_coordinates.latitude", "double"), - createFieldValueMapping("latest_status_update.geo_coordinates.longitude", "double"), - createFieldValueMapping("latest_status_update.address.postal_code", "string"), - createFieldValueMapping("latest_status_update.address.address_lines[]", "string-array"), - createFieldValueMapping("latest_status_update.address.city", "string"), - createFieldValueMapping("latest_status_update.address.state", "string"), - createFieldValueMapping("latest_status_update.address.country", "string"), - createFieldValueMapping("latest_status_update.stop_number", "int"), - createFieldValueMapping("latest_stop_statuses[].stop_number", "int"), + SerializerTestFixture.createFieldValueMapping("latest_status_update.geo_coordinates.latitude", "double"), + SerializerTestFixture.createFieldValueMapping("latest_status_update.geo_coordinates.longitude", "double"), + SerializerTestFixture.createFieldValueMapping("latest_status_update.address.postal_code", "string"), + SerializerTestFixture.createFieldValueMapping("latest_status_update.address.address_lines[]", "string-array"), + SerializerTestFixture.createFieldValueMapping("latest_status_update.address.city", "string"), + SerializerTestFixture.createFieldValueMapping("latest_status_update.address.state", "string"), + SerializerTestFixture.createFieldValueMapping("latest_status_update.address.country", "string"), + SerializerTestFixture.createFieldValueMapping("latest_status_update.stop_number", "int"), + SerializerTestFixture.createFieldValueMapping("latest_stop_statuses[].stop_number", "int"), FieldValueMapping.builder().fieldName("latest_stop_statuses[].status_code").fieldType("enum").valueLength(0) .fieldValueList( "[STOP_STATUS_CODE_UNSPECIFIED, STOP_STATUS_CODE_UNKNOWN, STOP_STATUS_CODE_EN_ROUTE, STOP_STATUS_CODE_ARRIVED, STOP_STATUS_CODE_DEPARTED]") .required(true) .isAncestorRequired(true).build(), - createFieldValueMapping("latest_stop_statuses[].arrival_estimate.estimated_arrival_window.start_date_time", "google.protobuf.Timestamp"), - createFieldValueMapping("latest_stop_statuses[].arrival_estimate.estimated_arrival_window.end_date_time", "google.protobuf.Timestamp"), - createFieldValueMapping("latest_stop_statuses[].arrival_estimate.last_calculated_date_time", "google.protobuf.Timestamp"), + SerializerTestFixture.createFieldValueMapping("latest_stop_statuses[].arrival_estimate.estimated_arrival_window.start_date_time", "google.protobuf.Timestamp"), + SerializerTestFixture.createFieldValueMapping("latest_stop_statuses[].arrival_estimate.estimated_arrival_window.end_date_time", "google.protobuf.Timestamp"), + SerializerTestFixture.createFieldValueMapping("latest_stop_statuses[].arrival_estimate.last_calculated_date_time", "google.protobuf.Timestamp"), FieldValueMapping.builder().fieldName("latest_stop_statuses[].arrival_code").fieldType("enum").valueLength(0) .fieldValueList("[ARRIVAL_CODE_UNSPECIFIED, ARRIVAL_CODE_UNKNOWN, ARRIVAL_CODE_EARLY, ARRIVAL_CODE_ON_TIME, ARRIVAL_CODE_LATE]").required(true) .isAncestorRequired(true).build(), - createFieldValueMapping("latest_stop_statuses[].additional_appointment_window_statuses[]", "string-array")) + SerializerTestFixture.createFieldValueMapping("latest_stop_statuses[].additional_appointment_window_statuses[]", "string-array")) ); static final Pair> TEST_EASY = new Pair<>( new FileHelper().getFile("/proto-files/easyTest.proto"), Arrays.asList( - createFieldValueMapping("street", "string"), - createFieldValueMapping("number[]", "int-array"), - createFieldValueMapping("zipcode", "long")) + SerializerTestFixture.createFieldValueMapping("street", "string"), + SerializerTestFixture.createFieldValueMapping("number[]", "int-array"), + SerializerTestFixture.createFieldValueMapping("zipcode", "long")) ); static final Pair> TEST_EMBEDDED_TYPE = new Pair<>( new FileHelper().getFile("/proto-files/embeddedTypeTest.proto"), Arrays.asList( - createFieldValueMapping("phones.addressesPhone[:].id[]", "string-array"), + SerializerTestFixture.createFieldValueMapping("phones.addressesPhone[:].id[]", "string-array"), FieldValueMapping.builder().fieldName("phones.phoneType").fieldType("enum").valueLength(0) .fieldValueList("[MOBILE, HOME, WORK]").required(true) .isAncestorRequired(true).build()) @@ -148,84 +146,87 @@ public class ProtobuffSerializerTestFixture { static final Pair> TEST_GOOGLE_TYPES = new Pair<>( new FileHelper().getFile("/proto-files/googleTypesTest.proto"), Arrays.asList( - createFieldValueMapping("id", "Int32Value"), - createFieldValueMapping("occurrence_id", "StringValue"), - createFieldValueMapping("load_number", "StringValue")) + SerializerTestFixture.createFieldValueMapping("id", "Int32Value"), + SerializerTestFixture.createFieldValueMapping("occurrence_id", "StringValue"), + SerializerTestFixture.createFieldValueMapping("load_number", "StringValue")) ); static final Pair> TEST_ISSUE_311 = new Pair<>( new FileHelper().getFile("/proto-files/issue311Test.proto"), Arrays.asList( - createFieldValueMapping("order_id", "int"), - createFieldValueMapping("order_number", "string"), - createFieldValueMapping("customer_account.billing_party.party_id", "string"), - createFieldValueMapping("customer_account.billing_party.address.address_line_one", "string"), - createFieldValueMapping("customer_account.billing_party.address.address_line_two", "string"), - createFieldValueMapping("customer_account.billing_party.party_contact[].contact_id", "int"), - createFieldValueMapping("customer_account.billing_party.party_contact[].contact_name", "string"), - createFieldValueMapping("details.in_details.customs_details.party.party_address.address_line_one", "string"), - createFieldValueMapping("details.in_details.customs_details.party.party_address.address_line_two", "string"), - createFieldValueMapping("details.in_details.customs_details.party.party_contact[].contact_id", "int"), - createFieldValueMapping("details.in_details.customs_details.party.party_contact[].contact_name", "string"), - createFieldValueMapping("details.in_details.customs_details.party.p_contact[].contact_id", "int"), - createFieldValueMapping("details.in_details.customs_details.party.p_contact[].contact_name", "string")) + SerializerTestFixture.createFieldValueMapping("order_id", "int"), + SerializerTestFixture.createFieldValueMapping("order_number", "string"), + SerializerTestFixture.createFieldValueMapping("customer_account.billing_party.party_id", "string"), + SerializerTestFixture.createFieldValueMapping("customer_account.billing_party.address.address_line_one", "string"), + SerializerTestFixture.createFieldValueMapping("customer_account.billing_party.address.address_line_two", "string"), + SerializerTestFixture.createFieldValueMapping("customer_account.billing_party.party_contact[].contact_id", "int"), + SerializerTestFixture.createFieldValueMapping("customer_account.billing_party.party_contact[].contact_name", "string"), + SerializerTestFixture.createFieldValueMapping("details.in_details.customs_details.party.party_address.address_line_one", "string"), + SerializerTestFixture.createFieldValueMapping("details.in_details.customs_details.party.party_address.address_line_two", "string"), + SerializerTestFixture.createFieldValueMapping("details.in_details.customs_details.party.party_contact[].contact_id", "int"), + SerializerTestFixture.createFieldValueMapping("details.in_details.customs_details.party.party_contact[].contact_name", "string"), + SerializerTestFixture.createFieldValueMapping("details.in_details.customs_details.party.p_contact[].contact_id", "int"), + SerializerTestFixture.createFieldValueMapping("details.in_details.customs_details.party.p_contact[].contact_name", "string")) ); static final Pair> TEST_MAP = new Pair<>( new FileHelper().getFile("/proto-files/mapTest.proto"), Arrays.asList( - createFieldValueMapping("name[:]", "string-map"), - createFieldValueMapping("addresses[:].street", "string"), - createFieldValueMapping("addresses[:].number", "int"), - createFieldValueMapping("addresses[:].zipcode", "int"), - createFieldValueMapping("addressesNoDot[:].street", "string"), - createFieldValueMapping("addressesNoDot[:].number", "int"), - createFieldValueMapping("addressesNoDot[:].zipcode", "int")) + SerializerTestFixture.createFieldValueMapping("name[:]", "string-map"), + SerializerTestFixture.createFieldValueMapping("addresses[:].street", "string"), + SerializerTestFixture.createFieldValueMapping("addresses[:].number", "int"), + SerializerTestFixture.createFieldValueMapping("addresses[:].zipcode", "int"), + SerializerTestFixture.createFieldValueMapping("addressesNoDot[:].street", "string"), + SerializerTestFixture.createFieldValueMapping("addressesNoDot[:].number", "int"), + SerializerTestFixture.createFieldValueMapping("addressesNoDot[:].zipcode", "int")) ); static final Pair> TEST_ONE_OF = new Pair<>( new FileHelper().getFile("/proto-files/oneOfTest.proto"), Arrays.asList( - createFieldValueMapping("type.street", "string"), - createFieldValueMapping("type.number", "int"), - createFieldValueMapping("type.test", "string"), - createFieldValueMapping("optionString", "string")) + SerializerTestFixture.createFieldValueMapping("type.street", "string"), + SerializerTestFixture.createFieldValueMapping("type.number", "int"), + SerializerTestFixture.createFieldValueMapping("type.test", "string"), + SerializerTestFixture.createFieldValueMapping("optionString", "string")) ); static final Pair> TEST_PROVIDED = new Pair<>( new FileHelper().getFile("/proto-files/providedTest.proto"), Arrays.asList( - createFieldValueMapping("id", "int"), - createFieldValueMapping("occurrence_id", "string"), - createFieldValueMapping("load_number", "string"), - createFieldValueMapping("claim_type.code", "string"), - createFieldValueMapping("claim_type.description", "string"), - createFieldValueMapping("collision_type.code", "string"), - createFieldValueMapping("collision_type.description", "string"), - createFieldValueMapping("incident_cause_type.code", "string"), - createFieldValueMapping("incident_cause_type.description", "string"), - createFieldValueMapping("incident_type.code", "string"), - createFieldValueMapping("incident_type.description", "string"), - createFieldValueMapping("review_status_type.code", "string"), - createFieldValueMapping("review_status_type.description", "string"), - createFieldValueMapping("incident_latitude", "double"), - createFieldValueMapping("incident_longitude", "double"), - createFieldValueMapping("incident_date", "string"), - createFieldValueMapping("incident_time", "google.protobuf.Timestamp"), - createFieldValueMapping("incident_city", "string"), - createFieldValueMapping("incident_state", "string"), - createFieldValueMapping("location_description", "string"), - createFieldValueMapping("incident_equipment_details[].equipment_number", "string"), - createFieldValueMapping("incident_equipment_details[].equipment_type", "string"), - createFieldValueMapping("incident_equipment_details[].equipment_prefix", "string"), - createFieldValueMapping("driver.driver_id", "int"), - createFieldValueMapping("driver.driver_first_name", "string"), - createFieldValueMapping("driver.driver_last_name", "string"), - createFieldValueMapping("dot_accident_indicator", "string"), - createFieldValueMapping("drug_test_required_indicator", "string"), - createFieldValueMapping("hazardous_material_indicator", "string"), - createFieldValueMapping("preventable_indicator", "string"), - createFieldValueMapping("report_by_name", "string"), - createFieldValueMapping("create_user_id", "string")) + SerializerTestFixture.createFieldValueMapping("id", "int"), + SerializerTestFixture.createFieldValueMapping("occurrence_id", "string"), + SerializerTestFixture.createFieldValueMapping("load_number", "string"), + SerializerTestFixture.createFieldValueMapping("claim_type.code", "string"), + SerializerTestFixture.createFieldValueMapping("claim_type.description", "string"), + SerializerTestFixture.createFieldValueMapping("collision_type.code", "string"), + SerializerTestFixture.createFieldValueMapping("collision_type.description", "string"), + SerializerTestFixture.createFieldValueMapping("incident_cause_type.code", "string"), + SerializerTestFixture.createFieldValueMapping("incident_cause_type.description", "string"), + SerializerTestFixture.createFieldValueMapping("incident_type.code", "string"), + SerializerTestFixture.createFieldValueMapping("incident_type.description", "string"), + SerializerTestFixture.createFieldValueMapping("review_status_type.code", "string"), + SerializerTestFixture.createFieldValueMapping("review_status_type.description", "string"), + SerializerTestFixture.createFieldValueMapping("incident_latitude", "double"), + SerializerTestFixture.createFieldValueMapping("incident_longitude", "double"), + SerializerTestFixture.createFieldValueMapping("incident_date", "string"), + SerializerTestFixture.createFieldValueMapping("incident_time", "google.protobuf.Timestamp"), + SerializerTestFixture.createFieldValueMapping("incident_city", "string"), + SerializerTestFixture.createFieldValueMapping("incident_state", "string"), + SerializerTestFixture.createFieldValueMapping("location_description", "string"), + SerializerTestFixture.createFieldValueMapping("incident_equipment_details[].equipment_number", "string"), + SerializerTestFixture.createFieldValueMapping("incident_equipment_details[].equipment_type", "string"), + SerializerTestFixture.createFieldValueMapping("incident_equipment_details[].equipment_prefix", "string"), + SerializerTestFixture.createFieldValueMapping("driver.driver_id", "int"), + SerializerTestFixture.createFieldValueMapping("driver.driver_first_name", "string"), + SerializerTestFixture.createFieldValueMapping("driver.driver_last_name", "string"), + SerializerTestFixture.createFieldValueMapping("dot_accident_indicator", "string"), + SerializerTestFixture.createFieldValueMapping("drug_test_required_indicator", "string"), + SerializerTestFixture.createFieldValueMapping("hazardous_material_indicator", "string"), + SerializerTestFixture.createFieldValueMapping("preventable_indicator", "string"), + SerializerTestFixture.createFieldValueMapping("report_by_name", "string"), + SerializerTestFixture.createFieldValueMapping("create_user_id", "string")) ); + + protected ProtobuffSerializerTestFixture() { + } } From 6a3d7e71174bf4f8b4857c0a09b8e6f90a493369 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jose=20Enrique=20Garc=C3=ADa=20Maci=C3=B1eiras?= Date: Tue, 3 Oct 2023 18:43:34 +0200 Subject: [PATCH 24/30] #332 Fix styles --- .../json/AbstractJsonExtractor.java | 78 ++++--- .../loadgen/impl/SRLoadGenerator.java | 1 + .../randomtool/random/RandomObject.java | 2 +- .../sngular/kloadgen/sampler/SamplerUtil.java | 41 ++-- .../schemaregistry/SchemaRegistryAdapter.java | 1 + .../impl/ConfluentSchemaRegistry.java | 3 - .../randomtool/random/RandomArrayTest.java | 6 +- .../kloadgen/sampler/SamplerUtilTest.java | 209 +++++++++--------- .../GenericRecordAvroSerializersTest.java | 64 +++--- .../kloadgen/testutil/SchemaParseUtil.java | 38 ++-- 10 files changed, 226 insertions(+), 217 deletions(-) diff --git a/src/main/java/com/sngular/kloadgen/extractor/extractors/json/AbstractJsonExtractor.java b/src/main/java/com/sngular/kloadgen/extractor/extractors/json/AbstractJsonExtractor.java index 07a7525b..538e757a 100644 --- a/src/main/java/com/sngular/kloadgen/extractor/extractors/json/AbstractJsonExtractor.java +++ b/src/main/java/com/sngular/kloadgen/extractor/extractors/json/AbstractJsonExtractor.java @@ -6,7 +6,6 @@ import com.sngular.kloadgen.exception.KLoadGenException; import com.sngular.kloadgen.extractor.extractors.SchemaExtractorUtil; -import com.sngular.kloadgen.extractor.parser.impl.JSONSchemaParser; import com.sngular.kloadgen.model.ConstraintTypeEnum; import com.sngular.kloadgen.model.FieldValueMapping; import com.sngular.kloadgen.model.json.ArrayField; @@ -19,15 +18,10 @@ import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.collections4.Transformer; import org.apache.commons.lang3.StringUtils; +import org.jetbrains.annotations.NotNull; public abstract class AbstractJsonExtractor { - - private static final JSONSchemaParser JSON_SCHEMA_PARSER = new JSONSchemaParser(); - - public final JSONSchemaParser getSchemaParser() { - return JSON_SCHEMA_PARSER; - } - + private static String extractFieldName(final String fieldName) { String fieldNameClean = fieldName; if (fieldName.endsWith("[][]") || fieldName.endsWith("[:][]")) { @@ -157,11 +151,7 @@ private List extractArrayInternalFields( completeFieldList.add( FieldValueMapping.builder() .fieldName(name) - .fieldType(value.getType() - + SchemaExtractorUtil.ARRAY_TYPE_POSTFIX - + ((StringUtils.isNotEmpty(breadCrumb) - && breadCrumb.endsWith("[]")) - ? SchemaExtractorUtil.ARRAY_TYPE_POSTFIX : (StringUtils.isNotEmpty(breadCrumb) && breadCrumb.endsWith("[:]")) ? "-map" : breadCrumb)) + .fieldType(calculateFieldType(breadCrumb, value)) .required(!name.endsWith("][]") && !name.endsWith("][:]") && innerField.isFieldRequired()) .isAncestorRequired(!isRootElement && isAncestorRequired != null && isAncestorRequired) .build()); @@ -170,6 +160,17 @@ private List extractArrayInternalFields( return completeFieldList; } + @NotNull + private static String calculateFieldType(final String breadCrumb, final Field value) { + return value.getType() + + SchemaExtractorUtil.ARRAY_TYPE_POSTFIX + + ((StringUtils.isNotEmpty(breadCrumb) + && breadCrumb.endsWith("[]")) ? SchemaExtractorUtil.ARRAY_TYPE_POSTFIX + : (StringUtils.isNotEmpty(breadCrumb) + && breadCrumb.endsWith("[:]")) ? "-map" + : breadCrumb); + } + private List extractMapInternalFields( final MapField innerField, final Boolean isRootElement, final Boolean isAncestorRequired, final String breadCrumb) { @@ -189,10 +190,7 @@ private List extractMapInternalFields( completeFieldList.add( FieldValueMapping.builder() .fieldName(name) - .fieldType(value.getType() + "-map" + ((StringUtils.isNotEmpty(breadCrumb) - && breadCrumb.endsWith("[:]")) ? "-map" - : (StringUtils.isNotEmpty(breadCrumb) && breadCrumb.endsWith("[]")) ? SchemaExtractorUtil.ARRAY_TYPE_POSTFIX - : breadCrumb)) + .fieldType(getFieldType(breadCrumb, value)) .required(!name.endsWith("][]") && !name.endsWith("][:]") && innerField.isFieldRequired()) .isAncestorRequired(!isRootElement && isAncestorRequired != null && isAncestorRequired) .build()); @@ -200,22 +198,36 @@ private List extractMapInternalFields( return completeFieldList; } - private void processRecordFieldList(final String fieldName, final String splitter, final List internalFields, - final List completeFieldList, final boolean isAncestorRequired) { - internalFields.forEach(internalField -> { - if (Objects.nonNull(internalField.getFieldName())) { - internalField.setFieldName(fieldName + splitter + internalField.getFieldName()); - } else { - internalField.setFieldName(fieldName); - } - final String[] splittedName = internalField.getFieldName().split("\\."); - String parentName = splittedName[splittedName.length - 2]; - parentName = parentName.replace("[]", ""); - if (fieldName.equals(parentName)) { - internalField.setAncestorRequired(isAncestorRequired); - } - completeFieldList.add(internalField); - }); + @NotNull + private static String getFieldType(final String breadCrumb, final Field value) { + return value.getType() + + "-map" + + ((StringUtils.isNotEmpty(breadCrumb) + && breadCrumb.endsWith("[:]")) ? "-map" + : (StringUtils.isNotEmpty(breadCrumb) + && breadCrumb.endsWith("[]")) ? SchemaExtractorUtil.ARRAY_TYPE_POSTFIX + : breadCrumb); + } + + private void processRecordFieldList(final String fieldName, final String splitter, final List internalFields, final List completeFieldList, + final boolean isAncestorRequired) { + internalFields.forEach(internalField -> extracted(fieldName, splitter, completeFieldList, isAncestorRequired, internalField)); + } + + private static void extracted(final String fieldName, final String splitter, final List completeFieldList, final boolean isAncestorRequired, + final FieldValueMapping internalField) { + if (Objects.nonNull(internalField.getFieldName())) { + internalField.setFieldName(fieldName + splitter + internalField.getFieldName()); + } else { + internalField.setFieldName(fieldName); + } + final String[] splitName = internalField.getFieldName().split("\\."); + String parentName = splitName[splitName.length - 2]; + parentName = parentName.replace("[]", ""); + if (fieldName.equals(parentName)) { + internalField.setAncestorRequired(isAncestorRequired); + } + completeFieldList.add(internalField); } private Boolean checkRequiredByType( diff --git a/src/main/java/com/sngular/kloadgen/loadgen/impl/SRLoadGenerator.java b/src/main/java/com/sngular/kloadgen/loadgen/impl/SRLoadGenerator.java index 735b6a2e..91f9789d 100644 --- a/src/main/java/com/sngular/kloadgen/loadgen/impl/SRLoadGenerator.java +++ b/src/main/java/com/sngular/kloadgen/loadgen/impl/SRLoadGenerator.java @@ -2,6 +2,7 @@ import java.io.IOException; import java.util.Map; + import com.sngular.kloadgen.schemaregistry.SchemaRegistryAdapter; import com.sngular.kloadgen.schemaregistry.SchemaRegistryManagerFactory; import com.sngular.kloadgen.schemaregistry.adapter.impl.BaseSchemaMetadata; diff --git a/src/main/java/com/sngular/kloadgen/randomtool/random/RandomObject.java b/src/main/java/com/sngular/kloadgen/randomtool/random/RandomObject.java index 3d791c60..02dbde14 100644 --- a/src/main/java/com/sngular/kloadgen/randomtool/random/RandomObject.java +++ b/src/main/java/com/sngular/kloadgen/randomtool/random/RandomObject.java @@ -159,7 +159,7 @@ private BigInteger getIntegerValueOrRandom(final Integer valueLength, final List Number maximum; if (valueLength == 0) { maximum = 1000; - int num = rand.nextInt((Integer) maximum); + final int num = rand.nextInt((Integer) maximum); value = new BigInteger(String.valueOf(num)); } else { maximum = calculateMaximum(valueLength, constraints); diff --git a/src/main/java/com/sngular/kloadgen/sampler/SamplerUtil.java b/src/main/java/com/sngular/kloadgen/sampler/SamplerUtil.java index 35c7fd9e..a8712639 100644 --- a/src/main/java/com/sngular/kloadgen/sampler/SamplerUtil.java +++ b/src/main/java/com/sngular/kloadgen/sampler/SamplerUtil.java @@ -17,6 +17,7 @@ import java.util.Objects; import java.util.Properties; import java.util.Set; + import com.sngular.kloadgen.exception.KLoadGenException; import com.sngular.kloadgen.loadgen.BaseLoadGenerator; import com.sngular.kloadgen.loadgen.impl.AvroSRLoadGenerator; @@ -100,15 +101,15 @@ public static Properties setupCommonProperties(final JavaSamplerContext context) final Properties props = new Properties(); props.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, context.getParameter(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG)); - if ("true".equals(context.getJMeterVariables().get(PropsKeysHelper.SCHEMA_KEYED_MESSAGE_KEY))) { - props.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, context.getJMeterVariables().get(PropsKeysHelper.KEY_SERIALIZER_CLASS_PROPERTY)); - } else if ("true".equals(context.getJMeterVariables().get(PropsKeysHelper.SIMPLE_KEYED_MESSAGE_KEY))) { - props.put(PropsKeysHelper.MESSAGE_KEY_KEY_TYPE, context.getJMeterVariables().get(PropsKeysHelper.KEY_TYPE)); - props.put(PropsKeysHelper.MESSAGE_KEY_KEY_VALUE, context.getJMeterVariables().get(PropsKeysHelper.KEY_VALUE)); - if (Objects.nonNull(context.getJMeterVariables().get(PropsKeysHelper.KEY_SCHEMA_TYPE))) { - props.put(PropsKeysHelper.KEY_SCHEMA_TYPE, context.getJMeterVariables().get(PropsKeysHelper.KEY_SCHEMA_TYPE)); + if ("true".equals(JavaSamplerContext.getJMeterVariables().get(PropsKeysHelper.SCHEMA_KEYED_MESSAGE_KEY))) { + props.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, JavaSamplerContext.getJMeterVariables().get(PropsKeysHelper.KEY_SERIALIZER_CLASS_PROPERTY)); + } else if ("true".equals(JavaSamplerContext.getJMeterVariables().get(PropsKeysHelper.SIMPLE_KEYED_MESSAGE_KEY))) { + props.put(PropsKeysHelper.MESSAGE_KEY_KEY_TYPE, JavaSamplerContext.getJMeterVariables().get(PropsKeysHelper.KEY_TYPE)); + props.put(PropsKeysHelper.MESSAGE_KEY_KEY_VALUE, JavaSamplerContext.getJMeterVariables().get(PropsKeysHelper.KEY_VALUE)); + if (Objects.nonNull(JavaSamplerContext.getJMeterVariables().get(PropsKeysHelper.KEY_SCHEMA_TYPE))) { + props.put(PropsKeysHelper.KEY_SCHEMA_TYPE, JavaSamplerContext.getJMeterVariables().get(PropsKeysHelper.KEY_SCHEMA_TYPE)); } - props.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, context.getJMeterVariables().get(PropsKeysHelper.KEY_SERIALIZER_CLASS_PROPERTY)); + props.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, JavaSamplerContext.getJMeterVariables().get(PropsKeysHelper.KEY_SERIALIZER_CLASS_PROPERTY)); } else { props.put(PropsKeysHelper.SCHEMA_KEYED_MESSAGE_KEY, Boolean.FALSE); } @@ -127,15 +128,15 @@ public static Properties setupCommonProperties(final JavaSamplerContext context) props.put(CommonClientConfigs.SECURITY_PROTOCOL_CONFIG, context.getParameter(CommonClientConfigs.SECURITY_PROTOCOL_CONFIG)); props.put(ProducerKeysHelper.SASL_MECHANISM, context.getParameter(ProducerKeysHelper.SASL_MECHANISM)); - final String schemaRegistryNameValue = context.getJMeterVariables().get(SchemaRegistryKeyHelper.SCHEMA_REGISTRY_NAME); + final String schemaRegistryNameValue = JavaSamplerContext.getJMeterVariables().get(SchemaRegistryKeyHelper.SCHEMA_REGISTRY_NAME); final String enableSchemaRegistrationValue = context.getParameter(SchemaRegistryKeyHelper.ENABLE_AUTO_SCHEMA_REGISTRATION_CONFIG); if (SchemaRegistryKeyHelper.SCHEMA_REGISTRY_APICURIO.equalsIgnoreCase(schemaRegistryNameValue)) { props.put(SerdeConfig.AUTO_REGISTER_ARTIFACT, enableSchemaRegistrationValue); - props.put(SchemaResolverConfig.REGISTRY_URL, context.getJMeterVariables().get(SchemaResolverConfig.REGISTRY_URL)); - props.put(SchemaRegistryKeyHelper.SCHEMA_REGISTRY_URL, context.getJMeterVariables().get(SchemaResolverConfig.REGISTRY_URL)); + props.put(SchemaResolverConfig.REGISTRY_URL, JavaSamplerContext.getJMeterVariables().get(SchemaResolverConfig.REGISTRY_URL)); + props.put(SchemaRegistryKeyHelper.SCHEMA_REGISTRY_URL, JavaSamplerContext.getJMeterVariables().get(SchemaResolverConfig.REGISTRY_URL)); } else { props.put(SchemaRegistryKeyHelper.ENABLE_AUTO_SCHEMA_REGISTRATION_CONFIG, enableSchemaRegistrationValue); - final String schemaRegistryURL = context.getJMeterVariables().get(SchemaRegistryKeyHelper.SCHEMA_REGISTRY_URL); + final String schemaRegistryURL = JavaSamplerContext.getJMeterVariables().get(SchemaRegistryKeyHelper.SCHEMA_REGISTRY_URL); if (StringUtils.isNotBlank(schemaRegistryURL)) { props.put(SchemaRegistryKeyHelper.SCHEMA_REGISTRY_URL, schemaRegistryURL); } @@ -266,14 +267,14 @@ public static Properties setupCommonConsumerProperties(final JavaSamplerContext props.put(CommonClientConfigs.CLIENT_ID_CONFIG, context.getParameter(CommonClientConfigs.CLIENT_ID_CONFIG)); - if (Objects.nonNull(context.getJMeterVariables().get(PropsKeysHelper.VALUE_SCHEMA))) { - props.put(PropsKeysHelper.VALUE_SCHEMA, context.getJMeterVariables().get(PropsKeysHelper.VALUE_SCHEMA)); + if (Objects.nonNull(JavaSamplerContext.getJMeterVariables().get(PropsKeysHelper.VALUE_SCHEMA))) { + props.put(PropsKeysHelper.VALUE_SCHEMA, JavaSamplerContext.getJMeterVariables().get(PropsKeysHelper.VALUE_SCHEMA)); } - if (Objects.nonNull(context.getJMeterVariables().get(PropsKeysHelper.KEY_SCHEMA))) { - props.put(PropsKeysHelper.KEY_SCHEMA, context.getJMeterVariables().get(PropsKeysHelper.KEY_SCHEMA)); + if (Objects.nonNull(JavaSamplerContext.getJMeterVariables().get(PropsKeysHelper.KEY_SCHEMA))) { + props.put(PropsKeysHelper.KEY_SCHEMA, JavaSamplerContext.getJMeterVariables().get(PropsKeysHelper.KEY_SCHEMA)); } - if (Objects.nonNull(context.getJMeterVariables().get(SchemaRegistryKeyHelper.SCHEMA_REGISTRY_URL))) { - props.put(SchemaRegistryKeyHelper.SCHEMA_REGISTRY_URL, context.getJMeterVariables().get(SchemaRegistryKeyHelper.SCHEMA_REGISTRY_URL)); + if (Objects.nonNull(JavaSamplerContext.getJMeterVariables().get(SchemaRegistryKeyHelper.SCHEMA_REGISTRY_URL))) { + props.put(SchemaRegistryKeyHelper.SCHEMA_REGISTRY_URL, JavaSamplerContext.getJMeterVariables().get(SchemaRegistryKeyHelper.SCHEMA_REGISTRY_URL)); } props.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, context.getParameter(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG)); @@ -353,9 +354,9 @@ public static BaseLoadGenerator configureValueGenerator(final Properties props) final String valueNameStrategy = jMeterVariables.get(ProducerKeysHelper.VALUE_NAME_STRATEGY); if (SchemaRegistryKeyHelper.SCHEMA_REGISTRY_APICURIO.equalsIgnoreCase(jMeterVariables.get(SchemaRegistryKeyHelper.SCHEMA_REGISTRY_NAME))) { - props.put(SchemaResolverConfig.ARTIFACT_RESOLVER_STRATEGY, (Objects.nonNull(valueNameStrategy) ? valueNameStrategy : ProducerKeysHelper.TOPIC_NAME_STRATEGY_APICURIO)); + props.put(SchemaResolverConfig.ARTIFACT_RESOLVER_STRATEGY, Objects.nonNull(valueNameStrategy) ? valueNameStrategy : ProducerKeysHelper.TOPIC_NAME_STRATEGY_APICURIO); } else if (SchemaRegistryKeyHelper.SCHEMA_REGISTRY_CONFLUENT.equalsIgnoreCase(jMeterVariables.get(SchemaRegistryKeyHelper.SCHEMA_REGISTRY_NAME))) { - props.put(ProducerKeysHelper.VALUE_NAME_STRATEGY, (Objects.nonNull(valueNameStrategy) ? valueNameStrategy : ProducerKeysHelper.TOPIC_NAME_STRATEGY_CONFLUENT)); + props.put(ProducerKeysHelper.VALUE_NAME_STRATEGY, Objects.nonNull(valueNameStrategy) ? valueNameStrategy : ProducerKeysHelper.TOPIC_NAME_STRATEGY_CONFLUENT); } generator = getLoadGenerator(jMeterVariables); diff --git a/src/main/java/com/sngular/kloadgen/schemaregistry/SchemaRegistryAdapter.java b/src/main/java/com/sngular/kloadgen/schemaregistry/SchemaRegistryAdapter.java index 916828a4..17571aa5 100644 --- a/src/main/java/com/sngular/kloadgen/schemaregistry/SchemaRegistryAdapter.java +++ b/src/main/java/com/sngular/kloadgen/schemaregistry/SchemaRegistryAdapter.java @@ -2,6 +2,7 @@ import java.util.Collection; import java.util.Map; + import com.sngular.kloadgen.schemaregistry.adapter.impl.BaseParsedSchema; import com.sngular.kloadgen.schemaregistry.adapter.impl.BaseSchemaMetadata; import com.sngular.kloadgen.schemaregistry.adapter.impl.SchemaMetadataAdapter; diff --git a/src/main/java/com/sngular/kloadgen/schemaregistry/impl/ConfluentSchemaRegistry.java b/src/main/java/com/sngular/kloadgen/schemaregistry/impl/ConfluentSchemaRegistry.java index 1f112843..3204aca1 100644 --- a/src/main/java/com/sngular/kloadgen/schemaregistry/impl/ConfluentSchemaRegistry.java +++ b/src/main/java/com/sngular/kloadgen/schemaregistry/impl/ConfluentSchemaRegistry.java @@ -30,9 +30,6 @@ public final class ConfluentSchemaRegistry implements SchemaRegistryAdapter { private SchemaRegistryClient schemaRegistryClient; - public ConfluentSchemaRegistry() { - } - @Override public String getSchemaRegistryUrlKey() { return AbstractKafkaSchemaSerDeConfig.SCHEMA_REGISTRY_URL_CONFIG; diff --git a/src/test/java/com/sngular/kloadgen/randomtool/random/RandomArrayTest.java b/src/test/java/com/sngular/kloadgen/randomtool/random/RandomArrayTest.java index a6a713fc..94f824d2 100644 --- a/src/test/java/com/sngular/kloadgen/randomtool/random/RandomArrayTest.java +++ b/src/test/java/com/sngular/kloadgen/randomtool/random/RandomArrayTest.java @@ -50,9 +50,9 @@ void generateArrayRandomValue(final String fieldType, final Integer valueLength, void generateArrayRandomValueZero(final String fieldType, final Integer valueLength, final List fieldValuesList) { Assertions.assertThat((List) new RandomArray().generateArray(fieldType, valueLength, fieldValuesList, 1, Collections.emptyMap())) .isNotNull(); - Object number1 = new RandomArray().generateArray(fieldType, valueLength, fieldValuesList, 1, Collections.emptyMap()); - Object number2 = new RandomArray().generateArray(fieldType, valueLength, fieldValuesList, 1, Collections.emptyMap()); - List number3 = (List) new RandomArray().generateArray(fieldType, valueLength, fieldValuesList, 1, Collections.emptyMap()); + final Object number1 = new RandomArray().generateArray(fieldType, valueLength, fieldValuesList, 1, Collections.emptyMap()); + final Object number2 = new RandomArray().generateArray(fieldType, valueLength, fieldValuesList, 1, Collections.emptyMap()); + final List number3 = (List) new RandomArray().generateArray(fieldType, valueLength, fieldValuesList, 1, Collections.emptyMap()); Assertions.assertThat(number1).isNotNull(); Assertions.assertThat(number2).isNotNull(); Assertions.assertThat(number3).isNotNull(); diff --git a/src/test/java/com/sngular/kloadgen/sampler/SamplerUtilTest.java b/src/test/java/com/sngular/kloadgen/sampler/SamplerUtilTest.java index c7fd1c3d..a1468c33 100644 --- a/src/test/java/com/sngular/kloadgen/sampler/SamplerUtilTest.java +++ b/src/test/java/com/sngular/kloadgen/sampler/SamplerUtilTest.java @@ -13,11 +13,9 @@ import java.util.Properties; import java.util.stream.Stream; -import static com.sngular.kloadgen.serializer.SerializerTestFixture.createFieldValueMapping; - -import com.sngular.kloadgen.extractor.SchemaExtractor; import com.sngular.kloadgen.model.FieldValueMapping; import com.sngular.kloadgen.processor.fixture.JsonSchemaFixturesConstants; +import com.sngular.kloadgen.serializer.SerializerTestFixture; import com.sngular.kloadgen.testutil.FileHelper; import com.sngular.kloadgen.testutil.SchemaParseUtil; import com.sngular.kloadgen.util.PropsKeysHelper; @@ -34,112 +32,111 @@ class SamplerUtilTest { - - private JMeterContext jmcx; - - private final FileHelper fileHelper = new FileHelper(); - - private final SchemaExtractor extractor = new SchemaExtractor(); - - private static Stream parametersForConfigureValueGeneratorTest() { - return Stream.of("localhost:8081", ""); - } - - private static Stream parametersForConfigureKeyGeneratorTest() { - return Stream.of("avro", "json","protobuf"); - } - - @BeforeEach - public final void setUp() throws IOException { - final File file = new File("src/test/resources"); - final String absolutePath = file.getAbsolutePath(); - JMeterUtils.loadJMeterProperties(absolutePath + "/kloadgen.properties"); - jmcx = JMeterContextService.getContext(); - jmcx.setVariables(new JMeterVariables()); - JMeterUtils.setLocale(Locale.ENGLISH); - } - - public JMeterVariables getVariablesAvro() throws IOException { - final File testFile = fileHelper.getFile("/avro-files/avros-example-with-sub-entity-array-test.avsc"); - final ParsedSchema parsedSchema = SchemaParseUtil.getParsedSchema(testFile, "AVRO"); - var variables = new JMeterVariables(); - variables.put(PropsKeysHelper.KEY_SCHEMA_TYPE, "avro"); - variables.put(PropsKeysHelper.VALUE_SUBJECT_NAME, "test"); - variables.put(PropsKeysHelper.KEY_SUBJECT_NAME, "test"); - variables.put(PropsKeysHelper.VALUE_SCHEMA, String.valueOf(parsedSchema)); - variables.put(PropsKeysHelper.KEY_SCHEMA, String.valueOf(parsedSchema)); - variables.putObject(PropsKeysHelper.VALUE_SCHEMA_PROPERTIES, Arrays.asList( - FieldValueMapping.builder().fieldName("subEntity.anotherLevel.subEntityIntArray[2]").fieldType("int-array").valueLength(0).fieldValueList("[1]").required(true) - .isAncestorRequired(true).build(), - FieldValueMapping.builder().fieldName("topLevelIntArray[3]").fieldType("int-array").valueLength(0).fieldValueList("[2]").required(true).isAncestorRequired(true).build()) - ); - return variables; + private final FileHelper fileHelper = new FileHelper(); + + private JMeterContext jmcx; + + private static Stream parametersForConfigureValueGeneratorTest() { + return Stream.of("localhost:8081", ""); + } + + private static Stream parametersForConfigureKeyGeneratorTest() { + return Stream.of("avro", "json", "protobuf"); + } + + @BeforeEach + public final void setUp() throws IOException { + final File file = new File("src/test/resources"); + final String absolutePath = file.getAbsolutePath(); + JMeterUtils.loadJMeterProperties(absolutePath + "/kloadgen.properties"); + jmcx = JMeterContextService.getContext(); + jmcx.setVariables(new JMeterVariables()); + JMeterUtils.setLocale(Locale.ENGLISH); + } + + @ParameterizedTest + @MethodSource("parametersForConfigureValueGeneratorTest") + void configureValueGeneratorTest(final String jmeterProps) throws IOException { + final Properties props = new Properties(); + props.put(SchemaRegistryKeyHelper.SCHEMA_REGISTRY_URL, jmeterProps); + jmcx.setVariables(getVariablesAvro()); + final var generator = SamplerUtil.configureValueGenerator(props); + Assertions.assertThat(generator.nextMessage()).isNotNull(); + } + + public JMeterVariables getVariablesAvro() throws IOException { + final File testFile = fileHelper.getFile("/avro-files/avros-example-with-sub-entity-array-test.avsc"); + final ParsedSchema parsedSchema = SchemaParseUtil.getParsedSchema(testFile, "AVRO"); + final var variables = new JMeterVariables(); + variables.put(PropsKeysHelper.KEY_SCHEMA_TYPE, "avro"); + variables.put(PropsKeysHelper.VALUE_SUBJECT_NAME, "test"); + variables.put(PropsKeysHelper.KEY_SUBJECT_NAME, "test"); + variables.put(PropsKeysHelper.VALUE_SCHEMA, String.valueOf(parsedSchema)); + variables.put(PropsKeysHelper.KEY_SCHEMA, String.valueOf(parsedSchema)); + variables.putObject(PropsKeysHelper.VALUE_SCHEMA_PROPERTIES, Arrays.asList( + FieldValueMapping.builder().fieldName("subEntity.anotherLevel.subEntityIntArray[2]").fieldType("int-array").valueLength(0).fieldValueList("[1]").required(true) + .isAncestorRequired(true).build(), + FieldValueMapping.builder().fieldName("topLevelIntArray[3]").fieldType("int-array").valueLength(0).fieldValueList("[2]").required(true).isAncestorRequired(true).build()) + ); + return variables; + } + + @ParameterizedTest + @MethodSource("parametersForConfigureKeyGeneratorTest") + void configureKeyGeneratorTest(final String jmeterProps) throws IOException { + final Properties props = new Properties(); + props.put(SchemaRegistryKeyHelper.SCHEMA_REGISTRY_URL, "localhost:8081"); + props.put(PropsKeysHelper.KEY_SCHEMA_TYPE, jmeterProps); + + if (jmeterProps.equalsIgnoreCase("json")) { + jmcx.setVariables(getVariablesJsonSchema()); } - - public JMeterVariables getVariablesProtobuf() throws IOException { - - final File testFile = fileHelper.getFile("/proto-files/easyTest.proto"); - final ParsedSchema parsedSchema = SchemaParseUtil.getParsedSchema(testFile, "PROTOBUF"); - var variables = new JMeterVariables(); - variables.put(PropsKeysHelper.KEY_SCHEMA_TYPE, "protobuf"); - variables.put(PropsKeysHelper.VALUE_SUBJECT_NAME, "protobufSubject"); - variables.put(PropsKeysHelper.VALUE_SCHEMA, String.valueOf(parsedSchema)); - variables.put(PropsKeysHelper.KEY_SCHEMA, String.valueOf(parsedSchema)); - variables.put(PropsKeysHelper.KEY_SUBJECT_NAME, "protobufSubject"); - variables.putObject(PropsKeysHelper.KEY_SCHEMA_PROPERTIES, Arrays.asList( - createFieldValueMapping("street", "string"), - createFieldValueMapping("number[]", "int-array"), - createFieldValueMapping("zipcode", "long"))); - variables.putObject(PropsKeysHelper.VALUE_SCHEMA_PROPERTIES, Arrays.asList( - createFieldValueMapping("street", "string"), - createFieldValueMapping("number[]", "int-array"), - createFieldValueMapping("zipcode", "long"))); - return variables; - } - - public JMeterVariables getVariablesJsonSchema() throws IOException { - - final File testFile = fileHelper.getFile("/jsonschema/basic.jcs"); - final ParsedSchema parsedSchema = SchemaParseUtil.getParsedSchema(testFile, "JSON"); - - var variables = new JMeterVariables(); - variables.put(PropsKeysHelper.KEY_SCHEMA_TYPE, "JSON"); - variables.put(PropsKeysHelper.VALUE_SUBJECT_NAME, "jsonSubject"); - variables.put(PropsKeysHelper.KEY_SUBJECT_NAME, "jsonSubject"); - variables.put(PropsKeysHelper.VALUE_SCHEMA, String.valueOf(parsedSchema)); - variables.put(PropsKeysHelper.KEY_SCHEMA, String.valueOf(parsedSchema)); - variables.putObject(PropsKeysHelper.VALUE_SCHEMA_PROPERTIES, JsonSchemaFixturesConstants.SIMPLE_SCHEMA); - return variables; + if (jmeterProps.equalsIgnoreCase("avro")) { + jmcx.setVariables(getVariablesAvro()); } - - @ParameterizedTest - @MethodSource("parametersForConfigureValueGeneratorTest") - void configureValueGeneratorTest(String jmeterProps) throws IOException { - Properties props = new Properties(); - props.put(SchemaRegistryKeyHelper.SCHEMA_REGISTRY_URL, jmeterProps); - jmcx.setVariables(getVariablesAvro()); - var generator = SamplerUtil.configureValueGenerator(props); - Assertions.assertThat(generator.nextMessage()).isNotNull(); + if (jmeterProps.equalsIgnoreCase("protobuf")) { + jmcx.setVariables(getVariablesProtobuf()); } - @ParameterizedTest - @MethodSource("parametersForConfigureKeyGeneratorTest") - void configureKeyGeneratorTest(String jmeterProps) throws IOException { - Properties props = new Properties(); - props.put(SchemaRegistryKeyHelper.SCHEMA_REGISTRY_URL, "localhost:8081"); - props.put(PropsKeysHelper.KEY_SCHEMA_TYPE, jmeterProps); - - if (jmeterProps.equalsIgnoreCase("json")) - jmcx.setVariables(getVariablesJsonSchema()); - if (jmeterProps.equalsIgnoreCase("avro")) - jmcx.setVariables(getVariablesAvro()); - if (jmeterProps.equalsIgnoreCase("protobuf")) - jmcx.setVariables(getVariablesProtobuf()); - - var generator = SamplerUtil.configureKeyGenerator(props); - Assertions.assertThat(generator.nextMessage()).isNotNull(); - - - } + final var generator = SamplerUtil.configureKeyGenerator(props); + Assertions.assertThat(generator.nextMessage()).isNotNull(); + + } + + public JMeterVariables getVariablesJsonSchema() throws IOException { + + final File testFile = fileHelper.getFile("/jsonschema/basic.jcs"); + final ParsedSchema parsedSchema = SchemaParseUtil.getParsedSchema(testFile, "JSON"); + + final var variables = new JMeterVariables(); + variables.put(PropsKeysHelper.KEY_SCHEMA_TYPE, "JSON"); + variables.put(PropsKeysHelper.VALUE_SUBJECT_NAME, "jsonSubject"); + variables.put(PropsKeysHelper.KEY_SUBJECT_NAME, "jsonSubject"); + variables.put(PropsKeysHelper.VALUE_SCHEMA, String.valueOf(parsedSchema)); + variables.put(PropsKeysHelper.KEY_SCHEMA, String.valueOf(parsedSchema)); + variables.putObject(PropsKeysHelper.VALUE_SCHEMA_PROPERTIES, JsonSchemaFixturesConstants.SIMPLE_SCHEMA); + return variables; + } + + public JMeterVariables getVariablesProtobuf() throws IOException { + + final File testFile = fileHelper.getFile("/proto-files/easyTest.proto"); + final ParsedSchema parsedSchema = SchemaParseUtil.getParsedSchema(testFile, "PROTOBUF"); + final var variables = new JMeterVariables(); + variables.put(PropsKeysHelper.KEY_SCHEMA_TYPE, "protobuf"); + variables.put(PropsKeysHelper.VALUE_SUBJECT_NAME, "protobufSubject"); + variables.put(PropsKeysHelper.VALUE_SCHEMA, String.valueOf(parsedSchema)); + variables.put(PropsKeysHelper.KEY_SCHEMA, String.valueOf(parsedSchema)); + variables.put(PropsKeysHelper.KEY_SUBJECT_NAME, "protobufSubject"); + variables.putObject(PropsKeysHelper.KEY_SCHEMA_PROPERTIES, Arrays.asList( + SerializerTestFixture.createFieldValueMapping("street", "string"), + SerializerTestFixture.createFieldValueMapping("number[]", "int-array"), + SerializerTestFixture.createFieldValueMapping("zipcode", "long"))); + variables.putObject(PropsKeysHelper.VALUE_SCHEMA_PROPERTIES, Arrays.asList( + SerializerTestFixture.createFieldValueMapping("street", "string"), + SerializerTestFixture.createFieldValueMapping("number[]", "int-array"), + SerializerTestFixture.createFieldValueMapping("zipcode", "long"))); + return variables; + } } diff --git a/src/test/java/com/sngular/kloadgen/serializer/GenericRecordAvroSerializersTest.java b/src/test/java/com/sngular/kloadgen/serializer/GenericRecordAvroSerializersTest.java index 290771ff..4aa2fee8 100644 --- a/src/test/java/com/sngular/kloadgen/serializer/GenericRecordAvroSerializersTest.java +++ b/src/test/java/com/sngular/kloadgen/serializer/GenericRecordAvroSerializersTest.java @@ -27,40 +27,46 @@ public class GenericRecordAvroSerializersTest { private static Stream getSerializerAndSchemaToTest() { final Builder builder = Stream.builder(); - AvroSerializersTestFixture.SERIALIZER_LIST.forEach(serializer -> { - final Named> serializerArgument = Named.of(serializer.getClass().getName(), serializer); - final File testSubentityArrayFile = AvroSerializersTestFixture.TEST_SUBENTITY_ARRAY.getFirst(); - builder.add(Arguments.arguments(serializerArgument, Named.of(testSubentityArrayFile.getName(), testSubentityArrayFile), AvroSerializersTestFixture.TEST_SUBENTITY_ARRAY.getSecond())); - final File testEmbededAvrosExampleFile = AvroSerializersTestFixture.TEST_EMBEDED_AVROS_EXAMPLE.getFirst(); - builder.add(Arguments.arguments(serializerArgument, Named.of(testEmbededAvrosExampleFile.getName(), testEmbededAvrosExampleFile), AvroSerializersTestFixture.TEST_EMBEDED_AVROS_EXAMPLE.getSecond())); - final File testIssueFile = AvroSerializersTestFixture.TEST_ISSUE.getFirst(); - builder.add(Arguments.arguments(serializerArgument, Named.of(testIssueFile.getName(), testIssueFile), AvroSerializersTestFixture.TEST_ISSUE.getSecond())); - final File testOptionalEnumFile = AvroSerializersTestFixture.TEST_OPTIONAL_ENUM.getFirst(); - builder.add(Arguments.arguments(serializerArgument, Named.of(testOptionalEnumFile.getName(), testOptionalEnumFile), AvroSerializersTestFixture.TEST_OPTIONAL_ENUM.getSecond())); - final File testFileIssueFile = AvroSerializersTestFixture.TEST_FILE_ISSUE.getFirst(); - builder.add(Arguments.arguments(serializerArgument, Named.of(testFileIssueFile.getName(), testFileIssueFile), AvroSerializersTestFixture.TEST_FILE_ISSUE.getSecond())); - final File testLogicalTypesFile = AvroSerializersTestFixture.TEST_LOGICAL_TYPES.getFirst(); - builder.add(Arguments.arguments(serializerArgument, Named.of(testLogicalTypesFile.getName(), testLogicalTypesFile), AvroSerializersTestFixture.TEST_LOGICAL_TYPES.getSecond())); - final File testMapFile = AvroSerializersTestFixture.TEST_MAP.getFirst(); - builder.add(Arguments.arguments(serializerArgument, Named.of(testMapFile.getName(), testMapFile), AvroSerializersTestFixture.TEST_MAP.getSecond())); - final File testNullOnOptionalFieldsFile = AvroSerializersTestFixture.TEST_NULL_ON_OPTIONAL_FIELDS.getFirst(); - builder.add( - Arguments.arguments(serializerArgument, Named.of(testNullOnOptionalFieldsFile.getName(), testNullOnOptionalFieldsFile), AvroSerializersTestFixture.TEST_NULL_ON_OPTIONAL_FIELDS.getSecond())); - final File testOptionalMapFile = AvroSerializersTestFixture.TEST_OPTIONAL_MAP.getFirst(); - builder.add(Arguments.arguments(serializerArgument, Named.of(testOptionalMapFile.getName(), testOptionalMapFile), AvroSerializersTestFixture.TEST_OPTIONAL_MAP.getSecond())); - final File testNullOnOptionalFields = AvroSerializersTestFixture.TEST_NULL_ON_OPTIONAL_FIELDS.getFirst(); - builder.add(Arguments.arguments(serializerArgument, Named.of(testNullOnOptionalFields.getName(), testNullOnOptionalFields), AvroSerializersTestFixture.TEST_NULL_ON_OPTIONAL_FIELDS.getSecond())); - final File testUnionRecordFile = AvroSerializersTestFixture.TEST_UNION_RECORD.getFirst(); - builder.add(Arguments.arguments(serializerArgument, Named.of(testUnionRecordFile.getName(), testUnionRecordFile), AvroSerializersTestFixture.TEST_UNION_RECORD.getSecond())); - final File testUserFile = AvroSerializersTestFixture.TEST_USER.getFirst(); - builder.add(Arguments.arguments(serializerArgument, Named.of(testUserFile.getName(), testUserFile), AvroSerializersTestFixture.TEST_USER.getSecond())); - }); + AvroSerializersTestFixture.SERIALIZER_LIST.forEach(serializer -> extracted(serializer, builder)); return builder.build(); } + private static void extracted(final Serializer serializer, final Builder builder) { + final Named> serializerArgument = Named.of(serializer.getClass().getName(), serializer); + final File testSubentityArrayFile = AvroSerializersTestFixture.TEST_SUBENTITY_ARRAY.getFirst(); + builder.add(Arguments.arguments(serializerArgument, Named.of(testSubentityArrayFile.getName(), testSubentityArrayFile), + AvroSerializersTestFixture.TEST_SUBENTITY_ARRAY.getSecond())); + final File testEmbededAvrosExampleFile = AvroSerializersTestFixture.TEST_EMBEDED_AVROS_EXAMPLE.getFirst(); + builder.add(Arguments.arguments(serializerArgument, Named.of(testEmbededAvrosExampleFile.getName(), testEmbededAvrosExampleFile), + AvroSerializersTestFixture.TEST_EMBEDED_AVROS_EXAMPLE.getSecond())); + final File testIssueFile = AvroSerializersTestFixture.TEST_ISSUE.getFirst(); + builder.add(Arguments.arguments(serializerArgument, Named.of(testIssueFile.getName(), testIssueFile), AvroSerializersTestFixture.TEST_ISSUE.getSecond())); + final File testOptionalEnumFile = AvroSerializersTestFixture.TEST_OPTIONAL_ENUM.getFirst(); + builder.add(Arguments.arguments(serializerArgument, Named.of(testOptionalEnumFile.getName(), testOptionalEnumFile), AvroSerializersTestFixture.TEST_OPTIONAL_ENUM.getSecond())); + final File testFileIssueFile = AvroSerializersTestFixture.TEST_FILE_ISSUE.getFirst(); + builder.add(Arguments.arguments(serializerArgument, Named.of(testFileIssueFile.getName(), testFileIssueFile), AvroSerializersTestFixture.TEST_FILE_ISSUE.getSecond())); + final File testLogicalTypesFile = AvroSerializersTestFixture.TEST_LOGICAL_TYPES.getFirst(); + builder.add(Arguments.arguments(serializerArgument, Named.of(testLogicalTypesFile.getName(), testLogicalTypesFile), AvroSerializersTestFixture.TEST_LOGICAL_TYPES.getSecond())); + final File testMapFile = AvroSerializersTestFixture.TEST_MAP.getFirst(); + builder.add(Arguments.arguments(serializerArgument, Named.of(testMapFile.getName(), testMapFile), AvroSerializersTestFixture.TEST_MAP.getSecond())); + final File testNullOnOptionalFieldsFile = AvroSerializersTestFixture.TEST_NULL_ON_OPTIONAL_FIELDS.getFirst(); + builder.add( + Arguments.arguments(serializerArgument, Named.of(testNullOnOptionalFieldsFile.getName(), testNullOnOptionalFieldsFile), + AvroSerializersTestFixture.TEST_NULL_ON_OPTIONAL_FIELDS.getSecond())); + final File testOptionalMapFile = AvroSerializersTestFixture.TEST_OPTIONAL_MAP.getFirst(); + builder.add(Arguments.arguments(serializerArgument, Named.of(testOptionalMapFile.getName(), testOptionalMapFile), AvroSerializersTestFixture.TEST_OPTIONAL_MAP.getSecond())); + final File testNullOnOptionalFields = AvroSerializersTestFixture.TEST_NULL_ON_OPTIONAL_FIELDS.getFirst(); + builder.add(Arguments.arguments(serializerArgument, Named.of(testNullOnOptionalFields.getName(), testNullOnOptionalFields), + AvroSerializersTestFixture.TEST_NULL_ON_OPTIONAL_FIELDS.getSecond())); + final File testUnionRecordFile = AvroSerializersTestFixture.TEST_UNION_RECORD.getFirst(); + builder.add(Arguments.arguments(serializerArgument, Named.of(testUnionRecordFile.getName(), testUnionRecordFile), AvroSerializersTestFixture.TEST_UNION_RECORD.getSecond())); + final File testUserFile = AvroSerializersTestFixture.TEST_USER.getFirst(); + builder.add(Arguments.arguments(serializerArgument, Named.of(testUserFile.getName(), testUserFile), AvroSerializersTestFixture.TEST_USER.getSecond())); + } + @ParameterizedTest @MethodSource("getSerializerAndSchemaToTest") - void genericAvroRecordSerializerTest(Serializer serializer, File schemaFile, List fieldValueMappings) throws Exception { + void genericAvroRecordSerializerTest(final Serializer serializer, final File schemaFile, final List fieldValueMappings) throws Exception { final var schemaStr = SerializerTestFixture.readSchema(schemaFile); final BaseSchemaMetadata confluentBaseSchemaMetadata = new BaseSchemaMetadata<>( diff --git a/src/test/java/com/sngular/kloadgen/testutil/SchemaParseUtil.java b/src/test/java/com/sngular/kloadgen/testutil/SchemaParseUtil.java index 181600a9..be52e15c 100644 --- a/src/test/java/com/sngular/kloadgen/testutil/SchemaParseUtil.java +++ b/src/test/java/com/sngular/kloadgen/testutil/SchemaParseUtil.java @@ -3,6 +3,7 @@ import java.io.File; import java.io.IOException; import java.nio.file.Files; + import com.sngular.kloadgen.common.SchemaTypeEnum; import com.sngular.kloadgen.exception.KLoadGenException; import io.confluent.kafka.schemaregistry.ParsedSchema; @@ -16,33 +17,26 @@ public final class SchemaParseUtil { private SchemaParseUtil() { } + public static ParsedSchema getParsedSchema(final File schema, final String type) throws IOException { + return getParsedSchema(new String(Files.readAllBytes(schema.toPath())), type); + } + public static ParsedSchema getParsedSchema(final String schema, final String type) { - SchemaTypeEnum schemaType = EnumUtils.isValidEnum(SchemaTypeEnum.class, type.toUpperCase()) ? - SchemaTypeEnum.valueOf(type.toUpperCase()) : null; - ParsedSchema schemaParsed = null; + final SchemaTypeEnum schemaType = EnumUtils.isValidEnum(SchemaTypeEnum.class, type.toUpperCase()) + ? SchemaTypeEnum.valueOf(type.toUpperCase()) : null; + final ParsedSchema schemaParsed; try { - switch (schemaType) { - case JSON: - schemaParsed = new JsonSchema(schema); - break; - case AVRO: - schemaParsed = new AvroSchema(schema); - break; - case PROTOBUF: - schemaParsed = new ProtobufSchema(schema); - break; - default: - throw new KLoadGenException("Unsupported schema type"); - } - } catch (KLoadGenException e) { + assert schemaType != null; + schemaParsed = switch (schemaType) { + case JSON -> new JsonSchema(schema); + case AVRO -> new AvroSchema(schema); + case PROTOBUF -> new ProtobufSchema(schema); + }; + } catch (final KLoadGenException e) { final String logMsg = "Please, make sure that the schema sources fed are correct"; throw new KLoadGenException("Error obtaining object creator factory. " + logMsg); } - return schemaParsed; - } - - public static ParsedSchema getParsedSchema(final File schema, final String type) throws IOException { - return getParsedSchema(new String(Files.readAllBytes(schema.toPath())), type); + return schemaParsed; } } From b10e556b7a4f404c2b7ccb375e399d49188a6250 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jose=20Enrique=20Garc=C3=ADa=20Maci=C3=B1eiras?= Date: Tue, 3 Oct 2023 23:51:29 +0200 Subject: [PATCH 25/30] #332 Try to process multi level protobuff dependencies --- pom-maven-central.xml | 75 ++++++++----------- .../processor/util/SchemaProcessorUtils.java | 56 ++++++++------ 2 files changed, 63 insertions(+), 68 deletions(-) diff --git a/pom-maven-central.xml b/pom-maven-central.xml index c710f436..e0380d1a 100644 --- a/pom-maven-central.xml +++ b/pom-maven-central.xml @@ -20,7 +20,7 @@ Mozilla Public License 2.0 https://github.com/sngular/kloadgen/blob/master/LICENSE repo - + @@ -321,10 +321,10 @@ - 3.22.0 + 3.24.2 1.2.0 1.5.1 - 1.11.0 + 1.11.2 1.9.4 4.4 3.12.0 @@ -333,12 +333,12 @@ 2.4.0-b180830.0359 17 provided - 5.5 + 5.6.2 2.6.0 5.8.2 7.1.1 - 2.4.1.Final - 3.1.0 + 2.4.3.Final + 3.5.1 1.18.22 0.9.5 4.2.0 @@ -346,21 +346,10 @@ 4.5.0 2.0.0-alpha1 1.3.1 - 2.35.0 + 2.35.1 - - org.apache.jmeter - jorphan - ${jmeter.version} - - - org.slf4j - slf4j-api - - - org.apache.kafka kafka-clients @@ -376,11 +365,6 @@ - - com.google.guava - guava - ${guava.version} - com.eclipsesource.minimal-json minimal-json @@ -400,7 +384,12 @@ com.fasterxml.jackson.core jackson-core - 2.13.1 + 2.15.2 + + + com.fasterxml.jackson.core + jackson-annotations + 2.15.2 org.projectlombok @@ -448,6 +437,12 @@ io.apicurio apicurio-registry-client ${apicurio-registry.version} + + + org.jboss.slf4j + slf4j-jboss-logmanager + + io.apicurio @@ -472,7 +467,7 @@ org.apache.avro avro-protobuf - 1.11.0 + 1.11.2 com.github.os72 @@ -485,10 +480,16 @@ 4.0.1 runtime + + com.google.api.grpc + proto-google-common-protos + 2.24.0 + org.slf4j slf4j-api ${slf4j-api.version} + provided org.apache.commons @@ -513,7 +514,7 @@ com.github.everit-org.json-schema org.everit.json.schema - 1.14.0 + 1.14.2 @@ -612,11 +613,6 @@ ${jaxb-api.version} compile - - com.google.api.grpc - proto-google-common-protos - 2.24.0 - org.junit.jupiter junit-jupiter-params @@ -674,7 +670,6 @@ maven-checkstyle-plugin checkstyle.xml - UTF-8 true true false @@ -705,7 +700,7 @@ org.apache.maven.plugins maven-compiler-plugin - 3.6.0 + 3.8.1 true true @@ -750,7 +745,7 @@ cobertura-maven-plugin 2.7 - + xml @@ -770,7 +765,7 @@ org.apache.maven.plugins maven-surefire-plugin - 2.22.2 + 3.1.2 org.sonatype.plugins @@ -821,10 +816,6 @@ ${jmeter.version} provided - - org.apache.jmeter - jorphan - org.apache.logging.log4j log4j-slf4j-impl @@ -856,7 +847,7 @@ org.apache.maven.plugins maven-shade-plugin - 2.4.2 + 3.2.4 @@ -894,10 +885,6 @@ ApacheJMeter_java ${jmeter.version} - - org.apache.jmeter - jorphan - org.apache.logging.log4j log4j-slf4j-impl diff --git a/src/main/java/com/sngular/kloadgen/processor/util/SchemaProcessorUtils.java b/src/main/java/com/sngular/kloadgen/processor/util/SchemaProcessorUtils.java index 2a5e3d22..8f035c90 100644 --- a/src/main/java/com/sngular/kloadgen/processor/util/SchemaProcessorUtils.java +++ b/src/main/java/com/sngular/kloadgen/processor/util/SchemaProcessorUtils.java @@ -15,6 +15,7 @@ import java.util.ListIterator; import java.util.Map; import java.util.Objects; +import java.util.concurrent.ConcurrentLinkedQueue; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -156,27 +157,9 @@ public static Descriptors.Descriptor buildProtoDescriptor(final ProtoFileElement throws Descriptors.DescriptorValidationException, IOException { final DynamicSchema.Builder schemaBuilder = DynamicSchema.newBuilder(); - final List imports = schema.getImports(); + final var imports = new ConcurrentLinkedQueue<>(schema.getImports()); for (final String importedClass : imports) { - try (final InputStream resourceStream = SchemaProcessorUtils.class.getClassLoader().getResourceAsStream(importedClass)) { - if (null != resourceStream) { - final String schemaToString = new String(resourceStream.readAllBytes()); - final var lines = new ArrayList<>(CollectionUtils.select(Arrays.asList(schemaToString.split("\\n")), isValid())); - if (!ProtobufHelper.NOT_ACCEPTED_IMPORTS.contains(importedClass)) { - final var importedSchema = processImported(lines); - schemaBuilder.addDependency(importedSchema.getFileDescriptorSet().getFile(0).getName()); - schemaBuilder.addSchema(importedSchema); - } - } else { - final AbstractParsedSchemaAdapter protoFileElement = JMeterHelper.getParsedSchema(getSubjectName(importedClass, metadata), - JMeterContextService.getContext().getProperties()).getParsedSchemaAdapter(); - final var importedProtobufSchema = new ProtobufSchema(protoFileElement.getRawSchema(), metadata.getSchemaMetadataAdapter().getReferences(), new HashMap<>()); - if (!ProtobufHelper.NOT_ACCEPTED_IMPORTS.contains(importedClass)) { - schemaBuilder.addDependency(importedProtobufSchema.toDescriptor().getFullName()); - schemaBuilder.addSchema(convertDynamicSchema(importedProtobufSchema)); - } - } - } + processImport(metadata, importedClass, schemaBuilder); } final MessageElement messageElement = (MessageElement) schema.getTypes().get(0); @@ -192,6 +175,29 @@ public static Descriptors.Descriptor buildProtoDescriptor(final ProtoFileElement return schemaBuilder.build().getMessageDescriptor(messageElement.getName()); } + private static void processImport(final BaseSchemaMetadata metadata, final String importedClass, final DynamicSchema.Builder schemaBuilder) + throws IOException, DescriptorValidationException { + try (final InputStream resourceStream = SchemaProcessorUtils.class.getClassLoader().getResourceAsStream(importedClass)) { + if (null != resourceStream) { + final String schemaToString = new String(resourceStream.readAllBytes()); + final var lines = new ArrayList<>(CollectionUtils.select(Arrays.asList(schemaToString.split("\\n")), isValid())); + if (!ProtobufHelper.NOT_ACCEPTED_IMPORTS.contains(importedClass)) { + final var importedSchema = processImported(lines, metadata); + schemaBuilder.addDependency(importedSchema.getFileDescriptorSet().getFile(0).getName()); + schemaBuilder.addSchema(importedSchema); + } + } else { + final AbstractParsedSchemaAdapter protoFileElement = JMeterHelper.getParsedSchema(getSubjectName(importedClass, metadata), + JMeterContextService.getContext().getProperties()).getParsedSchemaAdapter(); + final var importedProtobufSchema = new ProtobufSchema(protoFileElement.getRawSchema(), metadata.getSchemaMetadataAdapter().getReferences(), new HashMap<>()); + if (!ProtobufHelper.NOT_ACCEPTED_IMPORTS.contains(importedClass)) { + schemaBuilder.addDependency(importedProtobufSchema.toDescriptor().getFullName()); + schemaBuilder.addSchema(convertDynamicSchema(importedProtobufSchema, metadata)); + } + } + } + } + private static String getSubjectName(final String importedClass, final BaseSchemaMetadata metadata) { final List references = metadata.getSchemaMetadataAdapter().getReferences(); String subjectName = null; @@ -206,15 +212,17 @@ private static String getSubjectName(final String importedClass, final BaseSchem return Objects.requireNonNullElse(subjectName, importedClass); } - private static DynamicSchema convertDynamicSchema(final ProtobufSchema importSchema) throws DescriptorValidationException { - return processImported(Arrays.asList(importSchema.rawSchema().toSchema().split("\\n"))); + private static DynamicSchema convertDynamicSchema(final ProtobufSchema importSchema, final BaseSchemaMetadata metadata) + throws DescriptorValidationException, IOException { + return processImported(Arrays.asList(importSchema.rawSchema().toSchema().split("\\n")), metadata); } private static Predicate isValid() { return line -> !line.contains("//") && !line.isEmpty(); } - private static DynamicSchema processImported(final List importedLines) throws DescriptorValidationException { + private static DynamicSchema processImported(final List importedLines, final BaseSchemaMetadata metadata) + throws DescriptorValidationException, IOException { final DynamicSchema.Builder schemaBuilder = DynamicSchema.newBuilder(); @@ -234,7 +242,7 @@ private static DynamicSchema processImported(final List importedLines) t } if (fileLine.startsWith("import")) { - schemaBuilder.addDependency(fileLine.substring(6)); + processImport(metadata, fileLine.substring(6), schemaBuilder); } } From dced5f8534293c6df6773c3f5c0e74da751ccbcb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jose=20Enrique=20Garc=C3=ADa=20Maci=C3=B1eiras?= Date: Wed, 4 Oct 2023 00:05:52 +0200 Subject: [PATCH 26/30] #332 Try to process multi level protobuff dependencies --- .../extractor/extractors/json/AbstractJsonExtractor.java | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/main/java/com/sngular/kloadgen/extractor/extractors/json/AbstractJsonExtractor.java b/src/main/java/com/sngular/kloadgen/extractor/extractors/json/AbstractJsonExtractor.java index 538e757a..a1aa15f9 100644 --- a/src/main/java/com/sngular/kloadgen/extractor/extractors/json/AbstractJsonExtractor.java +++ b/src/main/java/com/sngular/kloadgen/extractor/extractors/json/AbstractJsonExtractor.java @@ -6,6 +6,7 @@ import com.sngular.kloadgen.exception.KLoadGenException; import com.sngular.kloadgen.extractor.extractors.SchemaExtractorUtil; +import com.sngular.kloadgen.extractor.parser.impl.JSONSchemaParser; import com.sngular.kloadgen.model.ConstraintTypeEnum; import com.sngular.kloadgen.model.FieldValueMapping; import com.sngular.kloadgen.model.json.ArrayField; @@ -21,7 +22,13 @@ import org.jetbrains.annotations.NotNull; public abstract class AbstractJsonExtractor { - + + private final JSONSchemaParser jsonSchemaParser = new JSONSchemaParser(); + + protected final JSONSchemaParser getSchemaParser() { + return jsonSchemaParser; + } + private static String extractFieldName(final String fieldName) { String fieldNameClean = fieldName; if (fieldName.endsWith("[][]") || fieldName.endsWith("[:][]")) { From 8a19eeeac3b15debbe293f2732505a7820657043 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jose=20Enrique=20Garc=C3=ADa=20Maci=C3=B1eiras?= Date: Wed, 4 Oct 2023 09:24:56 +0200 Subject: [PATCH 27/30] #332 Fix jdk build version --- .github/workflows/maven-central-push.yml | 4 ++-- .github/workflows/maven-pr.yml | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/maven-central-push.yml b/.github/workflows/maven-central-push.yml index b2665d60..3564e316 100644 --- a/.github/workflows/maven-central-push.yml +++ b/.github/workflows/maven-central-push.yml @@ -25,10 +25,10 @@ jobs: with: ref: ${{ github.event.pull_request.head.ref }} - - name: Set up JDK 11 + - name: Set up JDK 17 uses: actions/setup-java@v2 with: - java-version: "11" + java-version: "17" distribution: "adopt" server-id: ossrh server-username: MAVEN_USERNAME diff --git a/.github/workflows/maven-pr.yml b/.github/workflows/maven-pr.yml index 2b2873ea..0b3ba30a 100644 --- a/.github/workflows/maven-pr.yml +++ b/.github/workflows/maven-pr.yml @@ -36,10 +36,10 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 - - name: Set up JDK 11 + - name: Set up JDK 17 uses: actions/setup-java@v2 with: - java-version: "11" + java-version: "17" distribution: "adopt" - name: Build with Maven run: mvn -B package --file pom.xml From be0cdb6b98f3c11230a48d1d18bfab76f4a19ad0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jose=20Enrique=20Garc=C3=ADa=20Maci=C3=B1eiras?= Date: Wed, 4 Oct 2023 21:45:13 +0200 Subject: [PATCH 28/30] #332 Fix checkstyle issues --- .../java/com/sngular/kloadgen/sampler/SamplerUtilTest.java | 6 +++--- .../sngular/kloadgen/serializer/AvroDeserializerTest.java | 3 ++- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/src/test/java/com/sngular/kloadgen/sampler/SamplerUtilTest.java b/src/test/java/com/sngular/kloadgen/sampler/SamplerUtilTest.java index a1468c33..2fe87967 100644 --- a/src/test/java/com/sngular/kloadgen/sampler/SamplerUtilTest.java +++ b/src/test/java/com/sngular/kloadgen/sampler/SamplerUtilTest.java @@ -88,13 +88,13 @@ void configureKeyGeneratorTest(final String jmeterProps) throws IOException { props.put(SchemaRegistryKeyHelper.SCHEMA_REGISTRY_URL, "localhost:8081"); props.put(PropsKeysHelper.KEY_SCHEMA_TYPE, jmeterProps); - if (jmeterProps.equalsIgnoreCase("json")) { + if ("json".equalsIgnoreCase(jmeterProps)) { jmcx.setVariables(getVariablesJsonSchema()); } - if (jmeterProps.equalsIgnoreCase("avro")) { + if ("avro".equalsIgnoreCase(jmeterProps)) { jmcx.setVariables(getVariablesAvro()); } - if (jmeterProps.equalsIgnoreCase("protobuf")) { + if ("protobuf".equalsIgnoreCase(jmeterProps)) { jmcx.setVariables(getVariablesProtobuf()); } diff --git a/src/test/java/com/sngular/kloadgen/serializer/AvroDeserializerTest.java b/src/test/java/com/sngular/kloadgen/serializer/AvroDeserializerTest.java index 057bc851..4fbf94b8 100644 --- a/src/test/java/com/sngular/kloadgen/serializer/AvroDeserializerTest.java +++ b/src/test/java/com/sngular/kloadgen/serializer/AvroDeserializerTest.java @@ -6,6 +6,8 @@ import java.util.stream.Stream; import java.util.stream.Stream.Builder; +import javax.xml.bind.DatatypeConverter; + import com.sngular.kloadgen.common.SchemaTypeEnum; import com.sngular.kloadgen.model.FieldValueMapping; import com.sngular.kloadgen.processor.SchemaProcessor; @@ -14,7 +16,6 @@ import com.sngular.kloadgen.testutil.SchemaParseUtil; import com.sngular.kloadgen.util.PropsKeysHelper; import io.confluent.kafka.schemaregistry.ParsedSchema; -import javax.xml.bind.DatatypeConverter; import lombok.extern.slf4j.Slf4j; import org.assertj.core.api.Assertions; import org.junit.jupiter.api.BeforeEach; From eea30291a1d6400bb8bdc3782f1920922b6f78ce Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jose=20Enrique=20Garc=C3=ADa=20Maci=C3=B1eiras?= Date: Thu, 2 Nov 2023 11:49:58 +0100 Subject: [PATCH 29/30] #332 Fix Style --- .../SchemaRegistryConfigElement.java | 10 ++---- .../SchemaRegistryConfigElementBeanInfo.java | 13 +++---- .../SchemaRegistryConfigElementValue.java | 11 +++--- .../editor/KeyDeserializerPropertyEditor.java | 1 + .../editor/NameStrategyPropertyEditor.java | 5 +-- .../SchemaRegistryConfigPropertyEditor.java | 8 ++--- .../SchemaRegistryNamePropertyEditor.java | 1 + .../property/editor/SerDesPackageValue.java | 13 ++++--- .../ValueDeserializerPropertyEditor.java | 1 + .../editor/ValueSerializerPropertyEditor.java | 1 + .../SchemaRegistryConstants.java | 13 ------- .../extractors/json/JsonDefaultTest.java | 36 +++++++------------ .../serializer/SerializerTestFixture.java | 7 ++-- 13 files changed, 52 insertions(+), 68 deletions(-) delete mode 100644 src/main/java/com/sngular/kloadgen/schemaregistry/SchemaRegistryConstants.java diff --git a/src/main/java/com/sngular/kloadgen/config/schemaregistry/SchemaRegistryConfigElement.java b/src/main/java/com/sngular/kloadgen/config/schemaregistry/SchemaRegistryConfigElement.java index 52ec84dc..f586de45 100644 --- a/src/main/java/com/sngular/kloadgen/config/schemaregistry/SchemaRegistryConfigElement.java +++ b/src/main/java/com/sngular/kloadgen/config/schemaregistry/SchemaRegistryConfigElement.java @@ -11,10 +11,6 @@ import java.util.Map; import java.util.Objects; -import static com.sngular.kloadgen.config.schemaregistry.SchemaRegistryConfigElementValue.SCHEMA_REGISTRY_NAME; -import static com.sngular.kloadgen.config.schemaregistry.SchemaRegistryConfigElementValue.SCHEMA_REGISTRY_PROPERTIES; -import static com.sngular.kloadgen.config.schemaregistry.SchemaRegistryConfigElementValue.SCHEMA_REGISTRY_URL; - import com.sngular.kloadgen.model.PropertyMapping; import com.sngular.kloadgen.schemaregistry.SchemaRegistryAdapter; import com.sngular.kloadgen.schemaregistry.SchemaRegistryManagerFactory; @@ -83,7 +79,7 @@ private void serializeProperties() { private Map getProperties() { final Map result = new HashMap<>(); - final JMeterProperty property = getProperty(SCHEMA_REGISTRY_PROPERTIES); + final JMeterProperty property = getProperty(SchemaRegistryConfigElementValue.SCHEMA_REGISTRY_PROPERTIES); if (Objects.nonNull(property)) { result.putAll( this.fromTestElementToPropertiesMap((List) property.getObjectValue())); @@ -94,7 +90,7 @@ private Map getProperties() { } private String getRegistryName() { - String registryName = getPropertyAsString(SCHEMA_REGISTRY_NAME); + String registryName = getPropertyAsString(SchemaRegistryConfigElementValue.SCHEMA_REGISTRY_NAME); if (StringUtils.isBlank(registryName)) { registryName = this.schemaRegistryName; } @@ -102,7 +98,7 @@ private String getRegistryName() { } private String getRegistryUrl() { - String registryUrl = getPropertyAsString(SCHEMA_REGISTRY_URL); + String registryUrl = getPropertyAsString(SchemaRegistryConfigElementValue.SCHEMA_REGISTRY_URL); if (StringUtils.isBlank(registryUrl)) { registryUrl = this.schemaRegistryUrl; } diff --git a/src/main/java/com/sngular/kloadgen/config/schemaregistry/SchemaRegistryConfigElementBeanInfo.java b/src/main/java/com/sngular/kloadgen/config/schemaregistry/SchemaRegistryConfigElementBeanInfo.java index cc90a49c..772fe2c6 100644 --- a/src/main/java/com/sngular/kloadgen/config/schemaregistry/SchemaRegistryConfigElementBeanInfo.java +++ b/src/main/java/com/sngular/kloadgen/config/schemaregistry/SchemaRegistryConfigElementBeanInfo.java @@ -6,10 +6,6 @@ package com.sngular.kloadgen.config.schemaregistry; -import static com.sngular.kloadgen.config.schemaregistry.SchemaRegistryConfigElementValue.SCHEMA_REGISTRY_NAME; -import static com.sngular.kloadgen.config.schemaregistry.SchemaRegistryConfigElementValue.SCHEMA_REGISTRY_PROPERTIES; -import static com.sngular.kloadgen.config.schemaregistry.SchemaRegistryConfigElementValue.SCHEMA_REGISTRY_URL; - import java.beans.PropertyDescriptor; import com.sngular.kloadgen.model.PropertyMapping; @@ -26,22 +22,23 @@ public SchemaRegistryConfigElementBeanInfo() { super(SchemaRegistryConfigElement.class); - createPropertyGroup("schema_registry_config", new String[]{SCHEMA_REGISTRY_NAME, SCHEMA_REGISTRY_URL, SCHEMA_REGISTRY_PROPERTIES}); + createPropertyGroup("schema_registry_config", new String[]{SchemaRegistryConfigElementValue.SCHEMA_REGISTRY_NAME, SchemaRegistryConfigElementValue.SCHEMA_REGISTRY_PROPERTIES, + SchemaRegistryConfigElementValue.SCHEMA_REGISTRY_URL}); - final PropertyDescriptor schemaRegistryName = property(SCHEMA_REGISTRY_NAME); + final PropertyDescriptor schemaRegistryName = property(SchemaRegistryConfigElementValue.SCHEMA_REGISTRY_NAME); schemaRegistryName.setPropertyEditorClass(SchemaRegistryNamePropertyEditor.class); schemaRegistryName.setValue(NOT_UNDEFINED, Boolean.TRUE); schemaRegistryName.setValue(DEFAULT, SchemaRegistryKeyHelper.SCHEMA_REGISTRY_NAME_DEFAULT); schemaRegistryName.setValue(NOT_EXPRESSION, Boolean.FALSE); - final PropertyDescriptor schemaRegistryUrl = property(SCHEMA_REGISTRY_URL); + final PropertyDescriptor schemaRegistryUrl = property(SchemaRegistryConfigElementValue.SCHEMA_REGISTRY_URL); schemaRegistryUrl.setPropertyEditorClass(SchemaRegistryConfigPropertyEditor.class); schemaRegistryUrl.setValue(NOT_UNDEFINED, Boolean.TRUE); schemaRegistryUrl.setValue(DEFAULT, SchemaRegistryKeyHelper.SCHEMA_REGISTRY_URL_DEFAULT); schemaRegistryUrl.setValue(NOT_EXPRESSION, Boolean.FALSE); final TypeEditor tableEditor = TypeEditor.TableEditor; - final PropertyDescriptor tableProperties = property(SCHEMA_REGISTRY_PROPERTIES, tableEditor); + final PropertyDescriptor tableProperties = property(SchemaRegistryConfigElementValue.SCHEMA_REGISTRY_PROPERTIES, tableEditor); tableProperties.setValue(TableEditor.CLASSNAME, PropertyMapping.class.getName()); tableProperties.setValue(TableEditor.HEADERS, new String[]{"Property Name", "Property Value"}); tableProperties.setValue(TableEditor.OBJECT_PROPERTIES, new String[]{PropertyMapping.PROPERTY_NAME, PropertyMapping.PROPERTY_VALUE}); diff --git a/src/main/java/com/sngular/kloadgen/config/schemaregistry/SchemaRegistryConfigElementValue.java b/src/main/java/com/sngular/kloadgen/config/schemaregistry/SchemaRegistryConfigElementValue.java index 153418d4..8aeae711 100644 --- a/src/main/java/com/sngular/kloadgen/config/schemaregistry/SchemaRegistryConfigElementValue.java +++ b/src/main/java/com/sngular/kloadgen/config/schemaregistry/SchemaRegistryConfigElementValue.java @@ -1,8 +1,11 @@ package com.sngular.kloadgen.config.schemaregistry; -public interface SchemaRegistryConfigElementValue { +public final class SchemaRegistryConfigElementValue { - String SCHEMA_REGISTRY_NAME = "schemaRegistryName"; - String SCHEMA_REGISTRY_URL = "schemaRegistryUrl"; - String SCHEMA_REGISTRY_PROPERTIES = "schemaRegistryProperties"; + public static final String SCHEMA_REGISTRY_NAME = "schemaRegistryName"; + public static final String SCHEMA_REGISTRY_URL = "schemaRegistryUrl"; + public static final String SCHEMA_REGISTRY_PROPERTIES = "schemaRegistryProperties"; + + private SchemaRegistryConfigElementValue() { + } } diff --git a/src/main/java/com/sngular/kloadgen/property/editor/KeyDeserializerPropertyEditor.java b/src/main/java/com/sngular/kloadgen/property/editor/KeyDeserializerPropertyEditor.java index f8369ea6..2605bb84 100644 --- a/src/main/java/com/sngular/kloadgen/property/editor/KeyDeserializerPropertyEditor.java +++ b/src/main/java/com/sngular/kloadgen/property/editor/KeyDeserializerPropertyEditor.java @@ -16,6 +16,7 @@ import javax.swing.JComboBox; import javax.swing.JPanel; + import lombok.extern.slf4j.Slf4j; import org.apache.jmeter.gui.ClearGui; import org.apache.jmeter.testbeans.gui.TestBeanPropertyEditor; diff --git a/src/main/java/com/sngular/kloadgen/property/editor/NameStrategyPropertyEditor.java b/src/main/java/com/sngular/kloadgen/property/editor/NameStrategyPropertyEditor.java index dbd337a0..8fddcbcf 100644 --- a/src/main/java/com/sngular/kloadgen/property/editor/NameStrategyPropertyEditor.java +++ b/src/main/java/com/sngular/kloadgen/property/editor/NameStrategyPropertyEditor.java @@ -14,10 +14,11 @@ import java.beans.PropertyEditorSupport; import java.util.Objects; -import io.apicurio.registry.resolver.strategy.ArtifactReferenceResolverStrategy; -import io.confluent.kafka.serializers.subject.strategy.SubjectNameStrategy; import javax.swing.JComboBox; import javax.swing.JPanel; + +import io.apicurio.registry.resolver.strategy.ArtifactReferenceResolverStrategy; +import io.confluent.kafka.serializers.subject.strategy.SubjectNameStrategy; import lombok.extern.slf4j.Slf4j; import org.apache.jmeter.gui.ClearGui; import org.apache.jmeter.testbeans.gui.TestBeanPropertyEditor; diff --git a/src/main/java/com/sngular/kloadgen/property/editor/SchemaRegistryConfigPropertyEditor.java b/src/main/java/com/sngular/kloadgen/property/editor/SchemaRegistryConfigPropertyEditor.java index 54b40f9c..b8544e7b 100644 --- a/src/main/java/com/sngular/kloadgen/property/editor/SchemaRegistryConfigPropertyEditor.java +++ b/src/main/java/com/sngular/kloadgen/property/editor/SchemaRegistryConfigPropertyEditor.java @@ -139,7 +139,7 @@ public final void actionPerformed(final ActionEvent actionEvent) { //Retrieve TableEditor and set all fields with default values to it final var propertyEditors = (PropertyEditor[]) editors.get(testBeanCustomizer); - Map schemaProperties = new HashMap<>(); + final Map schemaProperties = new HashMap<>(); for (PropertyEditor propertyEditor : propertyEditors) { if (propertyEditor instanceof TableEditor) { //noinspection unchecked @@ -150,7 +150,7 @@ public final void actionPerformed(final ActionEvent actionEvent) { } final Map originals = new HashMap<>(); - String schemaRegistryName = schemaProperties.get(SchemaRegistryKeyHelper.SCHEMA_REGISTRY_NAME); + final String schemaRegistryName = schemaProperties.get(SchemaRegistryKeyHelper.SCHEMA_REGISTRY_NAME); JMeterContextService.getContext().getProperties().setProperty(SchemaRegistryKeyHelper.SCHEMA_REGISTRY_NAME, schemaRegistryName); final SchemaRegistryAdapter schemaRegistryManager = SchemaRegistryManagerFactory.getSchemaRegistry(schemaRegistryName); @@ -163,8 +163,8 @@ public final void actionPerformed(final ActionEvent actionEvent) { JMeterContextService.getContext().getProperties().setProperty(SchemaRegistryKeyHelper.SCHEMA_REGISTRY_AUTH_KEY, SchemaRegistryKeyHelper.SCHEMA_REGISTRY_AUTH_BASIC_TYPE); originals.put(AbstractKafkaSchemaSerDeConfig.BASIC_AUTH_CREDENTIALS_SOURCE, "USER_INFO"); - originals.put(AbstractKafkaSchemaSerDeConfig.USER_INFO_CONFIG, schemaProperties.get(SchemaRegistryKeyHelper.SCHEMA_REGISTRY_USERNAME_KEY) + ":" + - schemaProperties.get(SchemaRegistryKeyHelper.SCHEMA_REGISTRY_PASSWORD_KEY)); + originals.put(AbstractKafkaSchemaSerDeConfig.USER_INFO_CONFIG, schemaProperties.get(SchemaRegistryKeyHelper.SCHEMA_REGISTRY_USERNAME_KEY) + ":" + + schemaProperties.get(SchemaRegistryKeyHelper.SCHEMA_REGISTRY_PASSWORD_KEY)); } } diff --git a/src/main/java/com/sngular/kloadgen/property/editor/SchemaRegistryNamePropertyEditor.java b/src/main/java/com/sngular/kloadgen/property/editor/SchemaRegistryNamePropertyEditor.java index 19338290..6d2e4dff 100644 --- a/src/main/java/com/sngular/kloadgen/property/editor/SchemaRegistryNamePropertyEditor.java +++ b/src/main/java/com/sngular/kloadgen/property/editor/SchemaRegistryNamePropertyEditor.java @@ -16,6 +16,7 @@ import javax.swing.JComboBox; import javax.swing.JPanel; + import lombok.extern.slf4j.Slf4j; import org.apache.jmeter.gui.ClearGui; import org.apache.jmeter.testbeans.gui.TestBeanPropertyEditor; diff --git a/src/main/java/com/sngular/kloadgen/property/editor/SerDesPackageValue.java b/src/main/java/com/sngular/kloadgen/property/editor/SerDesPackageValue.java index ecc2e00f..cf55f1c9 100644 --- a/src/main/java/com/sngular/kloadgen/property/editor/SerDesPackageValue.java +++ b/src/main/java/com/sngular/kloadgen/property/editor/SerDesPackageValue.java @@ -2,13 +2,16 @@ public class SerDesPackageValue { - public static final String COM_SNGULAR_KLOADGEN_SERIALIZER = "com.sngular.kloadgen.serializer"; + public static final String COM_SNGULAR_KLOADGEN_SERIALIZER = "com.sngular.kloadgen.serializer"; - public static final String IO_CONFLUENT_KAFKA_SERIALIZERS = "io.confluent.kafka.serializers"; + public static final String IO_CONFLUENT_KAFKA_SERIALIZERS = "io.confluent.kafka.serializers"; - public static final String IO_APICURIO_REGISTRY_SERDE_AVRO_AVRO_KAFKA_DESERIALIZER = "io.apicurio.registry.serde.avro"; + public static final String IO_APICURIO_REGISTRY_SERDE_AVRO_AVRO_KAFKA_DESERIALIZER = "io.apicurio.registry.serde.avro"; - public static final String IO_APICURIO_REGISTRY_SERDE_JSONSCHEMA_JSON_SCHEMA_KAFKA_DESERIALIZER = "io.apicurio.registry.serde.jsonschema"; + public static final String IO_APICURIO_REGISTRY_SERDE_JSONSCHEMA_JSON_SCHEMA_KAFKA_DESERIALIZER = "io.apicurio.registry.serde.jsonschema"; - public static final String IO_APICURIO_REGISTRY_SERDE_PROTOBUF_PROTOBUF_KAFKA_DESERIALIZER = "io.apicurio.registry.serde.protobuf"; + public static final String IO_APICURIO_REGISTRY_SERDE_PROTOBUF_PROTOBUF_KAFKA_DESERIALIZER = "io.apicurio.registry.serde.protobuf"; + + private SerDesPackageValue() { + } } diff --git a/src/main/java/com/sngular/kloadgen/property/editor/ValueDeserializerPropertyEditor.java b/src/main/java/com/sngular/kloadgen/property/editor/ValueDeserializerPropertyEditor.java index b7d4f1ca..27457119 100644 --- a/src/main/java/com/sngular/kloadgen/property/editor/ValueDeserializerPropertyEditor.java +++ b/src/main/java/com/sngular/kloadgen/property/editor/ValueDeserializerPropertyEditor.java @@ -16,6 +16,7 @@ import javax.swing.JComboBox; import javax.swing.JPanel; + import lombok.extern.slf4j.Slf4j; import org.apache.jmeter.gui.ClearGui; import org.apache.jmeter.testbeans.gui.TestBeanPropertyEditor; diff --git a/src/main/java/com/sngular/kloadgen/property/editor/ValueSerializerPropertyEditor.java b/src/main/java/com/sngular/kloadgen/property/editor/ValueSerializerPropertyEditor.java index 704ec72e..3c2b83d0 100644 --- a/src/main/java/com/sngular/kloadgen/property/editor/ValueSerializerPropertyEditor.java +++ b/src/main/java/com/sngular/kloadgen/property/editor/ValueSerializerPropertyEditor.java @@ -16,6 +16,7 @@ import javax.swing.JComboBox; import javax.swing.JPanel; + import lombok.extern.slf4j.Slf4j; import org.apache.jmeter.gui.ClearGui; import org.apache.jmeter.testbeans.gui.TestBeanPropertyEditor; diff --git a/src/main/java/com/sngular/kloadgen/schemaregistry/SchemaRegistryConstants.java b/src/main/java/com/sngular/kloadgen/schemaregistry/SchemaRegistryConstants.java deleted file mode 100644 index d435b865..00000000 --- a/src/main/java/com/sngular/kloadgen/schemaregistry/SchemaRegistryConstants.java +++ /dev/null @@ -1,13 +0,0 @@ -package com.sngular.kloadgen.schemaregistry; - -public final class SchemaRegistryConstants { - - public static final String SCHEMA_REGISTRY_CONFLUENT = "CONFLUENT"; - - public static final String SCHEMA_REGISTRY_APICURIO = "APICURIO"; - - public static final String BASIC_AUTH_CREDENTIALS = "BASIC_AUTH_CREDENTIALS"; - public static final String USER_INFO_CONFIG = "USER_INFO_CONFIG"; - public static final String BEARER_AUTH_CREDENTIALS = "BEARER_AUTH_CREDENTIALS"; - public static final String BEARER_AUTH_TOKEN_CONFIG = "BEARER_AUTH_TOKEN_CONFIG"; -} diff --git a/src/test/java/com/sngular/kloadgen/extractor/extractors/json/JsonDefaultTest.java b/src/test/java/com/sngular/kloadgen/extractor/extractors/json/JsonDefaultTest.java index db6c0818..b4f0898e 100644 --- a/src/test/java/com/sngular/kloadgen/extractor/extractors/json/JsonDefaultTest.java +++ b/src/test/java/com/sngular/kloadgen/extractor/extractors/json/JsonDefaultTest.java @@ -33,12 +33,11 @@ void testBasic() throws Exception { final List fieldValueMappingList = jsonDefaultExtractor.processSchema(schema); Assertions.assertThat(fieldValueMappingList) - .hasSize(3) - .containsExactlyInAnyOrder( - FieldValueMapping.builder().fieldName("firstName").fieldType("string").constraints(constraints).required(false).isAncestorRequired(false).build(), - FieldValueMapping.builder().fieldName("lastName").fieldType("string").constraints(constraints).required(true).isAncestorRequired(false).build(), - FieldValueMapping.builder().fieldName("age").fieldType("number").required(true).isAncestorRequired(false).build() - ); + .hasSize(3) + .containsExactlyInAnyOrder( + FieldValueMapping.builder().fieldName("firstName").fieldType("string").constraints(constraints).required(false).isAncestorRequired(false).build(), + FieldValueMapping.builder().fieldName("lastName").fieldType("string").constraints(constraints).required(true).isAncestorRequired(false).build(), + FieldValueMapping.builder().fieldName("age").fieldType("number").required(true).isAncestorRequired(false).build()); } @Test @@ -56,8 +55,7 @@ void testBasicArray() throws Exception { .containsExactlyInAnyOrder( FieldValueMapping.builder().fieldName("fruits[]").fieldType("string-array").required(true).isAncestorRequired(false).build(), FieldValueMapping.builder().fieldName("vegetables[].veggieName").fieldType("string").constraints(constraints).required(true).isAncestorRequired(true).build(), - FieldValueMapping.builder().fieldName("vegetables[].veggieLike").fieldType("boolean").required(true).isAncestorRequired(true).build() - ); + FieldValueMapping.builder().fieldName("vegetables[].veggieLike").fieldType("boolean").required(true).isAncestorRequired(true).build()); } @Test @@ -81,11 +79,9 @@ void testBasicNumber() throws Exception { .hasSize(2) .containsExactlyInAnyOrder( FieldValueMapping.builder().fieldName("latitude").fieldType("number").constraints(constraintsLatitude).required(true).isAncestorRequired(false).build(), - FieldValueMapping.builder().fieldName("longitude").fieldType("number").constraints(constraintsLongitude).required(true).isAncestorRequired(false).build() - ); + FieldValueMapping.builder().fieldName("longitude").fieldType("number").constraints(constraintsLongitude).required(true).isAncestorRequired(false).build()); } - @Test @DisplayName("Should extract optional collections and optional collections inside objects") void testFlatPropertiesOptionalCollections() throws Exception { @@ -120,8 +116,7 @@ void testFlatPropertiesOptionalCollections() throws Exception { FieldValueMapping.builder().fieldName("objectOfCollectionsOfObject.mapOfObjectsDog[:].vetData.dogId").fieldType("number").required(false).isAncestorRequired(true) .build(), FieldValueMapping.builder().fieldName("objectOfCollectionsOfObject.mapOfObjectsDog[:].vetData.breedName").fieldType("string").constraints(constraints) - .required(false).isAncestorRequired(true).build() - ); + .required(false).isAncestorRequired(true).build()); } @Test @@ -149,8 +144,7 @@ void testComplexDefinitions() throws Exception { FieldValueMapping.builder().fieldName("mapOfMaps[:][:].stringControlObject").fieldType("string").constraints(constraints).required(true).isAncestorRequired(true) .build(), FieldValueMapping.builder().fieldName("mapOfMaps[:][:].arrayOfArraysOfStrings[][]").fieldType("string-array-array").required(false).isAncestorRequired(true) - .build() - ); + .build()); } @Test @@ -174,8 +168,7 @@ void testRequiredPropagationChildrenFields() throws Exception { FieldValueMapping.builder().fieldName("geopoliticalSubdivisions.level2.code").fieldType("string").constraints(constraintsCode).required(false) .isAncestorRequired(true).build(), FieldValueMapping.builder().fieldName("geopoliticalSubdivisions.level2.freeForm").fieldType("string").constraints(constraintsFreeForm).required(false) - .isAncestorRequired(true).build() - ); + .isAncestorRequired(true).build()); } @Test @@ -193,8 +186,7 @@ void testShouldExtractJsonSchemaDefinitions() throws Exception { .contains( FieldValueMapping.builder().fieldName("duty.amount.value").fieldType("number").required(false).isAncestorRequired(false).build(), FieldValueMapping.builder().fieldName("duty.amount.currency").fieldType("string").constraints(constraints).required(false).isAncestorRequired(false).build(), - FieldValueMapping.builder().fieldName("duty.amount.exponent").fieldType("number").required(false).isAncestorRequired(false).build() - ); + FieldValueMapping.builder().fieldName("duty.amount.exponent").fieldType("number").required(false).isAncestorRequired(false).build()); } @Test @@ -209,8 +201,7 @@ void testMultipleType() throws Exception { Assertions.assertThat(fieldValueMappingList) .hasSize(1) .satisfiesExactly( - fieldValueMapping -> Set.of("number", "uuid").contains(fieldValueMapping.getFieldType()) - ); + fieldValueMapping -> Set.of("number", "uuid").contains(fieldValueMapping.getFieldType())); } @Test @@ -238,8 +229,7 @@ void testFlatPropertiesOptionalNestedCollections() throws Exception { FieldValueMapping.builder().fieldName("mapOfObjectsOfCollections[:].arrayOfMapsOfObject[][:].stringControl").fieldType("string").constraints(constraints) .required(false).isAncestorRequired(true).build(), FieldValueMapping.builder().fieldName("mapOfObjectsOfCollections[:].arrayOfMapsOfObject[][:].numberControl").fieldType("number").required(false) - .isAncestorRequired(true).build() - ); + .isAncestorRequired(true).build()); } @Test diff --git a/src/test/java/com/sngular/kloadgen/serializer/SerializerTestFixture.java b/src/test/java/com/sngular/kloadgen/serializer/SerializerTestFixture.java index 3e7ec09f..ae6827a7 100644 --- a/src/test/java/com/sngular/kloadgen/serializer/SerializerTestFixture.java +++ b/src/test/java/com/sngular/kloadgen/serializer/SerializerTestFixture.java @@ -8,7 +8,10 @@ import com.sngular.kloadgen.model.FieldValueMapping; -public class SerializerTestFixture { +public final class SerializerTestFixture { + + private SerializerTestFixture() { + } public static FieldValueMapping createFieldValueMapping(final String name, final String fieldType) { return FieldValueMapping.builder().fieldName(name).fieldType(fieldType).valueLength(0).fieldValueList("[]").required(true) @@ -20,7 +23,7 @@ public static FieldValueMapping createFieldValueMapping(final String name, final .isAncestorRequired(true).build(); } - static String readSchema(final File file) throws IOException { + public static String readSchema(final File file) throws IOException { final StringBuilder contentBuilder = new StringBuilder(); try (Stream stream = Files.lines(file.toPath(), StandardCharsets.UTF_8)) { From 217c117da5be16b41c420b5a358dc7aa10fe77ef Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jose=20Enrique=20Garc=C3=ADa=20Maci=C3=B1eiras?= Date: Mon, 6 Nov 2023 20:01:43 +0100 Subject: [PATCH 30/30] #332 Fix Google dependencies --- pom-maven-central.xml | 5 +++++ pom.xml | 5 +++++ 2 files changed, 10 insertions(+) diff --git a/pom-maven-central.xml b/pom-maven-central.xml index e0380d1a..9463466a 100644 --- a/pom-maven-central.xml +++ b/pom-maven-central.xml @@ -485,6 +485,11 @@ proto-google-common-protos 2.24.0 + + com.google.protobuf + protobuf-java + 3.24.3 + org.slf4j slf4j-api diff --git a/pom.xml b/pom.xml index deb1aaac..16a05cb6 100644 --- a/pom.xml +++ b/pom.xml @@ -485,6 +485,11 @@ proto-google-common-protos 2.24.0 + + com.google.protobuf + protobuf-java + 3.24.3 + org.slf4j slf4j-api