diff --git a/metacat-client/src/main/java/com/netflix/metacat/client/api/ParentChildRelV1.java b/metacat-client/src/main/java/com/netflix/metacat/client/api/ParentChildRelV1.java index 546030f69..b946c50c4 100644 --- a/metacat-client/src/main/java/com/netflix/metacat/client/api/ParentChildRelV1.java +++ b/metacat-client/src/main/java/com/netflix/metacat/client/api/ParentChildRelV1.java @@ -7,7 +7,8 @@ import javax.ws.rs.PathParam; import javax.ws.rs.core.MediaType; -import com.netflix.metacat.common.dto.notifications.ChildInfoDto; +import com.netflix.metacat.common.dto.ChildInfoDto; +import com.netflix.metacat.common.dto.ParentInfoDto; import java.util.Set; @@ -40,4 +41,24 @@ Set getChildren( @PathParam("table-name") String tableName ); + + /** + * Return the list of parent. + * @param catalogName catalogName + * @param databaseName databaseName + * @param tableName tableName + * @return list of parent info dtos + */ + @GET + @Path("parents/catalog/{catalog-name}/database/{database-name}/table/{table-name}") + @Consumes(MediaType.APPLICATION_JSON) + @Produces(MediaType.APPLICATION_JSON) + Set getParents( + @PathParam("catalog-name") + String catalogName, + @PathParam("database-name") + String databaseName, + @PathParam("table-name") + String tableName + ); } diff --git a/metacat-common-server/src/main/java/com/netflix/metacat/common/server/converter/ConverterUtil.java b/metacat-common-server/src/main/java/com/netflix/metacat/common/server/converter/ConverterUtil.java index 5ad54ace8..9307a7602 100644 --- a/metacat-common-server/src/main/java/com/netflix/metacat/common/server/converter/ConverterUtil.java +++ b/metacat-common-server/src/main/java/com/netflix/metacat/common/server/converter/ConverterUtil.java @@ -33,7 +33,8 @@ import com.netflix.metacat.common.dto.Sort; import com.netflix.metacat.common.dto.StorageDto; import com.netflix.metacat.common.dto.TableDto; -import com.netflix.metacat.common.dto.notifications.ChildInfoDto; +import com.netflix.metacat.common.dto.ChildInfoDto; +import com.netflix.metacat.common.dto.ParentInfoDto; import com.netflix.metacat.common.server.connectors.ConnectorRequestContext; import com.netflix.metacat.common.server.connectors.model.AuditInfo; import com.netflix.metacat.common.server.connectors.model.CatalogInfo; @@ -48,6 +49,7 @@ import com.netflix.metacat.common.server.connectors.model.StorageInfo; import com.netflix.metacat.common.server.connectors.model.TableInfo; import com.netflix.metacat.common.server.model.ChildInfo; +import com.netflix.metacat.common.server.model.ParentInfo; import lombok.NonNull; import org.dozer.CustomConverter; import org.dozer.DozerBeanMapper; @@ -287,4 +289,14 @@ public PartitionsSaveResponseDto toPartitionsSaveResponseDto(final PartitionsSav public ChildInfoDto toChildInfoDto(final ChildInfo childInfo) { return mapper.map(childInfo, ChildInfoDto.class); } + + /** + * Convert ParentInfo to ParentInfoDto. + * + * @param parentInfo parentInfo + * @return parentInfo dto + */ + public ParentInfoDto toParentInfoDto(final ParentInfo parentInfo) { + return mapper.map(parentInfo, ParentInfoDto.class); + } } diff --git a/metacat-common-server/src/main/java/com/netflix/metacat/common/server/properties/Config.java b/metacat-common-server/src/main/java/com/netflix/metacat/common/server/properties/Config.java index a22ea5885..c9669e737 100644 --- a/metacat-common-server/src/main/java/com/netflix/metacat/common/server/properties/Config.java +++ b/metacat-common-server/src/main/java/com/netflix/metacat/common/server/properties/Config.java @@ -607,5 +607,40 @@ public interface Config { * @return True if it should be. */ boolean shouldFetchOnlyMetadataLocationEnabled(); + + /** + * Whether we allow parent child relationship to be created. + * + * @return True if it should be. + */ + boolean isParentChildCreateEnabled(); + + /** + * Whether we allow renaming parent child relationship. + * + * @return True if it should be. + */ + boolean isParentChildRenameEnabled(); + + /** + * Whether we allow getting parent child relationship in the getTable call. + * + * @return True if it should be. + */ + boolean isParentChildGetEnabled(); + + /** + * Whether we allow dropping tables that are either parent or child. + * + * @return True if it should be. + */ + boolean isParentChildDropEnabled(); + + /** + * Get the parentChildRelationshipProperties config. + * + * @return parentChildRelationshipProperties + */ + ParentChildRelationshipProperties getParentChildRelationshipProperties(); } diff --git a/metacat-common-server/src/main/java/com/netflix/metacat/common/server/properties/DefaultConfigImpl.java b/metacat-common-server/src/main/java/com/netflix/metacat/common/server/properties/DefaultConfigImpl.java index 65b4f2a8e..5f223ed7a 100644 --- a/metacat-common-server/src/main/java/com/netflix/metacat/common/server/properties/DefaultConfigImpl.java +++ b/metacat-common-server/src/main/java/com/netflix/metacat/common/server/properties/DefaultConfigImpl.java @@ -678,4 +678,29 @@ public boolean disablePartitionDefinitionMetadata() { public boolean shouldFetchOnlyMetadataLocationEnabled() { return this.metacatProperties.getHive().getIceberg().isShouldFetchOnlyMetadataLocationEnabled(); } + + @Override + public boolean isParentChildCreateEnabled() { + return this.metacatProperties.getParentChildRelationshipProperties().isCreateEnabled(); + } + + @Override + public boolean isParentChildRenameEnabled() { + return this.metacatProperties.getParentChildRelationshipProperties().isRenameEnabled(); + } + + @Override + public boolean isParentChildGetEnabled() { + return this.metacatProperties.getParentChildRelationshipProperties().isGetEnabled(); + } + + @Override + public boolean isParentChildDropEnabled() { + return this.metacatProperties.getParentChildRelationshipProperties().isDropEnabled(); + } + + @Override + public ParentChildRelationshipProperties getParentChildRelationshipProperties() { + return this.metacatProperties.getParentChildRelationshipProperties(); + } } diff --git a/metacat-common-server/src/main/java/com/netflix/metacat/common/server/properties/MetacatProperties.java b/metacat-common-server/src/main/java/com/netflix/metacat/common/server/properties/MetacatProperties.java index 877ead1cc..6e9505ce9 100644 --- a/metacat-common-server/src/main/java/com/netflix/metacat/common/server/properties/MetacatProperties.java +++ b/metacat-common-server/src/main/java/com/netflix/metacat/common/server/properties/MetacatProperties.java @@ -18,6 +18,7 @@ package com.netflix.metacat.common.server.properties; import lombok.NonNull; +import org.springframework.core.env.Environment; /** * Entry point to entire property tree of Metacat namespace. @@ -27,6 +28,8 @@ */ @lombok.Data public class MetacatProperties { + @NonNull + private Environment env; @NonNull private Data data = new Data(); @NonNull @@ -67,4 +70,16 @@ public class MetacatProperties { private AliasServiceProperties aliasServiceProperties = new AliasServiceProperties(); @NonNull private RateLimiterProperties rateLimiterProperties = new RateLimiterProperties(); + @NonNull + private ParentChildRelationshipProperties parentChildRelationshipProperties; + + /** + * Constructor for MetacatProperties. + * + * @param env Spring Environment + */ + public MetacatProperties(final Environment env) { + this.env = env; + this.parentChildRelationshipProperties = new ParentChildRelationshipProperties(env); + } } diff --git a/metacat-common-server/src/main/java/com/netflix/metacat/common/server/properties/ParentChildRelationshipProperties.java b/metacat-common-server/src/main/java/com/netflix/metacat/common/server/properties/ParentChildRelationshipProperties.java new file mode 100644 index 000000000..c8152595a --- /dev/null +++ b/metacat-common-server/src/main/java/com/netflix/metacat/common/server/properties/ParentChildRelationshipProperties.java @@ -0,0 +1,119 @@ +package com.netflix.metacat.common.server.properties; + +import lombok.Data; +import lombok.extern.slf4j.Slf4j; +import org.springframework.core.env.Environment; + +import javax.annotation.Nullable; +import java.util.Arrays; +import java.util.HashMap; +import java.util.Map; +import java.util.stream.Collectors; + +/** + * Parent Child Relationship service properties. + * + * @author yingjianw + */ +@Data +@Slf4j +public class ParentChildRelationshipProperties { + private static final String MAX_ALLOW_PER_TABLE_PER_REL_PROPERTY_NAME = + "metacat.parentChildRelationshipProperties.maxAllowPerTablePerRelConfig"; + private static final String MAX_ALLOW_PER_DB_PER_REL_PROPERTY_NAME = + "metacat.parentChildRelationshipProperties.maxAllowPerDBPerRelConfig"; + private static final String DEFAULT_MAX_ALLOW_PER_REL_PROPERTY_NAME = + "metacat.parentChildRelationshipProperties.defaultMaxAllowPerRelConfig"; + private boolean createEnabled; + private boolean getEnabled; + private boolean renameEnabled; + private boolean dropEnabled; + private int maxAllow = 5; + private Map> maxAllowPerTablePerRelType = new HashMap<>(); + private Map> maxAllowPerDBPerRelType = new HashMap<>(); + private Map defaultMaxAllowPerRelType = new HashMap<>(); + + /** + * Constructor. + * + * @param env Spring environment + */ + public ParentChildRelationshipProperties(@Nullable final Environment env) { + if (env != null) { + setMaxAllowPerTablePerRelType( + env.getProperty(MAX_ALLOW_PER_TABLE_PER_REL_PROPERTY_NAME, String.class, "") + ); + setMaxAllowPerDBPerRelType( + env.getProperty(MAX_ALLOW_PER_DB_PER_REL_PROPERTY_NAME, String.class, "") + ); + setDefaultMaxAllowPerRelType( + env.getProperty(DEFAULT_MAX_ALLOW_PER_REL_PROPERTY_NAME, String.class, "") + ); + } + } + + /** + * setMaxAllowPerTablePerRelType based on String config. + * + * @param configStr configString + */ + public void setMaxAllowPerTablePerRelType(@Nullable final String configStr) { + if (configStr == null || configStr.isEmpty()) { + return; + } + try { + this.maxAllowPerTablePerRelType = parseNestedConfigString(configStr); + } catch (Exception e) { + log.error("Fail to apply configStr = {} for maxAllowPerTablePerRelType", configStr, e); + } + } + + /** + * setMaxAllowPerDBPerRelType based on String config. + * + * @param configStr configString + */ + public void setMaxAllowPerDBPerRelType(@Nullable final String configStr) { + if (configStr == null || configStr.isEmpty()) { + return; + } + try { + this.maxAllowPerDBPerRelType = parseNestedConfigString(configStr); + } catch (Exception e) { + log.error("Fail to apply configStr = {} for maxCloneAllowPerDBPerRelType", configStr); + } + } + /** + * setMaxCloneAllowPerDBPerRelType based on String config. + * + * @param configStr configString + */ + public void setDefaultMaxAllowPerRelType(@Nullable final String configStr) { + if (configStr == null || configStr.isEmpty()) { + return; + } + try { + this.defaultMaxAllowPerRelType = + Arrays.stream(configStr.split(";")) + .map(entry -> entry.split(",")) + .collect(Collectors.toMap( + parts -> parts[0], + parts -> Integer.parseInt(parts[1]) + )); + } catch (Exception e) { + log.error("Fail to apply configStr = {} for defaultMaxAllowPerRelType", configStr); + } + } + + private Map> parseNestedConfigString(final String configStr) { + return Arrays.stream(configStr.split(";")) + .map(entry -> entry.split(",")) + .collect(Collectors.groupingBy( + parts -> parts[0], + Collectors.toMap( + parts -> parts[1], + parts -> Integer.parseInt(parts[2]) + ) + )); + } +} diff --git a/metacat-common-server/src/main/java/com/netflix/metacat/common/server/usermetadata/ParentChildRelMetadataService.java b/metacat-common-server/src/main/java/com/netflix/metacat/common/server/usermetadata/ParentChildRelMetadataService.java index b8e6220a2..0ddfcbad9 100644 --- a/metacat-common-server/src/main/java/com/netflix/metacat/common/server/usermetadata/ParentChildRelMetadataService.java +++ b/metacat-common-server/src/main/java/com/netflix/metacat/common/server/usermetadata/ParentChildRelMetadataService.java @@ -1,8 +1,10 @@ package com.netflix.metacat.common.server.usermetadata; import com.netflix.metacat.common.QualifiedName; -import com.netflix.metacat.common.dto.notifications.ChildInfoDto; +import com.netflix.metacat.common.dto.ChildInfoDto; +import com.netflix.metacat.common.dto.ParentInfoDto; import com.netflix.metacat.common.server.model.ChildInfo; import com.netflix.metacat.common.server.model.ParentInfo; +import com.netflix.metacat.common.server.properties.ParentChildRelationshipProperties; import java.util.Set; @@ -24,13 +26,15 @@ public interface ParentChildRelMetadataService { * @param childName the name of the child entity * @param childUUID the uuid of the child * @param relationType the type of the relationship + * @param prop properties config */ void createParentChildRelation( QualifiedName parentName, String parentUUID, QualifiedName childName, String childUUID, - String relationType + String relationType, + ParentChildRelationshipProperties prop ); /** @@ -98,9 +102,30 @@ Set getChildren( /** * get the set of children dto for the input name. * @param name name - * @return a set of ChildInfo + * @return a set of ChildInfo dto */ Set getChildrenDto( QualifiedName name ); + + /** + * get the set of parent dto for the input name. + * @param name name + * @return a set of parentInfo dto + */ + Set getParentsDto(QualifiedName name); + + /** + * return whether the table is a parent. + * @param tableName tableName + * @return true if it exists + */ + boolean isParentTable(final QualifiedName tableName); + + /** + * return whether the table is a child. + * @param tableName tableName + * @return true if it exists + */ + boolean isChildTable(final QualifiedName tableName); } diff --git a/metacat-common/src/main/java/com/netflix/metacat/common/dto/notifications/ChildInfoDto.java b/metacat-common/src/main/java/com/netflix/metacat/common/dto/ChildInfoDto.java similarity index 85% rename from metacat-common/src/main/java/com/netflix/metacat/common/dto/notifications/ChildInfoDto.java rename to metacat-common/src/main/java/com/netflix/metacat/common/dto/ChildInfoDto.java index bf150c4af..faadfdcdf 100644 --- a/metacat-common/src/main/java/com/netflix/metacat/common/dto/notifications/ChildInfoDto.java +++ b/metacat-common/src/main/java/com/netflix/metacat/common/dto/ChildInfoDto.java @@ -1,6 +1,5 @@ -package com.netflix.metacat.common.dto.notifications; +package com.netflix.metacat.common.dto; -import com.netflix.metacat.common.dto.BaseDto; import io.swagger.annotations.ApiModelProperty; import lombok.AllArgsConstructor; import lombok.Builder; @@ -9,7 +8,7 @@ import lombok.NoArgsConstructor; /** - * ChildInfo information. + * ChildInfo dto information. */ @SuppressWarnings("unused") @Data diff --git a/metacat-common/src/main/java/com/netflix/metacat/common/dto/ParentInfoDto.java b/metacat-common/src/main/java/com/netflix/metacat/common/dto/ParentInfoDto.java new file mode 100644 index 000000000..e917eb2c1 --- /dev/null +++ b/metacat-common/src/main/java/com/netflix/metacat/common/dto/ParentInfoDto.java @@ -0,0 +1,30 @@ +package com.netflix.metacat.common.dto; + +import io.swagger.annotations.ApiModelProperty; +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.EqualsAndHashCode; +import lombok.NoArgsConstructor; + +/** + * ParentInfo dto information. + */ +@SuppressWarnings("unused") +@Data +@EqualsAndHashCode(callSuper = false) +@Builder +@AllArgsConstructor +@NoArgsConstructor +public class ParentInfoDto extends BaseDto { + private static final long serialVersionUID = 8121239864203088788L; + /* Name of the parent */ + @ApiModelProperty(value = "name of the child") + private String name; + /* Type of the relation */ + @ApiModelProperty(value = "type of the relation") + private String relationType; + /* uuid of the table */ + @ApiModelProperty(value = "uuid of the table") + private String uuid; +} diff --git a/metacat-connector-polaris/src/test/java/com/netflix/metacat/connector/polaris/PolarisConnectorDatabaseServiceTest.java b/metacat-connector-polaris/src/test/java/com/netflix/metacat/connector/polaris/PolarisConnectorDatabaseServiceTest.java index 98fb0eca7..e297704f8 100644 --- a/metacat-connector-polaris/src/test/java/com/netflix/metacat/connector/polaris/PolarisConnectorDatabaseServiceTest.java +++ b/metacat-connector-polaris/src/test/java/com/netflix/metacat/connector/polaris/PolarisConnectorDatabaseServiceTest.java @@ -65,7 +65,7 @@ public class PolarisConnectorDatabaseServiceTest { @BeforeEach public void init() { connectorContext = new ConnectorContext(CATALOG_NAME, CATALOG_NAME, "polaris", - new DefaultConfigImpl(new MetacatProperties()), new NoopRegistry(), null, Maps.newHashMap()); + new DefaultConfigImpl(new MetacatProperties(null)), new NoopRegistry(), null, Maps.newHashMap()); polarisDBService = new PolarisConnectorDatabaseService(polarisStoreService, connectorContext); } diff --git a/metacat-connector-polaris/src/test/java/com/netflix/metacat/connector/polaris/PolarisConnectorTableServiceTest.java b/metacat-connector-polaris/src/test/java/com/netflix/metacat/connector/polaris/PolarisConnectorTableServiceTest.java index 22f95fdfd..27c4f918e 100644 --- a/metacat-connector-polaris/src/test/java/com/netflix/metacat/connector/polaris/PolarisConnectorTableServiceTest.java +++ b/metacat-connector-polaris/src/test/java/com/netflix/metacat/connector/polaris/PolarisConnectorTableServiceTest.java @@ -33,6 +33,7 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.test.autoconfigure.orm.jpa.AutoConfigureDataJpa; import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.core.env.Environment; import org.springframework.test.annotation.DirtiesContext; import org.springframework.test.context.ActiveProfiles; import org.springframework.test.context.junit.jupiter.SpringExtension; @@ -75,6 +76,9 @@ public class PolarisConnectorTableServiceTest { @Shared private PolarisConnectorTableService polarisTableService; + @Shared + private Environment env = Mockito.mock(Environment.class); + /** * Initialization. */ @@ -83,7 +87,7 @@ public void init() { final String location = "file://temp"; polarisStoreService.createDatabase(DB_NAME, location, "metacat_user"); connectorContext = new ConnectorContext(CATALOG_NAME, CATALOG_NAME, "polaris", - new DefaultConfigImpl(new MetacatProperties()), new NoopRegistry(), null, Maps.newHashMap()); + new DefaultConfigImpl(new MetacatProperties(null)), new NoopRegistry(), null, Maps.newHashMap()); polarisDBService = new PolarisConnectorDatabaseService(polarisStoreService, connectorContext); polarisTableService = new PolarisConnectorTableService( polarisStoreService, diff --git a/metacat-functional-tests/metacat-test-cluster/docker-compose.yml b/metacat-functional-tests/metacat-test-cluster/docker-compose.yml index 4e422c91f..bdf867b91 100644 --- a/metacat-functional-tests/metacat-test-cluster/docker-compose.yml +++ b/metacat-functional-tests/metacat-test-cluster/docker-compose.yml @@ -73,7 +73,11 @@ services: -Dmetacat.table.delete.noDeleteOnTags=do_not_drop,iceberg_migration_do_not_modify -Dmetacat.table.rename.noRenameOnTags=do_not_rename,iceberg_migration_do_not_modify -Dmetacat.table.update.noUpdateOnTags=iceberg_migration_do_not_modify - -Dmetacat.event.updateIcebergTablePostEventEnabled=true' + -Dmetacat.event.updateIcebergTablePostEventEnabled=true + -Dmetacat.parentChildRelationshipProperties.createEnabled=true + -Dmetacat.parentChildRelationshipProperties.getEnabled=true + -Dmetacat.parentChildRelationshipProperties.renameEnabled=true + -Dmetacat.parentChildRelationshipProperties.dropEnabled=true' labels: - "com.netflix.metacat.oss.test" - "com.netflix.metacat.oss.test.war" diff --git a/metacat-functional-tests/src/functionalTest/groovy/com/netflix/metacat/MetacatSmokeSpec.groovy b/metacat-functional-tests/src/functionalTest/groovy/com/netflix/metacat/MetacatSmokeSpec.groovy index 9ccfd1e70..c7193abd6 100644 --- a/metacat-functional-tests/src/functionalTest/groovy/com/netflix/metacat/MetacatSmokeSpec.groovy +++ b/metacat-functional-tests/src/functionalTest/groovy/com/netflix/metacat/MetacatSmokeSpec.groovy @@ -24,7 +24,6 @@ import com.netflix.metacat.client.api.PartitionV1 import com.netflix.metacat.client.api.TagV1 import com.netflix.metacat.common.QualifiedName import com.netflix.metacat.common.dto.* -import com.netflix.metacat.common.dto.notifications.ChildInfoDto import com.netflix.metacat.common.exception.MetacatAlreadyExistsException import com.netflix.metacat.common.exception.MetacatBadRequestException import com.netflix.metacat.common.exception.MetacatNotFoundException @@ -1992,12 +1991,15 @@ class MetacatSmokeSpec extends Specification { def parent1UUID = "p1_uuid" def renameParent1 = "rename_parent1" def parent1Uri = isLocalEnv ? String.format('file:/tmp/%s/%s', databaseName, parent1) : null + def renameParent1Uri = isLocalEnv ? String.format('file:/tmp/%s/%s', databaseName, parent1) : null def parent1FullName = catalogName + "/" + databaseName + "/" + parent1 + def renameParent1FullName = catalogName + "/" + databaseName + "/" + renameParent1 def child11 = "child11" def child11UUID = "c11_uuid" def renameChild11 = "rename_child11" def child11Uri = isLocalEnv ? String.format('file:/tmp/%s/%s', databaseName, child11) : null + def renameChild11Uri = isLocalEnv ? String.format('file:/tmp/%s/%s', databaseName, renameChild11) : null def child11FullName = catalogName + "/" + databaseName + "/" + child11 def child12 = "child12" @@ -2043,35 +2045,95 @@ class MetacatSmokeSpec extends Specification { def parent1Table = api.getTable(catalogName, databaseName, parent1, true, true, false) def child11Table = api.getTable(catalogName, databaseName, child11, true, true, false) - def child11ParentChildRelationInfo = child11Table.definitionMetadata.get("parentChildRelationInfo") then: // Test Parent 1 parentChildInfo assert parent1Table.definitionMetadata.get("parentChildRelationInfo").get("isParent").booleanValue() assert parentChildRelV1.getChildren(catalogName, databaseName, parent1) == [new ChildInfoDto("embedded-fast-hive-metastore/iceberg_db/child11", "CLONE", "c11_uuid")] as Set + assert parentChildRelV1.getParents(catalogName, databaseName, parent1).isEmpty() // Test Child11 parentChildInfo assert !child11Table.definitionMetadata.get("parentChildRelationInfo").has("isParent") assert child11Table.definitionMetadata.get("random_key").asText() == "random_value" - JSONAssert.assertEquals(child11ParentChildRelationInfo.toString(), + JSONAssert.assertEquals(child11Table.definitionMetadata.get("parentChildRelationInfo").toString(), '{"parentInfos":[{"name":"embedded-fast-hive-metastore/iceberg_db/parent1","relationType":"CLONE", "uuid":"p1_uuid"}]}', false) + assert parentChildRelV1.getParents(catalogName, databaseName, child11) == [new ParentInfoDto("embedded-fast-hive-metastore/iceberg_db/parent1", "CLONE", "p1_uuid")] as Set assert parentChildRelV1.getChildren(catalogName, databaseName, child11).isEmpty() /* - Step 2: create another table with the same child1 name but different uuid under the same parent should fail + Step 2: create another table with the same parent1 name but should fail because the table already exists + Test this should not impact the previous record + */ + when: + api.createTable(catalogName, databaseName, parent1, parent1TableDto) + then: + def e = thrown(Exception) + assert e.message.contains("already exists") + + when: + parent1Table = api.getTable(catalogName, databaseName, parent1, true, true, false) + child11Table = api.getTable(catalogName, databaseName, child11, true, true, false) + then: + // Test Parent 1 parentChildInfo + assert parent1Table.definitionMetadata.get("parentChildRelationInfo").get("isParent").booleanValue() + assert parentChildRelV1.getChildren(catalogName, databaseName, parent1) == [ + new ChildInfoDto("embedded-fast-hive-metastore/iceberg_db/child11", "CLONE", "c11_uuid"), + ] as Set + assert parentChildRelV1.getParents(catalogName, databaseName, parent1).isEmpty() + // Test Child11 parentChildInfo + assert !child11Table.definitionMetadata.get("parentChildRelationInfo").has("isParent") + assert child11Table.definitionMetadata.get("random_key").asText() == "random_value" + JSONAssert.assertEquals(child11Table.definitionMetadata.get("parentChildRelationInfo").toString(), + '{"parentInfos":[{"name":"embedded-fast-hive-metastore/iceberg_db/parent1","relationType":"CLONE", "uuid":"p1_uuid"}]}', + false) + assert parentChildRelV1.getChildren(catalogName, databaseName, child11).isEmpty() + assert parentChildRelV1.getParents(catalogName, databaseName, child11) == [new ParentInfoDto("embedded-fast-hive-metastore/iceberg_db/parent1", "CLONE", "p1_uuid")] as Set + + /* + Step 3: create another table with the same child1 name but different uuid under the same parent should fail */ when: child11TableDto = PigDataDtoProvider.getTable(catalogName, databaseName, child11, 'amajumdar', child11Uri) initializeParentChildRelDefinitionMetadata(child11TableDto, parent1FullName, parent1UUID, "random_uuid") api.createTable(catalogName, databaseName, child11, child11TableDto) then: - def e = thrown(Exception) + e = thrown(Exception) + assert e.message.contains("Cannot have a child table having more than one parent") + + /* + Step 4: create another table with the same child1 name but different uuid under a different parent that + does not exist + */ + when: + child11TableDto = PigDataDtoProvider.getTable(catalogName, databaseName, child11, 'amajumdar', child11Uri) + initializeParentChildRelDefinitionMetadata(child11TableDto, parent2FullName, parent2UUID, "random_uuid") + api.createTable(catalogName, databaseName, child11, child11TableDto) + then: + e = thrown(Exception) + assert e.message.contains("does not exist") + + /* + Step 5: create another table with the same child1 name but different uuid under a different parent that + does exists + */ + when: + def randomParentName = "randomParent" + def randomParentUUID = "randomParent_uuid" + def randomParentFullName = catalogName + "/" + databaseName + "/" + randomParentName + def randomParentUri = isLocalEnv ? String.format('file:/tmp/%s/%s', databaseName, randomParentName) : null + def randomParentTableDto = PigDataDtoProvider.getTable(catalogName, databaseName, randomParentName, 'amajumdar', randomParentUri) + api.createTable(catalogName, databaseName, randomParentName, randomParentTableDto) + child11TableDto = PigDataDtoProvider.getTable(catalogName, databaseName, child11, 'amajumdar', child11Uri) + initializeParentChildRelDefinitionMetadata(child11TableDto, randomParentFullName, randomParentUUID, "random_uuid") + api.createTable(catalogName, databaseName, child11, child11TableDto) + then: + e = thrown(Exception) assert e.message.contains("Cannot have a child table having more than one parent") /* - Step 3: create another table with the same name different uuid without specifying the parent child relation + Step 6: create another table with the same name different uuid without specifying the parent child relation but should fail because the table already exists - This test the revert should not impact the previous record + This test the failure during creation should not impact the previous record */ when: child11TableDto = PigDataDtoProvider.getTable(catalogName, databaseName, child11, 'amajumdar', child11Uri) @@ -2080,21 +2142,27 @@ class MetacatSmokeSpec extends Specification { e = thrown(Exception) assert e.message.contains("already exists") + when: + parent1Table = api.getTable(catalogName, databaseName, parent1, true, true, false) + child11Table = api.getTable(catalogName, databaseName, child11, true, true, false) + then: // Test Parent 1 parentChildInfo assert parent1Table.definitionMetadata.get("parentChildRelationInfo").get("isParent").booleanValue() assert parentChildRelV1.getChildren(catalogName, databaseName, parent1) == [new ChildInfoDto("embedded-fast-hive-metastore/iceberg_db/child11", "CLONE", "c11_uuid")] as Set + assert parentChildRelV1.getParents(catalogName, databaseName, parent1).isEmpty() // Test Child11 parentChildInfo assert !child11Table.definitionMetadata.get("parentChildRelationInfo").has("isParent") assert child11Table.definitionMetadata.get("random_key").asText() == "random_value" - JSONAssert.assertEquals(child11ParentChildRelationInfo.toString(), + JSONAssert.assertEquals(child11Table.definitionMetadata.get("parentChildRelationInfo").toString(), '{"parentInfos":[{"name":"embedded-fast-hive-metastore/iceberg_db/parent1","relationType":"CLONE", "uuid":"p1_uuid"}]}', false) assert parentChildRelV1.getChildren(catalogName, databaseName, child11).isEmpty() + assert parentChildRelV1.getParents(catalogName, databaseName, child11) == [new ParentInfoDto("embedded-fast-hive-metastore/iceberg_db/parent1", "CLONE", "p1_uuid")] as Set /* - Step 4: Create a second child (child12) pointing to parent = parent1 + Step 7: Create a second child (child12) pointing to parent = parent1 */ when: // Create Child2 Table @@ -2103,7 +2171,6 @@ class MetacatSmokeSpec extends Specification { api.createTable(catalogName, databaseName, child12, child12TableDto) parent1Table = api.getTable(catalogName, databaseName, parent1, true, true, false) def child12Table = api.getTable(catalogName, databaseName, child12, true, true, false) - def child12ParentChildRelationInfo = child12Table.definitionMetadata.get("parentChildRelationInfo") then: // Test Parent 1 parentChildInfo @@ -2112,16 +2179,18 @@ class MetacatSmokeSpec extends Specification { new ChildInfoDto("embedded-fast-hive-metastore/iceberg_db/child11", "CLONE", "c11_uuid"), new ChildInfoDto("embedded-fast-hive-metastore/iceberg_db/child12", "CLONE", "c12_uuid") ] as Set + assert parentChildRelV1.getParents(catalogName, databaseName, parent1).isEmpty() // Test Child12 parentChildInfo assert !child12Table.definitionMetadata.get("parentChildRelationInfo").has("isParent") - JSONAssert.assertEquals(child12ParentChildRelationInfo.toString(), + JSONAssert.assertEquals(child12Table.definitionMetadata.get("parentChildRelationInfo").toString(), '{"parentInfos":[{"name":"embedded-fast-hive-metastore/iceberg_db/parent1","relationType":"CLONE","uuid":"p1_uuid"}]}', false) assert parentChildRelV1.getChildren(catalogName, databaseName, child12).isEmpty() + assert parentChildRelV1.getParents(catalogName, databaseName, child12) == [new ParentInfoDto("embedded-fast-hive-metastore/iceberg_db/parent1", "CLONE", "p1_uuid")] as Set /* - Step 5: create a parent table on top of another parent table should fail + Step 8: create a parent table on top of another parent table should fail */ when: def grandParent1TableDto = PigDataDtoProvider.getTable(catalogName, databaseName, grandParent1, 'amajumdar', grantParent1Uri) @@ -2135,83 +2204,149 @@ class MetacatSmokeSpec extends Specification { assert e.message.contains("Cannot create a parent table on top of another parent") /* - Step 6: create another table with the same parent1 name but should fail because the table already exists - Test the revert should not impact the previous record + Step 9: Create one grandChild As a Parent of A child table should fail */ when: - api.createTable(catalogName, databaseName, parent1, parent1TableDto) + def grandchild121TableDto = PigDataDtoProvider.getTable(catalogName, databaseName, grandChild121, 'amajumdar', null) + initializeParentChildRelDefinitionMetadata(grandchild121TableDto, child11FullName, child11UUID, grandChild121UUID) + api.createTable(catalogName, databaseName, grandChild121, grandchild121TableDto) + assert parentChildRelV1.getChildren(catalogName, databaseName, grandChild121).isEmpty() + then: + e = thrown(Exception) + assert e.message.contains("Cannot create a child table as parent") + + /* + Step 10: Create another parent child that is disconnected with the above + */ + when: + // Create Parent2 + def parent2TableDto = PigDataDtoProvider.getTable(catalogName, databaseName, parent2, 'amajumdar', parent2Uri) + api.createTable(catalogName, databaseName, parent2, parent2TableDto) + + // Create child21 Table with parent = parent2 + def child21TableDto = PigDataDtoProvider.getTable(catalogName, databaseName, child21, 'amajumdar', child21Uri) + initializeParentChildRelDefinitionMetadata(child21TableDto, parent2FullName, parent2UUID, child21UUID) + api.createTable(catalogName, databaseName, child21, child21TableDto) + def parent2Table = api.getTable(catalogName, databaseName, parent2, true, true, false) + def child21Table = api.getTable(catalogName, databaseName, child21, true, true, false) + + then: + // Test Parent 2 parentChildInfo + assert parent2Table.definitionMetadata.get("parentChildRelationInfo").get("isParent").booleanValue() + assert parentChildRelV1.getChildren(catalogName, databaseName, parent2) == [ + new ChildInfoDto("embedded-fast-hive-metastore/iceberg_db/child21", "CLONE", "c21_uuid") + ] as Set + assert parentChildRelV1.getParents(catalogName, databaseName, parent2).isEmpty() + + // Test Child21 parentChildInfo + JSONAssert.assertEquals(child21Table.definitionMetadata.get("parentChildRelationInfo").toString(), + '{"parentInfos":[{"name":"embedded-fast-hive-metastore/iceberg_db/parent2","relationType":"CLONE","uuid":"p2_uuid"}]}', + false) + assert parentChildRelV1.getChildren(catalogName, databaseName, child21).isEmpty() + assert parentChildRelV1.getParents(catalogName, databaseName, child21) == [new ParentInfoDto("embedded-fast-hive-metastore/iceberg_db/parent2", "CLONE", "p2_uuid")] as Set + + /* + Step 11: Create a table newParent1 without any parent child rel info + and attempt to rename parent1 to newParent1 should fail + Test the parentChildRelationship record remain the same after the revert + */ + when: + def newParent1TableDto = PigDataDtoProvider.getTable(catalogName, databaseName, renameParent1, 'amajumdar', renameParent1Uri) + api.createTable(catalogName, databaseName, renameParent1, newParent1TableDto) + api.renameTable(catalogName, databaseName, parent1, renameParent1) then: e = thrown(Exception) assert e.message.contains("already exists") + when: + parent1Table = api.getTable(catalogName, databaseName, parent1, true, true, false) + child11Table = api.getTable(catalogName, databaseName, child11, true, true, false) + child12Table = api.getTable(catalogName, databaseName, child12, true, true, false) + + then: // Test Parent 1 parentChildInfo assert parent1Table.definitionMetadata.get("parentChildRelationInfo").get("isParent").booleanValue() assert parentChildRelV1.getChildren(catalogName, databaseName, parent1) == [ new ChildInfoDto("embedded-fast-hive-metastore/iceberg_db/child11", "CLONE", "c11_uuid"), new ChildInfoDto("embedded-fast-hive-metastore/iceberg_db/child12", "CLONE", "c12_uuid") ] as Set - + assert parentChildRelV1.getParents(catalogName, databaseName, parent1).isEmpty() // Test Child11 parentChildInfo assert !child11Table.definitionMetadata.get("parentChildRelationInfo").has("isParent") assert child11Table.definitionMetadata.get("random_key").asText() == "random_value" - JSONAssert.assertEquals(child11ParentChildRelationInfo.toString(), + JSONAssert.assertEquals(child11Table.definitionMetadata.get("parentChildRelationInfo").toString(), '{"parentInfos":[{"name":"embedded-fast-hive-metastore/iceberg_db/parent1","relationType":"CLONE", "uuid":"p1_uuid"}]}', false) assert parentChildRelV1.getChildren(catalogName, databaseName, child11).isEmpty() + assert parentChildRelV1.getParents(catalogName, databaseName, child11) == [new ParentInfoDto("embedded-fast-hive-metastore/iceberg_db/parent1", "CLONE", "p1_uuid")] as Set + // Test Child12 parentChildInfo + assert !child12Table.definitionMetadata.get("parentChildRelationInfo").has("isParent") + JSONAssert.assertEquals(child12Table.definitionMetadata.get("parentChildRelationInfo").toString(), + '{"parentInfos":[{"name":"embedded-fast-hive-metastore/iceberg_db/parent1","relationType":"CLONE","uuid":"p1_uuid"}]}', + false) + assert parentChildRelV1.getChildren(catalogName, databaseName, child12).isEmpty() + assert parentChildRelV1.getParents(catalogName, databaseName, child12) == [new ParentInfoDto("embedded-fast-hive-metastore/iceberg_db/parent1", "CLONE", "p1_uuid")] as Set /* - Step 7: Create one grandChild As a Parent of A child table should fail + Step 12: Attempt to rename parent1 to parent2 which has parent child relationship and should fail + Test the parentChildRelationship record remain the same after the revert */ when: - def grandchild121TableDto = PigDataDtoProvider.getTable(catalogName, databaseName, grandChild121, 'amajumdar', null) - initializeParentChildRelDefinitionMetadata(grandchild121TableDto, child11FullName, child11UUID, grandChild121UUID) - - api.createTable(catalogName, databaseName, grandChild121, grandchild121TableDto) - assert parentChildRelV1.getChildren(catalogName, databaseName, grandChild121).isEmpty() - + api.renameTable(catalogName, databaseName, parent1, parent2) then: e = thrown(Exception) - assert e.message.contains("Cannot create a child table as parent") + assert e.message.contains("is already a parent table") - /* - Step 8: Create another parent child that is disconnected with the above - */ when: - // Create Parent2 - def parent2TableDto = PigDataDtoProvider.getTable(catalogName, databaseName, parent2, 'amajumdar', parent2Uri) - api.createTable(catalogName, databaseName, parent2, parent2TableDto) - - // Create child21 Table with parent = parent2 - def child21TableDto = PigDataDtoProvider.getTable(catalogName, databaseName, child21, 'amajumdar', child21Uri) - initializeParentChildRelDefinitionMetadata(child21TableDto, parent2FullName, parent2UUID, child21UUID) - api.createTable(catalogName, databaseName, child21, child21TableDto) - def parent2Table = api.getTable(catalogName, databaseName, parent2, true, true, false) - def child21Table = api.getTable(catalogName, databaseName, child21, true, true, false) - def child21ParentChildRelationInfo = child21Table.definitionMetadata.get("parentChildRelationInfo") - + parent1Table = api.getTable(catalogName, databaseName, parent1, true, true, false) + child11Table = api.getTable(catalogName, databaseName, child11, true, true, false) + child12Table = api.getTable(catalogName, databaseName, child12, true, true, false) + parent2Table = api.getTable(catalogName, databaseName, parent2, true, true, false) + child21Table = api.getTable(catalogName, databaseName, child21, true, true, false) then: + // Test Parent 1 parentChildInfo + assert parent1Table.definitionMetadata.get("parentChildRelationInfo").get("isParent").booleanValue() + assert parentChildRelV1.getChildren(catalogName, databaseName, parent1) == [ + new ChildInfoDto("embedded-fast-hive-metastore/iceberg_db/child11", "CLONE", "c11_uuid"), + new ChildInfoDto("embedded-fast-hive-metastore/iceberg_db/child12", "CLONE", "c12_uuid") + ] as Set + assert parentChildRelV1.getParents(catalogName, databaseName, parent1).isEmpty() + // Test Child11 parentChildInfo + assert !child11Table.definitionMetadata.get("parentChildRelationInfo").has("isParent") + assert child11Table.definitionMetadata.get("random_key").asText() == "random_value" + JSONAssert.assertEquals(child11Table.definitionMetadata.get("parentChildRelationInfo").toString(), + '{"parentInfos":[{"name":"embedded-fast-hive-metastore/iceberg_db/parent1","relationType":"CLONE", "uuid":"p1_uuid"}]}', + false) + assert parentChildRelV1.getChildren(catalogName, databaseName, child11).isEmpty() + assert parentChildRelV1.getParents(catalogName, databaseName, child11) == [new ParentInfoDto("embedded-fast-hive-metastore/iceberg_db/parent1", "CLONE", "p1_uuid")] as Set + // Test Child12 parentChildInfo + assert !child12Table.definitionMetadata.get("parentChildRelationInfo").has("isParent") + JSONAssert.assertEquals(child12Table.definitionMetadata.get("parentChildRelationInfo").toString(), + '{"parentInfos":[{"name":"embedded-fast-hive-metastore/iceberg_db/parent1","relationType":"CLONE","uuid":"p1_uuid"}]}', + false) + assert parentChildRelV1.getChildren(catalogName, databaseName, child12).isEmpty() // Test Parent 2 parentChildInfo assert parent2Table.definitionMetadata.get("parentChildRelationInfo").get("isParent").booleanValue() assert parentChildRelV1.getChildren(catalogName, databaseName, parent2) == [ new ChildInfoDto("embedded-fast-hive-metastore/iceberg_db/child21", "CLONE", "c21_uuid") ] as Set - // Test Child21 parentChildInfo - JSONAssert.assertEquals(child21ParentChildRelationInfo.toString(), + JSONAssert.assertEquals(child21Table.definitionMetadata.get("parentChildRelationInfo").toString(), '{"parentInfos":[{"name":"embedded-fast-hive-metastore/iceberg_db/parent2","relationType":"CLONE","uuid":"p2_uuid"}]}', false) assert parentChildRelV1.getChildren(catalogName, databaseName, child21).isEmpty() + assert parentChildRelV1.getParents(catalogName, databaseName, child12) == [new ParentInfoDto("embedded-fast-hive-metastore/iceberg_db/parent1", "CLONE", "p1_uuid")] as Set + /* - Step 9: Rename parent1 to newParent1 + Step 13: First drop the newParent1 and then Rename parent1 to newParent1 should now succeed */ when: + api.deleteTable(catalogName, databaseName, renameParent1) api.renameTable(catalogName, databaseName, parent1, renameParent1) parent1Table = api.getTable(catalogName, databaseName, renameParent1, true, true, false) child11Table = api.getTable(catalogName, databaseName, child11, true, true, false) - child11ParentChildRelationInfo = child11Table.definitionMetadata.get("parentChildRelationInfo") child12Table = api.getTable(catalogName, databaseName, child12, true, true, false) - child12ParentChildRelationInfo = child12Table.definitionMetadata.get("parentChildRelationInfo") then: // Test Parent 1 parentChildInfo newName @@ -2221,19 +2356,22 @@ class MetacatSmokeSpec extends Specification { new ChildInfoDto("embedded-fast-hive-metastore/iceberg_db/child11", "CLONE", "c11_uuid"), new ChildInfoDto("embedded-fast-hive-metastore/iceberg_db/child12", "CLONE", "c12_uuid") ] as Set + assert parentChildRelV1.getParents(catalogName, databaseName, renameParent1).isEmpty() // Test Child11 parentChildInfo assert !child11Table.definitionMetadata.get("parentChildRelationInfo").has("isParent") - JSONAssert.assertEquals(child11ParentChildRelationInfo.toString(), + JSONAssert.assertEquals(child11Table.definitionMetadata.get("parentChildRelationInfo").toString(), '{"parentInfos":[{"name":"embedded-fast-hive-metastore/iceberg_db/rename_parent1","relationType":"CLONE","uuid":"p1_uuid"}]}', false) assert parentChildRelV1.getChildren(catalogName, databaseName, child11).isEmpty() + assert parentChildRelV1.getParents(catalogName, databaseName, child11) == [new ParentInfoDto("embedded-fast-hive-metastore/iceberg_db/rename_parent1", "CLONE", "p1_uuid")] as Set // Test Child12 parentChildInfo assert !child12Table.definitionMetadata.get("parentChildRelationInfo").has("isParent") - JSONAssert.assertEquals(child12ParentChildRelationInfo.toString(), + JSONAssert.assertEquals(child12Table.definitionMetadata.get("parentChildRelationInfo").toString(), '{"parentInfos":[{"name":"embedded-fast-hive-metastore/iceberg_db/rename_parent1","relationType":"CLONE","uuid":"p1_uuid"}]}', false) assert parentChildRelV1.getChildren(catalogName, databaseName, child12).isEmpty() + assert parentChildRelV1.getParents(catalogName, databaseName, child12) == [new ParentInfoDto("embedded-fast-hive-metastore/iceberg_db/rename_parent1", "CLONE", "p1_uuid")] as Set //get the parent oldName should fail as it no longer exists when: @@ -2243,13 +2381,109 @@ class MetacatSmokeSpec extends Specification { assert e.message.contains("Unable to locate for") /* - Step 10: Rename child11 to renameChild11 + Step 14: Create a table renameChild11 without parent childInfo and then try to rename child11 to renameChild11, which should fail + This test to make sure the revert works properly. + */ + when: + def newChild1TableDto = PigDataDtoProvider.getTable(catalogName, databaseName, renameChild11, 'amajumdar', renameChild11Uri) + api.createTable(catalogName, databaseName, renameChild11, newChild1TableDto) + api.renameTable(catalogName, databaseName, child11, renameChild11) + + then: + e = thrown(Exception) + assert e.message.contains("already exists") + + when: + parent1Table = api.getTable(catalogName, databaseName, renameParent1, true, true, false) + child11Table = api.getTable(catalogName, databaseName, child11, true, true, false) + child12Table = api.getTable(catalogName, databaseName, child12, true, true, false) + then: + // Test Parent 1 parentChildInfo newName + assert parent1Table.definitionMetadata.get("parentChildRelationInfo").get("isParent").booleanValue() + assert parentChildRelV1.getChildren(catalogName, databaseName, renameParent1) == + [ + new ChildInfoDto("embedded-fast-hive-metastore/iceberg_db/child11", "CLONE", "c11_uuid"), + new ChildInfoDto("embedded-fast-hive-metastore/iceberg_db/child12", "CLONE", "c12_uuid") + ] as Set + assert parentChildRelV1.getParents(catalogName, databaseName, renameParent1).isEmpty() + // Test Child11 parentChildInfo + assert !child11Table.definitionMetadata.get("parentChildRelationInfo").has("isParent") + JSONAssert.assertEquals(child11Table.definitionMetadata.get("parentChildRelationInfo").toString(), + '{"parentInfos":[{"name":"embedded-fast-hive-metastore/iceberg_db/rename_parent1","relationType":"CLONE","uuid":"p1_uuid"}]}', + false) + assert parentChildRelV1.getChildren(catalogName, databaseName, child11).isEmpty() + assert parentChildRelV1.getParents(catalogName, databaseName, child11) == [new ParentInfoDto("embedded-fast-hive-metastore/iceberg_db/rename_parent1", "CLONE", "p1_uuid")] as Set + + // Test Child12 parentChildInfo + assert !child12Table.definitionMetadata.get("parentChildRelationInfo").has("isParent") + JSONAssert.assertEquals(child12Table.definitionMetadata.get("parentChildRelationInfo").toString(), + '{"parentInfos":[{"name":"embedded-fast-hive-metastore/iceberg_db/rename_parent1","relationType":"CLONE","uuid":"p1_uuid"}]}', + false) + assert parentChildRelV1.getChildren(catalogName, databaseName, child12).isEmpty() + assert parentChildRelV1.getParents(catalogName, databaseName, child12) == [new ParentInfoDto("embedded-fast-hive-metastore/iceberg_db/rename_parent1", "CLONE", "p1_uuid")] as Set + + /* + Step 15: Create a table renameChild11 with parent childInfo and then try to rename child11 to renameChild11, which should fail + This test to make sure the revert works properly. + */ + when: + api.deleteTable(catalogName, databaseName, renameChild11) + newChild1TableDto = PigDataDtoProvider.getTable(catalogName, databaseName, renameChild11, 'amajumdar', renameChild11Uri) + initializeParentChildRelDefinitionMetadata(newChild1TableDto, renameParent1FullName, parent1UUID, "random_uuid") + api.createTable(catalogName, databaseName, renameChild11, newChild1TableDto) + api.renameTable(catalogName, databaseName, child11, renameChild11) + + then: + e = thrown(Exception) + assert e.message.contains("is already a child table") + + when: + def renameChild11Table = api.getTable(catalogName, databaseName, renameChild11, true, true, false) + parent1Table = api.getTable(catalogName, databaseName, renameParent1, true, true, false) + child11Table = api.getTable(catalogName, databaseName, child11, true, true, false) + child12Table = api.getTable(catalogName, databaseName, child12, true, true, false) + then: + // Test Parent 1 parentChildInfo newName + assert parent1Table.definitionMetadata.get("parentChildRelationInfo").get("isParent").booleanValue() + assert parentChildRelV1.getChildren(catalogName, databaseName, renameParent1) == + [ + new ChildInfoDto("embedded-fast-hive-metastore/iceberg_db/child11", "CLONE", "c11_uuid"), + new ChildInfoDto("embedded-fast-hive-metastore/iceberg_db/child12", "CLONE", "c12_uuid"), + new ChildInfoDto("embedded-fast-hive-metastore/iceberg_db/rename_child11", "CLONE", "random_uuid") + ] as Set + assert parentChildRelV1.getParents(catalogName, databaseName, renameParent1).isEmpty() + // Test Child11 parentChildInfo + assert !child11Table.definitionMetadata.get("parentChildRelationInfo").has("isParent") + JSONAssert.assertEquals(child11Table.definitionMetadata.get("parentChildRelationInfo").toString(), + '{"parentInfos":[{"name":"embedded-fast-hive-metastore/iceberg_db/rename_parent1","relationType":"CLONE","uuid":"p1_uuid"}]}', + false) + assert parentChildRelV1.getChildren(catalogName, databaseName, child11).isEmpty() + assert parentChildRelV1.getParents(catalogName, databaseName, child11) == [new ParentInfoDto("embedded-fast-hive-metastore/iceberg_db/rename_parent1", "CLONE", "p1_uuid")] as Set + // Test Child12 parentChildInfo + assert !child12Table.definitionMetadata.get("parentChildRelationInfo").has("isParent") + JSONAssert.assertEquals(child12Table.definitionMetadata.get("parentChildRelationInfo").toString(), + '{"parentInfos":[{"name":"embedded-fast-hive-metastore/iceberg_db/rename_parent1","relationType":"CLONE","uuid":"p1_uuid"}]}', + false) + assert parentChildRelV1.getChildren(catalogName, databaseName, child12).isEmpty() + assert parentChildRelV1.getParents(catalogName, databaseName, child12) == [new ParentInfoDto("embedded-fast-hive-metastore/iceberg_db/rename_parent1", "CLONE", "p1_uuid")] as Set + // Test renameChild11Table parentChildInfo + assert !renameChild11Table.definitionMetadata.get("parentChildRelationInfo").has("isParent") + JSONAssert.assertEquals(renameChild11Table.definitionMetadata.get("parentChildRelationInfo").toString(), + '{"parentInfos":[{"name":"embedded-fast-hive-metastore/iceberg_db/rename_parent1","relationType":"CLONE","uuid":"p1_uuid"}]}', + false) + assert parentChildRelV1.getChildren(catalogName, databaseName, renameChild11).isEmpty() + assert parentChildRelV1.getParents(catalogName, databaseName, renameChild11) == [new ParentInfoDto("embedded-fast-hive-metastore/iceberg_db/rename_parent1", "CLONE", "p1_uuid")] as Set + + + /* + Step 16: Drop previous renameChild11 and Rename child11 to renameChild11 should now succeed */ when: + api.deleteTable(catalogName, databaseName, renameChild11) api.renameTable(catalogName, databaseName, child11, renameChild11) parent1Table = api.getTable(catalogName, databaseName, renameParent1, true, true, false) child11Table = api.getTable(catalogName, databaseName, renameChild11, true, true, false) - child11ParentChildRelationInfo = child11Table.definitionMetadata.get("parentChildRelationInfo") + then: // Test parent1Table parentChildInfo with newName assert parent1Table.definitionMetadata.get("parentChildRelationInfo").get("isParent").booleanValue() @@ -2257,13 +2491,15 @@ class MetacatSmokeSpec extends Specification { new ChildInfoDto("embedded-fast-hive-metastore/iceberg_db/rename_child11", "CLONE", "c11_uuid"), new ChildInfoDto("embedded-fast-hive-metastore/iceberg_db/child12", "CLONE", "c12_uuid") ] as Set + assert parentChildRelV1.getParents(catalogName, databaseName, renameParent1).isEmpty() // Test Child11 parentChildInfo with newName assert !child11Table.definitionMetadata.get("parentChildRelationInfo").has("isParent") assert child11Table.definitionMetadata.get("random_key").asText() == "random_value" - JSONAssert.assertEquals(child11ParentChildRelationInfo.toString(), + JSONAssert.assertEquals(child11Table.definitionMetadata.get("parentChildRelationInfo").toString(), '{"parentInfos":[{"name":"embedded-fast-hive-metastore/iceberg_db/rename_parent1","relationType":"CLONE","uuid":"p1_uuid"}]}', false) assert parentChildRelV1.getChildren(catalogName, databaseName, renameChild11).isEmpty() + assert parentChildRelV1.getParents(catalogName, databaseName, renameChild11) == [new ParentInfoDto("embedded-fast-hive-metastore/iceberg_db/rename_parent1", "CLONE", "p1_uuid")] as Set //get the child oldName should fail as it no longer exists when: @@ -2273,7 +2509,7 @@ class MetacatSmokeSpec extends Specification { assert e.message.contains("Unable to locate for") /* - Step 11: Drop parent renameParent1 + Step 17: Drop parent renameParent1 */ when: api.deleteTable(catalogName, databaseName, renameParent1) @@ -2283,7 +2519,7 @@ class MetacatSmokeSpec extends Specification { assert e.message.contains("because it still has") /* - Step 8: Drop renameChild11 should succeed + Step 18: Drop renameChild11 should succeed */ when: api.deleteTable(catalogName, databaseName, renameChild11) @@ -2295,9 +2531,10 @@ class MetacatSmokeSpec extends Specification { assert parentChildRelV1.getChildren(catalogName, databaseName, renameParent1) == [ new ChildInfoDto("embedded-fast-hive-metastore/iceberg_db/child12", "CLONE", "c12_uuid") ] as Set + assert parentChildRelV1.getParents(catalogName, databaseName, renameParent1).isEmpty() /* - Step 12: Create renameChild11 and should expect random_key should appear at it is reattached + Step 19: Create renameChild11 and should expect random_key should appear at it is reattached but parent childInfo should not as it is always coming from the parent child relationship service which currently does not have any record */ @@ -2305,19 +2542,21 @@ class MetacatSmokeSpec extends Specification { child11TableDto = PigDataDtoProvider.getTable(catalogName, databaseName, renameChild11, 'amajumdar', child11Uri) api.createTable(catalogName, databaseName, renameChild11, child11TableDto) child11Table = api.getTable(catalogName, databaseName, renameChild11, true, true, false) + parent1Table = api.getTable(catalogName, databaseName, renameParent1, true, true, false) then: assert !child11Table.definitionMetadata.has("parentChildRelationInfo") assert child11Table.definitionMetadata.get("random_key").asText() == "random_value" - assert parentChildRelV1.getChildren(catalogName, databaseName, child11).isEmpty() - + assert parentChildRelV1.getChildren(catalogName, databaseName, renameChild11).isEmpty() + assert parentChildRelV1.getParents(catalogName, databaseName, renameChild11).isEmpty() // Test parent1 Table still only have child12 assert parent1Table.definitionMetadata.get("parentChildRelationInfo").get("isParent").booleanValue() assert parentChildRelV1.getChildren(catalogName, databaseName, renameParent1) == [ new ChildInfoDto("embedded-fast-hive-metastore/iceberg_db/child12", "CLONE", "c12_uuid") ] as Set + assert parentChildRelV1.getParents(catalogName, databaseName, renameParent1).isEmpty() /* - Step 13: Drop child12 should succeed + Step 20: Drop child12 should succeed */ when: api.deleteTable(catalogName, databaseName, child12) @@ -2325,17 +2564,20 @@ class MetacatSmokeSpec extends Specification { then: assert !parent1Table.definitionMetadata.has("parentChildRelationInfo") assert parentChildRelV1.getChildren(catalogName, databaseName, renameParent1).isEmpty() + assert parentChildRelV1.getParents(catalogName, databaseName, renameParent1).isEmpty() /* - Step 14: Drop renameParent1 should succeed as there is no more child under it + Step 21: Drop renameParent1 should succeed as there is no more child under it */ when: api.deleteTable(catalogName, databaseName, renameParent1) parent2Table = api.getTable(catalogName, databaseName, parent2, true, true, false) child21Table = api.getTable(catalogName, databaseName, child21, true, true, false) - child21ParentChildRelationInfo = child21Table.definitionMetadata.get("parentChildRelationInfo") then: + //Since renameParent1 table is dropped + assert parentChildRelV1.getChildren(catalogName, databaseName, renameParent1).isEmpty() + assert parentChildRelV1.getParents(catalogName, databaseName, renameParent1).isEmpty() // Since all the operations above are on the first connected relationship, the second connected relationship // should remain the same // Test Parent 2 parentChildInfo @@ -2343,16 +2585,18 @@ class MetacatSmokeSpec extends Specification { assert parentChildRelV1.getChildren(catalogName, databaseName, parent2) == [ new ChildInfoDto("embedded-fast-hive-metastore/iceberg_db/child21", "CLONE", "c21_uuid") ] as Set + assert parentChildRelV1.getParents(catalogName, databaseName, parent2).isEmpty() // Test Child21 parentChildInfo assert !child21Table.definitionMetadata.get("parentChildRelationInfo").has("isParent") - JSONAssert.assertEquals(child21ParentChildRelationInfo.toString(), + JSONAssert.assertEquals(child21Table.definitionMetadata.get("parentChildRelationInfo").toString(), '{"parentInfos":[{"name":"embedded-fast-hive-metastore/iceberg_db/parent2","relationType":"CLONE","uuid":"p2_uuid"}]}', false) assert parentChildRelV1.getChildren(catalogName, databaseName, child21).isEmpty() + assert parentChildRelV1.getParents(catalogName, databaseName, child21) == [new ParentInfoDto("embedded-fast-hive-metastore/iceberg_db/parent2", "CLONE", "p2_uuid")] as Set /* - Step 15: update parent2 with random parentChildRelationInfo to test immutability + Step 22: update parent2 with random parentChildRelationInfo to test immutability */ when: def updateParent2Dto = parent2Table @@ -2361,22 +2605,22 @@ class MetacatSmokeSpec extends Specification { parent2Table = api.getTable(catalogName, databaseName, parent2, true, true, false) child21Table = api.getTable(catalogName, databaseName, child21, true, true, false) - child21ParentChildRelationInfo = child21Table.definitionMetadata.get("parentChildRelationInfo") then: assert parent2Table.definitionMetadata.get("parentChildRelationInfo").get("isParent").booleanValue() assert parentChildRelV1.getChildren(catalogName, databaseName, parent2) == [ new ChildInfoDto("embedded-fast-hive-metastore/iceberg_db/child21", "CLONE", "c21_uuid") ] as Set - + assert parentChildRelV1.getParents(catalogName, databaseName, parent2).isEmpty() // Test Child21 parentChildInfo assert !child21Table.definitionMetadata.get("parentChildRelationInfo").has("isParent") - JSONAssert.assertEquals(child21ParentChildRelationInfo.toString(), + JSONAssert.assertEquals(child21Table.definitionMetadata.get("parentChildRelationInfo").toString(), '{"parentInfos":[{"name":"embedded-fast-hive-metastore/iceberg_db/parent2","relationType":"CLONE","uuid":"p2_uuid"}]}', false) assert parentChildRelV1.getChildren(catalogName, databaseName, child21).isEmpty() + assert parentChildRelV1.getParents(catalogName, databaseName, child21) == [new ParentInfoDto("embedded-fast-hive-metastore/iceberg_db/parent2", "CLONE", "p2_uuid")] as Set /* - Step 16: update child21 with random parentChildRelationInfo to test immutability + Step 23: update child21 with random parentChildRelationInfo to test immutability */ when: def updateChild21Dto = child21Table @@ -2385,19 +2629,20 @@ class MetacatSmokeSpec extends Specification { parent2Table = api.getTable(catalogName, databaseName, parent2, true, true, false) child21Table = api.getTable(catalogName, databaseName, child21, true, true, false) - child21ParentChildRelationInfo = child21Table.definitionMetadata.get("parentChildRelationInfo") then: // Test Parent 2 parentChildInfo assert parent2Table.definitionMetadata.get("parentChildRelationInfo").get("isParent").booleanValue() assert parentChildRelV1.getChildren(catalogName, databaseName, parent2) == [ new ChildInfoDto("embedded-fast-hive-metastore/iceberg_db/child21", "CLONE", "c21_uuid") ] as Set + assert parentChildRelV1.getParents(catalogName, databaseName, parent2).isEmpty() // Test Child21 parentChildInfo assert !child21Table.definitionMetadata.get("parentChildRelationInfo").has("isParent") - JSONAssert.assertEquals(child21ParentChildRelationInfo.toString(), + JSONAssert.assertEquals(child21Table.definitionMetadata.get("parentChildRelationInfo").toString(), '{"parentInfos":[{"name":"embedded-fast-hive-metastore/iceberg_db/parent2","relationType":"CLONE","uuid":"p2_uuid"}]}', false) assert parentChildRelV1.getChildren(catalogName, databaseName, child21).isEmpty() + assert parentChildRelV1.getParents(catalogName, databaseName, child21) == [new ParentInfoDto("embedded-fast-hive-metastore/iceberg_db/parent2", "CLONE", "p2_uuid")] as Set } } diff --git a/metacat-functional-tests/src/functionalTest/groovy/com/netflix/metacat/ParentChildRelMetadataServiceSpec.groovy b/metacat-functional-tests/src/functionalTest/groovy/com/netflix/metacat/ParentChildRelMetadataServiceSpec.groovy index 99f0d0284..c199b12fd 100644 --- a/metacat-functional-tests/src/functionalTest/groovy/com/netflix/metacat/ParentChildRelMetadataServiceSpec.groovy +++ b/metacat-functional-tests/src/functionalTest/groovy/com/netflix/metacat/ParentChildRelMetadataServiceSpec.groovy @@ -8,7 +8,9 @@ import com.netflix.metacat.common.server.converter.DozerTypeConverter import com.netflix.metacat.common.server.converter.TypeConverterFactory import com.netflix.metacat.common.server.model.ChildInfo import com.netflix.metacat.common.server.model.ParentInfo +import com.netflix.metacat.common.server.properties.ParentChildRelationshipProperties import com.netflix.metacat.common.server.usermetadata.ParentChildRelMetadataService +import com.netflix.metacat.common.server.usermetadata.ParentChildRelServiceException import com.netflix.metacat.metadata.mysql.MySqlParentChildRelMetaDataService import org.springframework.jdbc.core.JdbcTemplate import org.springframework.jdbc.datasource.DriverManagerDataSource @@ -29,6 +31,19 @@ class ParentChildRelMetadataServiceSpec extends Specification{ @Shared private String database = "testpc" + @Shared + private ParentChildRelationshipProperties props = new ParentChildRelationshipProperties(null); + + @Shared + static final String SQL_CREATE_PARENT_CHILD_RELATIONS = + "INSERT INTO parent_child_relation (parent, parent_uuid, child, child_uuid, relation_type) " + + "VALUES (?, ?, ?, ?, ?)" + + private void insertNewParentChildRecord(final String pName, final String pUUID, + final String child, final String childUUID, final String type) { + jdbcTemplate.update(SQL_CREATE_PARENT_CHILD_RELATIONS, pName, pUUID, child, childUUID, type) + } + def setupSpec() { String jdbcUrl = "jdbc:mysql://localhost:3306/metacat" String username = "metacat_user" @@ -61,16 +76,20 @@ class ParentChildRelMetadataServiceSpec extends Specification{ def parent_children_expected = [new ChildInfo(child.toString(), type, childUUID)] as Set when: - service.createParentChildRelation(parent, parentUUID, child, childUUID, type) + service.createParentChildRelation(parent, parentUUID, child, childUUID, type, props) then: // Test Parent assert service.getParents(parent).isEmpty() assert service.getChildren(parent) == parent_children_expected + assert !service.isChildTable(parent) + assert service.isParentTable(parent) // Test Child assert service.getParents(child) == child_parent_expected assert service.getParents(child) == child_parent_expected + assert service.isChildTable(child) + assert !service.isParentTable(child) when: service.deleteParentChildRelation(parent, parentUUID, child, childUUID, type) @@ -79,10 +98,14 @@ class ParentChildRelMetadataServiceSpec extends Specification{ // Test Parent assert service.getParents(parent).isEmpty() assert service.getChildren(parent).isEmpty() + assert !service.isChildTable(parent) + assert !service.isParentTable(parent) // Test Child assert service.getParents(child).isEmpty() assert service.getParents(child).isEmpty() + assert !service.isChildTable(parent) + assert !service.isParentTable(parent) } @@ -102,21 +125,26 @@ class ParentChildRelMetadataServiceSpec extends Specification{ def child_parent_expected = [new ParentInfo(parent.toString(), type, parentUUID)] as Set when: - service.createParentChildRelation(parent, parentUUID, child1, child1UUID, type) - service.createParentChildRelation(parent, parentUUID, child2, child2UUID, type) + service.createParentChildRelation(parent, parentUUID, child1, child1UUID, type, props) + service.createParentChildRelation(parent, parentUUID, child2, child2UUID, type, props) then: // Test Parent assert service.getParents(parent).isEmpty() assert parent_children_expected == service.getChildren(parent) + assert !service.isChildTable(parent) + assert service.isParentTable(parent) // Test Children - // Test Child 1 assert child_parent_expected == service.getParents(child1) assert service.getChildren(child1).isEmpty() + assert service.isChildTable(child1) + assert !service.isParentTable(child1) assert child_parent_expected == service.getParents(child2) assert service.getChildren(child2).isEmpty() + assert service.isChildTable(child2) + assert !service.isParentTable(child2) } def "Test Create - oneChildMultiParentException"() { @@ -128,10 +156,10 @@ class ParentChildRelMetadataServiceSpec extends Specification{ def child = QualifiedName.ofTable(catalog, database, "c") def childUUID = "c_uuid" def type = "clone" - service.createParentChildRelation(parent1, parent1UUID, child, childUUID, type) + service.createParentChildRelation(parent1, parent1UUID, child, childUUID, type, props) when: - service.createParentChildRelation(parent2, parent2UUID, child, childUUID, type) + service.createParentChildRelation(parent2, parent2UUID, child, childUUID, type, props) then: def e = thrown(RuntimeException) @@ -141,6 +169,8 @@ class ParentChildRelMetadataServiceSpec extends Specification{ def child_parent_expected = [new ParentInfo(parent1.toString(), type, parent1UUID)] as Set assert child_parent_expected == service.getParents(child) assert service.getChildren(child).isEmpty() + assert service.isChildTable(child) + assert !service.isParentTable(child) } def "Test Create - oneChildAsParentOfAnotherException"() { @@ -152,10 +182,10 @@ class ParentChildRelMetadataServiceSpec extends Specification{ def grandChild = QualifiedName.ofTable(catalog, database, "gc") def grandChildUUID = "gc_uuid" def type = "clone" - service.createParentChildRelation(parent, parentUUID, child, childUUID, type) + service.createParentChildRelation(parent, parentUUID, child, childUUID, type, props) when: - service.createParentChildRelation(child, childUUID, grandChild, grandChildUUID, type) + service.createParentChildRelation(child, childUUID, grandChild, grandChildUUID, type, props) then: def e = thrown(RuntimeException) @@ -165,6 +195,8 @@ class ParentChildRelMetadataServiceSpec extends Specification{ def child_parent_expected = [new ParentInfo(parent.toString(), type, parentUUID)] as Set assert service.getParents(child) == child_parent_expected assert service.getChildren(child).isEmpty() + assert service.isChildTable(child) + assert !service.isParentTable(child) } def "Test Create - oneParentAsChildOfAnother"() { @@ -176,10 +208,10 @@ class ParentChildRelMetadataServiceSpec extends Specification{ def grandChild = QualifiedName.ofTable(catalog, database, "gc") def grandChildUUID = "gc_uuid" def type = "clone" - service.createParentChildRelation(child, childUUID, grandChild, grandChildUUID, type) + service.createParentChildRelation(child, childUUID, grandChild, grandChildUUID, type, props) when: - service.createParentChildRelation(parent, parentUUID, child, childUUID, type) + service.createParentChildRelation(parent, parentUUID, child, childUUID, type, props) then: def e = thrown(RuntimeException) @@ -194,7 +226,7 @@ class ParentChildRelMetadataServiceSpec extends Specification{ def childUUID = "c_uuid" def type = "clone"; def newParent = QualifiedName.ofTable(catalog, database, "np") - service.createParentChildRelation(parent, parentUUID, child, childUUID, type) + service.createParentChildRelation(parent, parentUUID, child, childUUID, type, props) when: service.rename(parent, newParent) @@ -203,16 +235,22 @@ class ParentChildRelMetadataServiceSpec extends Specification{ // Test Old Parent Name assert service.getParents(parent).isEmpty() assert service.getChildren(parent).isEmpty() + assert !service.isChildTable(parent) + assert !service.isParentTable(parent) // Test New Parent Name assert service.getParents(newParent).isEmpty() def newParent_children_expected = [new ChildInfo(child.toString(), type, childUUID)] as Set assert service.getChildren(newParent) == newParent_children_expected + assert !service.isChildTable(newParent) + assert service.isParentTable(newParent) // Test Child def child_parent_expected = [new ParentInfo(newParent.toString(), type, parentUUID)] as Set assert child_parent_expected == service.getParents(child) assert service.getChildren(child).isEmpty() + assert service.isChildTable(child) + assert !service.isParentTable(child) // rename back when: @@ -223,14 +261,20 @@ class ParentChildRelMetadataServiceSpec extends Specification{ // Test new Parent Name assert service.getParents(newParent).isEmpty() assert service.getChildren(newParent).isEmpty() + assert !service.isChildTable(newParent) + assert !service.isParentTable(newParent) // Test old Parent Name assert service.getParents(parent).isEmpty() assert service.getChildren(parent) == newParent_children_expected + assert !service.isChildTable(parent) + assert service.isParentTable(parent) // Test Child assert child_parent_expected == service.getParents(child) assert service.getChildren(child).isEmpty() + assert service.isChildTable(child) + assert !service.isParentTable(child) } def "Test Rename Parent - Multi Child"() { @@ -242,19 +286,40 @@ class ParentChildRelMetadataServiceSpec extends Specification{ def child2 = QualifiedName.ofTable(catalog, database, "c2") def child2UUID = "c2_uuid" def type = "clone"; - service.createParentChildRelation(parent, parentUUID, child1, child1UUID, type) - service.createParentChildRelation(parent, parentUUID, child2, child2UUID, type) + service.createParentChildRelation(parent, parentUUID, child1, child1UUID, type, props) + service.createParentChildRelation(parent, parentUUID, child2, child2UUID, type, props) def newParent = QualifiedName.ofTable(catalog, database, "np") def child_parent_expected = [new ParentInfo(newParent.toString(), type, parentUUID)] as Set when: service.rename(parent, newParent) + then: + // Test Old Parent Name + assert service.getParents(parent).isEmpty() + assert service.getChildren(parent).isEmpty() + assert !service.isChildTable(parent) + assert !service.isParentTable(parent) + + // Test New Parent Name + assert service.getParents(newParent).isEmpty() + def newParent_children_expected = [ + new ChildInfo(child1.toString(), type, child1UUID), + new ChildInfo(child2.toString(), type, child2UUID), + ] as Set + assert service.getChildren(newParent) == newParent_children_expected + assert !service.isChildTable(newParent) + assert service.isParentTable(newParent) + then: // Test Child1 assert service.getParents(child1) == child_parent_expected + assert service.isChildTable(child1) + assert !service.isParentTable(child1) //Test Child2 assert service.getParents(child2) == child_parent_expected + assert service.isChildTable(child1) + assert !service.isParentTable(child1) } def "Test Rename Child"() { @@ -264,7 +329,7 @@ class ParentChildRelMetadataServiceSpec extends Specification{ def child = QualifiedName.ofTable(catalog, database, "c") def childUUID = "c_uuid" def type = "clone" - service.createParentChildRelation(parent, parentUUID, child, childUUID, type) + service.createParentChildRelation(parent, parentUUID, child, childUUID, type, props) def newChild = QualifiedName.ofTable(catalog, database, "nc") when: @@ -274,15 +339,21 @@ class ParentChildRelMetadataServiceSpec extends Specification{ assert service.getParents(parent).isEmpty() def parent_children_expected = [new ChildInfo(newChild.toString(), type, childUUID)] as Set assert parent_children_expected == service.getChildren(parent) + assert !service.isChildTable(parent) + assert service.isParentTable(parent) - // Test Child + // Test old Child assert service.getParents(child).isEmpty() assert service.getChildren(child).isEmpty() + assert !service.isChildTable(child) + assert !service.isParentTable(child) // Test New Child def child_parent_expected = [new ParentInfo(parent.toString(), type, parentUUID)] as Set assert child_parent_expected == service.getParents(newChild) assert service.getChildren(child).isEmpty() + assert service.isChildTable(newChild) + assert !service.isParentTable(newChild) // rename back when: @@ -293,14 +364,20 @@ class ParentChildRelMetadataServiceSpec extends Specification{ // Test Parent assert service.getParents(parent).isEmpty() assert parent_children_expected == service.getChildren(parent) + assert !service.isChildTable(parent) + assert service.isParentTable(parent) // Test New Child assert service.getParents(newChild).isEmpty() assert service.getChildren(newChild).isEmpty() + assert !service.isChildTable(newChild) + assert !service.isParentTable(newChild) // Test Child assert child_parent_expected == service.getParents(child) assert service.getChildren(child).isEmpty() + assert service.isChildTable(child) + assert !service.isParentTable(child) } @@ -311,7 +388,7 @@ class ParentChildRelMetadataServiceSpec extends Specification{ def child = QualifiedName.ofTable(catalog, database, "c") def childUUID = "c_uuid" def type = "clone"; - service.createParentChildRelation(parent, parentUUID, child, childUUID, type) + service.createParentChildRelation(parent, parentUUID, child, childUUID, type, props) when: service.drop(child) @@ -319,10 +396,14 @@ class ParentChildRelMetadataServiceSpec extends Specification{ // Test Parent assert service.getParents(parent).isEmpty() assert service.getChildren(parent).isEmpty() + assert !service.isChildTable(parent) + assert !service.isParentTable(parent) // Test Child assert service.getParents(child).isEmpty() assert service.getChildren(child).isEmpty() + assert !service.isChildTable(child) + assert !service.isParentTable(child) } def "Test Rename and Drop Child"() { @@ -333,7 +414,7 @@ class ParentChildRelMetadataServiceSpec extends Specification{ def newChild = QualifiedName.ofTable(catalog, database, "nc") def childUUID = "c_uuid" def type = "clone"; - service.createParentChildRelation(parent, parentUUID, child, childUUID, type) + service.createParentChildRelation(parent, parentUUID, child, childUUID, type, props) when: service.rename(child, newChild) @@ -343,13 +424,198 @@ class ParentChildRelMetadataServiceSpec extends Specification{ // Test Parent assert service.getParents(parent).isEmpty() assert service.getChildren(parent).isEmpty() + assert !service.isChildTable(parent) + assert !service.isParentTable(parent) // Test Child assert service.getParents(child).isEmpty() assert service.getChildren(child).isEmpty() + assert !service.isChildTable(child) + assert !service.isParentTable(child) // Test newChild assert service.getParents(newChild).isEmpty() assert service.getChildren(newChild).isEmpty() + assert !service.isChildTable(newChild) + assert !service.isParentTable(newChild) + } + + def "rename to an existing tableName in parent child relationship service"() { + setup: + def parent1 = QualifiedName.ofTable(catalog, database, "p1") + def parent1UUID = "p_uuid1" + def parent2 = QualifiedName.ofTable(catalog, database, "p2") + def parent2UUID = "p_uuid2" + def child1 = QualifiedName.ofTable(catalog, database, "c1") + def child1UUID = "c1_uuid" + def child2 = QualifiedName.ofTable(catalog, database, "c2") + def child2UUID = "c2_uuid" + def type = "clone"; + service.createParentChildRelation(parent1, parent1UUID, child1, child1UUID, type, props) + service.createParentChildRelation(parent2, parent2UUID, child2, child2UUID, type, props) + def child1Parent = [new ParentInfo(parent1.toString(), type, parent1UUID)] as Set + def parent1Children = [new ChildInfo(child1.toString(), type, child1UUID)] as Set + def child2Parent = [new ParentInfo(parent2.toString(), type, parent2UUID)] as Set + def parent2Children = [new ChildInfo(child2.toString(), type, child2UUID)] as Set + + // rename to an existing parent + when: + service.rename(parent1, parent2) + then: + def e = thrown(Exception) + assert e.message.contains("is already a parent table") + + // rename to an existing child + when: + service.rename(child2, child1) + then: + e = thrown(Exception) + assert e.message.contains("is already a child table") + + //Test p1 + assert service.getParents(parent1).isEmpty() + assert service.getChildren(parent1) == parent1Children + assert !service.isChildTable(parent1) + assert service.isParentTable(parent1) + + //Test c1 + assert service.getParents(child1) == child1Parent + assert service.getChildren(child1).isEmpty() + assert service.isChildTable(child1) + assert !service.isParentTable(child1) + + //Test p2 + assert service.getParents(parent2).isEmpty() + assert service.getChildren(parent2) == parent2Children + assert !service.isChildTable(parent2) + assert service.isParentTable(parent2) + + //Test c2 + assert service.getParents(child2) == child2Parent + assert service.getChildren(child2).isEmpty() + assert service.isChildTable(child2) + assert !service.isParentTable(child2) + } + + // This could happen in 2 cases: + // 1. Fail to create the table but did not clean up the parent child relationship + // 2: Successfully Drop the table but fail to clean up the parent child relationship + def "simulate a record is not cleaned up and a same parent or child name is created"() { + given: + def parent = QualifiedName.ofTable(catalog, database, "p") + def parentUUID = "parent_uuid" + def child = QualifiedName.ofTable(catalog, database, "c") + def childUUID = "child_uuid" + def type = "clone" + def randomParent = QualifiedName.ofTable(catalog, database, "rp") + def randomParentUUID = "random_parent_uuid" + def randomChild = QualifiedName.ofTable(catalog, database, "rc") + def randomChildUUID = "random_child_uuid" + + insertNewParentChildRecord(parent.toString(), parentUUID, child.toString(), childUUID, type) + + // Same parent name is created with a different uuid should fail + when: + service.createParentChildRelation(parent, randomParentUUID, randomChild, randomChildUUID, type, props) + then: + def e = thrown(RuntimeException) + assert e.message.contains("This normally means table prodhive/testpc/p already exists") + + // Same childName with a different uuid should fail + when: + service.createParentChildRelation(randomParent, randomParentUUID, child, randomChildUUID, type, props) + then: + e = thrown(RuntimeException) + assert e.message.contains("Cannot have a child table having more than one parent") + } + + def "Test maxCloneAllow"() { + given: + def catalog = 'testhive' + def targetParentDB = 'test' + def targetParentTable = "parent" + def targetType = "CLONE" + def parentQName = QualifiedName.ofTable(catalog, targetParentDB, targetParentTable) + + def targetChildDB = 'testChild' + def targetChildPrefix = 'testChild' + + def parentChildProps = new ParentChildRelationshipProperties(null) + parentChildProps.setMaxAllow(maxAllow) + parentChildProps.setDefaultMaxAllowPerRelType(defaultMaxAllowPerRelStr) + parentChildProps.setMaxAllowPerDBPerRelType(maxAllowPerDBPerRelTypeStr) + parentChildProps.setMaxAllowPerTablePerRelType(maxAllowPerTablePerRelTypeStr) + + // create expected amount of child should all succeed + when: + def String child_name = "" + def childQualifiedName = null + for (int i = 0; i < expectedChildAllowCount; i++) { + child_name = targetChildPrefix + i + childQualifiedName = QualifiedName.ofTable(catalog, targetChildDB, child_name) + service.createParentChildRelation(parentQName, targetParentTable, childQualifiedName, child_name, targetType, parentChildProps) + } + then: + noExceptionThrown() + service.getChildren(parentQName).size() == expectedChildAllowCount + + // create one more with the same type should fail + when: + child_name = targetChildPrefix + expectedChildAllowCount + childQualifiedName = QualifiedName.ofTable(catalog, targetChildDB, child_name) + service.createParentChildRelation(parentQName, targetParentTable, childQualifiedName, child_name, targetType, parentChildProps) + then: + def e = thrown(ParentChildRelServiceException) + assert e.message.contains("is not allow to have more than $expectedChildAllowCount child table") + service.getChildren(parentQName).size() == expectedChildAllowCount + + // create one more with different type should succeed + when: + service.createParentChildRelation(parentQName, targetParentTable, childQualifiedName, child_name, "random", parentChildProps) + then: + noExceptionThrown() + service.getChildren(parentQName).size() == (expectedChildAllowCount + 1) + + //change the config to -1, and now it should allow new creation + when: + if (!maxAllowPerTablePerRelTypeStr.isEmpty() && maxAllowPerTablePerRelTypeStr.contains("CLONE,testhive/test/parent")) { + def p = /CLONE,testhive\/test\/parent,\d+/ + maxAllowPerTablePerRelTypeStr = maxAllowPerTablePerRelTypeStr.replaceAll(p, "CLONE,testhive/test/parent,-1") + parentChildProps.setMaxAllowPerTablePerRelType(maxAllowPerTablePerRelTypeStr) + } else if (!maxAllowPerDBPerRelTypeStr.isEmpty() && maxAllowPerDBPerRelTypeStr.contains("CLONE,test")) { + def pattern = /CLONE,test,\d+/ + maxAllowPerDBPerRelTypeStr = maxAllowPerDBPerRelTypeStr.replaceAll(pattern, "CLONE,test,-1") + parentChildProps.setMaxAllowPerDBPerRelType(maxAllowPerDBPerRelTypeStr) + } else if (!defaultMaxAllowPerRelStr.isEmpty() && defaultMaxAllowPerRelStr.contains("CLONE")) { + def pattern = /CLONE,\d+/ + defaultMaxAllowPerRelStr = defaultMaxAllowPerRelStr.replaceAll(pattern, "CLONE,-1") + parentChildProps.setDefaultMaxAllowPerRelType(defaultMaxAllowPerRelStr) + } else { + parentChildProps.setMaxAllow(-1) + } + child_name = targetChildPrefix + (expectedChildAllowCount + 1) + childQualifiedName = QualifiedName.ofTable(catalog, targetChildDB, child_name) + service.createParentChildRelation(parentQName, targetParentTable, childQualifiedName, child_name, targetType, parentChildProps) + + then: + noExceptionThrown() + service.getChildren(parentQName).size() == (expectedChildAllowCount + 2) + assert (parentChildProps.getMaxAllow() == -1 ? 1 : 0) + + (defaultMaxAllowPerRelStr.contains("-1") ? 1 : 0) + + (maxAllowPerDBPerRelTypeStr.contains("-1") ? 1 : 0) + + (maxAllowPerTablePerRelTypeStr.contains("-1") ? 1 : 0) == 1 + + where: + maxAllow | defaultMaxAllowPerRelStr | maxAllowPerDBPerRelTypeStr | maxAllowPerTablePerRelTypeStr | expectedChildAllowCount + 3 | "" | "" | "" | 3 + 3 | "Other,5" | "Other,other,2" | "Other,testhive/test/other,2" | 3 + 1 | "CLONE,5" | "" | "" | 5 + 1 | "CLONE,5;Other,3" | "" | "" | 5 + 1 | "CLONE,5;Other,3" | "Other,other,2" | "Other,testhive/test/other,2" | 5 + 1 | "CLONE,5" | "CLONE,test,3" | "" | 3 + 1 | "CLONE,5;Other,3" | "CLONE,test,3;CLONE,other,2"| "" | 3 + 1 | "CLONE,5;Other,3" | "CLONE,test,3;OTHER,other,2"| "CLONE,testhive/test/other,2" | 3 + 1 | "CLONE,5" | "CLONE,test,3;OTHER,other,2"| "CLONE,testhive/test/parent,2" | 2 + 1 | "CLONE,5;Other,3" | "CLONE,test,3;CLONE,other,2"| "CLONE,testhive/test/parent,2;CLONE,testhive/test/other,2" | 2 } } diff --git a/metacat-main/src/main/java/com/netflix/metacat/main/api/v1/ParentChildRelController.java b/metacat-main/src/main/java/com/netflix/metacat/main/api/v1/ParentChildRelController.java index 6a43a70d2..6eff65f48 100644 --- a/metacat-main/src/main/java/com/netflix/metacat/main/api/v1/ParentChildRelController.java +++ b/metacat-main/src/main/java/com/netflix/metacat/main/api/v1/ParentChildRelController.java @@ -1,7 +1,8 @@ package com.netflix.metacat.main.api.v1; import com.netflix.metacat.common.QualifiedName; -import com.netflix.metacat.common.dto.notifications.ChildInfoDto; +import com.netflix.metacat.common.dto.ChildInfoDto; +import com.netflix.metacat.common.dto.ParentInfoDto; import com.netflix.metacat.common.server.usermetadata.ParentChildRelMetadataService; import com.netflix.metacat.main.api.RequestWrapper; import io.swagger.annotations.Api; @@ -72,4 +73,35 @@ public Set getChildren( ) ); } + + /** + * Return the list of children for a given table. + * @param catalogName catalogName + * @param databaseName databaseName + * @param tableName tableName + * @return list of childInfoDto + */ + @RequestMapping(method = RequestMethod.GET, + path = "/parents/catalog/{catalog-name}/database/{database-name}/table/{table-name}") + @ResponseStatus(HttpStatus.OK) + @ApiOperation( + position = 1, + value = "Returns the parents", + notes = "Returns the parents" + ) + public Set getParents( + @ApiParam(value = "The name of the catalog", required = true) + @PathVariable("catalog-name") final String catalogName, + @ApiParam(value = "The name of the database", required = true) + @PathVariable("database-name") final String databaseName, + @ApiParam(value = "The name of the table", required = true) + @PathVariable("table-name") final String tableName + ) { + return this.requestWrapper.processRequest( + "ParentChildRelV1Resource.getParents", + () -> this.parentChildRelMetadataService.getParentsDto( + QualifiedName.ofTable(catalogName, databaseName, tableName) + ) + ); + } } diff --git a/metacat-main/src/main/java/com/netflix/metacat/main/configs/PropertiesConfig.java b/metacat-main/src/main/java/com/netflix/metacat/main/configs/PropertiesConfig.java index 01a73c3c7..ed308c605 100644 --- a/metacat-main/src/main/java/com/netflix/metacat/main/configs/PropertiesConfig.java +++ b/metacat-main/src/main/java/com/netflix/metacat/main/configs/PropertiesConfig.java @@ -23,6 +23,7 @@ import org.springframework.boot.context.properties.ConfigurationProperties; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; +import org.springframework.core.env.Environment; /** * Configuration for binding Metacat properties. @@ -36,12 +37,13 @@ public class PropertiesConfig { /** * Static properties bindings. * + * @param env Spring environment * @return The metacat properties. */ @Bean @ConfigurationProperties("metacat") - public MetacatProperties metacatProperties() { - return new MetacatProperties(); + public MetacatProperties metacatProperties(final Environment env) { + return new MetacatProperties(env); } /** diff --git a/metacat-main/src/main/java/com/netflix/metacat/main/services/impl/TableServiceImpl.java b/metacat-main/src/main/java/com/netflix/metacat/main/services/impl/TableServiceImpl.java index 1dcd48668..df784eab8 100644 --- a/metacat-main/src/main/java/com/netflix/metacat/main/services/impl/TableServiceImpl.java +++ b/metacat-main/src/main/java/com/netflix/metacat/main/services/impl/TableServiceImpl.java @@ -154,8 +154,9 @@ public TableDto create(final QualifiedName name, final TableDto tableDto) { return dto; } - private ObjectNode createParentChildObjectNode(@Nullable final Set parentInfos, - @Nullable final Set childInfos) { + private ObjectNode createParentChildObjectNode(final QualifiedName name) { + final Set parentInfos = parentChildRelMetadataService.getParents(name); + final ObjectMapper objectMapper = new ObjectMapper(); final ObjectNode rootNode = objectMapper.createObjectNode(); @@ -173,7 +174,7 @@ private ObjectNode createParentChildObjectNode(@Nullable final Set p } // For parent table, if it has a child, we will put a field to indicate that the table is a parent table. - if (childInfos != null && !childInfos.isEmpty()) { + if (parentChildRelMetadataService.isParentTable(name)) { rootNode.put(ParentChildRelMetadataConstants.IS_PARENT, true); } return rootNode; @@ -184,28 +185,36 @@ private Optional saveParentChildRelationship(final QualifiedName child if (tableDto.getDefinitionMetadata() != null) { final ObjectNode definitionMetadata = tableDto.getDefinitionMetadata(); if (definitionMetadata.has(ParentChildRelMetadataConstants.PARENT_CHILD_RELINFO)) { + + if (!config.isParentChildCreateEnabled()) { + throw new RuntimeException("parent child creation is currently disabled"); + } + final JsonNode parentChildRelInfo = definitionMetadata.get(ParentChildRelMetadataConstants.PARENT_CHILD_RELINFO); String parentName; if (!parentChildRelInfo.has(ParentChildRelMetadataConstants.PARENT_NAME)) { - throw new RuntimeException("parent name is not specified"); + throw new IllegalArgumentException("parent name is not specified"); } parentName = parentChildRelInfo.path(ParentChildRelMetadataConstants.PARENT_NAME) .asText(); final QualifiedName parent = QualifiedName.fromString(parentName); validate(parent); + if (!exists(parent)) { + throw new IllegalArgumentException("Parent Table:" + parent + " does not exist"); + } // fetch parent and child uuid String parentUUID; String childUUID; if (!parentChildRelInfo.has(ParentChildRelMetadataConstants.PARENT_UUID)) { - throw new RuntimeException("parent_table_uuid is not specified for parent table=" + throw new IllegalArgumentException("parent_table_uuid is not specified for parent table=" + parentName); } if (!parentChildRelInfo.has(ParentChildRelMetadataConstants.CHILD_UUID)) { - throw new RuntimeException("child_table_uuid is not specified for child table=" + child); + throw new IllegalArgumentException("child_table_uuid is not specified for child table=" + child); } parentUUID = parentChildRelInfo.path(ParentChildRelMetadataConstants.PARENT_UUID).asText(); childUUID = parentChildRelInfo.path(ParentChildRelMetadataConstants.CHILD_UUID).asText(); @@ -220,7 +229,7 @@ private Optional saveParentChildRelationship(final QualifiedName child // Create parent child relationship parentChildRelMetadataService.createParentChildRelation(parent, parentUUID, - child, childUUID, relationType); + child, childUUID, relationType, config.getParentChildRelationshipProperties()); // Return a Runnable for deleting the relationship return Optional.of(() -> { @@ -367,6 +376,17 @@ public TableDto deleteAndReturn(final QualifiedName name, final boolean isMView) } } + if (!config.isParentChildDropEnabled()) { + if (parentChildRelMetadataService.isChildTable(name)) { + throw new RuntimeException(name + " is a child table and " + + "dropping a child table is currently disabled"); + } + if (parentChildRelMetadataService.isParentTable(name)) { + throw new RuntimeException(name + " is a parent table and " + + "dropping a parent table is currently disabled"); + } + } + final Set childInfos = parentChildRelMetadataService.getChildren(name); if (childInfos != null && !childInfos.isEmpty()) { final StringBuilder errorSb = new StringBuilder(); @@ -498,15 +518,15 @@ public Optional get(final QualifiedName name, final GetTableServicePar && definitionMetadata.get().has(ParentChildRelMetadataConstants.PARENT_CHILD_RELINFO)) { definitionMetadata.get().remove(ParentChildRelMetadataConstants.PARENT_CHILD_RELINFO); } - final Set parentInfo = parentChildRelMetadataService.getParents(name); - final Set childInfos = parentChildRelMetadataService.getChildren(name); - final ObjectNode parentChildRelObjectNode = createParentChildObjectNode(parentInfo, childInfos); - if (!parentChildRelObjectNode.isEmpty()) { - if (!definitionMetadata.isPresent()) { - definitionMetadata = Optional.of(new ObjectMapper().createObjectNode()); + if (config.isParentChildGetEnabled()) { + final ObjectNode parentChildRelObjectNode = createParentChildObjectNode(name); + if (!parentChildRelObjectNode.isEmpty()) { + if (!definitionMetadata.isPresent()) { + definitionMetadata = Optional.of(new ObjectMapper().createObjectNode()); + } + definitionMetadata.get().set(ParentChildRelMetadataConstants.PARENT_CHILD_RELINFO, + parentChildRelObjectNode); } - definitionMetadata.get().set(ParentChildRelMetadataConstants.PARENT_CHILD_RELINFO, - parentChildRelObjectNode); } definitionMetadata.ifPresent(table::setDefinitionMetadata); } @@ -568,6 +588,17 @@ public void rename( //Ignore if the operation is not supported, so that we can at least go ahead and save the user metadata eventBus.post(new MetacatRenameTablePreEvent(oldName, metacatRequestContext, this, newName)); + if (!config.isParentChildRenameEnabled()) { + if (parentChildRelMetadataService.isChildTable(oldName)) { + throw new RuntimeException(oldName + " is a child table and " + + "renaming on child table is currently disabled"); + } + if (parentChildRelMetadataService.isParentTable(oldName)) { + throw new RuntimeException(oldName + " is a parent table and " + + "renaming on parent table is currently disabled"); + } + } + // Before rename, first rename its parent child relation parentChildRelMetadataService.rename(oldName, newName); diff --git a/metacat-main/src/test/groovy/com/netflix/metacat/main/services/impl/TableServiceImplSpec.groovy b/metacat-main/src/test/groovy/com/netflix/metacat/main/services/impl/TableServiceImplSpec.groovy index 6ef85906e..5aedea827 100644 --- a/metacat-main/src/test/groovy/com/netflix/metacat/main/services/impl/TableServiceImplSpec.groovy +++ b/metacat-main/src/test/groovy/com/netflix/metacat/main/services/impl/TableServiceImplSpec.groovy @@ -36,8 +36,8 @@ import com.netflix.metacat.common.server.events.MetacatEventBus import com.netflix.metacat.common.server.events.MetacatUpdateTablePostEvent import com.netflix.metacat.common.server.events.MetacatUpdateTablePreEvent import com.netflix.metacat.common.server.model.ChildInfo -import com.netflix.metacat.common.server.model.ParentInfo import com.netflix.metacat.common.server.properties.Config +import com.netflix.metacat.common.server.properties.ParentChildRelationshipProperties import com.netflix.metacat.common.server.spi.MetacatCatalogConfig import com.netflix.metacat.common.server.usermetadata.DefaultAuthorizationService import com.netflix.metacat.common.server.usermetadata.ParentChildRelMetadataConstants @@ -106,6 +106,26 @@ class TableServiceImplSpec extends Specification { eventBus, registry, config, converterUtil, authorizationService, ownerValidationService, parentChildRelSvc) } + ObjectNode createParentChildRelMetadata(String rootTableName, String rootTableUuid, String childTableUuid) { + ObjectMapper mapper = new ObjectMapper() + ObjectNode node = mapper.createObjectNode() + + if (rootTableName != null) { + node.put(ParentChildRelMetadataConstants.PARENT_NAME, rootTableName) + } + if (rootTableUuid != null) { + node.put(ParentChildRelMetadataConstants.PARENT_UUID, rootTableUuid) + } + if (childTableUuid != null) { + node.put(ParentChildRelMetadataConstants.CHILD_UUID, childTableUuid) + } + + ObjectNode outerNode = mapper.createObjectNode() + outerNode.set(ParentChildRelMetadataConstants.PARENT_CHILD_RELINFO, node) + + return outerNode + } + def testTableGet() { when: service.get(name,GetTableServiceParameters.builder(). @@ -274,7 +294,36 @@ class TableServiceImplSpec extends Specification { 0 * ownerValidationService.enforceOwnerValidation(_, _, _) } - def "Test Create - Clone Table Fail to create table"() { + def 'Test invalid Clone parameters' () { + when: + def dto = new TableDto() + def tableName = QualifiedName.ofTable("prodhive", "clone", "child"); + dto.setName(tableName) + dto.setDefinitionMetadata(createParentChildRelMetadata(parentName, rootTableUuid, childTableUuid)) + service.create(tableName, dto) + + then: + 1 * config.isParentChildCreateEnabled() >> true + 0 * config.getParentChildRelationshipProperties() + 1 * ownerValidationService.extractPotentialOwners(_) >> ["cloneClient"] + 1 * ownerValidationService.isUserValid(_) >> true + 1 * ownerValidationService.extractPotentialOwnerGroups(_) >> ["cloneClientGroup"] + 1 * ownerValidationService.isGroupValid(_) >> true + _ * connectorTableServiceProxy.exists(_) >> true + + def e = thrown(IllegalArgumentException) + assert e.message.contains(message) + where: + parentName | rootTableUuid | childTableUuid | message + null | "parent_uuid" | "child_uuid" | "parent name is not specified" + "prodhive/clone/parent" | null | "child_uuid" | "parent_table_uuid is not specified for parent table=prodhive/clone/parent" + "prodhive/clone/parent" | "parent_uuid" | null | "child_table_uuid is not specified for child table=prodhive/clone/child" + "prodhive/parent" | "parent_uuid" | "child_uuid" | "does not refer to a table" + } + + + + def "Mock Parent Child Relationship Create"() { given: def childTableName = QualifiedName.ofTable("clone", "clone", "c") def parentTableName = QualifiedName.ofTable("clone", "clone", "p") @@ -292,56 +341,272 @@ class TableServiceImplSpec extends Specification { definitionMetadata: outerNode, serde: new StorageDto(uri: 's3:/clone/clone/c') ) + def parentChildProps = new ParentChildRelationshipProperties(null) + // mock case where create Table Fail as parent table does not exist when: service.create(childTableName, createTableDto) then: + 1 * config.isParentChildCreateEnabled() >> true + 0 * config.getParentChildRelationshipProperties() 1 * ownerValidationService.extractPotentialOwners(_) >> ["cloneClient"] 1 * ownerValidationService.isUserValid(_) >> true 1 * ownerValidationService.extractPotentialOwnerGroups(_) >> ["cloneClientGroup"] 1 * ownerValidationService.isGroupValid(_) >> true + 1 * connectorTableServiceProxy.exists(_) >> false - 1 * parentChildRelSvc.createParentChildRelation(parentTableName, "p_uuid", childTableName, "child_uuid", "CLONE") + 0 * parentChildRelSvc.createParentChildRelation(parentTableName, "p_uuid", childTableName, "child_uuid", "CLONE") + 0 * connectorTableServiceProxy.create(_, _) >> {throw new RuntimeException("Fail to create")} + 0 * parentChildRelSvc.deleteParentChildRelation(parentTableName, "p_uuid", childTableName, "child_uuid", "CLONE") + def e = thrown(RuntimeException) + assert e.message.contains("does not exist") + + // mock case where create Table Fail and revert function is triggerred + when: + service.create(childTableName, createTableDto) + then: + 1 * config.isParentChildCreateEnabled() >> true + 1 * config.getParentChildRelationshipProperties() >> parentChildProps + 1 * ownerValidationService.extractPotentialOwners(_) >> ["cloneClient"] + 1 * ownerValidationService.isUserValid(_) >> true + 1 * ownerValidationService.extractPotentialOwnerGroups(_) >> ["cloneClientGroup"] + 1 * ownerValidationService.isGroupValid(_) >> true + 1 * connectorTableServiceProxy.exists(_) >> true + + 1 * parentChildRelSvc.createParentChildRelation(parentTableName, "p_uuid", childTableName, "child_uuid", "CLONE", parentChildProps) 1 * connectorTableServiceProxy.create(_, _) >> {throw new RuntimeException("Fail to create")} 1 * parentChildRelSvc.deleteParentChildRelation(parentTableName, "p_uuid", childTableName, "child_uuid", "CLONE") thrown(RuntimeException) + + // mock case where parent child relationship creation is disabled + when: + service.create(childTableName, createTableDto) + + then: + 1 * config.isParentChildCreateEnabled() >> false + 0 * config.getParentChildRelationshipProperties() + 1 * ownerValidationService.extractPotentialOwners(_) >> ["cloneClient"] + 1 * ownerValidationService.isUserValid(_) >> true + 1 * ownerValidationService.extractPotentialOwnerGroups(_) >> ["cloneClientGroup"] + 1 * ownerValidationService.isGroupValid(_) >> true + 0 * connectorTableServiceProxy.exists(_) + + 0 * parentChildRelSvc.createParentChildRelation(parentTableName, "p_uuid", childTableName, "child_uuid", "CLONE", parentChildProps) + 0 * connectorTableServiceProxy.create(_, _) + e = thrown(RuntimeException) + assert e.message.contains("is currently disabled") + + // mock successful case + when: + service.create(childTableName, createTableDto) + then: + 1 * config.isParentChildCreateEnabled() >> true + 1 * config.getParentChildRelationshipProperties() >> parentChildProps + 1 * ownerValidationService.extractPotentialOwners(_) >> ["cloneClient"] + 1 * ownerValidationService.isUserValid(_) >> true + 1 * ownerValidationService.extractPotentialOwnerGroups(_) >> ["cloneClientGroup"] + 1 * ownerValidationService.isGroupValid(_) >> true + 1 * connectorTableServiceProxy.exists(_) >> true + + 1 * parentChildRelSvc.createParentChildRelation(parentTableName, "p_uuid", childTableName, "child_uuid", "CLONE", parentChildProps) + 1 * connectorTableServiceProxy.create(_, _) + 0 * parentChildRelSvc.deleteParentChildRelation(parentTableName, "p_uuid", childTableName, "child_uuid", "CLONE") + noExceptionThrown() } - def "Test Rename - Clone Table Fail to update parent child relation"() { + def "Mock Parent Child Relationship Rename"() { given: def oldName = QualifiedName.ofTable("clone", "clone", "oldChild") def newName = QualifiedName.ofTable("clone", "clone", "newChild") + // mock when rename fail and revert happen when: service.rename(oldName, newName, false) then: + 1 * config.isParentChildRenameEnabled() >> true 1 * config.getNoTableRenameOnTags() >> [] 1 * parentChildRelSvc.rename(oldName, newName) 1 * connectorTableServiceProxy.rename(oldName, newName, _) >> {throw new RuntimeException("Fail to rename")} 1 * parentChildRelSvc.rename(newName, oldName) thrown(RuntimeException) + + // mock when rename parent child relation is disabled and the table is a child table + when: + service.rename(oldName, newName, false) + then: + 1 * config.getNoTableRenameOnTags() >> [] + 1 * config.isParentChildRenameEnabled() >> false + 1 * parentChildRelSvc.isChildTable(oldName) >> true + 0 * parentChildRelSvc.isParentTable(oldName) + 0 * parentChildRelSvc.rename(oldName, newName) + 0 * connectorTableServiceProxy.rename(oldName, newName, _) + def e = thrown(RuntimeException) + assert e.message.contains("is currently disabled") + + // mock when rename parent child relation is disabled and the table is a parent table + when: + service.rename(oldName, newName, false) + then: + 1 * config.getNoTableRenameOnTags() >> [] + 1 * config.isParentChildRenameEnabled() >> false + 1 * parentChildRelSvc.isChildTable(oldName) >> false + 1 * parentChildRelSvc.isParentTable(oldName) >> true + 0 * parentChildRelSvc.rename(oldName, newName) + 0 * connectorTableServiceProxy.rename(oldName, newName, _) + e = thrown(RuntimeException) + assert e.message.contains("is currently disabled") + + // mock when rename parent child relation is disabled but the table is not a parent nor child table + when: + service.rename(oldName, newName, false) + then: + 1 * config.getNoTableRenameOnTags() >> [] + 1 * config.isParentChildRenameEnabled() >> false + 1 * parentChildRelSvc.isChildTable(oldName) >> false + 1 * parentChildRelSvc.isParentTable(oldName) >> false + 1 * parentChildRelSvc.rename(oldName, newName) + 1 * connectorTableServiceProxy.rename(oldName, newName, _) + noExceptionThrown() + + // mock normal success case + when: + service.rename(oldName, newName, false) + then: + 1 * config.getNoTableRenameOnTags() >> [] + 1 * config.isParentChildRenameEnabled() >> true + 0 * parentChildRelSvc.isChildTable(oldName) + 0 * parentChildRelSvc.isParentTable(oldName) + 1 * parentChildRelSvc.rename(oldName, newName) + 1 * connectorTableServiceProxy.rename(oldName, newName, _) + noExceptionThrown() } - def "Test Drop - Clone Table Fail to drop parent child relation"() { + def "Mock Parent Child Relationship Drop"() { given: def name = QualifiedName.ofTable("clone", "clone", "child") + // drop a parent table that has child when: service.delete(name) then: - 1 * parentChildRelSvc.getParents(name) >> {[new ParentInfo("parent", "clone", "parent_uuid")] as Set} - 2 * parentChildRelSvc.getChildren(name) >> {[new ChildInfo("child", "clone", "child_uuid")] as Set} + 1 * config.isParentChildGetEnabled() >> true + 1 * config.isParentChildDropEnabled() >> true + 1 * parentChildRelSvc.getParents(name) >> {[] as Set} + 1 * parentChildRelSvc.isParentTable(name) >> true + 1 * parentChildRelSvc.getChildren(name) >> {[new ChildInfo("child", "clone", "child_uuid")] as Set} 1 * config.getNoTableDeleteOnTags() >> [] - thrown(RuntimeException) + def e = thrown(RuntimeException) + assert e.message.contains("because it still has") + // mock failure to drop the table, should not trigger deletion in parentChildRelSvc when: service.delete(name) then: + 1 * config.isParentChildGetEnabled() >> true + 1 * config.isParentChildDropEnabled() >> true 1 * parentChildRelSvc.getParents(name) - 2 * parentChildRelSvc.getChildren(name) + 1 * parentChildRelSvc.isParentTable(name) >> true + 1 * parentChildRelSvc.getChildren(name) 1 * config.getNoTableDeleteOnTags() >> [] 1 * connectorTableServiceProxy.delete(_) >> {throw new RuntimeException("Fail to drop")} - 0 * parentChildRelSvc.drop(_, _) + 0 * parentChildRelSvc.drop(_) thrown(RuntimeException) + + // mock drop is not enabled and it is a child table + when: + service.delete(name) + then: + 1 * config.isParentChildGetEnabled() >> true + 1 * config.isParentChildDropEnabled() >> false + 1 * parentChildRelSvc.isChildTable(name) >> true + 1 * parentChildRelSvc.isParentTable(name) >> false + 1 * parentChildRelSvc.getParents(name) + 0 * parentChildRelSvc.getChildren(name) + 1 * config.getNoTableDeleteOnTags() >> [] + 0 * connectorTableServiceProxy.delete(_) + 0 * parentChildRelSvc.drop(_) + e = thrown(RuntimeException) + assert e.message.contains("is currently disabled") + + // mock drop is not enabled and it is a parent table + when: + service.delete(name) + then: + 1 * config.isParentChildGetEnabled() >> true + 1 * config.isParentChildDropEnabled() >> false + 1 * parentChildRelSvc.isChildTable(name) >> false + 2 * parentChildRelSvc.isParentTable(name) >> true + 1 * parentChildRelSvc.getParents(name) + 0 * parentChildRelSvc.getChildren(name) + 1 * config.getNoTableDeleteOnTags() >> [] + 0 * connectorTableServiceProxy.delete(_) + 0 * parentChildRelSvc.drop(_) + e = thrown(RuntimeException) + assert e.message.contains("is currently disabled") + + // mock drop is not enabled and it is not a parent nor child table + // and it doesn't have any children + when: + service.delete(name) + then: + 1 * config.isParentChildGetEnabled() >> true + 1 * config.isParentChildDropEnabled() >> false + 1 * parentChildRelSvc.isChildTable(name) >> false + 2 * parentChildRelSvc.isParentTable(name) >> false + 1 * parentChildRelSvc.getParents(name) >> {[] as Set} + 1 * parentChildRelSvc.getChildren(name) >> {[] as Set} + 1 * config.getNoTableDeleteOnTags() >> [] + 1 * connectorTableServiceProxy.delete(_) + 1 * parentChildRelSvc.drop(_) + 1 * config.canDeleteTableDefinitionMetadata() >> true + noExceptionThrown() + + // mock normal successful case + when: + service.delete(name) + then: + 1 * config.isParentChildGetEnabled() >> true + 1 * config.isParentChildDropEnabled() >> true + 0 * parentChildRelSvc.isChildTable(name) + 1 * parentChildRelSvc.isParentTable(name) + 1 * parentChildRelSvc.getParents(name) >> {[] as Set} + 1 * parentChildRelSvc.getChildren(name) >> {[] as Set} + 1 * config.getNoTableDeleteOnTags() >> [] + 1 * connectorTableServiceProxy.delete(_) + 1 * parentChildRelSvc.drop(_) + 1 * config.canDeleteTableDefinitionMetadata() >> true + noExceptionThrown() + } + + def "Mock Parent Child Relationship Get"() { + when: + service.get(name,GetTableServiceParameters.builder(). + disableOnReadMetadataIntercetor(false) + .includeDataMetadata(true) + .includeDefinitionMetadata(true) + .build()) + then: + 1 * connectorManager.getCatalogConfig(_) >> catalogConfig + 1 * usermetadataService.getDefinitionMetadataWithInterceptor(_,_) >> Optional.empty() + 1 * usermetadataService.getDataMetadata(_) >> Optional.empty() + 0 * usermetadataService.getDefinitionMetadata(_) >> Optional.empty() + 1 * config.isParentChildGetEnabled() >> false + 0 * parentChildRelSvc.getParents(name) + 0 * parentChildRelSvc.isParentTable(name) + + when: + service.get(name,GetTableServiceParameters.builder(). + disableOnReadMetadataIntercetor(false) + .includeDataMetadata(true) + .includeDefinitionMetadata(true) + .build()) + then: + 1 * connectorManager.getCatalogConfig(_) >> catalogConfig + 1 * usermetadataService.getDefinitionMetadataWithInterceptor(_,_) >> Optional.empty() + 1 * usermetadataService.getDataMetadata(_) >> Optional.empty() + 0 * usermetadataService.getDefinitionMetadata(_) >> Optional.empty() + 1 * config.isParentChildGetEnabled() >> true + 1 * parentChildRelSvc.getParents(name) + 1 * parentChildRelSvc.isParentTable(name) } def "Will not throw on Successful Table Update with Failed Get"() { diff --git a/metacat-metadata-mysql/src/main/java/com/netflix/metacat/metadata/mysql/MySqlParentChildRelMetaDataService.java b/metacat-metadata-mysql/src/main/java/com/netflix/metacat/metadata/mysql/MySqlParentChildRelMetaDataService.java index 89f0ef6d6..742d1503e 100644 --- a/metacat-metadata-mysql/src/main/java/com/netflix/metacat/metadata/mysql/MySqlParentChildRelMetaDataService.java +++ b/metacat-metadata-mysql/src/main/java/com/netflix/metacat/metadata/mysql/MySqlParentChildRelMetaDataService.java @@ -1,24 +1,33 @@ package com.netflix.metacat.metadata.mysql; import com.netflix.metacat.common.QualifiedName; -import com.netflix.metacat.common.dto.notifications.ChildInfoDto; +import com.netflix.metacat.common.dto.ChildInfoDto; +import com.netflix.metacat.common.dto.ParentInfoDto; import com.netflix.metacat.common.server.converter.ConverterUtil; import com.netflix.metacat.common.server.model.ChildInfo; import com.netflix.metacat.common.server.model.ParentInfo; +import com.netflix.metacat.common.server.properties.ParentChildRelationshipProperties; import com.netflix.metacat.common.server.usermetadata.ParentChildRelMetadataService; import com.netflix.metacat.common.server.usermetadata.ParentChildRelServiceException; import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; import lombok.extern.slf4j.Slf4j; +import org.springframework.jdbc.core.PreparedStatementSetter; +import org.springframework.jdbc.core.ResultSetExtractor; +import org.springframework.transaction.annotation.Isolation; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.jdbc.core.JdbcTemplate; import org.springframework.transaction.annotation.Transactional; import java.sql.PreparedStatement; +import java.util.Map; import java.util.Set; import java.util.List; import java.util.ArrayList; import java.util.HashSet; +import java.util.StringJoiner; import java.util.stream.Collectors; +import java.sql.ResultSet; +import java.sql.SQLException; /** * Parent Child Relationship Metadata Service. @@ -52,6 +61,18 @@ public class MySqlParentChildRelMetaDataService implements ParentChildRelMetadat static final String SQL_GET_CHILDREN = "SELECT child, child_uuid, relation_type " + "FROM parent_child_relation WHERE parent = ?"; + static final String SQL_IS_PARENT_TABLE = "SELECT 1 FROM parent_child_relation WHERE parent = ? LIMIT 1"; + static final String SQL_IS_CHILD_TABLE = "SELECT 1 FROM parent_child_relation WHERE child = ? LIMIT 1"; + + static final String SQL_GET_PARENT_UUIDS = "SELECT DISTINCT parent_uuid FROM parent_child_relation " + + "where parent = ?"; + + static final String SQL_GET_CHILDREN_UUIDS = "SELECT DISTINCT child_uuid FROM parent_child_relation " + + "where child = ?"; + + static final String SQL_GET_CHILDREN_SIZE_PER_REL = "SELECT COUNT(*) FROM parent_child_relation " + + "where parent = ? and relation_type = ?"; + private final JdbcTemplate jdbcTemplate; private final ConverterUtil converterUtil; @@ -67,12 +88,77 @@ public MySqlParentChildRelMetaDataService(final JdbcTemplate jdbcTemplate, final this.converterUtil = converterUtil; } - @Override - public void createParentChildRelation(final QualifiedName parentName, - final String parentUUID, - final QualifiedName childName, - final String childUUID, - final String type) { + private Integer getMaxAllowedFromNestedMap( + final QualifiedName parent, + final String relationType, + final Map> maxAllowPerResourcePerRelType, + final boolean isTable) { + Integer maxCloneAllow = null; + final Map maxAllowPerResource = maxAllowPerResourcePerRelType.get(relationType); + if (maxAllowPerResource != null) { + if (isTable) { + maxCloneAllow = maxAllowPerResource.get(parent.toString()); + } else { + maxCloneAllow = maxAllowPerResource.get(parent.getDatabaseName()); + } + } + return maxCloneAllow; + } + + private void validateMaxAllow(final QualifiedName parentName, + final String type, + final ParentChildRelationshipProperties props) { + // Validate max clone allow + // First check if the parent table have configured max allowed on the table config + Integer maxAllow = getMaxAllowedFromNestedMap( + parentName, + type, + props.getMaxAllowPerTablePerRelType(), + true + ); + + // Then check if the parent have configured max allowed on the db config + if (maxAllow == null) { + maxAllow = getMaxAllowedFromNestedMap( + parentName, + type, + props.getMaxAllowPerDBPerRelType(), + false + ); + } + + // If not specified in maxAllowPerDBPerRelType,check the default max Allow based on relationType + if (maxAllow == null) { + final Integer count = props.getDefaultMaxAllowPerRelType().get(type); + if (count != null) { + maxAllow = count; + } + } + + // Finally fallback to the default value for all types + if (maxAllow == null) { + maxAllow = props.getMaxAllow(); + } + + // if maxAllow < 0, this means we can create as many child table under the parent + if (maxAllow < 0) { + return; + } + + if (getChildrenCountPerType(parentName, type) >= maxAllow) { + final String errorMsg = String.format( + "Parent table: %s is not allow to have more than %s child table for %s relation type", + parentName, maxAllow, type); + throw new ParentChildRelServiceException(errorMsg); + } + } + + private void validateCreate(final QualifiedName parentName, + final String parentUUID, + final QualifiedName childName, + final String childUUID, + final String type, + final ParentChildRelationshipProperties props) { // Validation to prevent having a child have two parents final Set childParents = getParents(childName); if (!childParents.isEmpty()) { @@ -96,6 +182,26 @@ public void createParentChildRelation(final QualifiedName parentName, + "- child table: " + childName + " already have child"); } + // Validation to prevent creating a parent with an uuid that is different from the existing parent uuids + final Set existingParentUuids = getExistingUUIDS(parentName.toString()); + validateUUIDs(parentName.toString(), existingParentUuids, parentUUID, "Parent"); + + // Validation to prevent creating a child with an uuid that is different from the existing child uuids + final Set existingChildUuids = getExistingUUIDS(childName.toString()); + validateUUIDs(childName.toString(), existingChildUuids, childUUID, "Child"); + + // Validation to control how many children tables can be created per type + validateMaxAllow(parentName, type, props); + } + + @Override + public void createParentChildRelation(final QualifiedName parentName, + final String parentUUID, + final QualifiedName childName, + final String childUUID, + final String type, + final ParentChildRelationshipProperties props) { + validateCreate(parentName, parentUUID, childName, childUUID, type, props); try { jdbcTemplate.update(connection -> { final PreparedStatement ps = connection.prepareStatement(SQL_CREATE_PARENT_CHILD_RELATIONS); @@ -156,7 +262,15 @@ public void deleteParentChildRelation(final QualifiedName parentName, } @Override + @Transactional(isolation = Isolation.SERIALIZABLE) public void rename(final QualifiedName oldName, final QualifiedName newName) { + if (isChildTable(newName)) { + throw new ParentChildRelServiceException(newName + " is already a child table"); + } + if (isParentTable(newName)) { + throw new ParentChildRelServiceException(newName + " is already a parent table"); + } + renameParent(oldName, newName); renameChild(oldName, newName); log.info("Successfully rename parent child relationship for oldName={}, newName={}", @@ -261,4 +375,87 @@ public Set getChildrenDto(final QualifiedName name) { return getChildren(name).stream() .map(converterUtil::toChildInfoDto).collect(Collectors.toSet()); } + + @Override + public Set getParentsDto(final QualifiedName name) { + return getParents(name).stream() + .map(converterUtil::toParentInfoDto).collect(Collectors.toSet()); + } + + @Override + public boolean isParentTable(final QualifiedName tableName) { + return tableExist(tableName.toString(), SQL_IS_PARENT_TABLE); + } + + @Override + public boolean isChildTable(final QualifiedName tableName) { + return tableExist(tableName.toString(), SQL_IS_CHILD_TABLE); + } + + private boolean tableExist(final String tableName, final String sql) { + return jdbcTemplate.query(sql, + new PreparedStatementSetter() { + @Override + public void setValues(final PreparedStatement ps) throws SQLException { + ps.setString(1, tableName); + } + }, + new ResultSetExtractor() { + @Override + public Boolean extractData(final ResultSet rs) throws SQLException { + return rs.next(); + } + } + ); + } + + private Set getExistingUUIDS(final String tableName) { + final Set existingUUIDs = new HashSet<>(); + existingUUIDs.addAll(getParentUuidsForParent(tableName)); + existingUUIDs.addAll(getChildUuidsForChild(tableName)); + return existingUUIDs; + } + + private Set getParentUuidsForParent(final String parentName) { + return new HashSet<>(jdbcTemplate.query(connection -> { + final PreparedStatement ps = connection.prepareStatement(SQL_GET_PARENT_UUIDS); + ps.setString(1, parentName); + return ps; + }, (rs, rowNum) -> rs.getString("parent_uuid"))); + } + + private Set getChildUuidsForChild(final String childName) { + return new HashSet<>(jdbcTemplate.query(connection -> { + final PreparedStatement ps = connection.prepareStatement(SQL_GET_CHILDREN_UUIDS); + ps.setString(1, childName); + return ps; + }, (rs, rowNum) -> rs.getString("child_uuid"))); + } + + private void validateUUIDs(final String name, + final Set existingUUIDs, + final String inputUUID, + final String entity + ) throws ParentChildRelServiceException { + if (existingUUIDs.size() > 1 || (!existingUUIDs.isEmpty() && !existingUUIDs.contains(inputUUID))) { + final StringJoiner uuidJoiner = new StringJoiner(", "); + for (String uuid : existingUUIDs) { + uuidJoiner.add(uuid); + } + final String existingUuidsString = uuidJoiner.toString(); + + throw new ParentChildRelServiceException( + String.format("Cannot create parent-child relation: %s '%s' has existing UUIDs [%s] " + + "that differ from the input %s UUID '%s'. This normally means table %s already exists", + entity, name, existingUuidsString, entity, inputUUID, name) + ); + } + } + + private int getChildrenCountPerType(final QualifiedName parent, final String type) { + final List params = new ArrayList<>(); + params.add(parent.toString()); + params.add(type); + return jdbcTemplate.queryForObject(SQL_GET_CHILDREN_SIZE_PER_REL, params.toArray(), Integer.class); + } }