Skip to content

Commit

Permalink
Yingjianw/additional safety rollout (#602)
Browse files Browse the repository at this point in the history
* safety flag to control the parent child relationship rollout

* Add validaton to prevent creating another new name with a different uuid

* add search endpoint for getParents

* moveConfig out to oss

* default flags to false

* use limit 1 to speed up check exist query

---------

Co-authored-by: Yingjian Wu <[email protected]>
  • Loading branch information
stevie9868 and Yingjian Wu authored Jul 9, 2024
1 parent 6ea9332 commit b585cda
Show file tree
Hide file tree
Showing 19 changed files with 1,456 additions and 129 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,8 @@
import javax.ws.rs.PathParam;

import javax.ws.rs.core.MediaType;
import com.netflix.metacat.common.dto.notifications.ChildInfoDto;
import com.netflix.metacat.common.dto.ChildInfoDto;
import com.netflix.metacat.common.dto.ParentInfoDto;

import java.util.Set;

Expand Down Expand Up @@ -40,4 +41,24 @@ Set<ChildInfoDto> getChildren(
@PathParam("table-name")
String tableName
);

/**
* Return the list of parent.
* @param catalogName catalogName
* @param databaseName databaseName
* @param tableName tableName
* @return list of parent info dtos
*/
@GET
@Path("parents/catalog/{catalog-name}/database/{database-name}/table/{table-name}")
@Consumes(MediaType.APPLICATION_JSON)
@Produces(MediaType.APPLICATION_JSON)
Set<ParentInfoDto> getParents(
@PathParam("catalog-name")
String catalogName,
@PathParam("database-name")
String databaseName,
@PathParam("table-name")
String tableName
);
}
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,8 @@
import com.netflix.metacat.common.dto.Sort;
import com.netflix.metacat.common.dto.StorageDto;
import com.netflix.metacat.common.dto.TableDto;
import com.netflix.metacat.common.dto.notifications.ChildInfoDto;
import com.netflix.metacat.common.dto.ChildInfoDto;
import com.netflix.metacat.common.dto.ParentInfoDto;
import com.netflix.metacat.common.server.connectors.ConnectorRequestContext;
import com.netflix.metacat.common.server.connectors.model.AuditInfo;
import com.netflix.metacat.common.server.connectors.model.CatalogInfo;
Expand All @@ -48,6 +49,7 @@
import com.netflix.metacat.common.server.connectors.model.StorageInfo;
import com.netflix.metacat.common.server.connectors.model.TableInfo;
import com.netflix.metacat.common.server.model.ChildInfo;
import com.netflix.metacat.common.server.model.ParentInfo;
import lombok.NonNull;
import org.dozer.CustomConverter;
import org.dozer.DozerBeanMapper;
Expand Down Expand Up @@ -287,4 +289,14 @@ public PartitionsSaveResponseDto toPartitionsSaveResponseDto(final PartitionsSav
public ChildInfoDto toChildInfoDto(final ChildInfo childInfo) {
return mapper.map(childInfo, ChildInfoDto.class);
}

/**
* Convert ParentInfo to ParentInfoDto.
*
* @param parentInfo parentInfo
* @return parentInfo dto
*/
public ParentInfoDto toParentInfoDto(final ParentInfo parentInfo) {
return mapper.map(parentInfo, ParentInfoDto.class);
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -607,5 +607,40 @@ public interface Config {
* @return True if it should be.
*/
boolean shouldFetchOnlyMetadataLocationEnabled();

/**
* Whether we allow parent child relationship to be created.
*
* @return True if it should be.
*/
boolean isParentChildCreateEnabled();

/**
* Whether we allow renaming parent child relationship.
*
* @return True if it should be.
*/
boolean isParentChildRenameEnabled();

/**
* Whether we allow getting parent child relationship in the getTable call.
*
* @return True if it should be.
*/
boolean isParentChildGetEnabled();

/**
* Whether we allow dropping tables that are either parent or child.
*
* @return True if it should be.
*/
boolean isParentChildDropEnabled();

/**
* Get the parentChildRelationshipProperties config.
*
* @return parentChildRelationshipProperties
*/
ParentChildRelationshipProperties getParentChildRelationshipProperties();
}

Original file line number Diff line number Diff line change
Expand Up @@ -678,4 +678,29 @@ public boolean disablePartitionDefinitionMetadata() {
public boolean shouldFetchOnlyMetadataLocationEnabled() {
return this.metacatProperties.getHive().getIceberg().isShouldFetchOnlyMetadataLocationEnabled();
}

@Override
public boolean isParentChildCreateEnabled() {
return this.metacatProperties.getParentChildRelationshipProperties().isCreateEnabled();
}

@Override
public boolean isParentChildRenameEnabled() {
return this.metacatProperties.getParentChildRelationshipProperties().isRenameEnabled();
}

@Override
public boolean isParentChildGetEnabled() {
return this.metacatProperties.getParentChildRelationshipProperties().isGetEnabled();
}

@Override
public boolean isParentChildDropEnabled() {
return this.metacatProperties.getParentChildRelationshipProperties().isDropEnabled();
}

@Override
public ParentChildRelationshipProperties getParentChildRelationshipProperties() {
return this.metacatProperties.getParentChildRelationshipProperties();
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@
package com.netflix.metacat.common.server.properties;

import lombok.NonNull;
import org.springframework.core.env.Environment;

/**
* Entry point to entire property tree of Metacat namespace.
Expand All @@ -27,6 +28,8 @@
*/
@lombok.Data
public class MetacatProperties {
@NonNull
private Environment env;
@NonNull
private Data data = new Data();
@NonNull
Expand Down Expand Up @@ -67,4 +70,16 @@ public class MetacatProperties {
private AliasServiceProperties aliasServiceProperties = new AliasServiceProperties();
@NonNull
private RateLimiterProperties rateLimiterProperties = new RateLimiterProperties();
@NonNull
private ParentChildRelationshipProperties parentChildRelationshipProperties;

/**
* Constructor for MetacatProperties.
*
* @param env Spring Environment
*/
public MetacatProperties(final Environment env) {
this.env = env;
this.parentChildRelationshipProperties = new ParentChildRelationshipProperties(env);
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,119 @@
package com.netflix.metacat.common.server.properties;

import lombok.Data;
import lombok.extern.slf4j.Slf4j;
import org.springframework.core.env.Environment;

import javax.annotation.Nullable;
import java.util.Arrays;
import java.util.HashMap;
import java.util.Map;
import java.util.stream.Collectors;

/**
* Parent Child Relationship service properties.
*
* @author yingjianw
*/
@Data
@Slf4j
public class ParentChildRelationshipProperties {
private static final String MAX_ALLOW_PER_TABLE_PER_REL_PROPERTY_NAME =
"metacat.parentChildRelationshipProperties.maxAllowPerTablePerRelConfig";
private static final String MAX_ALLOW_PER_DB_PER_REL_PROPERTY_NAME =
"metacat.parentChildRelationshipProperties.maxAllowPerDBPerRelConfig";
private static final String DEFAULT_MAX_ALLOW_PER_REL_PROPERTY_NAME =
"metacat.parentChildRelationshipProperties.defaultMaxAllowPerRelConfig";
private boolean createEnabled;
private boolean getEnabled;
private boolean renameEnabled;
private boolean dropEnabled;
private int maxAllow = 5;
private Map<String, Map<String, Integer>> maxAllowPerTablePerRelType = new HashMap<>();
private Map<String, Map<String, Integer>> maxAllowPerDBPerRelType = new HashMap<>();
private Map<String, Integer> defaultMaxAllowPerRelType = new HashMap<>();

/**
* Constructor.
*
* @param env Spring environment
*/
public ParentChildRelationshipProperties(@Nullable final Environment env) {
if (env != null) {
setMaxAllowPerTablePerRelType(
env.getProperty(MAX_ALLOW_PER_TABLE_PER_REL_PROPERTY_NAME, String.class, "")
);
setMaxAllowPerDBPerRelType(
env.getProperty(MAX_ALLOW_PER_DB_PER_REL_PROPERTY_NAME, String.class, "")
);
setDefaultMaxAllowPerRelType(
env.getProperty(DEFAULT_MAX_ALLOW_PER_REL_PROPERTY_NAME, String.class, "")
);
}
}

/**
* setMaxAllowPerTablePerRelType based on String config.
*
* @param configStr configString
*/
public void setMaxAllowPerTablePerRelType(@Nullable final String configStr) {
if (configStr == null || configStr.isEmpty()) {
return;
}
try {
this.maxAllowPerTablePerRelType = parseNestedConfigString(configStr);
} catch (Exception e) {
log.error("Fail to apply configStr = {} for maxAllowPerTablePerRelType", configStr, e);
}
}

/**
* setMaxAllowPerDBPerRelType based on String config.
*
* @param configStr configString
*/
public void setMaxAllowPerDBPerRelType(@Nullable final String configStr) {
if (configStr == null || configStr.isEmpty()) {
return;
}
try {
this.maxAllowPerDBPerRelType = parseNestedConfigString(configStr);
} catch (Exception e) {
log.error("Fail to apply configStr = {} for maxCloneAllowPerDBPerRelType", configStr);
}
}
/**
* setMaxCloneAllowPerDBPerRelType based on String config.
*
* @param configStr configString
*/
public void setDefaultMaxAllowPerRelType(@Nullable final String configStr) {
if (configStr == null || configStr.isEmpty()) {
return;
}
try {
this.defaultMaxAllowPerRelType =
Arrays.stream(configStr.split(";"))
.map(entry -> entry.split(","))
.collect(Collectors.toMap(
parts -> parts[0],
parts -> Integer.parseInt(parts[1])
));
} catch (Exception e) {
log.error("Fail to apply configStr = {} for defaultMaxAllowPerRelType", configStr);
}
}

private Map<String, Map<String, Integer>> parseNestedConfigString(final String configStr) {
return Arrays.stream(configStr.split(";"))
.map(entry -> entry.split(","))
.collect(Collectors.groupingBy(
parts -> parts[0],
Collectors.toMap(
parts -> parts[1],
parts -> Integer.parseInt(parts[2])
)
));
}
}
Original file line number Diff line number Diff line change
@@ -1,8 +1,10 @@
package com.netflix.metacat.common.server.usermetadata;
import com.netflix.metacat.common.QualifiedName;
import com.netflix.metacat.common.dto.notifications.ChildInfoDto;
import com.netflix.metacat.common.dto.ChildInfoDto;
import com.netflix.metacat.common.dto.ParentInfoDto;
import com.netflix.metacat.common.server.model.ChildInfo;
import com.netflix.metacat.common.server.model.ParentInfo;
import com.netflix.metacat.common.server.properties.ParentChildRelationshipProperties;

import java.util.Set;

Expand All @@ -24,13 +26,15 @@ public interface ParentChildRelMetadataService {
* @param childName the name of the child entity
* @param childUUID the uuid of the child
* @param relationType the type of the relationship
* @param prop properties config
*/
void createParentChildRelation(
QualifiedName parentName,
String parentUUID,
QualifiedName childName,
String childUUID,
String relationType
String relationType,
ParentChildRelationshipProperties prop
);

/**
Expand Down Expand Up @@ -98,9 +102,30 @@ Set<ChildInfo> getChildren(
/**
* get the set of children dto for the input name.
* @param name name
* @return a set of ChildInfo
* @return a set of ChildInfo dto
*/
Set<ChildInfoDto> getChildrenDto(
QualifiedName name
);

/**
* get the set of parent dto for the input name.
* @param name name
* @return a set of parentInfo dto
*/
Set<ParentInfoDto> getParentsDto(QualifiedName name);

/**
* return whether the table is a parent.
* @param tableName tableName
* @return true if it exists
*/
boolean isParentTable(final QualifiedName tableName);

/**
* return whether the table is a child.
* @param tableName tableName
* @return true if it exists
*/
boolean isChildTable(final QualifiedName tableName);
}
Original file line number Diff line number Diff line change
@@ -1,6 +1,5 @@
package com.netflix.metacat.common.dto.notifications;
package com.netflix.metacat.common.dto;

import com.netflix.metacat.common.dto.BaseDto;
import io.swagger.annotations.ApiModelProperty;
import lombok.AllArgsConstructor;
import lombok.Builder;
Expand All @@ -9,7 +8,7 @@
import lombok.NoArgsConstructor;

/**
* ChildInfo information.
* ChildInfo dto information.
*/
@SuppressWarnings("unused")
@Data
Expand Down
Loading

0 comments on commit b585cda

Please sign in to comment.