[Feature] support resource of physical deletion (#3834)

Signed-off-by: Zzm0809 <[email protected]>
Co-authored-by: Zzm0809 <[email protected]>

dinky-admin/src/main/java/org/dinky/service/resource/impl/ResourceServiceImpl.java

@@ -24,6 +24,7 @@
 import org.dinky.data.enums.Status;
 import org.dinky.data.exception.BusException;
 import org.dinky.data.model.Resources;
+import org.dinky.data.model.SystemConfiguration;
 import org.dinky.data.result.Result;
 import org.dinky.mapper.ResourcesMapper;
 import org.dinky.resource.BaseResourceManager;
@@ -53,7 +54,9 @@
 import cn.hutool.core.lang.Assert;
 import cn.hutool.core.lang.Opt;
 import cn.hutool.core.util.StrUtil;
+import lombok.extern.slf4j.Slf4j;
 
+@Slf4j
 @Service
 public class ResourceServiceImpl extends ServiceImpl<ResourcesMapper, Resources> implements ResourcesService {
     private static final TimedCache<Integer, Resources> RESOURCES_CACHE = new TimedCache<>(30 * 1000);
@@ -322,7 +325,17 @@ public boolean remove(Integer id) {
                         == -1,
                 () -> new BusException(Status.ROOT_DIR_NOT_ALLOW_DELETE));
         try {
+            SystemConfiguration systemConfiguration = SystemConfiguration.getInstances();
+
             Resources byId = getById(id);
+
+            if (systemConfiguration.getPhysicalDeletion().getValue()) {
+                getBaseResourceManager().remove(byId.getFullName());
+                log.warn(
+                        "The resource type you have configured is [{}] and physical deletion is enabled. The File or Directory [{}] will be deleted",
+                        systemConfiguration.getResourcesModel().getValue().name(),
+                        byId.getFullName());
+            }
             if (isExistsChildren(id)) {
                 if (byId.getIsDirectory()) {
                     List<Resources> resourceByPidToChildren =

dinky-common/src/main/java/org/dinky/data/enums/Status.java

@@ -409,6 +409,8 @@ public enum Status {
     SYS_METRICS_SETTINGS_FLINK_GATHERTIMEOUT_NOTE(155, "sys.metrics.settings.flink.gatherTimeout.note"),
     SYS_RESOURCE_SETTINGS_ENABLE(156, "sys.resource.settings.base.enable"),
     SYS_RESOURCE_SETTINGS_ENABLE_NOTE(157, "sys.resource.settings.base.enable.note"),
+    SYS_RESOURCE_SETTINGS_PHYSICAL_DELETION(182, "sys.resource.settings.base.physicalDeletion"),
+    SYS_RESOURCE_SETTINGS_PHYSICAL_DELETION_NOTE(183, "sys.resource.settings.base.physicalDeletion.note"),
     SYS_RESOURCE_SETTINGS_UPLOAD_BASE_PATH(158, "sys.resource.settings.base.upload.base.path"),
     SYS_RESOURCE_SETTINGS_UPLOAD_BASE_PATH_NOTE(159, "sys.resource.settings.base.upload.base.path.note"),
     SYS_RESOURCE_SETTINGS_MODEL(160, "sys.resource.settings.base.model"),

dinky-common/src/main/java/org/dinky/data/model/SystemConfiguration.java

@@ -264,6 +264,11 @@ public static Configuration.OptionBuilder key(Status status) {
             .defaultValue(true)
             .note(Status.SYS_RESOURCE_SETTINGS_ENABLE_NOTE);
 
+    private final Configuration<Boolean> physicalDeletion = key(Status.SYS_RESOURCE_SETTINGS_PHYSICAL_DELETION)
+            .booleanType()
+            .defaultValue(false)
+            .note(Status.SYS_RESOURCE_SETTINGS_PHYSICAL_DELETION_NOTE);
+
     private final Configuration<ResourcesModelEnum> resourcesModel = key(Status.SYS_RESOURCE_SETTINGS_MODEL)
             .enumType(ResourcesModelEnum.class)
             .defaultValue(ResourcesModelEnum.LOCAL)

dinky-common/src/main/resources/i18n/messages_en_US.properties

@@ -256,6 +256,8 @@ sys.metrics.settings.flink.gatherTimeout=Flink Metrics collection time granulari
 sys.metrics.settings.flink.gatherTimeout.note=Flink Metrics collection timeout period, scheduled task interval trigger (this configuration item should be smaller than Flink Metrics collection time granularity)
 sys.resource.settings.base.enable=Whether to enable Resource
 sys.resource.settings.base.enable.note=If this function is not enabled, the resource management function in the registration center will not be available. If you switch storage mode/modify other configuration items, you need to enable this function again to reload the new configuration, otherwise it will not take effect. Please ensure that the configuration is correct
+sys.resource.settings.base.physicalDeletion=Whether to enable physical deletion
+sys.resource.settings.base.physicalDeletion.note=The default is logical deletion. If you need to enable physical deletion, please enable this configuration. Otherwise, only metadata will be deleted. This configuration applies to all storage modes, please use caution when enabling it!
 sys.resource.settings.base.upload.base.path=Root path of the upload directory
 sys.resource.settings.base.upload.base.path.note=Resources are stored on the HDFS/OSS(S3) path. Resource files will be stored in this base path. Configure it by yourself. Please ensure that the directory exists on the relevant storage system and has read capabilities. Write permission. recommend
 sys.resource.settings.base.model=Storage model

dinky-common/src/main/resources/i18n/messages_zh_CN.properties

@@ -256,6 +256,9 @@ sys.metrics.settings.flink.gatherTimeout=Flink Metrics 采集时间粒度，定
 sys.metrics.settings.flink.gatherTimeout.note=Flink Metrics 采集超时时长，定时任务间隔触发（此配置项应小于Flink Metrics 采集时间粒度）
 sys.resource.settings.base.enable=是否启用Resource
 sys.resource.settings.base.enable.note=如不启用该功能，注册中心中的资源管理功能将无法使用,如果切换存储模式/修改了其他配置项，需要重新启用该功能,来重新加载新配置,否则无法生效,请确保配置正确
+sys.resource.settings.base.physicalDeletion=是否开启物理删除
+sys.resource.settings.base.physicalDeletion.note=默认为逻辑删除，如你需要开启物理删除，请开启此配置，否则仅删除元数据,此配置适用于所有存储模式,请谨慎启用!
+
 sys.resource.settings.base.upload.base.path=上传目录的根路径
 sys.resource.settings.base.upload.base.path.note=资源存储在HDFS/OSS (S3)路径上，资源文件将存储到此基本路径，自行配置，请确保该目录存在于相关存储系统上并具有读写权限。
 sys.resource.settings.base.model=存储模式

dinky-web/src/locales/en-US/pages.ts

@@ -1085,6 +1085,9 @@ export default {
   'rc.resource.enable': 'Resource management function is not enabled',
   'rc.resource.enable.tips':
     'Please go to [Setting Center -> Global Settings -> Resource Configuration] to enable the resource management function!!!',
+  'rc.resource.delete': 'Delete Resource',
+  'rc.resource.deleteConfirm':
+    'We have detected that you have enabled physical deletion mode. Once executed, it cannot be restored and this operation is irreversible. Are you sure you want to delete this resource file?',
 
   'rc.template.codeType': 'Code Type',
   'rc.template.codeTypePlaceholder': 'Please select code type! ',

dinky-web/src/locales/zh-CN/pages.ts

@@ -1004,6 +1004,9 @@ export default {
     '复制成功，值为：[{fillValue}] 已复制到剪贴板。请粘贴到需要的地方进行使用',
   'rc.resource.enable': '未启用资源管理功能',
   'rc.resource.enable.tips': '请前往 [配置中心 -> 全局配置 -> Resource 配置] 中启用资源管理功能!!!',
+  'rc.resource.delete': '删除资源',
+  'rc.resource.deleteConfirm':
+    '检测到你开启了物理删除模式,一旦执行无法恢复,此操作不可逆, 确定删除该资源文件吗？',
 
   'rc.template.codeType': '代码类型',
   'rc.template.codeTypePlaceholder': '请选择代码类型！',

dinky-web/src/pages/RegCenter/Resource/components/ResourceOverView/index.tsx

@@ -54,7 +54,7 @@ import React, { useCallback, useEffect, useRef, useState } from 'react';
 import { connect } from 'umi';
 
 const ResourceOverView: React.FC<connect> = (props) => {
-  const { dispatch, enableResource } = props;
+  const { dispatch, enableResource, resourcePhysicalDelete } = props;
 
   const [resourceState, setResourceState] = useState<ResourceState>(InitResourceState);
 
@@ -73,7 +73,7 @@ const ResourceOverView: React.FC<connect> = (props) => {
     );
   };
 
-  useEffect(() => {
+  useAsyncEffect(() => {
     dispatch({
       type: CONFIG_MODEL_ASYNC.queryResourceConfig,
       payload: SettingConfigKeyEnum.RESOURCE.toLowerCase()
@@ -138,14 +138,26 @@ const ResourceOverView: React.FC<connect> = (props) => {
     }
   };
 
+  const realDelete = async () => {
+    await handleRemoveById(API_CONSTANTS.RESOURCE_REMOVE, resourceState.rightClickedNode.id);
+    await refreshTree();
+  };
+
   /**
    * the node right click event OF delete,
    */
   const handleDelete = async () => {
     if (resourceState.rightClickedNode) {
       setResourceState((prevState) => ({ ...prevState, contextMenuOpen: false }));
-      await handleRemoveById(API_CONSTANTS.RESOURCE_REMOVE, resourceState.rightClickedNode.id);
-      await refreshTree();
+      if (resourcePhysicalDelete) {
+        Modal.confirm({
+          title: l('rc.resource.delete'),
+          content: l('rc.resource.deleteConfirm'),
+          onOk: async () => realDelete()
+        });
+      } else {
+        await realDelete();
+      }
     }
   };
 
@@ -421,5 +433,6 @@ const ResourceOverView: React.FC<connect> = (props) => {
 };
 
 export default connect(({ SysConfig }: { SysConfig: SysConfigStateType }) => ({
-  enableResource: SysConfig.enableResource
+  enableResource: SysConfig.enableResource,
+  resourcePhysicalDelete: SysConfig.resourcePhysicalDelete
 }))(ResourceOverView);

dinky-web/src/pages/SettingCenter/GlobalSetting/model.ts

@@ -37,6 +37,7 @@ export type SysConfigStateType = {
   dsConfig: BaseConfigProperties[];
   enabledDs: boolean;
   enableResource: boolean;
+  resourcePhysicalDelete: boolean;
   taskOwnerLockingStrategy: TaskOwnerLockingStrategy;
 };
 
@@ -51,6 +52,7 @@ export type ConfigModelType = {
   reducers: {
     saveDsConfig: Reducer<SysConfigStateType>;
     updateEnabledDs: Reducer<SysConfigStateType>;
+    updateResourcePhysicalDelete: Reducer<SysConfigStateType>;
     updateEnableResource: Reducer<SysConfigStateType>;
     updateTaskOwnerLockingStrategy: Reducer<SysConfigStateType>;
   };
@@ -62,6 +64,7 @@ const ConfigModel: ConfigModelType = {
     dsConfig: [],
     enabledDs: false,
     enableResource: false,
+    resourcePhysicalDelete: false,
     taskOwnerLockingStrategy: TaskOwnerLockingStrategy.ALL
   },
 
@@ -90,7 +93,7 @@ const ConfigModel: ConfigModelType = {
       if (response && response.length > 0) {
         const enabledDs = response.some(
           (item: BaseConfigProperties) =>
-            item.key === GLOBAL_SETTING_KEYS.SYS_DOLPHINSETTINGS_ENABLE && item.value === true
+            item.key === GLOBAL_SETTING_KEYS.SYS_DOLPHINSETTINGS_ENABLE && item.value == true
         );
         yield put({
           type: 'updateEnabledDs',
@@ -113,6 +116,16 @@ const ConfigModel: ConfigModelType = {
           type: 'updateEnableResource',
           payload: enableResource
         });
+
+        const physicalDelete = response.some(
+          (item: BaseConfigProperties) =>
+            item.key === GLOBAL_SETTING_KEYS.SYS_RESOURCE_SETTINGS_BASE_PHYSICAL_DELETION &&
+            item.value == true
+        );
+        yield put({
+          type: 'updateResourcePhysicalDelete',
+          payload: physicalDelete
+        });
       }
     }
   },
@@ -141,6 +154,12 @@ const ConfigModel: ConfigModelType = {
         ...state,
         enableResource: payload
       };
+    },
+    updateResourcePhysicalDelete(state, { payload }) {
+      return {
+        ...state,
+        resourcePhysicalDelete: payload
+      };
     }
   }
 };

dinky-web/src/types/SettingCenter/data.d.ts

@@ -146,6 +146,7 @@ export enum GLOBAL_SETTING_KEYS {
   SYS_METRICS_SETTINGS_FLINK_GATHER_TIMING = 'sys.metrics.settings.flink.gatherTiming',
   SYS_METRICS_SETTINGS_FLINK_GATHER_TIMEOUT = 'sys.metrics.settings.flink.gatherTimeout',
   SYS_RESOURCE_SETTINGS_BASE_ENABLE = 'sys.resource.settings.base.enable',
+  SYS_RESOURCE_SETTINGS_BASE_PHYSICAL_DELETION = 'sys.resource.settings.base.physicalDeletion',
   SYS_RESOURCE_SETTINGS_BASE_UPLOAD_BASE_PATH = 'sys.resource.settings.base.upload.base.path',
   SYS_RESOURCE_SETTINGS_BASE_MODEL = 'sys.resource.settings.base.model',
   SYS_RESOURCE_SETTINGS_OSS_ENDPOINT = 'sys.resource.settings.oss.endpoint',