diff --git a/addOns/exim/CHANGELOG.md b/addOns/exim/CHANGELOG.md index 788dedb11ec..266ba587ece 100644 --- a/addOns/exim/CHANGELOG.md +++ b/addOns/exim/CHANGELOG.md @@ -6,6 +6,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). ## Unreleased ### Added - Add Automation Framework job to export data (e.g. HAR, URLs). +- Support for Sites Tree export and prune. ### Changed - Update dependency. diff --git a/addOns/exim/src/main/java/org/zaproxy/addon/exim/Exporter.java b/addOns/exim/src/main/java/org/zaproxy/addon/exim/Exporter.java index c3ec1d98a96..c5c92db4f50 100644 --- a/addOns/exim/src/main/java/org/zaproxy/addon/exim/Exporter.java +++ b/addOns/exim/src/main/java/org/zaproxy/addon/exim/Exporter.java @@ -19,6 +19,7 @@ */ package org.zaproxy.addon.exim; +import java.io.BufferedWriter; import java.io.IOException; import java.io.Writer; import java.nio.charset.StandardCharsets; @@ -30,7 +31,10 @@ import org.parosproxy.paros.db.DatabaseException; import org.parosproxy.paros.model.HistoryReference; import org.parosproxy.paros.model.Model; +import org.zaproxy.addon.exim.ExporterOptions.Source; +import org.zaproxy.addon.exim.ExporterOptions.Type; import org.zaproxy.addon.exim.har.HarExporter; +import org.zaproxy.addon.exim.sites.SitesTreeHandler; import org.zaproxy.zap.model.Context; import org.zaproxy.zap.utils.Stats; @@ -82,25 +86,24 @@ private ExporterResult exportImpl(ExporterOptions options) { StandardOpenOption.CREATE, StandardOpenOption.TRUNCATE_EXISTING)) { - List historyIds = - model.getDb() - .getTableHistory() - .getHistoryIdsOfHistType( - model.getSession().getSessionId(), getHistoryTypes(options)); - - ExporterType type = createExporterType(options); - type.begin(writer); - Context context = options.getContext(); - for (Integer id : historyIds) { - HistoryReference ref = new HistoryReference(id, true); - if (context != null && !context.isInContext(ref)) { - continue; + if (Source.SITESTREE.equals(options.getSource())) { + if (!Type.YAML.equals(options.getType())) { + result.addError( + Constant.messages.getString( + "exim.exporter.error.type.sitestree", options.getType())); + } else { + SitesTreeHandler.exportSitesTree(writer, result); + } + } else { + if (Type.YAML.equals(options.getType())) { + result.addError( + Constant.messages.getString( + "exim.exporter.error.type.messages", options.getSource())); + } else { + exportMessagesImpl(writer, result, options); } - - result.incrementCount(); - type.write(writer, ref); } - type.end(writer); + } catch (IOException e) { result.addError( Constant.messages.getString("exim.exporter.error.io", e.getLocalizedMessage()), @@ -114,6 +117,30 @@ private ExporterResult exportImpl(ExporterOptions options) { return result; } + private void exportMessagesImpl( + BufferedWriter writer, ExporterResult result, ExporterOptions options) + throws DatabaseException, IOException { + List historyIds = + model.getDb() + .getTableHistory() + .getHistoryIdsOfHistType( + model.getSession().getSessionId(), getHistoryTypes(options)); + + ExporterType type = createExporterType(options); + type.begin(writer); + Context context = options.getContext(); + for (Integer id : historyIds) { + HistoryReference ref = new HistoryReference(id, true); + if (context != null && !context.isInContext(ref)) { + continue; + } + + result.incrementCount(); + type.write(writer, ref); + } + type.end(writer); + } + private static boolean isValid(Path file, ExporterResult result) { if (Files.exists(file)) { if (!Files.isRegularFile(file)) { diff --git a/addOns/exim/src/main/java/org/zaproxy/addon/exim/ExporterOptions.java b/addOns/exim/src/main/java/org/zaproxy/addon/exim/ExporterOptions.java index 20a060c115f..2b3a0d629b9 100644 --- a/addOns/exim/src/main/java/org/zaproxy/addon/exim/ExporterOptions.java +++ b/addOns/exim/src/main/java/org/zaproxy/addon/exim/ExporterOptions.java @@ -168,7 +168,9 @@ public enum Type { /** The messages are exported as an HAR. */ HAR, /** The messages are exported as URLs. */ - URL; + URL, + /** The SiteTree will be exported as YAML. */ + YAML; private String id; private String name; @@ -200,6 +202,9 @@ public static Type fromString(String value) { if (URL.id.equalsIgnoreCase(value)) { return URL; } + if (YAML.id.equalsIgnoreCase(value)) { + return YAML; + } return HAR; } } @@ -209,7 +214,9 @@ public enum Source { /** Exports the messages proxied and manually accessed by the user. */ HISTORY, /** Exports all messages accessed, includes temporary messages. */ - ALL; + ALL, + /** Exports the Sites tree, only in yaml format */ + SITESTREE; private String id; private String name; @@ -241,6 +248,9 @@ public static Source fromString(String value) { if (ALL.id.equalsIgnoreCase(value)) { return ALL; } + if (SITESTREE.id.equalsIgnoreCase(value)) { + return SITESTREE; + } return HISTORY; } } diff --git a/addOns/exim/src/main/java/org/zaproxy/addon/exim/ExporterResult.java b/addOns/exim/src/main/java/org/zaproxy/addon/exim/ExporterResult.java index cb6bbac1fa5..388df7a73e2 100644 --- a/addOns/exim/src/main/java/org/zaproxy/addon/exim/ExporterResult.java +++ b/addOns/exim/src/main/java/org/zaproxy/addon/exim/ExporterResult.java @@ -44,7 +44,7 @@ public int getCount() { return count; } - void incrementCount() { + public void incrementCount() { this.count++; } diff --git a/addOns/exim/src/main/java/org/zaproxy/addon/exim/ExtensionExim.java b/addOns/exim/src/main/java/org/zaproxy/addon/exim/ExtensionExim.java index df262da5204..1b8998bd5dd 100644 --- a/addOns/exim/src/main/java/org/zaproxy/addon/exim/ExtensionExim.java +++ b/addOns/exim/src/main/java/org/zaproxy/addon/exim/ExtensionExim.java @@ -37,6 +37,8 @@ import org.zaproxy.addon.exim.har.PopupMenuItemSaveHarMessage; import org.zaproxy.addon.exim.log.MenuItemImportLogs; import org.zaproxy.addon.exim.pcap.MenuItemImportPcap; +import org.zaproxy.addon.exim.sites.MenuPruneSites; +import org.zaproxy.addon.exim.sites.MenuSaveSites; import org.zaproxy.addon.exim.urls.MenuItemImportUrls; public class ExtensionExim extends ExtensionAdaptor { @@ -110,6 +112,9 @@ public void hook(ExtensionHook extensionHook) { MainMenuBar menuBar = getView().getMainFrame().getMainMenuBar(); menuBar.add(getMenuExport(), menuBar.getMenuCount() - 2); // Before Online and Help + getMenuExport().add(new MenuSaveSites()); + extensionHook.getHookMenu().addToolsMenuItem(new MenuPruneSites()); + extensionHook.getHookMenu().addImportMenuItem(new MenuImportHar()); extensionHook.getHookMenu().addImportMenuItem(new MenuItemImportUrls()); extensionHook.getHookMenu().addImportMenuItem(new MenuItemImportLogs()); diff --git a/addOns/exim/src/main/java/org/zaproxy/addon/exim/ImportExportApi.java b/addOns/exim/src/main/java/org/zaproxy/addon/exim/ImportExportApi.java index 5f365a24a33..70248afc68b 100644 --- a/addOns/exim/src/main/java/org/zaproxy/addon/exim/ImportExportApi.java +++ b/addOns/exim/src/main/java/org/zaproxy/addon/exim/ImportExportApi.java @@ -26,7 +26,9 @@ import java.io.IOException; import java.nio.charset.StandardCharsets; import java.util.ArrayList; +import java.util.HashMap; import java.util.List; +import java.util.Map; import net.sf.json.JSONObject; import org.apache.commons.httpclient.URI; import org.apache.logging.log4j.LogManager; @@ -45,6 +47,8 @@ import org.zaproxy.addon.exim.har.HarImporter; import org.zaproxy.addon.exim.har.HarUtils; import org.zaproxy.addon.exim.log.LogsImporter; +import org.zaproxy.addon.exim.sites.PruneSiteResult; +import org.zaproxy.addon.exim.sites.SitesTreeHandler; import org.zaproxy.addon.exim.urls.UrlsImporter; import org.zaproxy.zap.extension.api.API; import org.zaproxy.zap.extension.api.ApiAction; @@ -54,6 +58,7 @@ import org.zaproxy.zap.extension.api.ApiOther; import org.zaproxy.zap.extension.api.ApiResponse; import org.zaproxy.zap.extension.api.ApiResponseElement; +import org.zaproxy.zap.extension.api.ApiResponseSet; import org.zaproxy.zap.network.HttpRedirectionValidator; import org.zaproxy.zap.network.HttpRequestConfig; import org.zaproxy.zap.utils.ApiUtils; @@ -76,6 +81,8 @@ public class ImportExportApi extends ApiImplementor { private static final String ACTION_IMPORT_URLS = "importUrls"; private static final String ACTION_IMPORT_ZAP_LOGS = "importZapLogs"; private static final String ACTION_IMPORT_MODSEC2_LOGS = "importModsec2Logs"; + private static final String ACTION_EXPORT_SITES = "exportSitesTree"; + private static final String ACTION_PRUNE_SITES = "pruneSitesTree"; private static final String OTHER_EXPORT_HAR = "exportHar"; private static final String OTHER_EXPORT_HAR_BY_ID = "exportHarById"; @@ -90,6 +97,8 @@ public ImportExportApi() { this.addApiAction(new ApiAction(ACTION_IMPORT_ZAP_LOGS, new String[] {PARAM_FILE_PATH})); this.addApiAction( new ApiAction(ACTION_IMPORT_MODSEC2_LOGS, new String[] {PARAM_FILE_PATH})); + this.addApiAction(new ApiAction(ACTION_EXPORT_SITES, new String[] {PARAM_FILE_PATH})); + this.addApiAction(new ApiAction(ACTION_PRUNE_SITES, new String[] {PARAM_FILE_PATH})); this.addApiOthers( new ApiOther( @@ -133,6 +142,24 @@ public ApiResponse handleApiAction(String name, JSONObject params) throws ApiExc LogsImporter logsImporter = new LogsImporter(file, LogsImporter.LogType.MOD_SECURITY_2); return handleFileImportResponse(logsImporter.isSuccess(), file); + case ACTION_EXPORT_SITES: + file = new File(ApiUtils.getNonEmptyStringParam(params, PARAM_FILE_PATH)); + try { + SitesTreeHandler.exportSitesTree(file, new ExporterResult()); + return handleFileImportResponse(true, file); + } catch (IOException e) { + LOGGER.error(e.getMessage(), e); + return handleFileImportResponse(false, file); + } + case ACTION_PRUNE_SITES: + file = new File(ApiUtils.getNonEmptyStringParam(params, PARAM_FILE_PATH)); + PruneSiteResult res = SitesTreeHandler.pruneSiteNodes(file); + + Map map = new HashMap<>(); + map.put("readNodes", Integer.toString(res.getReadNodes())); + map.put("deletedNodes", Integer.toString(res.getDeletedNodes())); + map.put("errors", res.getError() == null ? "" : res.getError()); + return new ApiResponseSet<>(name, map); default: throw new ApiException(Type.BAD_ACTION); } diff --git a/addOns/exim/src/main/java/org/zaproxy/addon/exim/automation/ExportJob.java b/addOns/exim/src/main/java/org/zaproxy/addon/exim/automation/ExportJob.java index 96c51116857..5d46fb5a0c1 100644 --- a/addOns/exim/src/main/java/org/zaproxy/addon/exim/automation/ExportJob.java +++ b/addOns/exim/src/main/java/org/zaproxy/addon/exim/automation/ExportJob.java @@ -20,6 +20,7 @@ package org.zaproxy.addon.exim.automation; import java.nio.charset.StandardCharsets; +import java.nio.file.Path; import java.util.LinkedHashMap; import java.util.Map; import lombok.Getter; @@ -79,6 +80,23 @@ public void verifyParameters(AutomationProgress progress) { this.getName(), null, progress); + + // Check for invalid combinations + if (Source.SITESTREE.equals(this.parameters.getSource()) + && !Type.YAML.equals(this.parameters.getType())) { + progress.error( + Constant.messages.getString( + "exim.automation.export.error.sitestree.type", + this.getName(), + this.parameters.getType())); + } else if (!Source.SITESTREE.equals(this.parameters.getSource()) + && Type.YAML.equals(this.parameters.getType())) { + progress.error( + Constant.messages.getString( + "exim.automation.export.error.messages.type", + this.getName(), + this.parameters.getSource())); + } } @Override @@ -112,10 +130,12 @@ public void runJob(AutomationEnvironment env, AutomationProgress progress) { return; } + Path path = JobUtils.getFile(fileName, getPlan()).toPath(); + ExporterOptions options = ExporterOptions.builder() .setContext(contextWrapper.getContext()) - .setOutputFile(JobUtils.getFile(fileName, getPlan()).toPath()) + .setOutputFile(path) .setType(getParameters().getType()) .setSource(getParameters().getSource()) .build(); @@ -123,7 +143,10 @@ public void runJob(AutomationEnvironment env, AutomationProgress progress) { ExporterResult result = extension.getExporter().export(options); progress.info( Constant.messages.getString( - "exim.automation.export.exportcount", getName(), result.getCount())); + "exim.automation.export.exportcount", + getName(), + result.getCount(), + path.toAbsolutePath())); result.getErrors() .forEach( error -> diff --git a/addOns/exim/src/main/java/org/zaproxy/addon/exim/automation/ExportJobDialog.java b/addOns/exim/src/main/java/org/zaproxy/addon/exim/automation/ExportJobDialog.java index 69f0c14e528..4d572c98d9b 100644 --- a/addOns/exim/src/main/java/org/zaproxy/addon/exim/automation/ExportJobDialog.java +++ b/addOns/exim/src/main/java/org/zaproxy/addon/exim/automation/ExportJobDialog.java @@ -24,6 +24,7 @@ import java.util.stream.Stream; import javax.swing.DefaultComboBoxModel; import javax.swing.JFileChooser; +import org.parosproxy.paros.Constant; import org.parosproxy.paros.view.View; import org.zaproxy.addon.exim.ExporterOptions; import org.zaproxy.zap.utils.DisplayUtils; @@ -89,7 +90,16 @@ public void save() { @Override public String validateFields() { - // Nothing to do + if (ExporterOptions.Source.SITESTREE.equals(sourceOptionModel.getSelectedItem()) + && !ExporterOptions.Type.YAML.equals(typeOptionModel.getSelectedItem())) { + return Constant.messages.getString( + "exim.automation.export.dialog.error.sitestree.type"); + } else if (!ExporterOptions.Source.SITESTREE.equals(sourceOptionModel.getSelectedItem()) + && ExporterOptions.Type.YAML.equals(typeOptionModel.getSelectedItem())) { + return Constant.messages.getString( + "exim.automation.export.dialog.error.messages.type", + sourceOptionModel.getSelectedItem()); + } return null; } } diff --git a/addOns/exim/src/main/java/org/zaproxy/addon/exim/automation/ExtensionEximAutomation.java b/addOns/exim/src/main/java/org/zaproxy/addon/exim/automation/ExtensionEximAutomation.java index 324f2753f19..31319ba63ec 100644 --- a/addOns/exim/src/main/java/org/zaproxy/addon/exim/automation/ExtensionEximAutomation.java +++ b/addOns/exim/src/main/java/org/zaproxy/addon/exim/automation/ExtensionEximAutomation.java @@ -37,6 +37,7 @@ public class ExtensionEximAutomation extends ExtensionAdaptor { private ImportJob importJob; private ExportJob exportJob; + private PruneJob pruneJob; public ExtensionEximAutomation() { super(NAME); @@ -55,6 +56,8 @@ public void hook(ExtensionHook extensionHook) { extAuto.registerAutomationJob(importJob); exportJob = new ExportJob(getExtension(ExtensionExim.class)); extAuto.registerAutomationJob(exportJob); + pruneJob = new PruneJob(); + extAuto.registerAutomationJob(pruneJob); } private static T getExtension(Class clazz) { @@ -72,6 +75,7 @@ public void unload() { extAuto.unregisterAutomationJob(importJob); extAuto.unregisterAutomationJob(exportJob); + extAuto.unregisterAutomationJob(pruneJob); } @Override diff --git a/addOns/exim/src/main/java/org/zaproxy/addon/exim/automation/PruneJob.java b/addOns/exim/src/main/java/org/zaproxy/addon/exim/automation/PruneJob.java new file mode 100644 index 00000000000..bbf1024a61a --- /dev/null +++ b/addOns/exim/src/main/java/org/zaproxy/addon/exim/automation/PruneJob.java @@ -0,0 +1,208 @@ +/* + * Zed Attack Proxy (ZAP) and its related class files. + * + * ZAP is an HTTP/HTTPS proxy for assessing web application security. + * + * Copyright 2024 The ZAP Development Team + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.zaproxy.addon.exim.automation; + +import java.io.File; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.util.LinkedHashMap; +import java.util.Map; +import org.apache.commons.io.IOUtils; +import org.apache.commons.lang3.StringUtils; +import org.parosproxy.paros.CommandLine; +import org.parosproxy.paros.Constant; +import org.zaproxy.addon.automation.AutomationData; +import org.zaproxy.addon.automation.AutomationEnvironment; +import org.zaproxy.addon.automation.AutomationJob; +import org.zaproxy.addon.automation.AutomationProgress; +import org.zaproxy.addon.automation.jobs.JobData; +import org.zaproxy.addon.automation.jobs.JobUtils; +import org.zaproxy.addon.exim.sites.PruneSiteResult; +import org.zaproxy.addon.exim.sites.SitesTreeHandler; + +public class PruneJob extends AutomationJob { + + private static final String JOB_NAME = "prune"; + private static final String RESOURCES_DIR = "/org/zaproxy/addon/exim/resources/"; + + private static final String PARAM_FILE_NAME = "fileName"; + + private Parameters parameters = new Parameters(); + private Data data; + + public PruneJob() { + this.data = new Data(this, parameters); + } + + @Override + public void verifyParameters(AutomationProgress progress) { + Map jobData = this.getJobData(); + if (jobData == null) { + return; + } + JobUtils.applyParamsToObject( + (LinkedHashMap) jobData.get("parameters"), + this.parameters, + this.getName(), + null, + progress); + } + + @Override + public void applyParameters(AutomationProgress progress) { + // Nothing to do + } + + @Override + public Map getCustomConfigParameters() { + Map map = super.getCustomConfigParameters(); + map.put(PARAM_FILE_NAME, ""); + return map; + } + + @Override + public void runJob(AutomationEnvironment env, AutomationProgress progress) { + + String fileName = this.getParameters().getFileName(); + + if (!StringUtils.isEmpty(fileName)) { + File file = JobUtils.getFile(fileName, getPlan()); + if (!file.exists() || file.canWrite()) { + PruneSiteResult res = SitesTreeHandler.pruneSiteNodes(file); + if (res.getError() == null) { + progress.info( + Constant.messages.getString( + "exim.automation.prune.ok.result", + this.getName(), + file.getAbsolutePath(), + Integer.valueOf(res.getReadNodes()), + Integer.valueOf(res.getDeletedNodes()))); + } else { + progress.error( + Constant.messages.getString( + "exim.automation.prune.fail.result", + this.getName(), + file.getAbsolutePath(), + Integer.valueOf(res.getReadNodes()), + Integer.valueOf(res.getDeletedNodes()), + res.getError())); + } + } else { + progress.error( + Constant.messages.getString( + "exim.automation.import.error.file", this.getName(), fileName)); + } + } + } + + @Override + public String getTemplateDataMin() { + return getResourceAsString(this.getType() + "-min.yaml"); + } + + @Override + public String getTemplateDataMax() { + return getResourceAsString(this.getType() + "-max.yaml"); + } + + private static String getResourceAsString(String name) { + try { + return IOUtils.toString( + PruneJob.class.getResourceAsStream(RESOURCES_DIR + name), + StandardCharsets.UTF_8); + } catch (IOException e) { + CommandLine.error( + Constant.messages.getString( + "exim.automation.import.error.nofile", RESOURCES_DIR + name)); + } + return ""; + } + + @Override + public Order getOrder() { + return Order.AFTER_EXPLORE; + } + + @Override + public String getType() { + return JOB_NAME; + } + + @Override + public Object getParamMethodObject() { + return null; + } + + @Override + public String getParamMethodName() { + return null; + } + + @Override + public Parameters getParameters() { + return parameters; + } + + @Override + public void showDialog() { + new PruneJobDialog(this).setVisible(true); + } + + @Override + public String getSummary() { + return Constant.messages.getString( + "exim.automation.prune.dialog.summary", + JobUtils.unBox(this.getParameters().getFileName(), "''")); + } + + @Override + public Data getData() { + return data; + } + + public static class Data extends JobData { + private Parameters parameters; + + public Data(AutomationJob job, Parameters parameters) { + super(job); + this.parameters = parameters; + } + + public Parameters getParameters() { + return parameters; + } + + public void setParameters(Parameters parameters) { + this.parameters = parameters; + } + } + + public static class Parameters extends AutomationData { + private String fileName; + + public String getFileName() { + return fileName; + } + + public void setFileName(String fileName) { + this.fileName = fileName; + } + } +} diff --git a/addOns/exim/src/main/java/org/zaproxy/addon/exim/automation/PruneJobDialog.java b/addOns/exim/src/main/java/org/zaproxy/addon/exim/automation/PruneJobDialog.java new file mode 100644 index 00000000000..0cb2a82590a --- /dev/null +++ b/addOns/exim/src/main/java/org/zaproxy/addon/exim/automation/PruneJobDialog.java @@ -0,0 +1,66 @@ +/* + * Zed Attack Proxy (ZAP) and its related class files. + * + * ZAP is an HTTP/HTTPS proxy for assessing web application security. + * + * Copyright 2024 The ZAP Development Team + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.zaproxy.addon.exim.automation; + +import java.io.File; +import javax.swing.JFileChooser; +import org.parosproxy.paros.view.View; +import org.zaproxy.zap.utils.DisplayUtils; +import org.zaproxy.zap.view.StandardFieldsDialog; + +@SuppressWarnings("serial") +public class PruneJobDialog extends StandardFieldsDialog { + + private static final long serialVersionUID = 1L; + + private static final String TITLE = "exim.automation.prune.dialog.title"; + private static final String NAME_PARAM = "exim.automation.prune.dialog.name"; + private static final String FILE_NAME_PARAM = "exim.automation.prune.dialog.filename"; + + private PruneJob job; + + public PruneJobDialog(PruneJob job) { + super(View.getSingleton().getMainFrame(), TITLE, DisplayUtils.getScaledDimension(500, 200)); + this.job = job; + + this.addTextField(NAME_PARAM, this.job.getData().getName()); + + String fileName = this.job.getData().getParameters().getFileName(); + File f = null; + if (fileName != null && !fileName.isEmpty()) { + f = new File(fileName); + } + this.addFileSelectField(FILE_NAME_PARAM, f, JFileChooser.FILES_AND_DIRECTORIES, null); + this.addPadding(); + } + + @Override + public void save() { + this.job.getData().setName(this.getStringValue(NAME_PARAM)); + this.job.getParameters().setFileName(this.getStringValue(FILE_NAME_PARAM)); + this.job.resetAndSetChanged(); + } + + @Override + public String validateFields() { + // Nothing to do + return null; + } +} diff --git a/addOns/exim/src/main/java/org/zaproxy/addon/exim/sites/ExImSiteNode.java b/addOns/exim/src/main/java/org/zaproxy/addon/exim/sites/ExImSiteNode.java new file mode 100644 index 00000000000..20bb64db480 --- /dev/null +++ b/addOns/exim/src/main/java/org/zaproxy/addon/exim/sites/ExImSiteNode.java @@ -0,0 +1,198 @@ +/* + * Zed Attack Proxy (ZAP) and its related class files. + * + * ZAP is an HTTP/HTTPS proxy for assessing web application security. + * + * Copyright 2024 The ZAP Development Team + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.zaproxy.addon.exim.sites; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.LinkedHashMap; +import java.util.List; +import org.parosproxy.paros.Constant; + +public class ExImSiteNode { + + public static final String NODE_KEY = "node"; + public static final String URL_KEY = "url"; + public static final String METHOD_KEY = "method"; + public static final String DATA_KEY = "data"; + public static final String RESPONSE_LENGTH_KEY = "responseLength"; + public static final String STATUS_CODE_KEY = "statusCode"; + public static final String CHILDREN_KEY = "children"; + + private String node; + private String url; + private String method; + private String data; + private int responseLength; + private int statusCode; + private List children = new ArrayList<>(); + private List errors; + + private static final List KEYS = + Arrays.asList( + NODE_KEY, + URL_KEY, + METHOD_KEY, + DATA_KEY, + RESPONSE_LENGTH_KEY, + STATUS_CODE_KEY, + CHILDREN_KEY); + + public ExImSiteNode() {} + + public ExImSiteNode(LinkedHashMap lhm) { + this(lhm, null); + } + + private ExImSiteNode(LinkedHashMap lhm, List errors) { + this.errors = errors; + if (this.errors == null) { + this.errors = new ArrayList<>(); + } + + node = getString(lhm, NODE_KEY); + url = getString(lhm, URL_KEY); + method = getString(lhm, METHOD_KEY); + data = getString(lhm, DATA_KEY); + responseLength = getInt(lhm, RESPONSE_LENGTH_KEY); + statusCode = getInt(lhm, STATUS_CODE_KEY); + + Object childrenObj = lhm.get(CHILDREN_KEY); + if (childrenObj != null) { + if (childrenObj instanceof ArrayList) { + ArrayList al = (ArrayList) childrenObj; + al.forEach( + childObj -> { + if (childObj instanceof LinkedHashMap) { + children.add( + new ExImSiteNode( + (LinkedHashMap) childObj, this.errors)); + } + }); + } + } + lhm.keySet() + .forEach( + key -> { + if (!KEYS.contains(key)) { + this.errors.add( + Constant.messages.getString( + "exim.sites.error.badkey", getName(), key)); + } + }); + } + + private String getName() { + String name = ""; + if (this.node != null) { + name = this.node; + } else if (this.url != null) { + name = this.url; + } + return name; + } + + private String getString(LinkedHashMap lhm, String key) { + if (lhm.containsKey(key)) { + Object obj = lhm.get(key); + if (obj instanceof String) { + return (String) obj; + } else { + this.errors.add( + Constant.messages.getString( + "exim.sites.error.badtype", getName(), key, obj)); + } + } + return null; + } + + private int getInt(LinkedHashMap lhm, String key) { + if (lhm.containsKey(key)) { + Object obj = lhm.get(key); + if (obj instanceof Integer) { + return (Integer) obj; + } else { + this.errors.add( + Constant.messages.getString( + "exim.sites.error.badtype", getName(), key, obj)); + } + } + return -1; + } + + public String getNode() { + return node; + } + + public void setNode(String node) { + this.node = node; + } + + public String getUrl() { + return url; + } + + public void setUrl(String url) { + this.url = url; + } + + public String getMethod() { + return method; + } + + public void setMethod(String method) { + this.method = method; + } + + public String getData() { + return data; + } + + public void setData(String data) { + this.data = data; + } + + public int getResponseLength() { + return responseLength; + } + + public void setResponseLength(int responseLength) { + this.responseLength = responseLength; + } + + public int getStatusCode() { + return statusCode; + } + + public void setStatusCode(int statusCode) { + this.statusCode = statusCode; + } + + public List getChildren() { + return children; + } + + public void setChildren(List children) { + this.children = children; + } + + public List getErrors() { + return errors; + } +} diff --git a/addOns/exim/src/main/java/org/zaproxy/addon/exim/sites/MenuPruneSites.java b/addOns/exim/src/main/java/org/zaproxy/addon/exim/sites/MenuPruneSites.java new file mode 100644 index 00000000000..ea5573ae08a --- /dev/null +++ b/addOns/exim/src/main/java/org/zaproxy/addon/exim/sites/MenuPruneSites.java @@ -0,0 +1,83 @@ +/* + * Zed Attack Proxy (ZAP) and its related class files. + * + * ZAP is an HTTP/HTTPS proxy for assessing web application security. + * + * Copyright 2024 The ZAP Development Team + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.zaproxy.addon.exim.sites; + +import java.io.File; +import javax.swing.JFileChooser; +import javax.swing.filechooser.FileNameExtensionFilter; +import org.parosproxy.paros.Constant; +import org.parosproxy.paros.model.Model; +import org.parosproxy.paros.view.View; +import org.zaproxy.addon.commonlib.ui.ReadableFileChooser; +import org.zaproxy.zap.view.ZapMenuItem; + +public class MenuPruneSites extends ZapMenuItem { + + private static final long serialVersionUID = -9207224834749823025L; + private static final String THREAD_PREFIX = "ZAP-Prune-Sites-"; + + private int threadId = 1; + + public MenuPruneSites() { + super("exim.sites.menu.prune"); + + this.setToolTipText(Constant.messages.getString("exim.sites.menu.prune.tooltip")); + this.addActionListener( + e -> { + FileNameExtensionFilter yamlFilesFilter = + new FileNameExtensionFilter( + Constant.messages.getString("exim.file.format.yaml"), + "yaml", + "yml"); + JFileChooser chooser = + new ReadableFileChooser( + Model.getSingleton().getOptionsParam().getUserDirectory()); + chooser.addChoosableFileFilter(yamlFilesFilter); + chooser.setFileFilter(yamlFilesFilter); + + int rc = chooser.showOpenDialog(View.getSingleton().getMainFrame()); + if (rc == JFileChooser.APPROVE_OPTION) { + Thread t = + new Thread() { + @Override + public void run() { + this.setName(THREAD_PREFIX + threadId++); + File file = chooser.getSelectedFile(); + PruneSiteResult result = + SitesTreeHandler.pruneSiteNodes(file); + + if (result.getError() != null) { + View.getSingleton() + .showWarningDialog(result.getError()); + } else { + View.getSingleton() + .showMessageDialog( + Constant.messages.getString( + "exim.sites.prune.result", + result.getReadNodes(), + result.getDeletedNodes())); + } + } + }; + t.start(); + } + }); + } +} diff --git a/addOns/exim/src/main/java/org/zaproxy/addon/exim/sites/MenuSaveSites.java b/addOns/exim/src/main/java/org/zaproxy/addon/exim/sites/MenuSaveSites.java new file mode 100644 index 00000000000..afa4726e90b --- /dev/null +++ b/addOns/exim/src/main/java/org/zaproxy/addon/exim/sites/MenuSaveSites.java @@ -0,0 +1,68 @@ +/* + * Zed Attack Proxy (ZAP) and its related class files. + * + * ZAP is an HTTP/HTTPS proxy for assessing web application security. + * + * Copyright 2024 The ZAP Development Team + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.zaproxy.addon.exim.sites; + +import java.io.File; +import javax.swing.JFileChooser; +import javax.swing.filechooser.FileNameExtensionFilter; +import org.parosproxy.paros.Constant; +import org.parosproxy.paros.model.Model; +import org.parosproxy.paros.view.View; +import org.zaproxy.addon.exim.ExporterResult; +import org.zaproxy.zap.view.ZapMenuItem; +import org.zaproxy.zap.view.widgets.WritableFileChooser; + +public class MenuSaveSites extends ZapMenuItem { + + private static final long serialVersionUID = -9207224834749823025L; + + public MenuSaveSites() { + super("exim.sites.menu.save"); + + this.setToolTipText(Constant.messages.getString("exim.sites.menu.save.tooltip")); + this.addActionListener( + e -> { + FileNameExtensionFilter yamlFilesFilter = + new FileNameExtensionFilter( + Constant.messages.getString("exim.file.format.yaml"), + "yaml", + "yml"); + JFileChooser chooser = + new WritableFileChooser( + Model.getSingleton().getOptionsParam().getUserDirectory()); + chooser.addChoosableFileFilter(yamlFilesFilter); + chooser.setFileFilter(yamlFilesFilter); + + int rc = chooser.showSaveDialog(View.getSingleton().getMainFrame()); + if (rc == JFileChooser.APPROVE_OPTION) { + File file = chooser.getSelectedFile(); + try { + SitesTreeHandler.exportSitesTree(file, new ExporterResult()); + } catch (Exception e1) { + View.getSingleton() + .showWarningDialog( + Constant.messages.getString( + "exim.menu.export.urls.save.error", + file.getAbsolutePath())); + } + } + }); + } +} diff --git a/addOns/exim/src/main/java/org/zaproxy/addon/exim/sites/PruneSiteResult.java b/addOns/exim/src/main/java/org/zaproxy/addon/exim/sites/PruneSiteResult.java new file mode 100644 index 00000000000..cb9c1abcb5c --- /dev/null +++ b/addOns/exim/src/main/java/org/zaproxy/addon/exim/sites/PruneSiteResult.java @@ -0,0 +1,57 @@ +/* + * Zed Attack Proxy (ZAP) and its related class files. + * + * ZAP is an HTTP/HTTPS proxy for assessing web application security. + * + * Copyright 2024 The ZAP Development Team + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.zaproxy.addon.exim.sites; + +import org.zaproxy.addon.exim.ExtensionExim; +import org.zaproxy.zap.utils.Stats; + +public class PruneSiteResult { + + private int readNodes; + private int deletedNodes; + private String error; + + public int getReadNodes() { + return readNodes; + } + + public void incReadNodes() { + Stats.incCounter(ExtensionExim.STATS_PREFIX + "prune.sites.read"); + this.readNodes++; + } + + public int getDeletedNodes() { + return deletedNodes; + } + + public void incDeletedNodes() { + Stats.incCounter(ExtensionExim.STATS_PREFIX + "prune.sites.deleted"); + this.deletedNodes++; + } + + public String getError() { + return error; + } + + public void setError(String error) { + Stats.incCounter(ExtensionExim.STATS_PREFIX + "prune.sites.error"); + this.error = error; + } +} diff --git a/addOns/exim/src/main/java/org/zaproxy/addon/exim/sites/SitesTreeHandler.java b/addOns/exim/src/main/java/org/zaproxy/addon/exim/sites/SitesTreeHandler.java new file mode 100644 index 00000000000..0763c8f2858 --- /dev/null +++ b/addOns/exim/src/main/java/org/zaproxy/addon/exim/sites/SitesTreeHandler.java @@ -0,0 +1,244 @@ +/* + * Zed Attack Proxy (ZAP) and its related class files. + * + * ZAP is an HTTP/HTTPS proxy for assessing web application security. + * + * Copyright 2024 The ZAP Development Team + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.zaproxy.addon.exim.sites; + +import java.io.BufferedWriter; +import java.io.File; +import java.io.FileInputStream; +import java.io.FileWriter; +import java.io.IOException; +import java.io.InputStream; +import java.io.Writer; +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import org.apache.commons.httpclient.URI; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.parosproxy.paros.Constant; +import org.parosproxy.paros.model.HistoryReference; +import org.parosproxy.paros.model.Model; +import org.parosproxy.paros.model.SiteMap; +import org.parosproxy.paros.model.SiteNode; +import org.parosproxy.paros.network.HtmlParameter.Type; +import org.parosproxy.paros.network.HttpHeader; +import org.parosproxy.paros.network.HttpMessage; +import org.parosproxy.paros.network.HttpRequestHeader; +import org.yaml.snakeyaml.DumperOptions; +import org.yaml.snakeyaml.LoaderOptions; +import org.yaml.snakeyaml.Yaml; +import org.yaml.snakeyaml.representer.Representer; +import org.zaproxy.addon.exim.ExporterResult; +import org.zaproxy.addon.exim.ExtensionExim; +import org.zaproxy.zap.model.NameValuePair; +import org.zaproxy.zap.utils.Stats; + +public class SitesTreeHandler { + + private static final Logger LOGGER = LogManager.getLogger(SitesTreeHandler.class); + + private static final Yaml YAML; + + static { + // YAML is used for encoding + DumperOptions options = new DumperOptions(); + options.setDefaultFlowStyle(DumperOptions.FlowStyle.BLOCK); + options.setPrettyFlow(true); + Representer representer = new Representer(options); + representer.setDefaultScalarStyle(DumperOptions.ScalarStyle.DOUBLE_QUOTED); + YAML = new Yaml(representer, options); + } + + public static void exportSitesTree(File file, ExporterResult result) throws IOException { + try (FileWriter fw = new FileWriter(file, false)) { + exportSitesTree(fw, result); + } + } + + public static void exportSitesTree(Writer fw, ExporterResult result) throws IOException { + exportSitesTree(fw, Model.getSingleton().getSession().getSiteTree(), result); + } + + public static void exportSitesTree(Writer fw, SiteMap sites, ExporterResult result) + throws IOException { + try (BufferedWriter bw = new BufferedWriter(fw)) { + outputNode(bw, sites.getRoot(), 0, result); + } + } + + private static void outputKV( + BufferedWriter fw, String indent, boolean first, String key, Object value) + throws IOException { + fw.write(indent); + if (first) { + fw.write("- "); + } else { + fw.write(" "); + } + fw.write(key); + fw.write(": "); + fw.write(YAML.dump(value)); + } + + private static void outputNode( + BufferedWriter fw, SiteNode node, int level, ExporterResult result) throws IOException { + // We could create a set of data structures and use snakeyaml, but the format is very simple + // and this is much more memory efficient - it still uses snakeyaml for encoding + String indent = " ".repeat(level * 2); + HistoryReference href = node.getHistoryReference(); + + outputKV(fw, indent, true, ExImSiteNode.NODE_KEY, level == 0 ? "Sites" : node.toString()); + + if (href != null) { + outputKV(fw, indent, false, ExImSiteNode.URL_KEY, href.getURI().toString()); + outputKV(fw, indent, false, ExImSiteNode.METHOD_KEY, href.getMethod()); + + if (href.getStatusCode() > 0) { + outputKV( + fw, + indent, + false, + ExImSiteNode.RESPONSE_LENGTH_KEY, + href.getResponseHeaderLength() + href.getResponseBodyLength() + 2); + outputKV(fw, indent, false, ExImSiteNode.STATUS_CODE_KEY, href.getStatusCode()); + } + + if (HttpRequestHeader.POST.equals(href.getMethod())) { + try { + HttpMessage msg = href.getHttpMessage(); + if (!msg.getRequestHeader() + .getHeader(HttpHeader.CONTENT_TYPE) + .startsWith(HttpHeader.FORM_MULTIPART_CONTENT_TYPE)) { + List params = + Model.getSingleton().getSession().getParameters(msg, Type.form); + StringBuilder sb = new StringBuilder(); + params.forEach( + nvp -> { + if (sb.length() > 0) { + sb.append('&'); + } + sb.append(nvp.getName()); + sb.append("="); + }); + outputKV(fw, indent, false, ExImSiteNode.DATA_KEY, sb.toString()); + } + } catch (Exception e) { + LOGGER.error(e.getMessage(), e); + } + } + } + result.incrementCount(); + Stats.incCounter(ExtensionExim.STATS_PREFIX + "save.sites.node"); + + if (node.getChildCount() > 0) { + fw.write(indent); + fw.write(" "); + fw.write(ExImSiteNode.CHILDREN_KEY); + fw.write(": "); + fw.newLine(); + node.children() + .asIterator() + .forEachRemaining( + c -> { + try { + outputNode(fw, (SiteNode) c, level + 1, result); + } catch (IOException e) { + LOGGER.error(e.getMessage(), e); + } + }); + } + } + + protected static void pruneSiteNodes( + ExImSiteNode node, PruneSiteResult result, SiteMap siteMap) { + // Delete children first + result.incReadNodes(); + node.getChildren().forEach(child -> pruneSiteNodes(child, result, siteMap)); + + try { + if (node.getUrl() != null) { + URI uri = new URI(node.getUrl(), true); + SiteNode sn; + if (node.getNode().contains("(" + HttpHeader.FORM_MULTIPART_CONTENT_TYPE + ")")) { + // Indicates this request used a multipart form POST + HttpMessage msg = new HttpMessage(uri); + msg.getRequestHeader().setMethod(node.getMethod()); + msg.getRequestHeader() + .setHeader( + HttpHeader.CONTENT_TYPE, + HttpHeader.FORM_MULTIPART_CONTENT_TYPE); + sn = siteMap.findNode(msg); + } else { + sn = siteMap.findNode(uri, node.getMethod(), node.getData()); + } + if (sn != null && sn.getChildCount() == 0) { + siteMap.removeNodeFromParent(sn); + result.incDeletedNodes(); + LOGGER.debug("Deleted node {}", sn.getHierarchicNodeName()); + } else if (sn == null) { + // findNode typically does not find non leaf nodes, even those which no longer + // have any children + sn = siteMap.findClosestParent(new URI(node.getUrl() + "/test", true)); + if (sn != null && sn.getChildCount() == 0) { + siteMap.removeNodeFromParent(sn); + result.incDeletedNodes(); + } + } else { + LOGGER.debug( + "Keeping node {} as it has {} children", + sn.getHierarchicNodeName(), + sn.getChildCount()); + } + } + } catch (Exception e) { + LOGGER.error(e.getMessage(), e); + } + } + + public static PruneSiteResult pruneSiteNodes(File file) { + try (FileInputStream is = new FileInputStream(file)) { + return pruneSiteNodes(is, Model.getSingleton().getSession().getSiteTree()); + } catch (Exception e) { + LOGGER.error(e.getMessage(), e); + PruneSiteResult res = new PruneSiteResult(); + res.setError( + Constant.messages.getString( + "exim.sites.error.prune.exception", e.getMessage())); + return res; + } + } + + protected static PruneSiteResult pruneSiteNodes(InputStream is, SiteMap siteMap) { + PruneSiteResult res = new PruneSiteResult(); + // Don't load yaml using the Constructor class - that throws exceptions that don't give + // enough info + Yaml yaml = new Yaml(new LoaderOptions()); + + Object obj = yaml.load(is); + if (obj instanceof ArrayList) { + ArrayList list = (ArrayList) obj; + ExImSiteNode rootNode = new ExImSiteNode((LinkedHashMap) list.get(0)); + pruneSiteNodes(rootNode, res, siteMap); + } else { + res.setError(Constant.messages.getString("exim.sites.error.prune.badformat")); + } + return res; + } +} diff --git a/addOns/exim/src/main/javahelp/help/contents/automation.html b/addOns/exim/src/main/javahelp/help/contents/automation.html index 6e5367916d0..8c74400f84f 100644 --- a/addOns/exim/src/main/javahelp/help/contents/automation.html +++ b/addOns/exim/src/main/javahelp/help/contents/automation.html @@ -13,7 +13,7 @@

Automation Framework Support



Job: import

-The import job allows you to import HAR(HTTP Archive File), ModSecurity2 Logs, ZAP Messages or a file containing URLs locally. +The import job allows you to import HAR (HTTP Archive File), ModSecurity2 Logs, ZAP Messages or a file containing URLs locally.
   - type: import                       # Import a file of requests
     parameters:
@@ -22,15 +22,30 @@ 

Job: import

Job: export

-The export job allows you to export messages in HAR format or as URLs. I also allows to choose which messages to export with source of History or All, meaning the manually/proxied messages or all messages. +The export job allows you to export messages in HAR format or as URLs as well as exporting the Site Tree in the Sites Tree format. +The supported sources are: +
    +
  • all: all messages, including those generated by ZAP, supports 'har' and 'url' +
  • history: the manually/proxied messages, supports 'har' and 'url' +
  • sitestree: the ZAP Sites Tree, supports 'yaml' +
   - type: export            # Exports data into a file
       parameters:
         context:            # String: Name of the context from which to export. Default: first context
-        type:               # String: One of 'har', 'url'. Default: 'har'
-        source:             # String: One of 'history', 'all'. Default: 'history'
+      type:                 # String: One of 'har', 'url', 'yaml'. Default: 'har'
+      source:               # String: One of 'history', 'sitestree', 'all'. Default: 'history'
         fileName:           # String: Name/path to the file
 
+

Job: prune

+The prune job allows you to remove nodes from the Sites Tree using data from a file. +The file should use the Sites Tree format. +
+  - type: prune           # Prunes nodes from the Sites Tree using Sites Tree data (YAML) from a file
+    parameters:
+      fileName:           # String: Name/path to the file
+
+ \ No newline at end of file diff --git a/addOns/exim/src/main/javahelp/help/contents/exim.html b/addOns/exim/src/main/javahelp/help/contents/exim.html index 6d8c2fdc415..845d03b4292 100644 --- a/addOns/exim/src/main/javahelp/help/contents/exim.html +++ b/addOns/exim/src/main/javahelp/help/contents/exim.html @@ -8,20 +8,28 @@ -

Copy URLs

+

Import/Export

+ +This add-on allows you to import and export ZAP data in a range of formats. +

+It supports the Automation Framework. + +

Menus

+ +

Copy URLs

A context menu item to Copy URLs to the system clipboard. -

Save Selected Entries as HAR (HTTP Archive File)

+

Save Selected Entries as HAR (HTTP Archive File)

A context menu item to save the selected HTTP messages in HAR format. -

Save Raw Message

+

Save Raw Message

Provides a context menu to save content of HTTP messages as binary. (While the files will probably open in a simple editor it may have null characters or malformed bytes.) -

Save XML Message

+

Save XML Message

Provides a context menu to save content of HTTP messages as XML. -

Import HAR (HTTP Archive File)

+

Import HAR (HTTP Archive File)

An option to import messages from a HTTP Archive (HAR), available via the 'Import' menu. Note: The following modifications may be made when importing a HAR (HTTP Archive File): @@ -29,57 +37,61 @@

Import HAR (HTTP Archive File)

  • Missing HTTP Version - If the message is missing the httpVersion attribute it will be set to "HTTP/1.1".
  • HTTP Version 3 - If the message has its httpVersion attribute set as "h3", "http/3", "http/3.0" it will be set to "HTTP/2".
  • Carriage return (CR) or Line feed (LF) in Headers - If the message contains headers with CR or LF, the CRLF(s) will be removed.
  • -
      +
    -

    Import Log File

    +

    Import Log File

    Allows you to import log files from ModSecurity and files previously exported from ZAP. -

    Import URLs

    +

    Import URLs

    An option to import a file of URLs is available via the 'Import' menu ('Import a File Containing URLs'). The file must be plain text with one URL per line. Blank lines and lines starting with # will be ignored. -

    -It also supports the Automation Framework. +

    Tools / Prune Sites Tree...

    +This allows you to prune URLs from the Sites Tree using a file in the Sites Tree format -

    Export

    +

    Export

    The add-on also adds a top level "Export" menu, providing the following functionality. -

    Export Messages to File...

    +

    Save Messages...

    This allows you to save requests and responses to a text file.
    Select the messages to save in the History tab (including multi-select). -

    Export Response to File... +

    Save Responses...

    This allows you to save a specific responses to a file.
    Select the relevant message in the History tab - note that binary responses (such as images) can be saved as well as text responses. -

    Export All URLs to File...

    +

    Save Sites Tree...

    +This allows you to save the Sites Tree in the Sites Tree format + +

    Save URLs...

    This allows you to save all of the URLs accessed to a text or HTML file.
    This can be used, amongst other things, to compare the URLs available to users with different roles or permissions on the same system. (Also consider leveraging the Access Control Testing add-on.) This functionality is also available via the right-click context menu. -

    Export Selected URLs to File...

    -Based on the selection (including multi-select) in the Sites tree all URLs and child URLs of selected -nodes are exported. -This functionality is also available via the right-click context menu. - -

    Export URLs for Context

    +

    Context: Save URLs...

    All URLs in the Sites tree that fall within the selected context are exported. This functionality is also available from the right-click menu when used on a Context node in the Sites tree panel. -

    ZAP API

    +

    ZAP API

    This add-on also exposes various ZAP API endpoints to facilitate programmatic use of the functionality. + +

    Actions

      -
    • /exim/action/importHar (filePath*)
    • -
    • /exim/action/importModsec2Logs (filePath*)
    • -
    • /exim/action/importUrls (filePath*)
    • -
    • /exim/action/importZapLogs (filePath*)
    • -
    • ---
    • -
    • /exim/other/exportHar (baseurl start count)
    • -
    • /exim/other/exportHarById (ids*)
    • -
    • /exim/other/sendHarRequest (request* followRedirects)
    • +
    • exportSitesTree (filePath* ) Exports the Sites Tree in the Site Tree YAML format.
    • +
    • importHar (filePath* ) Imports a HAR file.
    • +
    • importModsec2Logs (filePath* ) Imports ModSecurity2 logs from the file with the given file system path.
    • +
    • importUrls (filePath* ) Imports URLs (one per line) from the file with the given file system path.
    • +
    • importZapLogs (filePath* ) Imports previously exported ZAP messages from the file with the given file system path.
    • +
    • pruneSitesTree (filePath* ) Prunes the Sites Tree based on a file in the Site Tree YAML format.
    • +
    + +

    Others

    +
  • exportHar (baseurl start count ) Gets the HTTP messages sent through/by ZAP, in HAR format, optionally filtered by URL and paginated with 'start' position and 'count' of messages
  • +
  • exportHarById (ids* ) Gets the HTTP messages with the given IDs, in HAR format.
  • +
  • sendHarRequest (request* followRedirects ) Sends the first HAR request entry, optionally following redirections. Returns, in HAR format, the request sent and response received and followed redirections, if any. The Mode is enforced when sending the request (and following redirections), custom manual requests are not allowed in 'Safe' mode nor in 'Protected' mode if out of scope.
  • diff --git a/addOns/exim/src/main/javahelp/help/contents/sitestreeformat.html b/addOns/exim/src/main/javahelp/help/contents/sitestreeformat.html new file mode 100644 index 00000000000..5a1f3d357ea --- /dev/null +++ b/addOns/exim/src/main/javahelp/help/contents/sitestreeformat.html @@ -0,0 +1,67 @@ + + + + + + Sites Tree File Format + + + + +

    Sites Tree File Format

    + +The Sites Tree Format is a YAML representation of the ZAP Sites Tree.
    +It is a hierarchy of nodes, each of which represents all of the essential information needed to uniquely identify the corresponding node in the Sites tree. +

    +Each node has the following format: + +

    +  - node:              # The name of the node, as shown in the ZAP Sites Tree
    +    url:               # The URL it represents, present for all apart from the top node
    +    method:            # The HTTP method, present for all apart from the top node
    +    responseLength:    # The length of the response, where relevant
    +    statusCode:        # The HTTP status code, where relevant
    +    data:              # The names of the data parameters, if any, separated with '=&'s
    +    children:          # A list of child nodes, present for all nodes apart from the leaves
    +
    + +

    +This format is used by the Automation Framework export and prune jobs, and by the corresponding +desktop menus. +

    + +A full simple example: + +

    +- node: Sites
    +  children:
    +  - node: https://www.example.com
    +    url: https://www.example.com
    +    method: GET
    +    children:
    +    - node: missing
    +      url: https://www.example.com/missing
    +      method: GET
    +      responseLength: 1221
    +      statusCode: 404
    +    - node: path
    +      url: https://www.example.com/path
    +      method: GET
    +      responseLength: 1234
    +      statusCode: 200
    +      children:
    +      - node: GET:query(q)
    +        url: https://www.example.com/seq/query?q=search
    +        method: GET
    +        responseLength: 2345
    +        statusCode: 200
    +      - node: submit
    +        url: https://www.example.com/seq/submit()(field1,field2,field3)
    +        method: POST
    +        data: field1=&field2=&field3
    +        responseLength: 3456
    +        statusCode: 200
    +
    + + + \ No newline at end of file diff --git a/addOns/exim/src/main/javahelp/help/index.xml b/addOns/exim/src/main/javahelp/help/index.xml index 21820e751ec..f094f961ae9 100644 --- a/addOns/exim/src/main/javahelp/help/index.xml +++ b/addOns/exim/src/main/javahelp/help/index.xml @@ -6,4 +6,5 @@ + diff --git a/addOns/exim/src/main/javahelp/help/map.jhm b/addOns/exim/src/main/javahelp/help/map.jhm index c20a7a3fb95..01a307f5f98 100644 --- a/addOns/exim/src/main/javahelp/help/map.jhm +++ b/addOns/exim/src/main/javahelp/help/map.jhm @@ -6,4 +6,5 @@ + diff --git a/addOns/exim/src/main/javahelp/help/toc.xml b/addOns/exim/src/main/javahelp/help/toc.xml index 9dbcc4a2e7c..e92739fe6b3 100644 --- a/addOns/exim/src/main/javahelp/help/toc.xml +++ b/addOns/exim/src/main/javahelp/help/toc.xml @@ -8,6 +8,7 @@ + diff --git a/addOns/exim/src/main/resources/org/zaproxy/addon/exim/resources/Messages.properties b/addOns/exim/src/main/resources/org/zaproxy/addon/exim/resources/Messages.properties index 5774db5811b..5e0e7a5f43a 100644 --- a/addOns/exim/src/main/resources/org/zaproxy/addon/exim/resources/Messages.properties +++ b/addOns/exim/src/main/resources/org/zaproxy/addon/exim/resources/Messages.properties @@ -1,7 +1,9 @@ +exim.api.action.exportSitesTree = Exports the Sites Tree in the Site Tree YAML format. exim.api.action.importHar = Imports a HAR file. exim.api.action.importModsec2Logs = Imports ModSecurity2 logs from the file with the given file system path. exim.api.action.importUrls = Imports URLs (one per line) from the file with the given file system path. exim.api.action.importZapLogs = Imports previously exported ZAP messages from the file with the given file system path. +exim.api.action.pruneSitesTree = Prunes the Sites Tree based on a file in the Site Tree YAML format. exim.api.desc = Export/Import functionality. exim.api.other.exportHar = Gets the HTTP messages sent through/by ZAP, in HAR format, optionally filtered by URL and paginated with 'start' position and 'count' of messages exim.api.other.exportHar.param.baseurl = The URL below which messages should be included. @@ -18,12 +20,16 @@ exim.automation.dialog.filename = File: exim.automation.dialog.name = Job Name: exim.automation.dialog.type = Type: exim.automation.error.noresourcefile = Cannot access file: {0} +exim.automation.export.dialog.error.messages.type = Invalid Type for {0}, YAML is not supported +exim.automation.export.dialog.error.sitestree.type = Invalid Type for Sites Tree, only YAML is supported exim.automation.export.dialog.source = Source: exim.automation.export.dialog.summary = Type: {0}, Source: {1}, File: {2} exim.automation.export.dialog.title = Export Job exim.automation.export.error = Job {0} Error: {1} +exim.automation.export.error.messages.type = Job {0} Invalid type for {1}, YAML is not supported +exim.automation.export.error.sitestree.type = Job {0} Invalid type for Sites Tree, only YAML is supported: {1} exim.automation.export.error.type = Job {0} Invalid type: {1} -exim.automation.export.exportcount = Job {0}: Exported {1} message(s). +exim.automation.export.exportcount = Job {0}: Exported {1} message(s) / node(s) to {2}. exim.automation.export.nofile = Job {0}: No file specified, the export will be skipped. exim.automation.import.dialog.summary = Type: {0}, File: {1} exim.automation.import.dialog.title = Import Job @@ -31,6 +37,12 @@ exim.automation.import.error = Error importing the file {0} as {1} exim.automation.import.error.file = Job {0} cannot read file: {1} exim.automation.import.error.type = Job {0} Invalid type: {1} exim.automation.name = Import/Export Automation +exim.automation.prune.dialog.filename = File: +exim.automation.prune.dialog.name = Job Name: +exim.automation.prune.dialog.summary = File: {0} +exim.automation.prune.dialog.title = Prune Job +exim.automation.prune.fail.result = Job {0} Read {2} node(s) from {1}, deleted {3} node(s), and Failed with '{4}' +exim.automation.prune.ok.result = Job {0} Read {2} node(s) from {1} and deleted {3} node(s) exim.description = Import and Export functionality supporting multiple formats. exim.exporter.error.db = Failed to read from session: {0} @@ -40,11 +52,16 @@ exim.exporter.error.file.parent.notdir = Cannot write file to non-directory: {0} exim.exporter.error.file.parent.notexists = Cannot write file to nonexistent directory: {0} exim.exporter.error.file.parent.notwritable = Cannot write file to directory: {0} exim.exporter.error.io = Failed to write to file: {0} +exim.exporter.error.type.messages = Invalid type for {0}, YAML is not supported +exim.exporter.error.type.sitestree = Invalid type for SitesTree, only YAML is supported: {0} exim.exporter.source.all = All exim.exporter.source.history = History +exim.exporter.source.sitestree = Sites Tree exim.exporter.type.har = HAR exim.exporter.type.url = URLs +exim.exporter.type.yaml = YAML +exim.file.format.yaml = YAML File exim.file.save.error = Error saving file to {0}. exim.har.file.description = HTTP Archive File (*.har) @@ -89,6 +106,8 @@ exim.menu.export.popup = Save All URLs... exim.menu.export.popup.context.error = Please select a Context. exim.menu.export.popup.selected = Save URLs... exim.menu.export.responses.popup = Save Response(s)... + +exim.menu.export.savesite.popup = Save Sites Tree... exim.menu.export.urls.save.error = Error saving file to {0} exim.options.value.type.har = HAR (HTTP Archive File) @@ -114,5 +133,15 @@ exim.saveraw.popup.option = Save as Raw exim.savexml.file.description = XML with Base64 Encoded Components (*.xml) exim.savexml.popup.option = Save as XML +exim.sites.error.badkey = Invalid key for node {0}: {1} +exim.sites.error.badtype = Unexpected value type for key {0}: {1} +exim.sites.error.prune.badformat = Unexpected file format - expected a YAML list +exim.sites.error.prune.exception = Exception loading file, for more details see the log file: {0} +exim.sites.menu.prune = Prune Site Nodes... +exim.sites.menu.prune.tooltip = Prune nodes from the sites tree defined in a file in the ZAP Sites Tree format. +exim.sites.menu.save = Save Sites Tree... +exim.sites.menu.save.tooltip = Save all of the nodes in the sites tree in a file using the ZAP Sites Tree format. + +exim.sites.prune.result = Prune Site Nodes Result:\n Nodes read: {0}\n Nodes deleted: {1} exim.ui.name = Import/Export diff --git a/addOns/exim/src/main/resources/org/zaproxy/addon/exim/resources/export-max.yaml b/addOns/exim/src/main/resources/org/zaproxy/addon/exim/resources/export-max.yaml index 6fa3174d697..3ee8f505e45 100644 --- a/addOns/exim/src/main/resources/org/zaproxy/addon/exim/resources/export-max.yaml +++ b/addOns/exim/src/main/resources/org/zaproxy/addon/exim/resources/export-max.yaml @@ -1,6 +1,6 @@ - - type: export # Exports data into a file + - type: export # Exports data into a file parameters: context: # String: Name of the context from which to export. Default: first context - type: # String: One of 'har', 'url'. Default: 'har' - source: # String: One of 'history', 'all'. Default: 'history' + type: # String: One of 'har', 'url', 'yaml'. Default: 'har' + source: # String: One of 'history', 'sitestree', 'all'. Default: 'history' fileName: # String: Name/path to the file diff --git a/addOns/exim/src/main/resources/org/zaproxy/addon/exim/resources/export-min.yaml b/addOns/exim/src/main/resources/org/zaproxy/addon/exim/resources/export-min.yaml index 25a4ff0c8d7..978d191159f 100644 --- a/addOns/exim/src/main/resources/org/zaproxy/addon/exim/resources/export-min.yaml +++ b/addOns/exim/src/main/resources/org/zaproxy/addon/exim/resources/export-min.yaml @@ -1,3 +1,3 @@ - - type: export # Exports data into a file + - type: export # Exports data into a file parameters: fileName: # String: Name/path to the file diff --git a/addOns/exim/src/main/resources/org/zaproxy/addon/exim/resources/prune-max.yaml b/addOns/exim/src/main/resources/org/zaproxy/addon/exim/resources/prune-max.yaml new file mode 100644 index 00000000000..b12c5a939ed --- /dev/null +++ b/addOns/exim/src/main/resources/org/zaproxy/addon/exim/resources/prune-max.yaml @@ -0,0 +1,3 @@ + - type: prune # Prunes nodes from the Sites Tree using Sites Tree data (YAML) from a file + parameters: + fileName: # String: Name/path to the file diff --git a/addOns/exim/src/main/resources/org/zaproxy/addon/exim/resources/prune-min.yaml b/addOns/exim/src/main/resources/org/zaproxy/addon/exim/resources/prune-min.yaml new file mode 100644 index 00000000000..b12c5a939ed --- /dev/null +++ b/addOns/exim/src/main/resources/org/zaproxy/addon/exim/resources/prune-min.yaml @@ -0,0 +1,3 @@ + - type: prune # Prunes nodes from the Sites Tree using Sites Tree data (YAML) from a file + parameters: + fileName: # String: Name/path to the file diff --git a/addOns/exim/src/test/java/org/zaproxy/addon/exim/ExporterUnitTest.java b/addOns/exim/src/test/java/org/zaproxy/addon/exim/ExporterUnitTest.java index e774269e1c7..dcba484c7a0 100644 --- a/addOns/exim/src/test/java/org/zaproxy/addon/exim/ExporterUnitTest.java +++ b/addOns/exim/src/test/java/org/zaproxy/addon/exim/ExporterUnitTest.java @@ -287,7 +287,9 @@ void shouldErrorIfParentOutputFileNotValid() { } @ParameterizedTest - @EnumSource(value = Type.class) + @EnumSource( + value = Type.class, + names = {"HAR", "URL"}) void shouldIncludeMessageInContext(Type type) throws Exception { // Given optionsWithType(type); @@ -306,7 +308,9 @@ void shouldIncludeMessageInContext(Type type) throws Exception { } @ParameterizedTest - @EnumSource(value = Type.class) + @EnumSource( + value = Type.class, + names = {"HAR", "URL"}) void shouldNotIncludeMessageNotInContext(Type type) throws Exception { // Given optionsWithType(type); @@ -323,4 +327,42 @@ void shouldNotIncludeMessageNotInContext(Type type) throws Exception { assertThat(Files.readString(outputFile), not(containsString("http://example.com/1"))); verify(context).isInContext(any(HistoryReference.class)); } + + @ParameterizedTest + @EnumSource( + value = Type.class, + names = {"HAR", "URL"}) + void shouldFailSiteTreeExportWithNonYamlFormat(Type type) throws Exception { + // Given + optionsWithType(type); + optionsWithSource(Source.SITESTREE); + // When + ExporterResult result = exporter.export(options); + // Then + assertCount(result, 0); + assertThat(result.getErrors().size(), is(1)); + assertThat( + result.getErrors().get(0), + is("Invalid type for SitesTree, only YAML is supported: " + type)); + assertThat(result.getCause(), is(nullValue())); + } + + @ParameterizedTest + @EnumSource( + value = Source.class, + names = {"HISTORY", "ALL"}) + void shouldFailMessageExportWithYamlFormat(Source source) throws Exception { + // Given + optionsWithType(Type.YAML); + optionsWithSource(source); + // When + ExporterResult result = exporter.export(options); + // Then + assertCount(result, 0); + assertThat(result.getErrors().size(), is(1)); + assertThat( + result.getErrors().get(0), + is("Invalid type for " + source + ", YAML is not supported")); + assertThat(result.getCause(), is(nullValue())); + } } diff --git a/addOns/exim/src/test/java/org/zaproxy/addon/exim/automation/ExportJobUnitTest.java b/addOns/exim/src/test/java/org/zaproxy/addon/exim/automation/ExportJobUnitTest.java index 691ca7bc4c9..9c3768655aa 100644 --- a/addOns/exim/src/test/java/org/zaproxy/addon/exim/automation/ExportJobUnitTest.java +++ b/addOns/exim/src/test/java/org/zaproxy/addon/exim/automation/ExportJobUnitTest.java @@ -38,8 +38,11 @@ import java.util.LinkedHashMap; import java.util.List; import java.util.Map; +import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.EnumSource; import org.mockito.quality.Strictness; import org.parosproxy.paros.model.Model; import org.yaml.snakeyaml.Yaml; @@ -50,6 +53,8 @@ import org.zaproxy.addon.automation.ContextWrapper; import org.zaproxy.addon.exim.Exporter; import org.zaproxy.addon.exim.ExporterOptions; +import org.zaproxy.addon.exim.ExporterOptions.Source; +import org.zaproxy.addon.exim.ExporterOptions.Type; import org.zaproxy.addon.exim.ExporterResult; import org.zaproxy.addon.exim.ExtensionExim; import org.zaproxy.zap.model.Context; @@ -62,6 +67,11 @@ class ExportJobUnitTest extends TestUtils { private Exporter exporter; private ExportJob job; + @BeforeAll + static void setupMessages() { + mockMessages(new ExtensionExim()); + } + @BeforeEach void setUp() { mockMessages(new ExtensionExim()); @@ -166,7 +176,9 @@ void shouldReportExporterCount() { // Then assertThat(progress.hasWarnings(), is(equalTo(false))); assertThat(progress.hasErrors(), is(equalTo(false))); - assertThat(progress.getInfos(), hasItem("Job export: Exported 42 message(s).")); + assertThat( + progress.getInfos(), + hasItem("Job export: Exported 42 message(s) / node(s) to /some/file.")); } @Test @@ -198,6 +210,83 @@ void shouldReportExporterErrors() { assertThat(progress.getErrors(), contains("Job export Error: Error while exporting")); } + @ParameterizedTest + @EnumSource( + value = Type.class, + names = {"HAR", "URL"}) + void shouldReportErrorSiteTreeExportWithNonYamlFormat(Type type) { + // Given + AutomationPlan plan = new AutomationPlan(); + AutomationProgress progress = plan.getProgress(); + AutomationEnvironment env = mock(AutomationEnvironment.class); + ContextWrapper contextWrapper = new ContextWrapper(mock(Context.class)); + given(env.getContextWrapper(any())).willReturn(contextWrapper); + String yamlStr = + "parameters:\n" + + " source: SitesTree\n" + + " type: " + + type.getId() + + "\n" + + " fileName: /some/file"; + Yaml yaml = new Yaml(); + Object data = yaml.load(yamlStr); + ExporterResult result = mock(); + given(exporter.export(any())).willReturn(result); + + job.setJobData(((LinkedHashMap) data)); + job.setPlan(plan); + + // When + job.verifyParameters(progress); + job.runJob(env, progress); + + // Then + assertThat(progress.hasWarnings(), is(equalTo(false))); + assertThat(progress.hasErrors(), is(equalTo(true))); + assertThat( + progress.getErrors(), + contains( + "Job export Invalid type for Sites Tree, only YAML is supported: " + type)); + } + + @ParameterizedTest + @EnumSource( + value = Source.class, + names = {"HISTORY", "ALL"}) + void shouldReportErrorNonSitesTreeExportWithYamlFormat(Source source) { + // Given + AutomationPlan plan = new AutomationPlan(); + AutomationProgress progress = plan.getProgress(); + AutomationEnvironment env = mock(AutomationEnvironment.class); + ContextWrapper contextWrapper = new ContextWrapper(mock(Context.class)); + given(env.getContextWrapper(any())).willReturn(contextWrapper); + String yamlStr = + "parameters:\n" + + " source: " + + source.getId() + + "\n" + + " type: YAML\n" + + " fileName: /some/file"; + Yaml yaml = new Yaml(); + Object data = yaml.load(yamlStr); + ExporterResult result = mock(); + given(exporter.export(any())).willReturn(result); + + job.setJobData(((LinkedHashMap) data)); + job.setPlan(plan); + + // When + job.verifyParameters(progress); + job.runJob(env, progress); + + // Then + assertThat(progress.hasWarnings(), is(equalTo(false))); + assertThat(progress.hasErrors(), is(equalTo(true))); + assertThat( + progress.getErrors(), + contains("Job export Invalid type for " + source + ", YAML is not supported")); + } + private static void assertValidTemplate(String value) { assertThat(value, is(not(equalTo("")))); assertDoesNotThrow(() -> new Yaml().load(value)); diff --git a/addOns/exim/src/test/java/org/zaproxy/addon/exim/automation/PruneJobUnitTest.java b/addOns/exim/src/test/java/org/zaproxy/addon/exim/automation/PruneJobUnitTest.java new file mode 100644 index 00000000000..b7a668b2233 --- /dev/null +++ b/addOns/exim/src/test/java/org/zaproxy/addon/exim/automation/PruneJobUnitTest.java @@ -0,0 +1,148 @@ +/* + * Zed Attack Proxy (ZAP) and its related class files. + * + * ZAP is an HTTP/HTTPS proxy for assessing web application security. + * + * Copyright 2024 The ZAP Development Team + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.zaproxy.addon.exim.automation; + +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.not; +import static org.hamcrest.Matchers.nullValue; +import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; +import static org.mockito.BDDMockito.given; +import static org.mockito.Mockito.CALLS_REAL_METHODS; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.withSettings; + +import java.util.LinkedHashMap; +import java.util.Locale; +import java.util.Map; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.mockito.quality.Strictness; +import org.parosproxy.paros.Constant; +import org.parosproxy.paros.control.Control; +import org.parosproxy.paros.extension.ExtensionLoader; +import org.parosproxy.paros.model.Model; +import org.yaml.snakeyaml.Yaml; +import org.zaproxy.addon.automation.AutomationEnvironment; +import org.zaproxy.addon.automation.AutomationJob; +import org.zaproxy.addon.automation.AutomationPlan; +import org.zaproxy.addon.automation.AutomationProgress; +import org.zaproxy.addon.exim.ExtensionExim; +import org.zaproxy.zap.utils.I18N; + +class PruneJobUnitTest { + private ExtensionExim extExim; + + @BeforeEach + void setUp() { + Constant.messages = new I18N(Locale.ENGLISH); + + Model model = mock(Model.class, withSettings().defaultAnswer(CALLS_REAL_METHODS)); + Model.setSingletonForTesting(model); + ExtensionLoader extensionLoader = + mock(ExtensionLoader.class, withSettings().strictness(Strictness.LENIENT)); + extExim = mock(ExtensionExim.class, withSettings().strictness(Strictness.LENIENT)); + given(extensionLoader.getExtension(ExtensionExim.class)).willReturn(extExim); + + Control.initSingletonForTesting(Model.getSingleton(), extensionLoader); + } + + @Test + void shouldReturnDefaultFields() { + // Given / When + PruneJob job = new PruneJob(); + + // Then + assertThat(job.getType(), is(equalTo("prune"))); + assertThat(job.getName(), is(equalTo("prune"))); + assertThat(job.getOrder(), is(equalTo(AutomationJob.Order.AFTER_EXPLORE))); + assertValidTemplate(job.getTemplateDataMin()); + assertValidTemplate(job.getTemplateDataMax()); + assertThat(job.getParamMethodObject(), is(nullValue())); + assertThat(job.getParamMethodName(), is(nullValue())); + } + + @Test + void shouldReturnCustomConfigParams() { + // Given + PruneJob job = new PruneJob(); + + // When + Map params = job.getCustomConfigParameters(); + + // Then + assertThat(params.size(), is(equalTo(1))); + assertThat(params.get("fileName"), is(equalTo(""))); + } + + @Test + void shouldApplyCustomConfigParams() { + // Given + Constant.messages = new I18N(Locale.ENGLISH); + AutomationProgress progress = new AutomationProgress(); + String fileName = "C:\\Users\\ZAPBot\\Documents\\test file.har"; + String yamlStr = "parameters:\n" + " fileName: " + fileName; + Yaml yaml = new Yaml(); + Object data = yaml.load(yamlStr); + + PruneJob job = new PruneJob(); + job.setJobData(((LinkedHashMap) data)); + + // When + job.verifyParameters(progress); + job.applyParameters(progress); + + // Then + assertThat(job.getParameters().getFileName(), is(equalTo(fileName))); + assertThat(progress.hasWarnings(), is(equalTo(false))); + assertThat(progress.hasErrors(), is(equalTo(false))); + } + + @Test + void shouldFailIfInvalidFile() { + // Given + Constant.messages = new I18N(Locale.ENGLISH); + AutomationPlan plan = new AutomationPlan(); + AutomationProgress progress = plan.getProgress(); + AutomationEnvironment env = plan.getEnv(); + String yamlStr = "parameters:\n" + " fileName: 'Invalid file path'"; + Yaml yaml = new Yaml(); + Object data = yaml.load(yamlStr); + + PruneJob job = new PruneJob(); + job.setJobData(((LinkedHashMap) data)); + job.setPlan(plan); + + // When + job.verifyParameters(progress); + job.runJob(env, progress); + + // Then + assertThat(progress.hasWarnings(), is(equalTo(false))); + assertThat(progress.hasErrors(), is(equalTo(true))); + assertThat(progress.getErrors().get(0), is(equalTo("!exim.automation.prune.fail.result!"))); + } + + private static void assertValidTemplate(String value) { + assertThat(value, is(not(equalTo("")))); + assertDoesNotThrow(() -> new Yaml().load(value)); + } +} diff --git a/addOns/exim/src/test/java/org/zaproxy/addon/exim/sites/EximSiteNodeUnitTest.java b/addOns/exim/src/test/java/org/zaproxy/addon/exim/sites/EximSiteNodeUnitTest.java new file mode 100644 index 00000000000..62807180375 --- /dev/null +++ b/addOns/exim/src/test/java/org/zaproxy/addon/exim/sites/EximSiteNodeUnitTest.java @@ -0,0 +1,199 @@ +/* + * Zed Attack Proxy (ZAP) and its related class files. + * + * ZAP is an HTTP/HTTPS proxy for assessing web application security. + * + * Copyright 2024 The ZAP Development Team + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.zaproxy.addon.exim.sites; + +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.is; + +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Locale; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.parosproxy.paros.Constant; +import org.yaml.snakeyaml.LoaderOptions; +import org.yaml.snakeyaml.Yaml; +import org.zaproxy.zap.utils.I18N; + +/** Unit test for {@link EximSiteNode}. */ +class EximSiteNodeUnitTest { + private static Yaml yaml; + + @BeforeAll + static void setup() { + Constant.messages = new I18N(Locale.ENGLISH); + yaml = new Yaml(new LoaderOptions()); + } + + @AfterAll + static void cleanup() { + Constant.messages = null; + } + + @Test + void shouldImportSimpleNode() { + // Given + String yamlStr = + "- node: www.example.com\n" + + " url: https://www.example.com\n" + + " method: GET\n" + + " responseLength: 1234\n" + + " statusCode: 200\n"; + + // When + List list = (ArrayList) yaml.load(yamlStr); + ExImSiteNode node = new ExImSiteNode((LinkedHashMap) list.get(0)); + + // Then + assertThat(node.getNode(), is(equalTo("www.example.com"))); + assertThat(node.getUrl(), is(equalTo("https://www.example.com"))); + assertThat(node.getMethod(), is(equalTo("GET"))); + assertThat(node.getResponseLength(), is(equalTo(1234))); + assertThat(node.getStatusCode(), is(equalTo(200))); + assertThat(node.getChildren().size(), is(equalTo(0))); + assertThat(node.getErrors().size(), is(equalTo(0))); + } + + @Test + void shouldImportNodeHierarchy() { + // Given + String yamlStr = + "- node: www.example.com\n" + + " url: https://www.example.com\n" + + " method: GET\n" + + " responseLength: 1234\n" + + " statusCode: 200\n" + + " children:\n" + + " - node: GET:/\n" + + " url: https://www.example.com/\n" + + " method: GET\n" + + " responseLength: 2345\n" + + " statusCode: 200\n" + + " - node: GET:aaa\n" + + " url: https://www.example.com/aaa\n" + + " method: GET\n" + + " responseLength: 3456\n" + + " statusCode: 201\n" + + " children:\n" + + " - node: POST:bbb\n" + + " url: https://www.example.com/aaa/bbb\n" + + " method: POST\n" + + " responseLength: 101\n" + + " statusCode: 401\n"; + + // When + List list = (ArrayList) yaml.load(yamlStr); + ExImSiteNode node = new ExImSiteNode((LinkedHashMap) list.get(0)); + + // Then + assertThat(node.getNode(), is(equalTo("www.example.com"))); + assertThat(node.getUrl(), is(equalTo("https://www.example.com"))); + assertThat(node.getMethod(), is(equalTo("GET"))); + assertThat(node.getResponseLength(), is(equalTo(1234))); + assertThat(node.getStatusCode(), is(equalTo(200))); + assertThat(node.getChildren().size(), is(equalTo(2))); + + ExImSiteNode child1 = node.getChildren().get(0); + assertThat(child1.getNode(), is(equalTo("GET:/"))); + assertThat(child1.getUrl(), is(equalTo("https://www.example.com/"))); + assertThat(child1.getMethod(), is(equalTo("GET"))); + assertThat(child1.getResponseLength(), is(equalTo(2345))); + assertThat(child1.getStatusCode(), is(equalTo(200))); + assertThat(child1.getChildren().size(), is(equalTo(0))); + + ExImSiteNode child2 = node.getChildren().get(1); + assertThat(child2.getNode(), is(equalTo("GET:aaa"))); + assertThat(child2.getUrl(), is(equalTo("https://www.example.com/aaa"))); + assertThat(child2.getMethod(), is(equalTo("GET"))); + assertThat(child2.getResponseLength(), is(equalTo(3456))); + assertThat(child2.getStatusCode(), is(equalTo(201))); + assertThat(child2.getChildren().size(), is(equalTo(1))); + + ExImSiteNode child3 = child2.getChildren().get(0); + assertThat(child3.getNode(), is(equalTo("POST:bbb"))); + assertThat(child3.getUrl(), is(equalTo("https://www.example.com/aaa/bbb"))); + assertThat(child3.getMethod(), is(equalTo("POST"))); + assertThat(child3.getResponseLength(), is(equalTo(101))); + assertThat(child3.getStatusCode(), is(equalTo(401))); + assertThat(child3.getChildren().size(), is(equalTo(0))); + + assertThat(node.getErrors().size(), is(equalTo(0))); + } + + @Test + void shouldReportErrorIfBadTypes() { + // Given + String yamlStr = + "- node: www.example.com\n" + + " url: https://www.example.com\n" + + " method: true\n" + + " responseLength: nan\n" + + " statusCode: 200\n"; + + // When + List list = (ArrayList) yaml.load(yamlStr); + ExImSiteNode node = new ExImSiteNode((LinkedHashMap) list.get(0)); + + // Then + assertThat(node.getNode(), is(equalTo("www.example.com"))); + assertThat(node.getUrl(), is(equalTo("https://www.example.com"))); + assertThat(node.getMethod(), is(equalTo(null))); + assertThat(node.getResponseLength(), is(equalTo(-1))); + assertThat(node.getStatusCode(), is(equalTo(200))); + assertThat(node.getChildren().size(), is(equalTo(0))); + assertThat(node.getErrors().size(), is(equalTo(2))); + assertThat(node.getErrors().get(0), is(equalTo("!exim.sites.error.badtype!"))); + assertThat(node.getErrors().get(1), is(equalTo("!exim.sites.error.badtype!"))); + } + + @Test + void shouldReportErrorIfUnknownKeys() { + // Given + String yamlStr = + "- node: www.example.com\n" + + " url: https://www.example.com\n" + + " method: GET\n" + + " responseLength: 101\n" + + " badKey1: true\n" + + " badKey2: 666\n" + + " statusCode: 200\n"; + + LoaderOptions loadingConfig = new LoaderOptions(); + Yaml yaml = new Yaml(loadingConfig); + + // When + List list = (ArrayList) yaml.load(yamlStr); + ExImSiteNode node = new ExImSiteNode((LinkedHashMap) list.get(0)); + + // Then + assertThat(node.getNode(), is(equalTo("www.example.com"))); + assertThat(node.getUrl(), is(equalTo("https://www.example.com"))); + assertThat(node.getMethod(), is(equalTo("GET"))); + assertThat(node.getResponseLength(), is(equalTo(101))); + assertThat(node.getStatusCode(), is(equalTo(200))); + assertThat(node.getChildren().size(), is(equalTo(0))); + assertThat(node.getErrors().size(), is(equalTo(2))); + assertThat(node.getErrors().get(0), is(equalTo("!exim.sites.error.badkey!"))); + assertThat(node.getErrors().get(1), is(equalTo("!exim.sites.error.badkey!"))); + } +} diff --git a/addOns/exim/src/test/java/org/zaproxy/addon/exim/sites/SiteTreeHandlerUnitTest.java b/addOns/exim/src/test/java/org/zaproxy/addon/exim/sites/SiteTreeHandlerUnitTest.java new file mode 100644 index 00000000000..4692378dddd --- /dev/null +++ b/addOns/exim/src/test/java/org/zaproxy/addon/exim/sites/SiteTreeHandlerUnitTest.java @@ -0,0 +1,477 @@ +/* + * Zed Attack Proxy (ZAP) and its related class files. + * + * ZAP is an HTTP/HTTPS proxy for assessing web application security. + * + * Copyright 2024 The ZAP Development Team + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.zaproxy.addon.exim.sites; + +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.notNullValue; +import static org.hamcrest.Matchers.nullValue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyInt; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.BDDMockito.given; +import static org.mockito.Mockito.mock; + +import java.io.ByteArrayInputStream; +import java.io.File; +import java.io.StringWriter; +import java.nio.charset.StandardCharsets; +import java.util.Collections; +import java.util.List; +import java.util.Locale; +import java.util.regex.Pattern; +import org.apache.commons.httpclient.URI; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.parosproxy.paros.Constant; +import org.parosproxy.paros.control.Control; +import org.parosproxy.paros.core.scanner.VariantMultipartFormParameters; +import org.parosproxy.paros.db.RecordHistory; +import org.parosproxy.paros.db.TableAlert; +import org.parosproxy.paros.db.TableHistory; +import org.parosproxy.paros.model.HistoryReference; +import org.parosproxy.paros.model.Model; +import org.parosproxy.paros.model.Session; +import org.parosproxy.paros.model.SiteMap; +import org.parosproxy.paros.model.SiteNode; +import org.parosproxy.paros.network.HtmlParameter; +import org.parosproxy.paros.network.HttpMessage; +import org.zaproxy.addon.exim.ExporterResult; +import org.zaproxy.zap.extension.ascan.VariantFactory; +import org.zaproxy.zap.model.Context; +import org.zaproxy.zap.model.StandardParameterParser; +import org.zaproxy.zap.model.StructuralNodeModifier; +import org.zaproxy.zap.model.StructuralNodeModifier.Type; +import org.zaproxy.zap.utils.I18N; + +/** Unit test for {@link SitesTreeHandler}. */ +class SiteTreeHandlerUnitTest { + + private SiteMap siteMap; + private Session session; + private StandardParameterParser spp; + + @BeforeEach + void setup() throws Exception { + Constant.messages = new I18N(Locale.ENGLISH); + + TableHistory tableHistory = mock(TableHistory.class); + session = mock(Session.class); + spp = new StandardParameterParser(); + given(session.getUrlParamParser(any(String.class))).willReturn(spp); + given(session.getFormParamParser(any(String.class))).willReturn(spp); + given(session.getParameters(any(HttpMessage.class), any(HtmlParameter.Type.class))) + .willCallRealMethod(); + Long sessionId = 1234L; + + given(session.getSessionId()).willReturn(sessionId); + given( + tableHistory.write( + any(Long.class), + eq(HistoryReference.TYPE_TEMPORARY), + any(HttpMessage.class))) + .willReturn(mock(RecordHistory.class)); + HistoryReference.setTableHistory(tableHistory); + + TableAlert tableAlert = mock(TableAlert.class); + given(tableAlert.getAlertsBySourceHistoryId(anyInt())).willReturn(Collections.emptyList()); + HistoryReference.setTableAlert(tableAlert); + + Model model = mock(Model.class); + Model.setSingletonForTesting(model); + Control.initSingletonForTesting(model); + given(model.getSession()).willReturn(session); + + VariantFactory factory = new VariantFactory(); + factory.addVariant(VariantMultipartFormParameters.class); + given(model.getVariantFactory()).willReturn(factory); + + SiteNode rootNode = new SiteNode(null, -1, "Root Node"); + siteMap = new SiteMap(rootNode, model); + } + + @Test + void shouldReportErrorIfNoFile() { + // Given + File f = new File("should-not-exist"); + + // When + PruneSiteResult result = SitesTreeHandler.pruneSiteNodes(f); + + // Then + assertThat(result.getError(), is(equalTo("!exim.sites.error.prune.exception!"))); + } + + private HistoryReference getHref(String url, String method) throws Exception { + HttpMessage msg = new HttpMessage(new URI(url, true)); + msg.getRequestHeader().setMethod(method); + return getHref(msg); + } + + private HistoryReference getHref(HttpMessage msg) throws Exception { + HistoryReference href = mock(HistoryReference.class); + given(href.getURI()).willReturn(msg.getRequestHeader().getURI()); + given(href.getMethod()).willReturn(msg.getRequestHeader().getMethod()); + given(href.getStatusCode()).willReturn(msg.getResponseHeader().getStatusCode()); + given(href.getResponseHeaderLength()) + .willReturn(msg.getResponseHeader().toString().length()); + given(href.getResponseBodyLength()).willReturn(msg.getResponseBody().length()); + + msg.setHistoryRef(href); + given(href.getHttpMessage()).willReturn(msg); + return href; + } + + @Test + void shouldOutputNodeWithData() throws Exception { + // Given + String expectedYaml = + "- node: Sites\n" + + " children: \n" + + " - node: https://www.example.com\n" + + " url: https://www.example.com?aa=bb&cc=dd\n" + + " method: POST\n" + + " responseLength: 61\n" + + " statusCode: 200\n" + + " data: eee=&ggg=\n"; + HttpMessage msg = + new HttpMessage( + "POST https://www.example.com?aa=bb&cc=dd HTTP/1.1\r\n" + + "Content-Type: text/html; charset=UTF-8\r\n", + "eee=fff&ggg=hhh".getBytes(), + "HTTP/1.1 200 OK\r\n" + "content-length: 20", + "12345678901234567890".getBytes()); + siteMap.addPath(getHref(msg)); + StringWriter sw = new StringWriter(); + ExporterResult result = new ExporterResult(); + + // When + SitesTreeHandler.exportSitesTree(sw, siteMap, result); + + // Then + assertThat(sw.toString(), is(expectedYaml)); + assertThat(result.getCount(), is(2)); + } + + @Test + void shouldOutputNodes() throws Exception { + // Given + String expectedYaml = + "- node: Sites\n" + + " children: \n" + + " - node: https://www.example.com\n" + + " url: https://www.example.com\n" + + " method: GET\n" + + " children: \n" + + " - node: POST:/()\n" + + " url: https://www.example.com/\n" + + " method: POST\n" + + " - node: PUT:aaa\n" + + " url: https://www.example.com/aaa\n" + + " method: PUT\n"; + siteMap.addPath(getHref("https://www.example.com/", "POST")); + siteMap.addPath(getHref("https://www.example.com/aaa", "PUT")); + + StringWriter sw = new StringWriter(); + ExporterResult result = new ExporterResult(); + + // When + SitesTreeHandler.exportSitesTree(sw, siteMap, result); + + // Then + assertThat(sw.toString(), is(expectedYaml)); + assertThat(result.getCount(), is(4)); + } + + @Test + void shouldOutputNodeWithMultipartFormData() throws Exception { + // Given + String expectedYaml = + "- node: Sites\n" + + " children: \n" + + " - node: https://www.example.com\n" + + " url: https://www.example.com\n" + + " method: GET\n" + + " children: \n" + + " - node: POST:/(bb,dd)(multipart/form-data)\n" + + " url: https://www.example.com/?bb=bcc&dd=ee\n" + + " method: POST\n" + + " responseLength: 61\n" + + " statusCode: 200\n"; + HttpMessage msg = + new HttpMessage( + "POST https://www.example.com/?bb=bcc&dd=ee HTTP/1.1\r\n" + + "Content-Type: multipart/form-data; boundary=-----------12345\r\n", + "-----------12345\nThis doesnt really matter\n-----------12345--" + .getBytes(), + "HTTP/1.1 200 OK\r\n" + "content-length: 20", + "12345678901234567890".getBytes()); + siteMap.addPath(getHref(msg)); + StringWriter sw = new StringWriter(); + ExporterResult result = new ExporterResult(); + + // When + SitesTreeHandler.exportSitesTree(sw, siteMap, result); + + // Then + assertThat(sw.toString(), is(expectedYaml)); + assertThat(result.getCount(), is(3)); + } + + @Test + void shouldOutputDdnNode() throws Exception { + // Given + Context context = new Context(session, 1); + context.addIncludeInContextRegex("https://www.example.com.*"); + Pattern p = Pattern.compile("https://www.example.com/(app/)(.+?)(/.*)"); + StructuralNodeModifier ddn = new StructuralNodeModifier(Type.DataDrivenNode, p, "DDN1"); + context.addDataDrivenNodes(ddn); + spp.setContext(context); + String expectedYaml = + "- node: Sites\n" + + " children: \n" + + " - node: https://www.example.com\n" + + " url: https://www.example.com\n" + + " method: GET\n" + + " children: \n" + + " - node: app\n" + + " url: https://www.example.com/app\n" + + " method: GET\n" + + " children: \n" + + " - node: «DDN1»\n" + + " url: https://www.example.com/app/company1\n" + + " method: GET\n" + + " children: \n" + + " - node: GET:aaa?ddd=eee(ddd)\n" + + " url: https://www.example.com/app/company1/aaa?ddd=eee\n" + + " method: GET\n"; + siteMap.addPath(getHref("https://www.example.com/app/company1/aaa?ddd=eee", "GET")); + siteMap.addPath(getHref("https://www.example.com/app/company2/aaa?ddd=eee", "GET")); + siteMap.addPath(getHref("https://www.example.com/app/company3/aaa?ddd=eee", "GET")); + + StringWriter sw = new StringWriter(); + ExporterResult result = new ExporterResult(); + + // When + SitesTreeHandler.exportSitesTree(sw, siteMap, result); + + // Then + assertThat(sw.toString(), is(expectedYaml)); + assertThat(result.getCount(), is(5)); + } + + @Test + void shoulErrorIfBadYaml() throws Exception { + // Given / When + PruneSiteResult res = + SitesTreeHandler.pruneSiteNodes( + new ByteArrayInputStream( + "This is not yaml".getBytes(StandardCharsets.UTF_8)), + siteMap); + + // Check the results + assertThat(res.getReadNodes(), is(0)); + assertThat(res.getDeletedNodes(), is(0)); + assertThat(res.getError(), is("!exim.sites.error.prune.badformat!")); + } + + @Test + void shouldPruneOneNode() throws Exception { + // Given + SiteNode exNode = siteMap.addPath(getHref("https://www.example.com/", "GET")); + + int rootCount = 0; + int exCount = 0; + if (exNode != null) { + rootCount = siteMap.getRoot().getChildCount(); + exCount = siteMap.getRoot().getChildAt(0).getChildCount(); + } + PruneSiteResult res = new PruneSiteResult(); + + // When + SitesTreeHandler.pruneSiteNodes( + getExImSiteNode("https://www.example.com/", "GET"), res, siteMap); + + // Check it did get setup correctly + assertThat(exNode, is(notNullValue())); + assertThat(rootCount, is(1)); + assertThat(exCount, is(1)); + + // Check the results + assertThat(res.getReadNodes(), is(1)); + assertThat(res.getDeletedNodes(), is(1)); + assertThat(res.getError(), is(nullValue())); + + // And that the node really was deleted + assertThat(siteMap.getRoot().getChildCount(), is(1)); + assertThat(siteMap.getRoot().getChildAt(0).getChildCount(), is(0)); + } + + private ExImSiteNode getExImSiteNode(String url, String method) { + ExImSiteNode node = new ExImSiteNode(); + node.setMethod(method); + node.setUrl(url); + return getExImSiteNode(url, method, method + ":" + url); + } + + private ExImSiteNode getExImSiteNode(String url, String method, String name) { + ExImSiteNode node = new ExImSiteNode(); + node.setNode(name); + node.setMethod(method); + node.setUrl(url); + return node; + } + + @Test + void shouldPruneAllNodes() throws Exception { + // Given + siteMap.addPath(getHref("https://www.example.com/", "GET")); + siteMap.addPath(getHref("https://www.example.com/aaa", "GET")); + siteMap.addPath(getHref("https://www.example.com/bbb", "GET")); + siteMap.addPath(getHref("https://www.example.com/ccc", "GET")); + + PruneSiteResult res = new PruneSiteResult(); + + ExImSiteNode exNode = getExImSiteNode("https://www.example.com", "GET"); + ExImSiteNode slNode = getExImSiteNode("https://www.example.com/", "GET"); + ExImSiteNode aaaNode = getExImSiteNode("https://www.example.com/aaa", "GET"); + ExImSiteNode bbbNode = getExImSiteNode("https://www.example.com/bbb", "GET"); + ExImSiteNode cccNode = getExImSiteNode("https://www.example.com/ccc", "GET"); + exNode.setChildren(List.of(slNode, aaaNode, bbbNode, cccNode)); + + // When + SitesTreeHandler.pruneSiteNodes(exNode, res, siteMap); + + // Check the results + assertThat(res.getReadNodes(), is(5)); + assertThat(res.getDeletedNodes(), is(5)); + assertThat(res.getError(), is(nullValue())); + + // And that the node really was deleted + assertThat(siteMap.getRoot().getChildCount(), is(0)); + } + + @Test + void shouldLeaveNodes() throws Exception { + // Given + siteMap.addPath(getHref("https://www.example.com/", "GET")); + siteMap.addPath(getHref("https://www.example.com/aaa", "GET")); + siteMap.addPath(getHref("https://www.example.com/bbb", "GET")); + siteMap.addPath(getHref("https://www.example.com/ccc", "GET")); + + PruneSiteResult res = new PruneSiteResult(); + + ExImSiteNode exNode = getExImSiteNode("https://www.example.com", "GET"); + ExImSiteNode slNode = getExImSiteNode("https://www.example.com/", "GET"); + ExImSiteNode aaaNode = getExImSiteNode("https://www.example.com/aaa", "GET"); + ExImSiteNode bbbNode = getExImSiteNode("https://www.example.com/bbb", "GET"); + exNode.setChildren(List.of(slNode, aaaNode, bbbNode)); + + // When + SitesTreeHandler.pruneSiteNodes(exNode, res, siteMap); + + // Check the results + assertThat(res.getReadNodes(), is(4)); + assertThat(res.getDeletedNodes(), is(3)); + assertThat(res.getError(), is(nullValue())); + + // And that the node really was deleted + assertThat(siteMap.getRoot().getChildCount(), is(1)); + assertThat(siteMap.getRoot().getChildAt(0).toString(), is("https://www.example.com")); + assertThat(siteMap.getRoot().getChildAt(0).getChildCount(), is(1)); + assertThat(siteMap.getRoot().getChildAt(0).getChildAt(0).toString(), is("GET:ccc")); + assertThat(siteMap.getRoot().getChildAt(0).getChildAt(0).getChildCount(), is(0)); + } + + @Test + void shouldPruneNodeWithMultipartFormData() throws Exception { + // Given + HttpMessage msg = + new HttpMessage( + "POST https://www.example.com/?bb=bcc&dd=ee HTTP/1.1\r\n" + + "Content-Type: multipart/form-data; boundary=-----------12345\r\n", + "-----------12345\nThis doesnt really matter\n-----------12345--" + .getBytes(), + "HTTP/1.1 200 OK\r\n" + "content-length: 20", + "12345678901234567890".getBytes()); + siteMap.addPath(getHref(msg)); + PruneSiteResult res = new PruneSiteResult(); + + // When + SitesTreeHandler.pruneSiteNodes( + getExImSiteNode( + "https://www.example.com/?bb=bcc&dd=ee", + "POST", + "POST:/(bb,dd)(multipart/form-data)"), + res, + siteMap); + + // Then + assertThat(res.getReadNodes(), is(1)); + assertThat(res.getDeletedNodes(), is(1)); + assertThat(res.getError(), is(nullValue())); + } + + @Test + void shoulPrubeDdnNode() throws Exception { + // Given + Context context = new Context(session, 1); + context.addIncludeInContextRegex("https://www.example.com.*"); + Pattern p = Pattern.compile("https://www.example.com/(app/)(.+?)(/.*)"); + StructuralNodeModifier ddn = new StructuralNodeModifier(Type.DataDrivenNode, p, "DDN1"); + context.addDataDrivenNodes(ddn); + spp.setContext(context); + String yaml = + "- node: Sites\n" + + " children: \n" + + " - node: https://www.example.com\n" + + " url: https://www.example.com\n" + + " method: GET\n" + + " children: \n" + + " - node: app\n" + + " url: https://www.example.com/app\n" + + " method: GET\n" + + " children: \n" + + " - node: «DDN1»\n" + + " url: https://www.example.com/app/company1\n" + + " method: GET\n" + + " children: \n" + + " - node: GET:aaa?ddd=eee(ddd)\n" + + " url: https://www.example.com/app/company1/aaa?ddd=eee\n" + + " method: GET\n"; + siteMap.addPath(getHref("https://www.example.com/app/company1/aaa?ddd=eee", "GET")); + siteMap.addPath(getHref("https://www.example.com/app/company2/aaa?ddd=eee", "GET")); + siteMap.addPath(getHref("https://www.example.com/app/company3/aaa?ddd=eee", "GET")); + + // When + PruneSiteResult res = + SitesTreeHandler.pruneSiteNodes( + new ByteArrayInputStream(yaml.getBytes(StandardCharsets.UTF_8)), siteMap); + + // Check the results + assertThat(res.getReadNodes(), is(5)); + assertThat(res.getDeletedNodes(), is(4)); // Root will be read but not deleted + assertThat(res.getError(), is(nullValue())); + + // And that the node hierarchy really was deleted + assertThat(siteMap.getRoot().getChildCount(), is(0)); + } +}