From 4f923bffd64d8d28a0a28df996f9a237fc55c968 Mon Sep 17 00:00:00 2001 From: "Tobias S. Keller" Date: Wed, 29 Nov 2023 10:22:37 +0100 Subject: [PATCH 1/2] feat: add state to ddl as well --- .../zendesk/maxwell/scripting/Scripting.java | 4 +-- .../maxwell/scripting/ScriptingTest.java | 33 +++++++++++++++++-- .../resources/scripting/test-set-state.js | 5 +++ 3 files changed, 38 insertions(+), 4 deletions(-) diff --git a/src/main/java/com/zendesk/maxwell/scripting/Scripting.java b/src/main/java/com/zendesk/maxwell/scripting/Scripting.java index 57b57cc4c..967aab83b 100644 --- a/src/main/java/com/zendesk/maxwell/scripting/Scripting.java +++ b/src/main/java/com/zendesk/maxwell/scripting/Scripting.java @@ -54,9 +54,9 @@ public Scripting(String filename) throws IOException, ScriptException, NoSuchMet public void invoke(RowMap row) { if ( row instanceof HeartbeatRowMap && processHeartbeatFunc != null ) - processHeartbeatFunc.call(null, new WrappedHeartbeatMap((HeartbeatRowMap) row)); + processHeartbeatFunc.call(null, new WrappedHeartbeatMap((HeartbeatRowMap) row), globalJavascriptState); else if ( row instanceof DDLMap && processDDLFunc != null ) - processDDLFunc.call(null, new WrappedDDLMap((DDLMap) row)); + processDDLFunc.call(null, new WrappedDDLMap((DDLMap) row), globalJavascriptState); else if ( row instanceof RowMap && processRowFunc != null ) processRowFunc.call(null, new WrappedRowMap(row), globalJavascriptState); } diff --git a/src/test/java/com/zendesk/maxwell/scripting/ScriptingTest.java b/src/test/java/com/zendesk/maxwell/scripting/ScriptingTest.java index a72fcf539..486addf68 100644 --- a/src/test/java/com/zendesk/maxwell/scripting/ScriptingTest.java +++ b/src/test/java/com/zendesk/maxwell/scripting/ScriptingTest.java @@ -4,6 +4,8 @@ import com.zendesk.maxwell.replication.Position; import com.zendesk.maxwell.row.RowMap; +import com.zendesk.maxwell.schema.ddl.DDLMap; +import com.zendesk.maxwell.schema.ddl.ResolvedTableAlter; import java.lang.reflect.Field; import java.util.ArrayList; @@ -26,7 +28,7 @@ private T getPrivateFieldOrFail(Object obj, String fieldName) throws Excepti @Test public void TestScripting() throws Exception { // Write a simple scripting file - Scripting scripting = new Scripting("src/test/resources/scripting/test.js"); + Scripting scripting = new Scripting("src/test/resources/scripting/test-set-state.js"); // String type, String database, String table, Long timestampMillis, List pkColumns, Position position, Position nextPosition, String rowQuery RowMap row = new RowMap( @@ -48,10 +50,37 @@ public void TestScripting() throws Exception { assertEquals(globalJavascriptState.get("mykey"), "myvalue"); } + @Test + public void TestScriptingDDL() throws Exception { + // Write a simple scripting file + Scripting scripting = new Scripting("src/test/resources/scripting/test-set-state.js"); + + // String type, String database, String table, Long timestampMillis, List pkColumns, Position position, Position nextPosition, String rowQuery + RowMap row = new DDLMap( + new ResolvedTableAlter(), + 123L, + "INSERT INTO mytable VALUES (1, 2, 3)", + new Position(null, 0), + new Position(null, 0), + 13L + ); + + + // Access the private globalJavascriptState field + LinkedHashMap globalJavascriptState = + getPrivateFieldOrFail(scripting, "globalJavascriptState"); + + globalJavascriptState.put("number", "1"); + + scripting.invoke(row); + + assertEquals(globalJavascriptState.get("number"), "2"); + } + @Test public void DontFailIfScriptHasNoStateParameter() throws Exception { // Write a simple scripting file - Scripting scripting = new Scripting("src/test/resources/scripting/test-no-state.js"); + Scripting scripting = new Scripting("src/test/resources/scripting/test-no-state-suppress-row.js"); // String type, String database, String table, Long timestampMillis, List pkColumns, Position position, Position nextPosition, String rowQuery RowMap row = new RowMap( diff --git a/src/test/resources/scripting/test-set-state.js b/src/test/resources/scripting/test-set-state.js index f2b2bd249..ca56b856d 100644 --- a/src/test/resources/scripting/test-set-state.js +++ b/src/test/resources/scripting/test-set-state.js @@ -1,3 +1,8 @@ function process_row(r, state) { state.put("mykey", "myvalue"); } + +function process_ddl(ddl, state) { + var num = parseInt(state.get("number")); + state.put("number", Number(num + 1).toFixed(0)); +} \ No newline at end of file From a21f3eef8db0e1dc776211f3643c5c6f43917e6e Mon Sep 17 00:00:00 2001 From: "Tobias S. Keller" Date: Wed, 29 Nov 2023 10:22:57 +0100 Subject: [PATCH 2/2] fix: remove accidentally duplicated test --- .../zendesk/maxwell/scripting/FilterTest.java | 163 ------------------ 1 file changed, 163 deletions(-) delete mode 100644 src/test/java/com/zendesk/maxwell/scripting/FilterTest.java diff --git a/src/test/java/com/zendesk/maxwell/scripting/FilterTest.java b/src/test/java/com/zendesk/maxwell/scripting/FilterTest.java deleted file mode 100644 index 4bc67e2db..000000000 --- a/src/test/java/com/zendesk/maxwell/scripting/FilterTest.java +++ /dev/null @@ -1,163 +0,0 @@ -package com.zendesk.maxwell.filtering; - -import org.junit.Test; - -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.regex.Pattern; - -import static org.junit.Assert.*; - -public class ScriptingTest { - private List filters; - - private List runParserTest(String input) throws Exception { - return new FilterParser(input).parse(); - } - - @Test - public void TestParser() throws Exception { - filters = runParserTest("include:/foo.*/.bar"); - - assertEquals(1, filters.size()); - assertEquals(FilterPatternType.INCLUDE, filters.get(0).getType()); - assertEquals(Pattern.compile("foo.*").toString(), filters.get(0).getDatabasePattern().toString()); - assertEquals(Pattern.compile("^bar$").toString(), filters.get(0).getTablePattern().toString()); - } - - @Test - public void TestBlacklists() throws Exception { - filters = runParserTest("blacklist:foo.*"); - assertEquals(1, filters.size()); - assertEquals(FilterPatternType.BLACKLIST, filters.get(0).getType()); - assertEquals(Pattern.compile("^foo$").toString(), filters.get(0).getDatabasePattern().toString()); - } - - @Test - public void TestQuoting() throws Exception { - String tests[] = { - "include:`foo`.*", - "include:'foo'.*", - "include:\"foo\".*" - }; - for ( String test : tests ) { - filters = runParserTest(test); - assertEquals(1, filters.size()); - assertEquals(Pattern.compile("^foo$").toString(), filters.get(0).getDatabasePattern().toString()); - } - } - - @Test - public void TestOddNames() throws Exception { - runParserTest("include: 1foo.bar"); - runParserTest("include: _foo._bar"); - - } - - @Test - public void TestAdvancedRegexp() throws Exception { - String pattern = "\\w+ \\/[a-z]*1"; - filters = runParserTest("include: /" + pattern + "/.*"); - assertEquals(1, filters.size()); - assertEquals(Pattern.compile(pattern).toString(), filters.get(0).getDatabasePattern().toString()); - } - - @Test - public void TestColumnFilterParse() throws Exception { - String input = "include: 1foo.bar.column=foo"; - filters = runParserTest(input); - assertEquals(1, filters.size()); - assertEquals(input, filters.get(0).toString()); - } - - @Test - public void TestExcludeAll() throws Exception { - Filter f = new Filter("exclude: *.*, include: foo.bar"); - assertTrue(f.includes("foo", "bar")); - assertFalse(f.includes("anything", "else")); - } - - @Test - public void TestBlacklist() throws Exception { - Filter f = new Filter("blacklist: seria.*"); - assertTrue(f.includes("foo", "bar")); - assertFalse(f.includes("seria", "var")); - assertTrue(f.isDatabaseBlacklisted("seria")); - assertTrue(f.isTableBlacklisted("seria", "anything")); - } - - @Test - public void TestColumnFiltersAreIgnoredByIncludes() throws Exception { - Filter f = new Filter("exclude: *.*, include: foo.bar.col=val"); - assertFalse(f.includes("foo", "bar")); - } - - @Test - public void TestColumnFilters() throws Exception { - Map map = new HashMap<>(); - map.put("col", "val"); - Filter f = new Filter("exclude: *.*, include: foo.bar.col=val"); - assertFalse(f.includes("foo", "bar")); - assertTrue(f.includes("foo", "bar", map)); - } - - @Test - public void TestColumnFiltersFromOtherTables() throws Exception { - Map map = new HashMap<>(); - map.put("col", "val"); - Filter f = new Filter("exclude: *.*, include: no.go.col=val"); - assertFalse(f.includes("foo", "bar")); - assertFalse(f.includes("foo", "bar", map)); - } - - @Test - public void TestNullValuesInData() throws Exception { - Map map = new HashMap<>(); - map.put("col", null); - Filter f = new Filter("exclude: *.*, include: foo.bar.col=null"); - assertFalse(f.includes("foo", "bar")); - assertTrue(f.includes("foo", "bar", map)); - - map.put("col", "null"); - assertTrue(f.includes("foo", "bar", map)); - - map.remove("col"); - assertFalse(f.includes("foo", "bar", map)); - - } - - @Test - public void TestSetValidFilter() throws Exception { - Filter f = new Filter("exclude: *.*, include: foo.bar.col=null"); - assertEquals(f.toString(), "exclude: *.*, include: foo.bar.col=null"); - f.set("blacklist: seria.*"); - assertEquals(f.toString(), "blacklist: seria.*"); - } - - @Test - public void TestSetInvalidFilter() throws Exception { - Filter f = new Filter("exclude: *.*, include: foo.bar.col=null"); - assertEquals(f.toString(), "exclude: *.*, include: foo.bar.col=null"); - try { - f.set("bl: seria.*"); - } catch (InvalidFilterException e) { - // do nothing - } - assertEquals(f.toString(), "exclude: *.*, include: foo.bar.col=null"); - } - - @Test - public void TestToString() throws Exception { - Filter f = new Filter("exclude: *.*, include: foo.bar.col=null"); - assertEquals(f.toString(), "exclude: *.*, include: foo.bar.col=null"); - } - - @Test - public void TestEmptyToString() throws Exception { - Filter f = new Filter(""); - assertEquals(f.toString(), ""); - } -} -