From 43346183530f33818e553b206090ff075241d0d9 Mon Sep 17 00:00:00 2001 From: tmixell Date: Wed, 7 Oct 2020 23:01:37 -0400 Subject: [PATCH 1/2] catching redis project up w/ incremental changes; added support for redis 6/redisearch 2 --- QueryableCache.cfc | 247 ++++++++++++++++++++--------------- tests/TestQueryableCache.cfc | 111 ++++++++++++---- 2 files changed, 226 insertions(+), 132 deletions(-) diff --git a/QueryableCache.cfc b/QueryableCache.cfc index a42b785..df611f7 100644 --- a/QueryableCache.cfc +++ b/QueryableCache.cfc @@ -1,9 +1,10 @@ component accessors = "true" extends = "lib.sql.QueryableCache" { property name = "importBatchSize" type = "numeric" default = "1000"; - property name = "name" type = "string" setter = "false"; + property name = "language" type = "string" default = "english"; property name = "maxResults" type = "numeric" default = "1000"; property name = "minPrefixLength" type = "numeric" default = "2"; + property name = "name" type = "string" setter = "false"; // https://oss.redislabs.com/redisearch/Stopwords.html variables.DEFAULT_STOP_WORDS = "a is the an and are as at be but by for if in into it no not of on or such that their then there these they this to was will with"; @@ -37,7 +38,8 @@ component accessors = "true" extends = "lib.sql.QueryableCache" { } } - local.indexOptions = createObject("java", "io.redisearch.client.Client$IndexOptions").init(0); + local.indexOptions = createObject("java", "io.redisearch.client.Client$IndexOptions"); + local.indexOptions.init(local.indexOptions.KEEP_FIELD_FLAGS + local.indexOptions.USE_TERM_OFFSETS); if(len(variables.stopWords) == 0) { local.indexOptions.setNoStopwords(); @@ -48,6 +50,15 @@ component accessors = "true" extends = "lib.sql.QueryableCache" { local.indexOptions.setStopwords(listToArray(variables.stopWords, " ")); } + try { + local.indexDefinition = createObject("java", "io.redisearch.client.IndexDefinition") + .setPrefixes([ getName() ]); + + local.indexOptions.setDefinition(local.indexDefinition); + } catch(Object e) { + // IndexDefinition isn't included in the version of JRediSearch being used + } + getClient().createIndex( local.schema, local.indexOptions @@ -76,14 +87,16 @@ component accessors = "true" extends = "lib.sql.QueryableCache" { local.parameters = arguments.selectStatement.getParameters(); for(local.i = 1; local.i <= arrayLen(local.criteria); local.i++) { - local.clause = "@#local.criteria[local.i].field#:"; // reset this on every iteration + structDelete(local, "clause"); structDelete(local, "value"); if(isRedisNumeric(local.criteria[local.i].field)) { + local.clause = "@#local.criteria[local.i].field#:"; + if(getQueryable().getFieldSQLType(local.criteria[local.i].field) == "bit") { local.value = local.parameters[local.i].value ? 1 : 0; - } else if(isRedisNumericDate(local.criteria[local.i].field)) { + } else if(isRedisDate(local.criteria[local.i].field)) { local.value = parseDateTime(local.parameters[local.i].value).getTime(); } @@ -124,27 +137,25 @@ component accessors = "true" extends = "lib.sql.QueryableCache" { local.clause = "(-" & local.clause & ")"; } break; - default: + case "=": local.clause &= "[#local.value# #local.value#]"; break; + default: + throw(type = "lib.redis.UnsupportedOperatorException", message = "#local.criteria[local.i].operator# is not supported for this field (#local.criteria[local.i].field#)"); + break; } } else { local.clause = "@_NFD_#local.criteria[local.i].field#:"; - local.value = stripAccents(local.parameters[local.i].value) - // remove wildcard operators - will re-add where appropriate - .replace("%", " ", "all") - // escape dashes - .replace("-", "\-", "all") - .trim(); + local.value = normalize(local.parameters[local.i].value); switch(local.criteria[local.i].operator) { case "LIKE": - // strip out stop words and format the value + // fuzzy operate our normalized values local.value = listReduce( local.value, function(result, item) { - if(!listFindNoCase(variables.stopWords, arguments.item, " ") && len(arguments.item) >= getMinPrefixLength()) { + if(len(arguments.item) >= getMinPrefixLength()) { arguments.result = listAppend(arguments.result, clause & arguments.item & "*", " "); } @@ -166,20 +177,37 @@ component accessors = "true" extends = "lib.sql.QueryableCache" { break; case "IN": case "NOT IN": - local.clause &= "(" & replace('"' & local.value & '"', chr(31), '"|"', "all") & ")"; + local.clause = local.clause + & "(" + & listReduce( + local.value, + function(result, item) { + if(find(" ", arguments.item)) { + arguments.item = '"' & arguments.item & '"'; + } + + return listAppend(arguments.result, arguments.item, "|"); + }, + "", + chr(31) + ) + & ")"; if(local.criteria[local.i].operator == "NOT IN") { local.clause = "(-" & local.clause & ")"; } break; - default: + case "=": local.clause &= '"' & local.value & '"'; break; + default: + throw(type = "lib.redis.UnsupportedOperatorException", message = "#local.criteria[local.i].operator# is not supported for this field (#local.criteria[local.i].field#)"); + break; } } // replace our placeholders w/ the cache-friendly values - local.queryString = replace(local.queryString, local.criteria[local.i].statement, local.clause, "one"); + local.queryString = replace(local.queryString, local.criteria[local.i].statement, "(" & local.clause & ")", "one"); } } @@ -195,7 +223,7 @@ component accessors = "true" extends = "lib.sql.QueryableCache" { // why aggregation? redisearch only supports multiple sorts via the `aggregate` command local.ab = createObject("java", "io.redisearch.aggregation.AggregationBuilder") .init("'" & local.queryString & "'") - .apply("@#getIdentifierField()#", "id"); + .apply((isRedisNumeric(getIdentifierField()) ? "@" : "@_NFD_") & getIdentifierField(), "id"); // sort local.sortedField = createObject("java", "io.redisearch.aggregation.SortedField"); @@ -204,10 +232,12 @@ component accessors = "true" extends = "lib.sql.QueryableCache" { local.sortFields = arrayReduce( arguments.selectStatement.getOrderCriteria(), function(result, item) { - if(isRedisNumeric(listFirst(arguments.item, " "))) { - local.f = "@" & listFirst(arguments.item, " "); + local.f = listFirst(arguments.item, " "); + + if(isRedisNumeric(local.f)) { + local.f = "@" & local.f; } else { - local.f = "@_NFD_" & listFirst(arguments.item, " "); + local.f = "@_NFD_" & local.f; } local.o = listLast(arguments.item, " "); @@ -223,7 +253,7 @@ component accessors = "true" extends = "lib.sql.QueryableCache" { [] ); } else { - local.sortFields = [ local.sortedField.asc("@" & getIdentifierField()) ]; + local.sortFields = [ local.sortedField.asc("@id") ]; } local.maxResults = getMaxResults(); @@ -248,8 +278,6 @@ component accessors = "true" extends = "lib.sql.QueryableCache" { } } - local.searchClient = getClient(); - // convert the command to a string for debugging purposes if(structKeyExists(arguments, "debug") && arguments.debug == "query") { local.byteArray = [ createObject("java", "java.lang.String").init(javaCast("string", "")).getBytes() ]; @@ -265,7 +293,7 @@ component accessors = "true" extends = "lib.sql.QueryableCache" { throw(type = "lib.redis.DebugException", message = "ft.aggregate " & getName() & " " & local.string); } - local.ids = local.searchClient.aggregate(local.ab); + local.ids = getClient().aggregate(local.ab); local.resultsLength = local.ids.totalResults; for(local.i = 0; local.i < local.ids.getResults().size(); local.i++) { @@ -310,7 +338,7 @@ component accessors = "true" extends = "lib.sql.QueryableCache" { local.query = queryNew(arguments.selectStatement.getSelect(), local.fieldSQLTypes); if(arrayLen(local.docIDs) > 0) { - local.documents = local.searchClient.getDocuments(local.docIDs); + local.documents = getClient().getDocuments(local.docIDs); for(local.i = 0; local.i < local.documents.size(); local.i++) { local.document = local.documents.get(local.i); @@ -318,7 +346,7 @@ component accessors = "true" extends = "lib.sql.QueryableCache" { if(!isNull(local.document)) { queryAddRow( local.query, - fromRedisearchDocument(local.document, arguments.selectStatement.getSelect()) + fromRediSearchDocument(local.document, arguments.selectStatement.getSelect()) ); } } @@ -336,7 +364,7 @@ component accessors = "true" extends = "lib.sql.QueryableCache" { return local.query; } - struct function fromRedisearchDocument(required any document, string fieldFilter = "") { + struct function fromRediSearchDocument(required any document, string fieldFilter = "") { if(arguments.fieldFilter == "") { arguments.fieldFilter = getFieldList(); } @@ -345,28 +373,29 @@ component accessors = "true" extends = "lib.sql.QueryableCache" { return listReduce( arguments.fieldFilter, function(result, item) { - if(isRedisNumeric(arguments.item)) { - if(isRedisNumericDate(arguments.item)) { - if(document.hasProperty(arguments.item)) { - arguments.result[arguments.item] = createObject("java", "java.util.Date").init(javaCast("long", document.getString(arguments.item))); - } else { - arguments.result[arguments.item] = javaCast("null", ""); - } - } else { - if(document.hasProperty(arguments.item)) { - arguments.result[arguments.item] = val(document.getString(arguments.item)); + if(document.hasProperty(arguments.item)) { + local.value = document.get(arguments.item); + if(!structKeyExists(local, "value")) { + local.value = ""; + } + + if(isRedisDate(arguments.item) && isNumeric(val(local.value))) { + arguments.result[arguments.item] = createObject("java", "java.util.Date").init(javaCast("long", val(local.value))); + } else if(isRedisNumeric(arguments.item) && isNumeric(val(local.value))) { + if(listFindNoCase("bigint,bit,integer,smallint,tinyint", getQueryable().getFieldSQLType(arguments.item))) { + arguments.result[arguments.item] = val(local.value); } else { - arguments.result[arguments.item] = javaCast("null", ""); + arguments.result[arguments.item] = javaCast("double", local.value); } - } - } else { - if(document.hasProperty(arguments.item)) { - arguments.result[arguments.item] = document.getString(arguments.item); - } else { - arguments.result[arguments.item] = javaCast("null", ""); + } else if(len(local.value) > 0) { + arguments.result[arguments.item] = local.value; } } + if(!structKeyExists(arguments.result, arguments.item)) { + arguments.result[arguments.item] = javaCast("null", ""); + } + return arguments.result; }, {} @@ -379,11 +408,15 @@ component accessors = "true" extends = "lib.sql.QueryableCache" { return createObject("java", "io.redisearch.client.Client").init(variables.name, variables.connectionPool.getJedisPool()); } + struct function getInfo() { + return getClient().getInfo(); + } + any function getRow() { local.document = getClient().getDocument(getRowKey(argumentCollection = arguments)); if(!isNull(local.document)) { - local.result = fromRedisearchDocument(local.document); + local.result = fromRediSearchDocument(local.document); for(local.key in getFieldList()) { if(!structKeyExists(local.result, local.key)) { @@ -395,7 +428,19 @@ component accessors = "true" extends = "lib.sql.QueryableCache" { } } - private boolean function isRedisNumeric(required string field) { + boolean function isRedisDate(required string field) { + switch(getQueryable().getFieldSQLType(arguments.field)) { + case "date": + case "time": + case "timestamp": + return true; + break; + }; + + return false; + } + + boolean function isRedisNumeric(required string field) { switch(getQueryable().getFieldSQLType(arguments.field)) { case "bigint": case "bit": @@ -418,37 +463,41 @@ component accessors = "true" extends = "lib.sql.QueryableCache" { return false; } - private boolean function isRedisNumericDate(required string field) { - switch(getQueryable().getFieldSQLType(arguments.field)) { - case "date": - case "time": - case "timestamp": - return true; - break; - }; + private string function normalize(required string input) { + arguments.input = lCase(arguments.input); + + // https://oss.redislabs.com/redisearch/Escaping.html + arguments.input = listReduce( + variables.normalizer.normalize(javaCast("string", arguments.input), variables.normalizerForm).replaceAll("\p{InCombiningDiacriticalMarks}+", ""), + function(outerResult, outerItem) { + arguments.outerItem = listReduce( + trim(REReplace(arguments.outerItem, "\W+", " ", "all")), + function(innerResult, innerItem) { + // these values have all been lCase'd at this point + if(!listFind(variables.stopWords, arguments.innerItem, " ")) { + arguments.innerResult = listAppend(arguments.innerResult, arguments.innerItem, " "); + } - return false; - } + return arguments.innerResult; + }, + "", + " " + ); - private boolean function isRedisNumericInt(required string field) { - switch(getQueryable().getFieldSQLType(arguments.field)) { - case "bigint": - case "bit": - case "integer": - case "smallint": - case "tinyint": - return true; - break; - }; + return listAppend(arguments.outerResult, arguments.outerItem, chr(31)); + }, + "", + chr(31) + ); - return false; + return trim(arguments.input); } void function putRow(required struct row) { getClient().addDocument( - toRedisearchDocument(arguments.row), + toRediSearchDocument(arguments.row), createObject("java", "io.redisearch.client.AddOptions") - .setLanguage("english") + .setLanguage(getLanguage()) .setReplacementPolicy(createObject("java", "io.redisearch.client.AddOptions$ReplacementPolicy").FULL) ); } @@ -457,15 +506,15 @@ component accessors = "true" extends = "lib.sql.QueryableCache" { getClient().deleteDocument(javaCast("string", getRowKey(argumentCollection = arguments)), true); } - void function seedFromQueryable(boolean overwrite = false) { + void function seedFromQueryable(boolean overwrite = false, string where = "") { local.documents = []; local.replacementPolicy = createObject("java", "io.redisearch.client.AddOptions$ReplacementPolicy"); local.addOptions = createObject("java", "io.redisearch.client.AddOptions") - .setLanguage("english") + .setLanguage(getLanguage()) .setReplacementPolicy(arguments.overwrite ? local.replacementPolicy.FULL : local.replacementPolicy.NONE); - for(local.row in getQueryable().select().execute()) { - arrayAppend(local.documents, toRedisearchDocument(local.row)); + for(local.row in getQueryable().select().where(arguments.where).execute()) { + arrayAppend(local.documents, toRediSearchDocument(local.row)); if(arrayLen(local.documents) == getImportBatchSize()) { getClient().addDocuments(local.addOptions, local.documents); @@ -509,48 +558,40 @@ component accessors = "true" extends = "lib.sql.QueryableCache" { return this; } - string function stripAccents(required string input) { - return variables.normalizer.normalize(javaCast("string", arguments.input), variables.normalizerForm).replaceAll("\p{InCombiningDiacriticalMarks}+", ""); - } - - any function toRedisearchDocument(required struct row) { + any function toRediSearchDocument(required struct row) { if(!structKeyExists(arguments.row, getIdentifierField())) { throw(type = "lib.redis.MissingIdentifierException", message = "the identifier field #getIdentifierField()# must be provided"); } - local.fields = createObject("java", "java.util.HashMap").init(); + local.document = createObject("java", "io.redisearch.Document").init(getRowKey(argumentCollection = arguments.row)); for(local.field in getQueryable().getFieldList()) { - if(isRedisNumeric(local.field)) { - if(isRedisNumericDate(local.field) - && structKeyExists(arguments.row, local.field) - && isDate(arguments.row[local.field]) - ) { - local.fields.put(local.field, javaCast("long", arguments.row[local.field].getTime())); - } else if(isRedisNumericInt(local.field) - && structKeyExists(arguments.row, local.field) - && (isNumeric(arguments.row[local.field]) || isBoolean(arguments.row[local.field])) - ) { - local.fields.put(local.field, javaCast("int", arguments.row[local.field])); - } else if(structKeyExists(arguments.row, local.field) && isNumeric(arguments.row[local.field])) { - local.fields.put(local.field, javaCast("double", arguments.row[local.field])); + if(!structKeyExists(arguments.row, local.field)) { + arguments.row[local.field] = ""; + } + + if(isRedisDate(local.field)) { + if(isDate(arguments.row[local.field])) { + local.document.set(local.field, javaCast("long", arguments.row[local.field].getTime())); } - } else { - if(structKeyExists(arguments.row, local.field) && len(arguments.row[local.field]) > 0) { - local.fields.put(local.field, javaCast("string", arguments.row[local.field])); - if(getQueryable().fieldIsFilterable(local.field)) { - // for filterable fields, store normalized value, escape dashes for guids/uuids - local.fields.put("_NFD_" & local.field, javaCast("string", stripAccents(arguments.row[local.field]).replace("-", "\-", "all"))); + } else if(isRedisNumeric(local.field)) { + if(isNumeric(arguments.row[local.field]) || isBoolean(arguments.row[local.field])) { + if(listFindNoCase("bigint,bit,integer,smallint,tinyint", getQueryable().getFieldSQLType(local.field))) { + local.document.set(local.field, javaCast("long", arguments.row[local.field])); + } else { + local.document.set(local.field, javaCast("double", arguments.row[local.field])); } } + } else { + local.document.set(local.field, javaCast("string", arguments.row[local.field])); + + if(getQueryable().fieldIsFilterable(local.field)) { + local.document.set("_NFD_" & local.field, javaCast("string", normalize(arguments.row[local.field]))); + } } } - return createObject("java", "io.redisearch.Document").init( - getRowKey(argumentCollection = arguments.row), - local.fields, - javaCast("double", 1) - ); + return local.document; } } \ No newline at end of file diff --git a/tests/TestQueryableCache.cfc b/tests/TestQueryableCache.cfc index 2c53b5c..78348bf 100644 --- a/tests/TestQueryableCache.cfc +++ b/tests/TestQueryableCache.cfc @@ -13,8 +13,8 @@ component extends = "mxunit.framework.TestCase" { variables.cache = new lib.redis.QueryableCache(connectionPool = variables.connectionPool, name = "mxunit:indexed"); variables.query = queryNew( - "id, createdTimestamp, createdDate, createdTime, foo, bar, letter", - "integer, timestamp, date, time, varchar, bit, varchar" + "id, createdTimestamp, createdDate, createdTime, foo, bar, letter, floatie", + "integer, timestamp, date, time, varchar, bit, varchar, double" ); queryAddRow( @@ -27,7 +27,8 @@ component extends = "mxunit.framework.TestCase" { "createdTimestamp": parseDateTime("2019-12-12T00:00:00.123Z"), "createdDate": createDate(2019, 12, 12), "createdTime": createTime(0, 0, 0), - "letter": "A" + "letter": "A", + "floatie": 10000000.00 }, { "id": 5002, @@ -36,7 +37,8 @@ component extends = "mxunit.framework.TestCase" { "createdTimestamp": parseDateTime("2019-12-25T00:00:00.001Z"), "createdDate": createDate(2019, 12, 25), "createdTime": createTime(0, 0, 0), - "letter": "B" + "letter": "B", + "floatie": 1.00 }, { "id": 5003, @@ -45,7 +47,8 @@ component extends = "mxunit.framework.TestCase" { "createdTimestamp": parseDateTime("2019-12-31T00:00:00.000Z"), "createdDate": createDate(2019, 12, 31), "createdTime": createTime(0, 0, 0), - "letter": "C" + "letter": "C", + "floatie": -1.300001 } ] ); @@ -57,11 +60,12 @@ component extends = "mxunit.framework.TestCase" { { "id": 5000 + local.i, "bar": (!randRange(1, 3) % 2 ? local.i % 2 : javaCast("null", "")), - "foo": (local.i == 500 ? "Šťŕĭńġ" : local.i == 501 ? "the rain in spain" : createUUID()), + "foo": (local.i == 500 ? "Šťŕĭńġ" : local.i == 501 ? "the rain in spain" : local.i == 502 ? "6k" : createUUID()), "createdTimestamp": (!randRange(1, 3) % 2 ? dateAdd("s", -(local.i), variables.now) : javaCast("null", "")), "createdDate": (!randRange(1, 3) % 2 ? variables.now : javaCast("null", "")), "createdTime": (!randRange(1, 3) % 2 ? variables.now : javaCast("null", "")), - "letter": chr(64 + randRange(1, 25) + (local.i % 2 ? 32 : 0)) + "letter": chr(64 + randRange(1, 26) + (local.i % 2 ? 32 : 0)), + "floatie": 0.01 } ); } @@ -76,8 +80,6 @@ component extends = "mxunit.framework.TestCase" { function test_0_createIndex() { // debug(variables.exception); return; - // indexes are prefixed w/ idx: - local.queryableCache = new lib.redis.Cache(variables.connectionPool, "idx"); try{ // create the index @@ -86,7 +88,9 @@ component extends = "mxunit.framework.TestCase" { // index exists already, do nothing } - assertTrue(local.queryableCache.containsKey("mxunit:indexed")); + local.info = variables.cache.getInfo(); +// debug(local.info); + assertEquals("mxunit:indexed", local.info["index_name"]); } function test_0_seedFromQueryable() { @@ -137,6 +141,24 @@ component extends = "mxunit.framework.TestCase" { assertEquals(0, dateCompare(local.compareDate, local.compare.createdTimestamp)); } + function test_putRow_getRow_DDMAINT_26680() { + local.queryRow = queryGetRow(variables.query, 1); + local.queryRow.foo = ""; + variables.cache.putRow(local.queryRow); + local.compare = variables.cache.getRow(argumentCollection = local.queryRow); + + assertEquals("", local.compare.foo); + } + + function test_putRow_getRow_DDMAINT_27043() { + local.queryRow = queryGetRow(variables.query, 1); + local.queryRow.createdTimestamp = ""; + variables.cache.putRow(local.queryRow); + local.compare = variables.cache.getRow(argumentCollection = local.queryRow); + + assertEquals("", local.compare.createdTimestamp); + } + function test_removeRow() { local.queryRow = queryGetRow(variables.query, 1); local.queryRow.foo = createUUID(); @@ -147,7 +169,7 @@ component extends = "mxunit.framework.TestCase" { assertFalse(variables.cache.containsRow(id = local.queryRow.id)); } - function test_seedFromQueryable_overwrite() { + function test_seedFromQueryable_overwrite_where() { variables.cache.seedFromQueryable(); local.row = queryGetRow(variables.query, 2); @@ -157,7 +179,7 @@ component extends = "mxunit.framework.TestCase" { // debug(local.row); - variables.cache.seedFromQueryable(overwrite = true); + variables.cache.seedFromQueryable(overwrite = true, where = "id = #local.row.id#"); local.overwriteElement = variables.cache.getRow(id = local.row.id); @@ -228,7 +250,7 @@ component extends = "mxunit.framework.TestCase" { } function test_select_where_compound_limit() { - local.result = variables.cache.select().where("id < 5100 AND createdTimestamp >= '#dateTimeFormat(dateAdd("d", -1, variables.now), "yyyy-mm-dd HH:nn:ss.l")#'").execute(limit = 10); + local.result = variables.cache.select().where("letter IN (A,B,C) AND createdTimestamp >= '#dateTimeFormat(dateAdd("d", -1, variables.now), "yyyy-mm-dd HH:nn:ss.l")#'").execute(limit = 10); // debug(local.result); assertEquals(10, local.result.recordCount); @@ -240,13 +262,6 @@ component extends = "mxunit.framework.TestCase" { debug(local.result); } - function test_select_where_DD_13660() { - local.result = variables.cache.select("letter, id").where("id > 5990.00").execute(); - - debug(local.result); - assertEquals(10, local.result.recordCount) - } - function test_select_where_DD_13763() { local.where = "foo LIKE '%#listFirst(lCase(variables.query.foo[1]), '-')#%'"; local.result = variables.cache.select().where(local.where).orderBy("id ASC").execute(); @@ -297,6 +312,42 @@ component extends = "mxunit.framework.TestCase" { debug(local.result); } + + function test_select_where_DDMAINT_27088_KEEP_FIELD_FLAGS() { + local.where = "letter = 'Šťŕĭńġ'"; + local.result = variables.cache.select().where(local.where).execute(); + + assertEquals(0, local.result.recordCount); + + debug(local.where); + debug(local.result); + } + + function test_select_where_DDMAINT_27088_KEEP_FIELD_FLAGS_2() { + local.where = "foo IN (6k)"; + local.result = variables.cache.select().where(local.where).execute(); + + assertEquals(1, local.result.recordCount); + + debug(local.where); + debug(local.result); + } + + function test_select_where_DDMAINT_27088_USE_TERM_OFFSETS() { + local.where = "foo = 'the rain in spain'"; + local.result = variables.cache.select().where(local.where).execute(); + + assertEquals(1, local.result.recordCount); + + debug(local.where); + debug(local.result); + + local.where = "foo = 'the spain in rain'"; + local.result = variables.cache.select().where(local.where).execute(); + + assertEquals(0, local.result.recordCount); + } + function test_select_where_in() { local.result = variables.cache.select("id, foo").where("foo IN ('#variables.query.foo[1]#', '#variables.query.foo[2]#', '#variables.query.foo[3]#') OR id IN (5005, 5010, 5015)").execute(); @@ -320,25 +371,26 @@ component extends = "mxunit.framework.TestCase" { function test_select_where_not_in() { // numeric filtering - local.result = variables.cache.select("id, foo").where("id NOT IN (5005, 5010, 5015) AND id < 5015").execute(); + local.result = variables.cache.select("id, foo").where("id NOT IN (5005, 5010, 5012)").orderBy("id ASC").execute(limit = 10); // debug(local.result); assertEquals("foo,id", listSort(local.result.columnList, "textnocase")); - assertEquals("5001,5002,5003,5004,5006,5007,5008,5009,5011,5012,5013,5014", listSort(valueList(local.result.id), "numeric")); + assertEquals("5001,5002,5003,5004,5006,5007,5008,5009,5011,5013", listSort(valueList(local.result.id), "numeric")); // string filtering - local.result = variables.cache.select("id, foo").where("foo NOT IN ('#variables.query.foo[1]#', '#variables.query.foo[2]#', '#variables.query.foo[3]#') AND id < 5015").execute(); + local.result = variables.cache.select("id, foo").where("foo NOT IN ('#variables.query.foo[1]#', '#variables.query.foo[2]#', '#variables.query.foo[3]#')").orderBy("id ASC").execute(limit = 10); debug(local.result); assertEquals("foo,id", listSort(local.result.columnList, "textnocase")); - assertEquals("5004,5005,5006,5007,5008,5009,5010,5011,5012,5013,5014", listSort(valueList(local.result.id), "numeric")); + assertEquals("5004,5005,5006,5007,5008,5009,5010,5011,5012,5013", listSort(valueList(local.result.id), "numeric")); // test negation of a single record - local.result = variables.cache.select("id, foo").where("foo NOT IN ('#variables.query.foo[1]#'").execute(); + local.result = variables.cache.select("id, foo").where("foo NOT IN ('#variables.query.foo[1]#'").orderBy("id ASC").execute(limit = 10); // debug(local.result); assertEquals("foo,id", listSort(local.result.columnList, "textnocase")); - assertEquals(999, local.result.recordCount); + assertEquals("5002,5003,5004,5005,5006,5007,5008,5009,5010,5011", listSort(valueList(local.result.id), "numeric")); + assertEquals(10, local.result.recordCount); } function test_select_where_orderBy_limit() { @@ -349,15 +401,16 @@ component extends = "mxunit.framework.TestCase" { assertEquals(10, local.result.recordCount); } - function test_toRedisearchDocument_fromRedisearchDocument() { + function test_toRediSearchDocument_fromRediSearchDocument() { local.row = queryGetRow(variables.query, 1); - local.document = variables.cache.toRedisearchDocument(local.row); + local.document = variables.cache.toRediSearchDocument(local.row); assertEquals(local.row.letter, local.document.getString("letter")); - local.rowFromDocument = variables.cache.fromRedisearchDocument(local.document); + local.rowFromDocument = variables.cache.fromRediSearchDocument(local.document); +// debug(local.row); // debug(local.rowFromDocument); assertEquals(local.row, local.rowFromDocument); } From 2c55165c182c628f9f7c25818a5a336384a8ed1b Mon Sep 17 00:00:00 2001 From: tmixell Date: Sun, 25 Jul 2021 12:22:55 -0400 Subject: [PATCH 2/2] adding redis 6/redisearch2 support --- QueryableCache.cfc | 201 +++++++++++++++++++++++++---------- tests/TestQueryableCache.cfc | 46 ++++++++ 2 files changed, 190 insertions(+), 57 deletions(-) diff --git a/QueryableCache.cfc b/QueryableCache.cfc index df611f7..ac023f5 100644 --- a/QueryableCache.cfc +++ b/QueryableCache.cfc @@ -1,4 +1,4 @@ -component accessors = "true" extends = "lib.sql.QueryableCache" { +component accessors = "true" extends = "lib.sql.QueryableCache" implements = "lib.sql.IWritable" { property name = "importBatchSize" type = "numeric" default = "1000"; property name = "language" type = "string" default = "english"; @@ -29,7 +29,7 @@ component accessors = "true" extends = "lib.sql.QueryableCache" { local.schema = createObject("java", "io.redisearch.Schema"); for(local.field in getQueryable().getFieldList()) { - if(getQueryable().fieldIsFilterable(local.field)) { + if(getQueryable().fieldIsFilterable(local.field) || local.field == getIdentifierField()) { if(isRedisNumeric(local.field)) { local.schema.addSortableNumericField(local.field); } else { @@ -65,8 +65,36 @@ component accessors = "true" extends = "lib.sql.QueryableCache" { ); } + lib.sql.DeleteStatement function delete() { + return new lib.sql.DeleteStatement(this); + } + void function dropIndex() { - getClient().dropIndex(); + getClient().dropIndex(true); + } + + void function executeDelete(required lib.sql.DeleteStatement deleteStatement) { + if(len(arguments.deleteStatement.getWhere()) == 0) { + dropIndex(true); + createIndex(); + } else { + do { + local.targetRecords = this.select(getIdentifierField()).where(arguments.deleteStatement.getWhere()).execute(); + local.keys = []; + + for(local.row in local.targetRecords) { + arrayAppend(local.keys, getRowKey(argumentCollection = local.row)); + } + + if(arrayLen(local.keys)) { + getClient().deleteDocuments(true, local.keys); + } + } while(local.targetRecords.recordCount > 0); + } + } + + void function executeInsert(required lib.sql.InsertStatement insertStatement) { + throw(type = "lib.redis.UnsupportedOperationException", message = "Insert is not supported in this implementation"); } query function executeSelect(required lib.sql.SelectStatement selectStatement, required numeric limit, required numeric offset) { @@ -211,8 +239,6 @@ component accessors = "true" extends = "lib.sql.QueryableCache" { } } - local.docIDs = []; - if(!structKeyExists(local, "searchException")) { try { local.queryString = local.queryString @@ -220,15 +246,15 @@ component accessors = "true" extends = "lib.sql.QueryableCache" { .replace(" OR ", " | ", "all") .trim(); - // why aggregation? redisearch only supports multiple sorts via the `aggregate` command - local.ab = createObject("java", "io.redisearch.aggregation.AggregationBuilder") - .init("'" & local.queryString & "'") - .apply((isRedisNumeric(getIdentifierField()) ? "@" : "@_NFD_") & getIdentifierField(), "id"); + if(arrayLen(arguments.selectStatement.getOrderCriteria()) > 1) { + // why aggregation? redisearch only supports multiple sorts via the `aggregate` command + local.ab = createObject("java", "io.redisearch.aggregation.AggregationBuilder") + .init("'" & local.queryString & "'") + .apply(variables.aggregateApply, "id"); - // sort - local.sortedField = createObject("java", "io.redisearch.aggregation.SortedField"); + // sort + local.sortedField = createObject("java", "io.redisearch.aggregation.SortedField"); - if(arrayLen(arguments.selectStatement.getOrderCriteria()) > 0) { local.sortFields = arrayReduce( arguments.selectStatement.getOrderCriteria(), function(result, item) { @@ -252,56 +278,80 @@ component accessors = "true" extends = "lib.sql.QueryableCache" { }, [] ); - } else { - local.sortFields = [ local.sortedField.asc("@id") ]; - } - local.maxResults = getMaxResults(); + local.maxResults = getMaxResults(); - if(arguments.limit > 0) { - if(arguments.offset > 0) { - local.maxResults = arguments.offset + arguments.limit; - } else { - local.maxResults = arguments.limit; + if(arguments.limit > 0) { + if(arguments.offset > 0) { + local.maxResults = arguments.offset + arguments.limit; + } else { + local.maxResults = arguments.limit; + } } - } - local.ab.sortBy(javaCast("int", local.maxResults), local.sortFields); + local.ab.sortBy(javaCast("int", local.maxResults), local.sortFields); - // pagination - // look familiar? - this is the same logic as above... just not good way to re-use the condition, as the syntax must be assembled in order - if(arguments.limit > 0) { - if(arguments.offset > 0) { - local.ab.limit(arguments.offset, arguments.limit); - } else { - local.ab.limit(arguments.limit); + // pagination + // look familiar? - this is the same logic as above... just not good way to re-use the condition, as the syntax must be assembled in order + if(arguments.limit > 0) { + if(arguments.offset > 0) { + local.ab.limit(arguments.offset, arguments.limit); + } else { + local.ab.limit(arguments.limit); + } } - } - // convert the command to a string for debugging purposes - if(structKeyExists(arguments, "debug") && arguments.debug == "query") { - local.byteArray = [ createObject("java", "java.lang.String").init(javaCast("string", "")).getBytes() ]; - local.ab.serializeRedisArgs(local.byteArray); - local.string = arrayReduce( - local.byteArray, - function(result, item) { - return listAppend(arguments.result, charsetEncode(arguments.item, "UTF-8"), " "); - }, - "" - ); + // convert the command to a string for debugging purposes + if(structKeyExists(arguments, "debug") && arguments.debug == "query") { + local.byteArray = [ createObject("java", "java.lang.String").init(javaCast("string", "")).getBytes() ]; + local.ab.serializeRedisArgs(local.byteArray); + local.string = arrayReduce( + local.byteArray, + function(result, item) { + return listAppend(arguments.result, charsetEncode(arguments.item, "UTF-8"), " "); + }, + "" + ); + + throw(type = "lib.redis.DebugException", message = "ft.aggregate " & getName() & " " & local.string); + } - throw(type = "lib.redis.DebugException", message = "ft.aggregate " & getName() & " " & local.string); - } + local.docIDs = []; + local.ids = getClient().aggregate(local.ab); + local.resultsLength = local.ids.totalResults; - local.ids = getClient().aggregate(local.ab); - local.resultsLength = local.ids.totalResults; + for(local.i = 0; local.i < local.ids.getResults().size(); local.i++) { + arrayAppend( + local.docIDs, + // GUID/UUID get their dashes stripped during indexing + replace(local.ids.getRow(javaCast("int", local.i)).getString("id"), " ", "", "all") + ); + } - for(local.i = 0; local.i < local.ids.getResults().size(); local.i++) { - arrayAppend( - local.docIDs, - // strip escape characters back out to get the ID - getRowKey(argumentCollection = { "#getIdentifierField()#": replace(local.ids.getRow(javaCast("int", local.i)).getString("id"), "\-", "-", "all") }) - ); + local.documents = getClient().getDocuments(local.docIDs); + } else { + local.q = createObject("java", "io.redisearch.Query") + .init("'" & local.queryString & "'") + .returnFields(listToArray(arguments.selectStatement.getSelect())); + + if(arrayLen(arguments.selectStatement.getOrderCriteria()) == 0) { + local.q.setSortBy(getIdentifierField(), true); + } else { + local.q.setSortBy( + listFirst(arguments.selectStatement.getOrderBy(), " "), + find("ASC", arguments.selectStatement.getOrderBy()) + ); + } + + if(arguments.limit > 0) { + local.q.limit(arguments.offset, arguments.limit); + } else { + local.q.limit(0, getMaxResults()); + } + + local.results = getClient().search(local.q); + local.resultsLength = local.results.totalResults; + local.documents = local.results.docs; } } catch(redis.clients.jedis.exceptions.JedisDataException e) { // nada @@ -337,9 +387,7 @@ component accessors = "true" extends = "lib.sql.QueryableCache" { local.query = queryNew(arguments.selectStatement.getSelect(), local.fieldSQLTypes); - if(arrayLen(local.docIDs) > 0) { - local.documents = getClient().getDocuments(local.docIDs); - + if(structKeyExists(local, "documents") && local.documents.size() > 0) { for(local.i = 0; local.i < local.documents.size(); local.i++) { local.document = local.documents.get(local.i); @@ -357,13 +405,21 @@ component accessors = "true" extends = "lib.sql.QueryableCache" { .setExtendedMetadata({ cached: !structKeyExists(local, "searchException"), engine: "redis", - recordCount: arrayLen(local.docIDs), + recordCount: structKeyExists(local, "documents") ? local.documents.size() : 0, totalRecordCount: (structKeyExists(local, "resultsLength") ? local.resultsLength : 0) }); return local.query; } + void function executeUpdate(required lib.sql.UpdateStatement updateStatement) { + throw(type = "lib.redis.UnsupportedOperationException", message = "Update is not supported in this implementation"); + } + + void function executeUpsert(required lib.sql.UpsertStatement upsertStatement) { + throw(type = "lib.redis.UnsupportedOperationException", message = "Upsert is not supported in this implementation"); + } + struct function fromRediSearchDocument(required any document, string fieldFilter = "") { if(arguments.fieldFilter == "") { arguments.fieldFilter = getFieldList(); @@ -428,6 +484,10 @@ component accessors = "true" extends = "lib.sql.QueryableCache" { } } + lib.sql.InsertStatement function insert(required struct fields) { + throw(type = "lib.redis.UnsupportedOperationException", message = "Insert is not supported in this implementation"); + } + boolean function isRedisDate(required string field) { switch(getQueryable().getFieldSQLType(arguments.field)) { case "date": @@ -548,6 +608,25 @@ component accessors = "true" extends = "lib.sql.QueryableCache" { return this; } + lib.sql.QueryableCache function setRowKeyMask(required string rowKeyMask) { + super.setRowKeyMask(argumentCollection = arguments); + + local.fl = getFieldList(); + + // parse the rowKeyMask to determine what properties we need + variables.aggregateApply = arrayReduce( + getKeyFields(), + function(result, field) { + return listAppend(replace(arguments.result, "{#lCase(arguments.field)#}", "%s"), (isRedisNumeric(arguments.field) ? "@" : "@_NFD_") & listGetAt(fl, listFindNoCase(fl, arguments.field))); + }, + "'#getRowKeyMask()#'" + ); + + variables.aggregateApply = "format(#variables.aggregateApply#)"; + + return this; + } + lib.sql.QueryableCache function setStopWords(required string stopWords) { if(structKeyExists(variables, "queryable")) { throw(type = "lib.redis.QueryableDefinedException", message = "an IQueryable has been furnished already"); @@ -585,7 +664,7 @@ component accessors = "true" extends = "lib.sql.QueryableCache" { } else { local.document.set(local.field, javaCast("string", arguments.row[local.field])); - if(getQueryable().fieldIsFilterable(local.field)) { + if(getQueryable().fieldIsFilterable(local.field) || local.field == getIdentifierField()) { local.document.set("_NFD_" & local.field, javaCast("string", normalize(arguments.row[local.field]))); } } @@ -594,4 +673,12 @@ component accessors = "true" extends = "lib.sql.QueryableCache" { return local.document; } + lib.sql.UpdateStatement function update(required struct fields) { + throw(type = "lib.redis.UnsupportedOperationException", message = "Update is not supported in this implementation"); + } + + lib.sql.UpsertStatement function upsert(required struct fields) { + throw(type = "lib.redis.UnsupportedOperationException", message = "Upsert is not supported in this implementation"); + } + } \ No newline at end of file diff --git a/tests/TestQueryableCache.cfc b/tests/TestQueryableCache.cfc index 78348bf..06c0d1f 100644 --- a/tests/TestQueryableCache.cfc +++ b/tests/TestQueryableCache.cfc @@ -102,6 +102,22 @@ component extends = "mxunit.framework.TestCase" { assertFalse(variables.cache.containsRow(id = "boo-boo-butt")); } + function test_delete() { + assertTrue(variables.cache.select().where("id > 5970 AND id < 5980").execute().recordCount > 0); + variables.cache.delete().where("id > 5970 AND id < 5980").execute(); + local.result = variables.cache.select().where("id > 5970 AND id < 5980").execute(); + assertEquals(0, local.result.recordCount); + } + + function test_delete_all() { + assertNotEquals(0, variables.cache.select().execute().recordCount); + variables.cache.delete().execute(); + assertEquals(0, variables.cache.select().execute().recordCount); + + // re-add all the things + variables.cache.seedFromQueryable(); + } + function test_fieldExists() { assertTrue(variables.cache.fieldExists("id")); assertFalse(variables.cache.fieldExists("asdfasfasdf")); @@ -131,6 +147,16 @@ component extends = "mxunit.framework.TestCase" { assertEquals("mxunit:indexed:id:5001", variables.cache.getRowKey(id = 5001)); } + function test_insert() { + try { + variables.cache.insert({ id: 10001, foo: "f00" }).execute(); + } catch(lib.redis.UnsupportedOperationException e) { + return; + } + + fail("exception not thrown"); + } + function test_putRow_getRow() { local.compareDate = createDateTime(1948, 12, 10, 10, 0, 0); local.queryRow = queryGetRow(variables.query, 1); @@ -415,4 +441,24 @@ component extends = "mxunit.framework.TestCase" { assertEquals(local.row, local.rowFromDocument); } + function test_update() { + try { + variables.cache.update({ id: 10001, foo: "f00" }).execute(); + } catch(lib.redis.UnsupportedOperationException e) { + return; + } + + fail("exception not thrown"); + } + + function test_upsert() { + try { + variables.cache.upsert({ id: 10001, foo: "f00" }).execute(); + } catch(lib.redis.UnsupportedOperationException e) { + return; + } + + fail("exception not thrown"); + } + } \ No newline at end of file