Support cases like "the day after" and "the day before" (#956)

* Support cases like "the day after" and "the day before"
* Refactor the Code-Writers and Generate resources in JavaScript and Python

.NET/Microsoft.Recognizers.Definitions/English/DateTimeDefinitions.cs

@@ -42,7 +42,7 @@ public static class DateTimeDefinitions
 		public const string SingleWeekDayRegex = @"\b(?<weekday>sunday|monday|tuesday|wednesday|thursday|friday|saturday|mon|tue|tues|wedn|weds|wed|thurs|thur|thu|fri|((?<=on\s+)(sat|sun)))\b";
 		public static readonly string RelativeMonthRegex = $@"(?<relmonth>(of\s+)?{RelativeRegex}\s+month)\b";
 		public const string WrittenMonthRegex = @"(((the\s+)?month of\s+)?(?<month>april|apr|august|aug|december|dec|february|feb|january|jan|july|jul|june|jun|march|mar|may|november|nov|october|oct|september|sept|sep))";
-		public static readonly string MonthSuffixRegex = $@"(?<msuf>(in\s+|of\s+|on\s+)?({RelativeMonthRegex}|{WrittenMonthRegex}))";
+		public static readonly string MonthSuffixRegex = $@"(?<msuf>((in|of|on)\s+)?({RelativeMonthRegex}|{WrittenMonthRegex}))";
 		public const string DateUnitRegex = @"(?<unit>decades?|years?|months?|weeks?|(?<business>business\s+)?days?)\b";
 		public const string DateTokenPrefix = "on ";
 		public const string TimeTokenPrefix = "at ";
@@ -86,7 +86,7 @@ public static class DateTimeDefinitions
 		public static readonly string ThisRegex = $@"\b((this(\s*week)?(\s*on)?\s+){WeekDayRegex})|({WeekDayRegex}((\s+of)?\s+this\s*week))\b";
 		public static readonly string LastDateRegex = $@"\b({PastPrefixRegex}(\s*week)?\s+{WeekDayRegex})|({WeekDayRegex}(\s+last\s*week))\b";
 		public static readonly string NextDateRegex = $@"\b({NextPrefixRegex}(\s*week(\s*,?\s*on)?)?\s+{WeekDayRegex})|((on\s+)?{WeekDayRegex}((\s+of)?\s+(the\s+following|(the\s+)?next)\s*week))\b";
-		public static readonly string SpecialDayRegex = $@"\b((the\s+)?day before yesterday|(the\s+)?day after (tomorrow|tmr)|((the\s+)?({RelativeRegex}|my)\s+day)|yesterday|tomorrow|tmr|today)\b";
+		public static readonly string SpecialDayRegex = $@"\b((the\s+)?day before yesterday|(the\s+)?day after (tomorrow|tmr)|the\s+day\s+(before|after)(?!=\s+day)|((the\s+)?({RelativeRegex}|my)\s+day)|yesterday|tomorrow|tmr|today)\b";
 		public static readonly string SpecialDayWithNumRegex = $@"\b((?<number>{WrittenNumRegex})\s+days?\s+from\s+(?<day>yesterday|tomorrow|tmr|today))\b";
 		public static readonly string RelativeDayRegex = $@"\b(((the\s+)?{RelativeRegex}\s+day))\b";
 		public const string SetWeekDayRegex = @"\b(?<prefix>on\s+)?(?<weekday>morning|afternoon|evening|night|sunday|monday|tuesday|wednesday|thursday|friday|saturday)s\b";
@@ -649,5 +649,29 @@ public static class DateTimeDefinitions
 		{
 			"night"
 		};
+		public static readonly IList<string> SameDayTerms = new List<string>
+		{
+			"today"
+		};
+		public static readonly IList<string> PlusOneDayTerms = new List<string>
+		{
+			"tomorrow",
+			"tmr",
+			"day after"
+		};
+		public static readonly IList<string> MinusOneDayTerms = new List<string>
+		{
+			"yesterday",
+			"day before"
+		};
+		public static readonly IList<string> PlusTwoDayTerms = new List<string>
+		{
+			"day after tomorrow",
+			"day after tmr"
+		};
+		public static readonly IList<string> MinusTwoDayTerms = new List<string>
+		{
+			"day before yesterday"
+		};
 	}
 }

.NET/Microsoft.Recognizers.Definitions/French/DateTimeDefinitions.cs

@@ -1,6 +1,4 @@
-
-
-//------------------------------------------------------------------------------
+//------------------------------------------------------------------------------
 // <auto-generated>
 //     This code was generated by a tool.
 //     Changes to this file may cause incorrect behavior and will be lost if
@@ -196,6 +194,9 @@ public static class DateTimeDefinitions
 		public static readonly string RelativeTimeUnitRegex = $@"(((({ThisPrefixRegex})?)\s+({TimeUnitRegex}(\s*{NextSuffixRegex}|{PastSuffixRegex})?))|((le))\s+({RestrictedTimeUnitRegex}))";
 		public static readonly string RelativeDurationUnitRegex = $@"(((?<=({ThisPrefixRegex})\s+)?\b({DurationUnitRegex})(\s+{NextSuffixRegex}|{PastSuffixRegex})?)|((le|my))\s+({RestrictedTimeUnitRegex}))";
 		public const string ReferenceDatePeriodRegex = @"^[.]";
+		public const string NextPrefixRegex = @"^[\.]";
+		public const string PastPrefixRegex = @"^[\.]";
+		public const string RelativeDayRegex = @"^[\.]";
 		public const string ConnectorRegex = @"^(,|pour|t|vers)$";
 		public const string ConnectorAndRegex = @"\b(et\s*(le|la(s)?)?)\b.+";
 		public const string FromRegex = @"((de|du)?)$";
@@ -699,5 +700,33 @@ public static class DateTimeDefinitions
 		{
 			"nuit"
 		};
+		public static readonly IList<string> SameDayTerms = new List<string>
+		{
+			"aujourd'hui",
+			"auj"
+		};
+		public static readonly IList<string> PlusOneDayTerms = new List<string>
+		{
+			"demain",
+			"a2m1",
+			"lendemain",
+			"jour suivant"
+		};
+		public static readonly IList<string> MinusOneDayTerms = new List<string>
+		{
+			"hier",
+			"dernier"
+		};
+		public static readonly IList<string> PlusTwoDayTerms = new List<string>
+		{
+			"après demain",
+			"après-demain",
+			"apres-demain"
+		};
+		public static readonly IList<string> MinusTwoDayTerms = new List<string>
+		{
+			"avant-hier",
+			"avant hier"
+		};
 	}
 }

.NET/Microsoft.Recognizers.Definitions/German/DateTimeDefinitions.cs

@@ -677,5 +677,26 @@ public static class DateTimeDefinitions
 		{
 			"nacht"
 		};
+		public static readonly IList<string> SameDayTerms = new List<string>
+		{
+			"heute"
+		};
+		public static readonly IList<string> PlusOneDayTerms = new List<string>
+		{
+			"morgen",
+			"tmr"
+		};
+		public static readonly IList<string> MinusOneDayTerms = new List<string>
+		{
+			"gestern"
+		};
+		public static readonly IList<string> PlusTwoDayTerms = new List<string>
+		{
+			"übermorgen"
+		};
+		public static readonly IList<string> MinusTwoDayTerms = new List<string>
+		{
+			"vorgestern"
+		};
 	}
 }

.NET/Microsoft.Recognizers.Definitions/Italian/DateTimeDefinitions.cs

@@ -69,6 +69,8 @@ public static class DateTimeDefinitions
 		public static readonly string WeekDayOfMonthRegex = $@"(?<wom>(la\s+)?(?<cardinal>prima|seconda|terza|quarta|quinta|ultima)\s+{WeekDayRegex}\s+{MonthSuffixRegex})";
 		public const string RelativeWeekDayRegex = @"^[.]";
 		public const string NumberEndingPattern = @"^[.]";
+		public const string NextPrefixRegex = @"^[.]";
+		public const string PastPrefixRegex = @"^[.]";
 		public static readonly string SpecialDate = $@"(?=\b(di|al)\s+il\s+){DayRegex}\b";
 		public static readonly string DateExtractor1 = $@"\b({WeekDayRegex}(\s+|\s*,\s*))?{MonthRegex}\s*[/\\\.\-]?\s*{DayRegex}\b";
 		public static readonly string DateExtractor2 = $@"\b({WeekDayRegex}(\s+|\s*,\s*))?{DayRegex}(\s+|\s*,\s*|\s+){MonthRegex}\s*[\.\-]?\s*{DateYearRegex}\b";
@@ -590,5 +592,10 @@ public static class DateTimeDefinitions
 		{
 			"nuit"
 		};
+		public static readonly string[] SameDayTerms = {  };
+		public static readonly string[] PlusOneDayTerms = {  };
+		public static readonly string[] MinusOneDayTerms = {  };
+		public static readonly string[] PlusTwoDayTerms = {  };
+		public static readonly string[] MinusTwoDayTerms = {  };
 	}
 }

.NET/Microsoft.Recognizers.Definitions/Portuguese/DateTimeDefinitions.cs

@@ -507,5 +507,35 @@ public static class DateTimeDefinitions
 		{
 			"noite"
 		};
+		public static readonly IList<string> SameDayTerms = new List<string>
+		{
+			"hoje",
+			"este dia",
+			"esse dia",
+			"o dia"
+		};
+		public static readonly IList<string> PlusOneDayTerms = new List<string>
+		{
+			"amanha",
+			"de amanha",
+			"dia seguinte",
+			"o dia de amanha",
+			"proximo dia"
+		};
+		public static readonly IList<string> MinusOneDayTerms = new List<string>
+		{
+			"ontem",
+			"ultimo dia"
+		};
+		public static readonly IList<string> PlusTwoDayTerms = new List<string>
+		{
+			"depois de amanha",
+			"dia depois de amanha"
+		};
+		public static readonly IList<string> MinusTwoDayTerms = new List<string>
+		{
+			"anteontem",
+			"dia antes de ontem"
+		};
 	}
 }

.NET/Microsoft.Recognizers.Definitions/Spanish/DateTimeDefinitions.cs

@@ -481,5 +481,32 @@ public static class DateTimeDefinitions
 		{
 			"noche"
 		};
+		public static readonly IList<string> SameDayTerms = new List<string>
+		{
+			"hoy",
+			"el dia"
+		};
+		public static readonly IList<string> PlusOneDayTerms = new List<string>
+		{
+			"mañana",
+			"dia siguiente",
+			"el dia de mañana",
+			"proximo dia"
+		};
+		public static readonly IList<string> MinusOneDayTerms = new List<string>
+		{
+			"ayer",
+			"ultimo dia"
+		};
+		public static readonly IList<string> PlusTwoDayTerms = new List<string>
+		{
+			"pasado mañana",
+			"dia despues de mañana"
+		};
+		public static readonly IList<string> MinusTwoDayTerms = new List<string>
+		{
+			"anteayer",
+			"dia antes de ayer"
+		};
 	}
 }

.NET/Microsoft.Recognizers.Text.DateTime/English/Extractors/EnglishDateExtractorConfiguration.cs

@@ -120,16 +120,27 @@ public EnglishDateExtractorConfiguration(IOptionsConfiguration config) : base(co
 
             ImplicitDateList = new List<Regex>
             {
+                // extract "12" from "on 12"
                 OnRegex,
+                // extract "12th" from "on/at/in 12th"
                 RelaxedOnRegex,
+                // "the day before yesterday", "previous day", "today", "yesterday", "tomorrow"
                 SpecialDayRegex,
+                // "this Monday", "Tuesday of this week"
                 ThisRegex,
+                // "last/previous Monday", "Monday of last week"
                 LastDateRegex,
+                // "next/following Monday", "Monday of next week"
                 NextDateRegex,
+                // "Sunday", "Weds"
                 SingleWeekDayRegex,
+                // "2nd Monday of April"
                 WeekDayOfMonthRegex,
+                // "on the 12th"
                 SpecialDate,
+                // "two days from today", "five days from tomorrow"
                 SpecialDayWithNumRegex,
+                // "three Monday from now"
                 RelativeWeekDayRegex
             };
 

.NET/Microsoft.Recognizers.Text.DateTime/English/Parsers/EnglishDateParserConfiguration.cs

@@ -59,19 +59,11 @@ public class EnglishDateParserConfiguration : BaseOptionsConfiguration, IDatePar
 
         public Regex RelativeWeekDayRegex { get; }
 
-        /*
-         The following three regexes only used in this configuration
-         They are not used in the base parser, therefore they are not extracted
-         If the spanish date parser need the same regexes, they should be extracted 
-        */
-        public static readonly Regex RelativeDayRegex= 
-            new Regex(DateTimeDefinitions.RelativeDayRegex, RegexOptions.Singleline);
+        public Regex RelativeDayRegex { get; }
 
-        public static readonly Regex NextPrefixRegex = 
-            new Regex(DateTimeDefinitions.NextPrefixRegex, RegexOptions.Singleline);
+        public Regex NextPrefixRegex { get; }
 
-        public static readonly Regex PastPrefixRegex = 
-            new Regex(DateTimeDefinitions.PastPrefixRegex, RegexOptions.Singleline);
+        public Regex PastPrefixRegex { get; }
 
         public IImmutableDictionary<string, int> DayOfMonth { get; }
 
@@ -81,6 +73,16 @@ The following three regexes only used in this configuration
 
         public IImmutableDictionary<string, int> CardinalMap { get; }
 
+        public IImmutableList<string> SameDayTerms { get; }
+
+        public IImmutableList<string> PlusOneDayTerms { get; }
+
+        public IImmutableList<string> MinusOneDayTerms { get; }
+
+        public IImmutableList<string> PlusTwoDayTerms { get; }
+
+        public IImmutableList<string> MinusTwoDayTerms { get; }
+
         public IDateTimeUtilityConfiguration UtilityConfiguration { get; }
 
         public EnglishDateParserConfiguration(ICommonDateTimeParserConfiguration config) : base(config)
@@ -109,65 +111,33 @@ public EnglishDateParserConfiguration(ICommonDateTimeParserConfiguration config)
             RelativeMonthRegex = EnglishDateExtractorConfiguration.RelativeMonthRegex;
             YearSuffix = EnglishDateExtractorConfiguration.YearSuffix;
             RelativeWeekDayRegex = EnglishDateExtractorConfiguration.RelativeWeekDayRegex;
+            RelativeDayRegex = new Regex(DateTimeDefinitions.RelativeDayRegex, RegexOptions.Singleline);
+            NextPrefixRegex = new Regex(DateTimeDefinitions.NextPrefixRegex, RegexOptions.Singleline);
+            PastPrefixRegex = new Regex(DateTimeDefinitions.PastPrefixRegex, RegexOptions.Singleline);
             DayOfMonth = config.DayOfMonth;
             DayOfWeek = config.DayOfWeek;
             MonthOfYear = config.MonthOfYear;
             CardinalMap = config.CardinalMap;
             UnitMap = config.UnitMap;
             UtilityConfiguration = config.UtilityConfiguration;
-        }
-
-        public int GetSwiftDay(string text)
-        {
-            var trimmedText = text.Trim().ToLowerInvariant();
-            var swift = 0;
-
-            var match = RelativeDayRegex.Match(text);
-
-            if (trimmedText.Equals("today"))
-            {
-                swift = 0;
-            }
-            else if (trimmedText.Equals("tomorrow") || trimmedText.Equals("tmr"))
-            {
-                swift = 1;
-            }
-            else if (trimmedText.Equals("yesterday"))
-            {
-                swift = -1;
-            }
-            else if (trimmedText.EndsWith("day after tomorrow") ||
-                     trimmedText.EndsWith("day after tmr"))
-            {
-                swift = 2;
-            }
-            else if (trimmedText.EndsWith("day before yesterday"))
-            {
-                swift = -2;
-            }
-            else if (match.Success)
-            {
-                swift = GetSwift(text);
-            }
-
-            return swift;
+            SameDayTerms = DateTimeDefinitions.SameDayTerms.ToImmutableList();
+            PlusOneDayTerms = DateTimeDefinitions.PlusOneDayTerms.ToImmutableList();
+            PlusTwoDayTerms = DateTimeDefinitions.PlusTwoDayTerms.ToImmutableList();
+            MinusOneDayTerms = DateTimeDefinitions.MinusOneDayTerms.ToImmutableList();
+            MinusTwoDayTerms = DateTimeDefinitions.MinusTwoDayTerms.ToImmutableList();
         }
 
         public int GetSwiftMonth(string text)
-        {
-            return GetSwift(text);
-        }
-
-        public int GetSwift(string text)
         {
             var trimmedText = text.Trim().ToLowerInvariant();
-
             var swift = 0;
+
             if (NextPrefixRegex.IsMatch(trimmedText))
             {
                 swift = 1;
             }
-            else if (PastPrefixRegex.IsMatch(trimmedText))
+
+            if (PastPrefixRegex.IsMatch(trimmedText))
             {
                 swift = -1;
             }
@@ -180,5 +150,10 @@ public bool IsCardinalLast(string text)
             var trimmedText = text.Trim().ToLowerInvariant();
             return trimmedText.Equals("last");
         }
+
+        public string Normalize(string text)
+        {
+            return text;
+        }
     }
 }