Skip to content

Commit

Permalink
[EN DateTimeV2] Support for longer span date time (#3166)
Browse files Browse the repository at this point in the history 
* Datetime for longer span - local draft commit

* DateTimeForLongerSpan - Implement for from

---------

Co-authored-by: Michael Wang (Centific Technologies Inc) <[email protected]>
  • Loading branch information
@MichaelMWW
MichaelMWW and Michael Wang (Centific Technologies Inc) authored Sep 25, 2024
1 parent 47f6d25 commit cdb178a
Show file tree
Hide file tree
Showing 10 changed files with 281 additions and 3 deletions.
3 changes: 2 additions & 1 deletion .NET/Microsoft.Recognizers.Definitions.Common/English/DateTimeDefinitions.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -260,12 +260,13 @@ public static class DateTimeDefinitions
public static readonly string SinceRegexExp = $@"({SinceRegex}|\bfrom(\s+the)?\b)";
public const string AgoRegex = @"\b(ago|earlier|before\s+(?<day>yesterday|today))\b";
public static readonly string LaterRegex = $@"\b(?:later(?!((\s+in)?\s*{OneWordPeriodRegex})|(\s+{TimeOfDayRegex})|\s+than\b)|from now|(from|after)\s+(?<day>tomorrow|tmrw?|today))\b";
public const string BeforeAfterRegex = @"\b((?<before>before)|(?<after>from|after))\b";
public const string BeforeAfterRegex = @"(,?\s*)\b((?<before>before)|(?<after>from|after))\b";
public static readonly string ModPrefixRegex = $@"\b({RelativeRegex}|{AroundRegex}|{BeforeRegex}|{AfterRegex}|{SinceRegex})\b";
public static readonly string ModSuffixRegex = $@"\b({AgoRegex}|{LaterRegex}|{BeforeAfterRegex}|{FutureSuffixRegex}|{PastSuffixRegex})\b";
public const string InConnectorRegex = @"\b(in)\b";
public static readonly string SinceYearSuffixRegex = $@"(^\s*{SinceRegex}(\s*(the\s+)?year\s*)?{YearSuffix})";
public static readonly string WithinNextPrefixRegex = $@"\b(within(\s+the)?(\s+(?<next>{NextPrefixRegex}))?)\b";
public const string ForPrefixRegex = @"((?<forfrom>for.*from.*)|(?<from>\bfrom\b)|(?<for>\bfor\b))";
public const string TodayNowRegex = @"\b(today|now|current (date|time))\b";
public static readonly string MorningStartEndRegex = $@"(^(morning|{AmDescRegex}))|((morning|{AmDescRegex})$)";
public static readonly string AfternoonStartEndRegex = $@"(^(afternoon|{PmDescRegex}))|((afternoon|{PmDescRegex})$)";
Expand Down
2 changes: 2 additions & 0 deletions .NET/Microsoft.Recognizers.Text.DateTime/Constants.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -245,6 +245,8 @@ public static class Constants
public const string StartGroupName = "start";
public const string EndGroupName = "end";
public const string WithinGroupName = "within";
public const string ForGroupName = "for";
public const string FromGroupName = "from";
public const string NumberGroupName = "number";
public const string OrdinalGroupName = "ordinal";
public const string OrderGroupName = "order";
Expand Down
3 changes: 3 additions & 0 deletions ...t.Recognizers.Text.DateTime/English/Extractors/EnglishDatePeriodExtractorConfiguration.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -124,6 +124,9 @@ public class EnglishDatePeriodExtractorConfiguration : BaseDateTimeOptionsConfig
public static readonly Regex WithinNextPrefixRegex =
new Regex(DateTimeDefinitions.WithinNextPrefixRegex, RegexFlags, RegexTimeOut);

public static readonly Regex ForPrefixRegex =
new Regex(DateTimeDefinitions.ForPrefixRegex, RegexFlags, RegexTimeOut);

public static readonly Regex RestOfDateRegex =
new Regex(DateTimeDefinitions.RestOfDateRegex, RegexFlags, RegexTimeOut);

Expand Down
3 changes: 3 additions & 0 deletions ...crosoft.Recognizers.Text.DateTime/English/Parsers/EnglishDatePeriodParserConfiguration.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -95,6 +95,7 @@ public EnglishDatePeriodParserConfiguration(ICommonDateTimeParserConfiguration c
RelativeDecadeRegex = EnglishDatePeriodExtractorConfiguration.RelativeDecadeRegex;
InConnectorRegex = config.UtilityConfiguration.InConnectorRegex;
WithinNextPrefixRegex = EnglishDatePeriodExtractorConfiguration.WithinNextPrefixRegex;
ForPrefixRegex = EnglishDatePeriodExtractorConfiguration.ForPrefixRegex;
ReferenceDatePeriodRegex = EnglishDatePeriodExtractorConfiguration.ReferenceDatePeriodRegex;
AgoRegex = EnglishDatePeriodExtractorConfiguration.AgoRegex;
LaterRegex = EnglishDatePeriodExtractorConfiguration.LaterRegex;
Expand Down Expand Up @@ -186,6 +187,8 @@ public EnglishDatePeriodParserConfiguration(ICommonDateTimeParserConfiguration c

public Regex WithinNextPrefixRegex { get; }

public Regex ForPrefixRegex { get; }

public Regex RestOfDateRegex { get; }

public Regex LaterEarlyPeriodRegex { get; }
Expand Down
14 changes: 13 additions & 1 deletion .NET/Microsoft.Recognizers.Text.DateTime/Extractors/BaseDatePeriodExtractor.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@
using System.Collections.Generic;
using System.Linq;
using System.Text.RegularExpressions;

using Microsoft.Recognizers.Text.DateTime.English;
using Microsoft.Recognizers.Text.InternalCache;
using Microsoft.Recognizers.Text.Utilities;
using DateObject = System.DateTime;
Expand Down Expand Up @@ -734,6 +734,18 @@ private List<Token> SingleTimePointWithPatterns(string text, List<ExtractResult>
}
}
}

// For cases like "for 1 week from today", "for 3 days from 20th May" etc..
if (EnglishDatePeriodExtractorConfiguration.ForPrefixRegex != null)
{
Match prefixMatchFor = EnglishDatePeriodExtractorConfiguration.ForPrefixRegex.Match(beforeString);
Match datepointMatchFrom = EnglishDatePeriodExtractorConfiguration.ForPrefixRegex.Match(extractionResult.Text);
if (prefixMatchFor.Success && prefixMatchFor.Groups[Constants.ForGroupName].Success
&& datepointMatchFrom.Success && datepointMatchFrom.Groups[Constants.FromGroupName].Success)
{
ret.AddRange(GetTokenForRegexMatching(beforeString, EnglishDatePeriodExtractorConfiguration.ForPrefixRegex, extractionResult, inPrefix: true));
}
}
}
}

Expand Down
66 changes: 66 additions & 0 deletions .NET/Microsoft.Recognizers.Text.DateTime/Parsers/BaseDatePeriodParser.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@
using System.Globalization;
using System.Linq;
using System.Text.RegularExpressions;
using Microsoft.Recognizers.Text.DateTime.English;
using Microsoft.Recognizers.Text.DateTime.Utilities;
using Microsoft.Recognizers.Text.Utilities;
using DateObject = System.DateTime;
Expand Down Expand Up @@ -476,6 +477,12 @@ private DateTimeResolutionResult ParseBaseDatePeriod(string text, DateObject ref
innerResult = ParseDatePointWithAgoAndLater(text, referenceDate);
}

// Cases like "for x weeks/days from today/12 sep etc."
if (!innerResult.Success)
{
innerResult = ParseDatePointWithForPrefix(text, referenceDate);
}

// Parse duration should be at the end since it will extract "the last week" from "the last week of July"
if (!innerResult.Success)
{
Expand Down Expand Up @@ -620,6 +627,65 @@ private DateTimeResolutionResult ParseDatePointWithAgoAndLater(string text, Date
return ret;
}

// Only handle cases like "for x weeks/days from today/tomorrow/some day"
private DateTimeResolutionResult ParseDatePointWithForPrefix(string text, DateObject referenceDate)
{
var ret = new DateTimeResolutionResult();
var er = this.config.DateExtractor.Extract(text, referenceDate).FirstOrDefault();

if (er != null)
{
var beforeString = text.Substring(0, (int)er.Start);
var isAgo = this.config.AgoRegex.Match(er.Text).Success;
var config = this.config as EnglishDatePeriodParserConfiguration;

if (!string.IsNullOrEmpty(beforeString) && config != null)
{
var matchFor = config.ForPrefixRegex.Match(beforeString);

if (matchFor.Success && matchFor.Groups[Constants.ForGroupName].Success)
{
var pr = this.config.DateParser.Parse(er, referenceDate);
var durationExtractionResult = this.config.DurationExtractor.Extract(er.Text, referenceDate).FirstOrDefault();

if (durationExtractionResult != null)
{
var duration = this.config.DurationParser.Parse(durationExtractionResult);
var durationInSeconds = (double)((DateTimeResolutionResult)duration.Value).PastValue;

DateObject startDate;
DateObject endDate;

if (isAgo)
{
startDate = (DateObject)((DateTimeResolutionResult)pr.Value).PastValue;
endDate = startDate.AddSeconds(durationInSeconds);
}
else
{
endDate = (DateObject)((DateTimeResolutionResult)pr.Value).FutureValue;
startDate = endDate.AddSeconds(-durationInSeconds);
}

if (startDate != DateObject.MinValue)
{
var startLuisStr = DateTimeFormatUtil.LuisDate(startDate);
var endLuisStr = DateTimeFormatUtil.LuisDate(endDate);
var durationTimex = ((DateTimeResolutionResult)duration.Value).Timex;

ret.Timex = $"({startLuisStr},{endLuisStr},{durationTimex})";
ret.FutureValue = new Tuple<DateObject, DateObject>(startDate, endDate);
ret.PastValue = new Tuple<DateObject, DateObject>(startDate, endDate);
ret.Success = true;
}
}
}
}
}

return ret;
}

private DateTimeResolutionResult ParseSingleTimePoint(string text, DateObject referenceDate, DateContext dateContext = null)
{
var ret = new DateTimeResolutionResult();
Expand Down
4 changes: 3 additions & 1 deletion Patterns/English/English-DateTime.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -624,7 +624,7 @@ LaterRegex: !nestedRegex
def: \b(?:later(?!((\s+in)?\s*{OneWordPeriodRegex})|(\s+{TimeOfDayRegex})|\s+than\b)|from now|(from|after)\s+(?<day>tomorrow|tmrw?|today))\b
references: [ OneWordPeriodRegex, TimeOfDayRegex ]
BeforeAfterRegex: !simpleRegex
def: \b((?<before>before)|(?<after>from|after))\b
def: (,?\s*)\b((?<before>before)|(?<after>from|after))\b
ModPrefixRegex: !nestedRegex
def: \b({RelativeRegex}|{AroundRegex}|{BeforeRegex}|{AfterRegex}|{SinceRegex})\b
references: [RelativeRegex, AroundRegex, BeforeRegex, AfterRegex, SinceRegex ]
Expand All @@ -639,6 +639,8 @@ SinceYearSuffixRegex: !nestedRegex
WithinNextPrefixRegex: !nestedRegex
def: \b(within(\s+the)?(\s+(?<next>{NextPrefixRegex}))?)\b
references: [ NextPrefixRegex ]
ForPrefixRegex: !simpleRegex
def: ((?<forfrom>for.*from.*)|(?<from>\bfrom\b)|(?<for>\bfor\b))
TodayNowRegex: !simpleRegex # Added to remove hard coded strings in BaseDatePeriodParser
def: \b(today|now|current (date|time))\b
# "next" group here is used to judge uncommon unsupported cases like "within the next 5 days before today"
Expand Down
36 changes: 36 additions & 0 deletions Specs/DateTime/English/DatePeriodExtractor.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3572,6 +3572,42 @@
}
]
},
{
"Input": "set OOO for 1 week, from 20th dec",
"NotSupported": "python, javascript, java",
"Results": [
{
"Text": "for 1 week, from 20th dec",
"Type": "daterange",
"Start": 8,
"Length": 25
}
]
},
{
"Input": "set OOO for 3 days, from today",
"NotSupported": "python, javascript, java",
"Results": [
{
"Text": "for 3 days, from today",
"Type": "daterange",
"Start": 8,
"Length": 22
}
]
},
{
"Input": "set OOO for 3 days from today",
"NotSupported": "python, javascript, java",
"Results": [
{
"Text": "for 3 days from today",
"Type": "daterange",
"Start": 8,
"Length": 21
}
]
},
{
"Input": "I will come back less than 2 weeks from today",
"NotSupported": "python, javascript",
Expand Down
78 changes: 78 additions & 0 deletions Specs/DateTime/English/DatePeriodParser.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5266,6 +5266,84 @@
}
]
},
{
"Input": "set OOO for 1 week, from 20th dec",
"Context": {
"ReferenceDateTime": "2018-05-29T00:00:00"
},
"NotSupported": "python, javascript, java",
"Results": [
{
"Text": "for 1 week, from 20th dec",
"Type": "daterange",
"Value": {
"Timex": "(2018-12-20,2018-12-27,P1W)",
"FutureResolution": {
"startDate": "2018-12-20",
"endDate": "2018-12-27"
},
"PastResolution": {
"startDate": "2018-12-20",
"endDate": "2018-12-27"
}
},
"Start": 8,
"Length": 25
}
]
},
{
"Input": "set OOO for 3 days, from today",
"Context": {
"ReferenceDateTime": "2018-05-23T00:00:00"
},
"NotSupported": "python, javascript, java",
"Results": [
{
"Text": "for 3 days, from today",
"Type": "daterange",
"Value": {
"Timex": "(2018-05-23,2018-05-26,P3D)",
"FutureResolution": {
"startDate": "2018-05-23",
"endDate": "2018-05-26"
},
"PastResolution": {
"startDate": "2018-05-23",
"endDate": "2018-05-26"
}
},
"Start": 8,
"Length": 22
}
]
},
{
"Input": "set OOO for 3 days from today",
"Context": {
"ReferenceDateTime": "2018-05-23T00:00:00"
},
"NotSupported": "python, javascript, java",
"Results": [
{
"Text": "for 3 days from today",
"Type": "daterange",
"Value": {
"Timex": "(2018-05-23,2018-05-26,P3D)",
"FutureResolution": {
"startDate": "2018-05-23",
"endDate": "2018-05-26"
},
"PastResolution": {
"startDate": "2018-05-23",
"endDate": "2018-05-26"
}
},
"Start": 8,
"Length": 21
}
]
},
{
"Input": "I have already finished all my work more than 2 weeks before today",
"Context": {
Expand Down
75 changes: 75 additions & 0 deletions Specs/DateTime/English/DateTimeModel.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20951,6 +20951,81 @@
}
]
},
{
"Input": "set OOO for 1 week, from 20th dec",
"Context": {
"ReferenceDateTime": "2018-05-29T00:00:00"
},
"NotSupported": "javascript, python, java",
"Results": [
{
"Text": "for 1 week, from 20th dec",
"Start": 8,
"End": 32,
"TypeName": "datetimeV2.daterange",
"Resolution": {
"values": [
{
"timex": "(2018-12-20,2018-12-27,P1W)",
"type": "daterange",
"start": "2018-12-20",
"end": "2018-12-27"
}
]
}
}
]
},
{
"Input": "set OOO for 3 days, from today",
"Context": {
"ReferenceDateTime": "2018-05-23T00:00:00"
},
"NotSupported": "javascript, python, java",
"Results": [
{
"Text": "for 3 days, from today",
"Start": 8,
"End": 29,
"TypeName": "datetimeV2.daterange",
"Resolution": {
"values": [
{
"timex": "(2018-05-23,2018-05-26,P3D)",
"type": "daterange",
"start": "2018-05-23",
"end": "2018-05-26"
}
]
}
}
]
},
{
"Input": "set OOO for 3 days from today",
"Context": {
"ReferenceDateTime": "2018-05-23T00:00:00"
},
"NotSupported": "javascript, python, java",
"Results": [
{
"Text": "for 3 days from today",
"Start": 8,
"End": 28,
"TypeName": "datetimeV2.daterange",
"Resolution": {
"values": [
{
"timex": "(2018-05-23,2018-05-26,P3D)",
"type": "daterange",
"start": "2018-05-23",
"end": "2018-05-26"
}
]
}
}
]
},
{
"Input": "The project was submitted last month and 3 weeks later it was approved",
"Context": {
Expand Down

0 comments on commit cdb178a

Please sign in to comment.