tyler/Libation
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request #656 from Mbucari/master

Browse Source 
Fix query parsing tags with underscores (#655)
This commit is contained in:
rmcrackan 2023-07-06 09:16:20 -04:00 committed by GitHub
commit 22a3dcbc1f
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 26 additions and 49 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

46
Source/LibationSearchEngine/QuerySanitizer.cs
View File

@ -2,10 +2,11 @@
using Lucene.Net.Analysis.Tokenattributes;
using System.Collections.Generic;
using System.Linq;
using System.Text.RegularExpressions;
namespace LibationSearchEngine
{
internal static class QuerySanitizer
internal static partial class QuerySanitizer
{
private static readonly HashSet<string> idTerms
= SearchEngine.FieldIndexRules.IdFieldNames
@ -23,11 +24,17 @@ namespace LibationSearchEngine
.Select(n => n.ToLowerInvariant())
.ToHashSet();
private static readonly Regex tagRegex = TagRegex();
internal static string Sanitize(string searchString, StandardAnalyzer analyzer)
{
if (string.IsNullOrWhiteSpace(searchString))
return SearchEngine.ALL_QUERY;
//Replace a block tags with tags with proper tag query syntax
//eg: [foo] -> tags:foo
searchString = tagRegex.Replace(searchString, $"{SearchEngine.TAGS}:$1 ");
// range operator " TO " and bool operators " AND " and " OR " must be uppercase
searchString
= searchString
@ -76,11 +83,6 @@ namespace LibationSearchEngine
addUnalteredToken(offset);
previousIsTags = false;
}
else if (tryParseBlockTag(offset, partList, searchString, out var tagName))
{
//The term is a block tag. add it to the part list
partList.Add($"{SearchEngine.TAGS}:{tagName}");
}
else if (double.TryParse(term, out var num))
{
//Term is a number so pad it with zeros
@ -117,35 +119,7 @@ namespace LibationSearchEngine
partList.Add(searchString.Substring(offset.StartOffset, offset.EndOffset - offset.StartOffset));
}
private static bool tryParseBlockTag(IOffsetAttribute offset, List<string> partList, string searchString, out string tagName)
{
tagName = null;
if (partList.Count == 0) return false;
var previous = partList[^1].TrimEnd();
//cannot be preceeded by an escaping \
if (previous.Length == 0) return false;
if (previous[^1] != '[' || (previous.Length > 1 && previous[^2] == '\\')) return false;
var next = searchString.Substring(offset.EndOffset);
if (next.Length == 0 || !next.TrimStart().StartsWith(']')) return false;
tagName = searchString.Substring(offset.StartOffset, offset.EndOffset - offset.StartOffset);
//Only legal tag characters are letters, numbers and underscores
//Per DataLayer.UserDefinedItem.IllegalCharacterRegex()
foreach (var c in tagName)
{
if (!char.IsLetterOrDigit(c) && c != '_')
return false;
}
//Remove the leading '['
partList[^1] = previous[..^1];
//Ignore the trailing ']'
offset.SetOffset(offset.StartOffset, searchString.IndexOf(']', offset.EndOffset) + 1);
return true;
}
[GeneratedRegex(@"(?<!\\)\[\u0020*(\w+)\u0020*\]", RegexOptions.Compiled)]
private static partial Regex TagRegex();
}
}

19
Source/_Tests/LibationSearchEngine.Tests/SearchEngineTests.cs
View File

@ -30,18 +30,21 @@ namespace SearchEngineTests
[DataRow(" ", "*:*")]
// tag surrounded by spaces
[DataRow("[foo]", "tags:foo")]
[DataRow(" [foo]", " tags:foo")]
[DataRow(" [ foo ]", " tags:foo")]
[DataRow("[foo]", "tags:foo ")]
[DataRow(" [foo]", " tags:foo ")]
[DataRow(" [ foo ]", " tags:foo ")]
[DataRow("[foo] ", "tags:foo ")]
[DataRow(" [foo] ", " tags:foo ")]
[DataRow("-[foo]", "-tags:foo")]
[DataRow(" -[foo]", " -tags:foo")]
[DataRow("-[foo]", "-tags:foo ")]
[DataRow(" -[foo]", " -tags:foo ")]
[DataRow("-[foo] ", "-tags:foo ")]
[DataRow(" -[foo] ", " -tags:foo ")]
[DataRow("[foo_bar]", "tags:foo_bar ")]
[DataRow("-[foo_bar]", "-tags:foo_bar ")]
[DataRow("[foo_bar] [foo_bar2]", "tags:foo_bar tags:foo_bar2 ")]
// tag case irrelevant
[DataRow("[FoO]", "tags:FoO")]
[DataRow("[FoO]", "tags:FoO ")]
// bool keyword surrounded by spaces
[DataRow("israted", "israted:True")]
@ -69,9 +72,9 @@ namespace SearchEngineTests
[DataRow("liberated AND isRated:false", "liberated:True AND israted:false")]
// tag which happens to be a bool keyword >> parse as tag
[DataRow("[israted]", "tags:israted")]
[DataRow("[israted]", "tags:israted ")]
[DataRow("[tags] [israted] [tags] [tags] [isliberated] [israted] ", "tags:tags tags:israted tags:tags tags:tags tags:isliberated tags:israted ")]
[DataRow("[tags][israted]", "tags:tagstags:israted")]
[DataRow("[tags][israted]", "tags:tags tags:israted ")]
// numbers with "to". TO all caps, numbers [8.2] format
[DataRow("1 to 10", "00000001.00 TO 00000010.00")]