需要使用c#和Nest从ElasticSearch返回Distinct字段，而不需要完整的搜索结果文档
本文关键字：不需要 字段 文档 搜索结果 Distinct 返回 ElasticSearch Nest | 更新日期: 2023-09-27 18:17:24
我有日志存储在ElasticSearch和使用c#和Nest的Windows应用程序中，该应用程序正在执行针对ElasticSearch的搜索。ElasticSearch中的映射如下所示:
"mappings": {
"qns": {
"properties": {
"@timestamp": {
"format": "strict_date_optional_time||epoch_millis",
"type": "date"
},
"Error_Description": {
"index": "not_analyzed",
"type": "string"
},
"Thread_Id": {
"index": "not_analyzed",
"type": "string"
},
"Error_Description_Analyzed": {
"type": "string"
},
"Error_Source": {
"index": "not_analyzed",
"type": "string"
},
"record": {
"type": "string"
},
"@version": {
"type": "string"
},
"Log_Level": {
"type": "string"
},
"Record": {
"type": "string"
},
"id": {
"type": "long"
},
"Error_Source_Analyzed": {
"type": "string"
},
"Timestamp": {
"format": "strict_date_optional_time||epoch_millis",
"type": "date"
}
}
}
}
[ElasticsearchType(IdProperty = "Id", Name = "qns")]
public class QNS
{
[Number(NumberType.Long, Name = "id")]
public long Id { get; set; }
[Date(Name = "Timestamp")]
public DateTime Timestamp { get; set; }
[String(Name = "Error_Description", Index = FieldIndexOption.NotAnalyzed)]
public string ErrorDescriptionKeyword { get; set; }
[String(Name = "Error_Description_Analyzed")]
public string ErrorDescriptionAnalyzed { get; set; }
[String(Name = "Error_Source", Index = FieldIndexOption.NotAnalyzed)]
public string ErrorSourceKeyword { get; set; }
[String(Name = "Error_Source_Analyzed")]
public string ErrorSourceAnalyzed { get; set; }
[String(Name = "Thread_Id", Index = FieldIndexOption.NotAnalyzed)]
public string ThreadId { get; set; }
[String(Name = "Log_Level")]
public string LogLevel { get; set; }
[String(Index = FieldIndexOption.NotAnalyzed)]
public string Record { get; set; }
}
我需要一种方法来搜索不同的错误记录落在一个日期时间范围内，并匹配一定范围的模式。虽然我能够得到结果，但我也得到了满足搜索的所有文档，而我只需要不同的错误字符串。对于不同的查询，我使用FluentNest(https://github.com/hoonzis/fluentnest)。检索结果的代码如下:
private List<string> FindDistinctErrorsByPatternAndTimeRangeInternal(DateTime fromDateTime, DateTime toDateTime, List<pattern> patterns, string indexName, string type)
{
var documents = new List<QNS>();
var fromTime = fromDateTime.ToString(Constants.IndexSearch.ES_DATETIME_FORMAT);
var toTime = toDateTime.ToString(Constants.IndexSearch.ES_DATETIME_FORMAT);
var patternQueries = new List<QueryContainer>();
foreach (var p in patterns)
{
var pType = PatternType.unknown;
if (Enum.TryParse<PatternType>(p.Pattern_Type.ToLowerInvariant(), out pType))
{
switch (pType)
{
case PatternType.word:
patternQueries.Add(Query<QNS>.Regexp(r =>
r.Field(f =>
f.ErrorDescriptionAnalyzed)
.Value(p.Pattern_Description)
)
);
break;
case PatternType.phrase:
patternQueries.Add(Query<QNS>.MatchPhrase(m =>
m.Field(f =>
f.ErrorDescriptionAnalyzed)
.Query(p.Pattern_Description)
)
);
break;
case PatternType.unknown:
default:
break;
}
}
}
var datetimeQuery = Query<QNS>.QueryString(q =>
q.DefaultField(f =>
f.Timestamp).Query($"[{fromTime} TO {toTime}]")
);
var searchResults = client.Search<QNS>(s => s.Index(indexName)
.Type(type)
.Query(q =>
q.Filtered(f =>
f.Filter(fq =>
fq.Bool(b =>
b.MinimumShouldMatch(1).Should(patternQueries.ToArray())
)
)
.Query(qd =>
qd.Bool(b =>
b.Must(datetimeQuery)
)
)
)
)
.Sort(sort => sort.Ascending(SortSpecialField.DocumentIndexOrder))
.Aggregations(agg => agg.DistinctBy(q => q.ErrorDescriptionKeyword)));
var results = searchResults.Aggs.AsContainer<QNS>().GetDistinct(d => d.ErrorDescriptionKeyword);
return results.ToList();
}
我需要修改这段代码，使其只返回不同的错误字符串，而不是整个结果集。来自查询的点击数大约是3500，只有2个不同的错误字符串存在。所以把这些记录都拿回来是没有意义的因为我不会用到它。是否有人可以帮助我使用日期范围和模式正则表达式/短语匹配获得正确的聚合查询，仅返回使用Nest或Nest/FluentNest的不同错误记录。
我想你是在找术语聚合。
但是你的整个问题有点奇怪。您是否有一些遗留需求。
第一件事你有两个字段ErrorDescriptionAnalyzed和ErrorDescriptionKeyword你做一个不同的字段只是有一个分析和一个没有?为什么不使用多字段
第二过滤方法已经过时一段时间了。
这里是一个快速的例子，我希望能帮助你。
ElasticClient db = new ElasticClient(uri);
db.DeleteIndex(indexName);
var mappings = new CreateIndexDescriptor(indexName).Mappings(ms => ms.Map<A>(map => map.
AutoMap().
Properties(props =>
props.String(p =>
p.Name(a => a.Text).
Fields(fields =>
fields.String(pr => pr.Name("raw").NotAnalyzed()))))));
db.CreateIndex(mappings);
foreach (var item in Enumerable.Range(0, 10).Select(i => new A
{
Price1 = random.NextDouble() * 1000,
Date = i % 3 == 0 ? new DateTime(1900, 1, 1) : DateTime.Now,
Text = i % 2 == 0 ? "ABC" : "EFG"
}))
{
db.Index(item, inx => inx.Index(indexName));
}
var toDate = DateTime.Now + TimeSpan.FromDays(1);
var fromDate = DateTime.Now - TimeSpan.FromDays(30);
var data = db.Search<A>(s =>
s.Index(indexName)
.Query(q=>
q.DateRange(r => r.Field(f => f.Date).GreaterThan(fromDate).LessThanOrEquals(toDate))
&&
(
//term query is for finding words by default all words are lowercase but you can set a different analyzer
q.Term(t => t.Field(f => f.Text).Value("ABC".ToLower()))
||
//Raw field is not analysed so no need to lower case you can use you query here if you want
q.Term(t => t.Field("text.raw").Value("EFG"))
)
).Aggregations(aggr => aggr.Terms("distinct", aterm => aterm.Field("text.raw"))));