fix: keyword search - add text_content field and CJK support

- Added text_content field to SearchResult and SemanticSearchResult
- Added get_chunk_by_id_no_embedding for keyword results without embedding requirement
- Fixed search_bm25 to use position-based ranking for CJK/Korean content
- Fixed sqlx column mapping with explicit alias
- Skip text_match filter for keyword-only results
- Use text_content as fallback when summary is empty
This commit is contained in:
Accusys
2026-07-02 21:16:38 +08:00
parent 5a9d4325d8
commit 78364afc51
2 changed files with 119 additions and 47 deletions
+47 -31
View File
@@ -34,6 +34,7 @@ pub struct SearchResult {
pub end_time: f64,
pub raw_text: Option<String>,
pub summary: Option<String>,
pub text_content: Option<String>,
pub metadata: Option<serde_json::Value>,
pub similarity: Option<f64>,
pub file_name: Option<String>,
@@ -82,6 +83,7 @@ async fn enrich_from_pg(
end_time: p.end_time,
raw_text: None,
summary: Some(p.summary),
text_content: p.text_content.clone(),
metadata: p.metadata.clone(),
similarity: Some(qdrant_score as f64),
file_name: None,
@@ -109,6 +111,7 @@ fn pg_result_to_search(p: &SemanticSearchResult) -> SearchResult {
end_time: p.end_time,
raw_text: None,
summary: Some(p.summary.clone()),
text_content: p.text_content.clone(),
metadata: p.metadata.clone(),
similarity: p.similarity,
file_name: None,
@@ -381,43 +384,55 @@ pub async fn smart_search(
let mut final_results = Vec::new();
for mr in ranked.iter().take(limit * 3) {
// 取更多結果以便過濾
if let Some(pg) = db
.get_chunk_by_file_and_chunk_id(&mr.file_uuid, &mr.chunk_id)
.await
.ok()
.flatten()
{
// 關鍵字過濾: CJK 用子字串匹配,英文用單詞邊界匹配
let summary_lower = pg.summary.to_lowercase();
let query_words: Vec<String> = query_lower
.split_whitespace()
.map(|s| s.to_string())
.collect();
// Use no_embedding version for keyword results, regular for semantic
let pg_opt = if mr.keyword_score.is_some() && mr.semantic_score.is_none() {
db.get_chunk_by_id_no_embedding(&mr.file_uuid, &mr.chunk_id).await
} else {
db.get_chunk_by_file_and_chunk_id(&mr.file_uuid, &mr.chunk_id).await
};
if let Some(pg) = pg_opt.ok().flatten() {
// 關鍵字結果跳過 text_match 過濾(search_bm25 已經匹配過)
let is_keyword_only = mr.keyword_score.is_some() && mr.semantic_score.is_none();
if !is_keyword_only {
// 關鍵字過濾: CJK 用子字串匹配,英文用單詞邊界匹配
let summary_lower = pg.summary.to_lowercase();
let query_words: Vec<String> = query_lower
.split_whitespace()
.map(|s| s.to_string())
.collect();
let text_match = !pg.summary.is_empty() && {
let has_cjk = |s: &str| -> bool {
s.chars().any(|c| {
('\u{4E00}'..='\u{9FFF}').contains(&c)
|| ('\u{3040}'..='\u{309F}').contains(&c)
|| ('\u{30A0}'..='\u{30FF}').contains(&c)
|| ('\u{AC00}'..='\u{D7AF}').contains(&c)
})
let text_match = !pg.summary.is_empty() && {
let has_cjk = |s: &str| -> bool {
s.chars().any(|c| {
('\u{4E00}'..='\u{9FFF}').contains(&c)
|| ('\u{3040}'..='\u{309F}').contains(&c)
|| ('\u{30A0}'..='\u{30FF}').contains(&c)
|| ('\u{AC00}'..='\u{D7AF}').contains(&c)
})
};
if has_cjk(&query_lower) || has_cjk(&summary_lower) {
query_words.iter().all(|w| summary_lower.contains(w))
} else {
let bordered = format!(" {} ", summary_lower);
query_words
.iter()
.all(|w| bordered.contains(&format!(" {} ", w)))
}
};
if has_cjk(&query_lower) || has_cjk(&summary_lower) {
query_words.iter().all(|w| summary_lower.contains(w))
} else {
let bordered = format!(" {} ", summary_lower);
query_words
.iter()
.all(|w| bordered.contains(&format!(" {} ", w)))
if !text_match {
continue;
}
};
if !text_match && mr.semantic_score.is_none() {
continue;
}
// 使用 text_content 如果 summary 為空
let display_text = if pg.summary.is_empty() {
pg.text_content.clone().unwrap_or_default()
} else {
pg.summary.clone()
};
final_results.push(SearchResult {
id: 0,
file_uuid: pg.file_uuid.clone(),
@@ -430,6 +445,7 @@ pub async fn smart_search(
end_time: pg.end_time,
raw_text: None,
summary: Some(pg.summary),
text_content: pg.text_content.clone(),
metadata: pg.metadata.clone(),
similarity: Some(mr.score),
file_name: None,