fix: keyword search - add text_content field and CJK support
- Added text_content field to SearchResult and SemanticSearchResult - Added get_chunk_by_id_no_embedding for keyword results without embedding requirement - Fixed search_bm25 to use position-based ranking for CJK/Korean content - Fixed sqlx column mapping with explicit alias - Skip text_match filter for keyword-only results - Use text_content as fallback when summary is empty
This commit is contained in:
+47
-31
@@ -34,6 +34,7 @@ pub struct SearchResult {
|
||||
pub end_time: f64,
|
||||
pub raw_text: Option<String>,
|
||||
pub summary: Option<String>,
|
||||
pub text_content: Option<String>,
|
||||
pub metadata: Option<serde_json::Value>,
|
||||
pub similarity: Option<f64>,
|
||||
pub file_name: Option<String>,
|
||||
@@ -82,6 +83,7 @@ async fn enrich_from_pg(
|
||||
end_time: p.end_time,
|
||||
raw_text: None,
|
||||
summary: Some(p.summary),
|
||||
text_content: p.text_content.clone(),
|
||||
metadata: p.metadata.clone(),
|
||||
similarity: Some(qdrant_score as f64),
|
||||
file_name: None,
|
||||
@@ -109,6 +111,7 @@ fn pg_result_to_search(p: &SemanticSearchResult) -> SearchResult {
|
||||
end_time: p.end_time,
|
||||
raw_text: None,
|
||||
summary: Some(p.summary.clone()),
|
||||
text_content: p.text_content.clone(),
|
||||
metadata: p.metadata.clone(),
|
||||
similarity: p.similarity,
|
||||
file_name: None,
|
||||
@@ -381,43 +384,55 @@ pub async fn smart_search(
|
||||
let mut final_results = Vec::new();
|
||||
for mr in ranked.iter().take(limit * 3) {
|
||||
// 取更多結果以便過濾
|
||||
if let Some(pg) = db
|
||||
.get_chunk_by_file_and_chunk_id(&mr.file_uuid, &mr.chunk_id)
|
||||
.await
|
||||
.ok()
|
||||
.flatten()
|
||||
{
|
||||
// 關鍵字過濾: CJK 用子字串匹配,英文用單詞邊界匹配
|
||||
let summary_lower = pg.summary.to_lowercase();
|
||||
let query_words: Vec<String> = query_lower
|
||||
.split_whitespace()
|
||||
.map(|s| s.to_string())
|
||||
.collect();
|
||||
// Use no_embedding version for keyword results, regular for semantic
|
||||
let pg_opt = if mr.keyword_score.is_some() && mr.semantic_score.is_none() {
|
||||
db.get_chunk_by_id_no_embedding(&mr.file_uuid, &mr.chunk_id).await
|
||||
} else {
|
||||
db.get_chunk_by_file_and_chunk_id(&mr.file_uuid, &mr.chunk_id).await
|
||||
};
|
||||
if let Some(pg) = pg_opt.ok().flatten() {
|
||||
// 關鍵字結果跳過 text_match 過濾(search_bm25 已經匹配過)
|
||||
let is_keyword_only = mr.keyword_score.is_some() && mr.semantic_score.is_none();
|
||||
if !is_keyword_only {
|
||||
// 關鍵字過濾: CJK 用子字串匹配,英文用單詞邊界匹配
|
||||
let summary_lower = pg.summary.to_lowercase();
|
||||
let query_words: Vec<String> = query_lower
|
||||
.split_whitespace()
|
||||
.map(|s| s.to_string())
|
||||
.collect();
|
||||
|
||||
let text_match = !pg.summary.is_empty() && {
|
||||
let has_cjk = |s: &str| -> bool {
|
||||
s.chars().any(|c| {
|
||||
('\u{4E00}'..='\u{9FFF}').contains(&c)
|
||||
|| ('\u{3040}'..='\u{309F}').contains(&c)
|
||||
|| ('\u{30A0}'..='\u{30FF}').contains(&c)
|
||||
|| ('\u{AC00}'..='\u{D7AF}').contains(&c)
|
||||
})
|
||||
let text_match = !pg.summary.is_empty() && {
|
||||
let has_cjk = |s: &str| -> bool {
|
||||
s.chars().any(|c| {
|
||||
('\u{4E00}'..='\u{9FFF}').contains(&c)
|
||||
|| ('\u{3040}'..='\u{309F}').contains(&c)
|
||||
|| ('\u{30A0}'..='\u{30FF}').contains(&c)
|
||||
|| ('\u{AC00}'..='\u{D7AF}').contains(&c)
|
||||
})
|
||||
};
|
||||
|
||||
if has_cjk(&query_lower) || has_cjk(&summary_lower) {
|
||||
query_words.iter().all(|w| summary_lower.contains(w))
|
||||
} else {
|
||||
let bordered = format!(" {} ", summary_lower);
|
||||
query_words
|
||||
.iter()
|
||||
.all(|w| bordered.contains(&format!(" {} ", w)))
|
||||
}
|
||||
};
|
||||
|
||||
if has_cjk(&query_lower) || has_cjk(&summary_lower) {
|
||||
query_words.iter().all(|w| summary_lower.contains(w))
|
||||
} else {
|
||||
let bordered = format!(" {} ", summary_lower);
|
||||
query_words
|
||||
.iter()
|
||||
.all(|w| bordered.contains(&format!(" {} ", w)))
|
||||
if !text_match {
|
||||
continue;
|
||||
}
|
||||
};
|
||||
|
||||
if !text_match && mr.semantic_score.is_none() {
|
||||
continue;
|
||||
}
|
||||
|
||||
// 使用 text_content 如果 summary 為空
|
||||
let display_text = if pg.summary.is_empty() {
|
||||
pg.text_content.clone().unwrap_or_default()
|
||||
} else {
|
||||
pg.summary.clone()
|
||||
};
|
||||
|
||||
final_results.push(SearchResult {
|
||||
id: 0,
|
||||
file_uuid: pg.file_uuid.clone(),
|
||||
@@ -430,6 +445,7 @@ pub async fn smart_search(
|
||||
end_time: pg.end_time,
|
||||
raw_text: None,
|
||||
summary: Some(pg.summary),
|
||||
text_content: pg.text_content.clone(),
|
||||
metadata: pg.metadata.clone(),
|
||||
similarity: Some(mr.score),
|
||||
file_name: None,
|
||||
|
||||
Reference in New Issue
Block a user