feat: add PostgreSQL schema isolation for playground environment

- Create schema.rs utility module with table_name() function
- Add schema prefix to all SQL queries in postgres_db.rs
- Support dev schema for playground, public for production
- Add DATABASE_SCHEMA, MONGODB_DATABASE, QDRANT_COLLECTION config
- Fix 40+ functions including videos, chunks, frames, vectors, etc.
- Update Cargo dependencies
This commit is contained in:
Warren
2026-03-31 10:30:33 +08:00
parent 95b44f1e55
commit 37d2b66c56
6 changed files with 1118 additions and 308 deletions

409
Cargo.lock generated
View File

@@ -8,6 +8,12 @@ version = "2.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa"
[[package]]
name = "adler32"
version = "1.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "aae1277d39aeec15cb388266ecc24b11c80469deae6067e17a1a7aa9e5c1f234"
[[package]]
name = "aead"
version = "0.5.2"
@@ -279,6 +285,7 @@ dependencies = [
"matchit",
"memchr",
"mime",
"multer",
"percent-encoding",
"pin-project-lite",
"rustversion",
@@ -348,6 +355,26 @@ version = "1.8.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2af50177e190e07a26ab74f8b1efbfe2ef87da2116221318cb1c2e82baf7de06"
[[package]]
name = "bincode"
version = "2.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "36eaf5d7b090263e8150820482d5d93cd964a81e4019913c972f4edcc6edb740"
dependencies = [
"bincode_derive",
"serde",
"unty",
]
[[package]]
name = "bincode_derive"
version = "2.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bf95709a440f45e986983918d0e8a1f30a9b1df04918fc828670606804ac3c09"
dependencies = [
"virtue",
]
[[package]]
name = "bitflags"
version = "1.3.2"
@@ -441,6 +468,25 @@ dependencies = [
"rustversion",
]
[[package]]
name = "cbindgen"
version = "0.29.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "befbfd072a8e81c02f8c507aefce431fe5e7d051f83d48a23ffc9b9fe5a11799"
dependencies = [
"clap",
"heck 0.5.0",
"indexmap 2.13.0",
"log",
"proc-macro2",
"quote",
"serde",
"serde_json",
"syn 2.0.117",
"tempfile",
"toml",
]
[[package]]
name = "cc"
version = "1.2.57"
@@ -448,9 +494,20 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7a0dd1ca384932ff3641c8718a02769f1698e7563dc6974ffd03346116310423"
dependencies = [
"find-msvc-tools",
"jobserver",
"libc",
"shlex",
]
[[package]]
name = "cedarwood"
version = "0.4.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6d910bedd62c24733263d0bed247460853c9d22e8956bd4cd964302095e04e90"
dependencies = [
"smallvec",
]
[[package]]
name = "cfg-if"
version = "1.0.4"
@@ -608,6 +665,15 @@ version = "0.8.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b"
[[package]]
name = "core2"
version = "0.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b49ba7ef1ad6107f8824dbe97de947cbaac53c44e7f9756a1fba0d37c1eec505"
dependencies = [
"memchr",
]
[[package]]
name = "cpufeatures"
version = "0.2.17"
@@ -823,6 +889,12 @@ dependencies = [
"syn 2.0.117",
]
[[package]]
name = "dary_heap"
version = "0.3.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "06d2e3287df1c007e74221c49ca10a95d557349e54b3a75dc2fb14712c751f04"
[[package]]
name = "data-encoding"
version = "2.10.0"
@@ -1057,6 +1129,37 @@ version = "2.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be"
[[package]]
name = "ferrous-opencc"
version = "0.3.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "800f9a39ad26b200b08673c059c2dbf78d1f63901a873e9ac425ba471e690410"
dependencies = [
"anyhow",
"bincode",
"cbindgen",
"ferrous-opencc-compiler",
"fst",
"phf",
"phf_codegen",
"serde",
"serde_json",
"thiserror 2.0.18",
"zstd",
]
[[package]]
name = "ferrous-opencc-compiler"
version = "0.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6ed87f42a19a234e791d6e769928819382c3d210da2c9049f832a1ae772d7082"
dependencies = [
"anyhow",
"bincode",
"fst",
"zstd",
]
[[package]]
name = "filetime"
version = "0.2.27"
@@ -1107,6 +1210,12 @@ version = "0.1.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2"
[[package]]
name = "foldhash"
version = "0.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "77ce24cb58228fbb8aa041425bb1050850ac19177686ea6e0f41a70416f56fdb"
[[package]]
name = "foreign-types"
version = "0.3.2"
@@ -1140,6 +1249,12 @@ dependencies = [
"libc",
]
[[package]]
name = "fst"
version = "0.4.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7ab85b9b05e3978cc9a9cf8fea7f01b494e1a09ed3037e16ba39edc7a29eb61a"
[[package]]
name = "funty"
version = "2.0.0"
@@ -1338,7 +1453,7 @@ checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1"
dependencies = [
"allocator-api2",
"equivalent",
"foldhash",
"foldhash 0.1.5",
]
[[package]]
@@ -1346,6 +1461,11 @@ name = "hashbrown"
version = "0.16.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100"
dependencies = [
"allocator-api2",
"equivalent",
"foldhash 0.2.0",
]
[[package]]
name = "hashlink"
@@ -1698,6 +1818,39 @@ dependencies = [
"icu_properties",
]
[[package]]
name = "include-flate"
version = "0.3.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8a05fb00d9abc625268e0573a519506b264a7d6965de09bac13201bfb44e723d"
dependencies = [
"include-flate-codegen",
"include-flate-compress",
]
[[package]]
name = "include-flate-codegen"
version = "0.3.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "92c3c319a7527668538a8530c541e74e881e94c4f41e1425622d0a41c16468af"
dependencies = [
"include-flate-compress",
"proc-macro-error2",
"proc-macro2",
"quote",
"syn 2.0.117",
]
[[package]]
name = "include-flate-compress"
version = "0.3.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ed0bd9ea81b94169d61c5a397e9faef02153d3711fc62d3270bcde3ac85380d9"
dependencies = [
"libflate",
"zstd",
]
[[package]]
name = "indexmap"
version = "1.9.3"
@@ -1829,6 +1982,29 @@ version = "1.0.17"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "92ecc6618181def0457392ccd0ee51198e065e016d1d527a7ac1b6dc7c1f09d2"
[[package]]
name = "jieba-macros"
version = "0.8.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "348294e44ee7e3c42685da656490f8febc7359632544019621588902216da95c"
dependencies = [
"phf_codegen",
]
[[package]]
name = "jieba-rs"
version = "0.8.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "766bd7012aa5ba49411ebdf4e93bddd59b182d2918e085d58dec5bb9b54b7105"
dependencies = [
"cedarwood",
"include-flate",
"jieba-macros",
"phf",
"regex",
"rustc-hash",
]
[[package]]
name = "jiff"
version = "0.2.23"
@@ -1853,6 +2029,16 @@ dependencies = [
"syn 2.0.117",
]
[[package]]
name = "jobserver"
version = "0.1.34"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9afb3de4395d6b3e67a780b6de64b51c978ecf11cb9a462c66be7d4ca9039d33"
dependencies = [
"getrandom 0.3.4",
"libc",
]
[[package]]
name = "js-sys"
version = "0.3.91"
@@ -1904,6 +2090,30 @@ version = "0.2.183"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b5b646652bf6661599e1da8901b3b9522896f01e736bad5f723fe7a3a27f899d"
[[package]]
name = "libflate"
version = "2.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e3248b8d211bd23a104a42d81b4fa8bb8ac4a3b75e7a43d85d2c9ccb6179cd74"
dependencies = [
"adler32",
"core2",
"crc32fast",
"dary_heap",
"libflate_lz77",
]
[[package]]
name = "libflate_lz77"
version = "2.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a599cb10a9cd92b1300debcef28da8f70b935ec937f44fcd1b70a7c986a11c5c"
dependencies = [
"core2",
"hashbrown 0.16.1",
"rle-decode-fast",
]
[[package]]
name = "libm"
version = "0.2.16"
@@ -2116,8 +2326,10 @@ dependencies = [
"crossterm",
"dotenv",
"env_logger",
"ferrous-opencc",
"futures-util",
"hex",
"jieba-rs",
"libc",
"md5",
"moka",
@@ -2190,6 +2402,23 @@ dependencies = [
"webpki-roots 0.25.4",
]
[[package]]
name = "multer"
version = "3.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "83e87776546dc87511aa5ee218730c92b666d7264ab6ed41f9d215af9cd5224b"
dependencies = [
"bytes",
"encoding_rs",
"futures-util",
"http",
"httparse",
"memchr",
"mime",
"spin",
"version_check",
]
[[package]]
name = "native-tls"
version = "0.2.18"
@@ -2440,6 +2669,59 @@ version = "2.3.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220"
[[package]]
name = "phf"
version = "0.13.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c1562dc717473dbaa4c1f85a36410e03c047b2e7df7f45ee938fbef64ae7fadf"
dependencies = [
"phf_macros",
"phf_shared",
"serde",
]
[[package]]
name = "phf_codegen"
version = "0.13.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "49aa7f9d80421bca176ca8dbfebe668cc7a2684708594ec9f3c0db0805d5d6e1"
dependencies = [
"phf_generator",
"phf_shared",
]
[[package]]
name = "phf_generator"
version = "0.13.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "135ace3a761e564ec88c03a77317a7c6b80bb7f7135ef2544dbe054243b89737"
dependencies = [
"fastrand",
"phf_shared",
]
[[package]]
name = "phf_macros"
version = "0.13.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "812f032b54b1e759ccd5f8b6677695d5268c588701effba24601f6932f8269ef"
dependencies = [
"phf_generator",
"phf_shared",
"proc-macro2",
"quote",
"syn 2.0.117",
]
[[package]]
name = "phf_shared"
version = "0.13.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e57fef6bc5981e38c2ce2d63bfa546861309f875b8a75f092d1d54ae2d64f266"
dependencies = [
"siphasher",
]
[[package]]
name = "pin-project"
version = "1.1.11"
@@ -2599,6 +2881,28 @@ dependencies = [
"version_check",
]
[[package]]
name = "proc-macro-error-attr2"
version = "2.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "96de42df36bb9bba5542fe9f1a054b8cc87e172759a1868aa05c1f3acc89dfc5"
dependencies = [
"proc-macro2",
"quote",
]
[[package]]
name = "proc-macro-error2"
version = "2.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "11ec05c52be0a07b08061f7dd003e7d7092e0472bc731b4af7bb1ef876109802"
dependencies = [
"proc-macro-error-attr2",
"proc-macro2",
"quote",
"syn 2.0.117",
]
[[package]]
name = "proc-macro2"
version = "1.0.106"
@@ -2966,6 +3270,12 @@ dependencies = [
"windows-sys 0.52.0",
]
[[package]]
name = "rle-decode-fast"
version = "1.0.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3582f63211428f83597b51b2ddb88e2a91a9d52d12831f9d08f5e624e8977422"
[[package]]
name = "rsa"
version = "0.9.10"
@@ -3323,6 +3633,15 @@ dependencies = [
"serde_core",
]
[[package]]
name = "serde_spanned"
version = "1.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "876ac351060d4f882bb1032b6369eb0aef79ad9df1ea8bc404874d8cc3d0cd98"
dependencies = [
"serde_core",
]
[[package]]
name = "serde_urlencoded"
version = "0.7.1"
@@ -3458,6 +3777,12 @@ version = "0.3.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e320a6c5ad31d271ad523dcf3ad13e2767ad8b1cb8f047f75a8aeaf8da139da2"
[[package]]
name = "siphasher"
version = "1.0.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b2aa850e253778c88a04c3d7323b043aeda9d3e30d5971937c1855769763678e"
[[package]]
name = "slab"
version = "0.4.12"
@@ -4059,6 +4384,30 @@ dependencies = [
"tokio",
]
[[package]]
name = "toml"
version = "0.9.12+spec-1.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cf92845e79fc2e2def6a5d828f0801e29a2f8acc037becc5ab08595c7d5e9863"
dependencies = [
"indexmap 2.13.0",
"serde_core",
"serde_spanned",
"toml_datetime 0.7.5+spec-1.1.0",
"toml_parser",
"toml_writer",
"winnow 0.7.15",
]
[[package]]
name = "toml_datetime"
version = "0.7.5+spec-1.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "92e1cfed4a3038bc5a127e35a2d360f145e1f4b971b551a2ba5fd7aedf7e1347"
dependencies = [
"serde_core",
]
[[package]]
name = "toml_datetime"
version = "1.0.1+spec-1.1.0"
@@ -4075,9 +4424,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8ca1a40644a28bce036923f6a431df0b34236949d111cc07cb6dca830c9ef2e1"
dependencies = [
"indexmap 2.13.0",
"toml_datetime",
"toml_datetime 1.0.1+spec-1.1.0",
"toml_parser",
"winnow",
"winnow 1.0.0",
]
[[package]]
@@ -4086,9 +4435,15 @@ version = "1.0.10+spec-1.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7df25b4befd31c4816df190124375d5a20c6b6921e2cad937316de3fccd63420"
dependencies = [
"winnow",
"winnow 1.0.0",
]
[[package]]
name = "toml_writer"
version = "1.1.0+spec-1.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d282ade6016312faf3e41e57ebbba0c073e4056dab1232ab1cb624199648f8ed"
[[package]]
name = "tonic"
version = "0.12.3"
@@ -4393,6 +4748,12 @@ version = "0.9.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1"
[[package]]
name = "unty"
version = "0.0.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6d49784317cd0d1ee7ec5c716dd598ec5b4483ea832a2dced265471cc0f690ae"
[[package]]
name = "url"
version = "2.5.8"
@@ -4491,6 +4852,12 @@ version = "0.9.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a"
[[package]]
name = "virtue"
version = "0.0.18"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "051eb1abcf10076295e815102942cc58f9d5e3b4560e46e53c21e8ff6f3af7b1"
[[package]]
name = "walkdir"
version = "2.5.0"
@@ -5029,6 +5396,12 @@ version = "0.53.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d6bbff5f0aada427a1e5a6da5f1f98158182f26556f345ac9e04d36d0ebed650"
[[package]]
name = "winnow"
version = "0.7.15"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "df79d97927682d2fd8adb29682d1140b343be4ac0f08fd68b7765d9c059d3945"
[[package]]
name = "winnow"
version = "1.0.0"
@@ -5281,3 +5654,31 @@ name = "zmij"
version = "1.0.21"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b8848ee67ecc8aedbaf3e4122217aff892639231befc6a1b58d29fff4c2cabaa"
[[package]]
name = "zstd"
version = "0.13.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e91ee311a569c327171651566e07972200e76fcfe2242a4fa446149a3881c08a"
dependencies = [
"zstd-safe",
]
[[package]]
name = "zstd-safe"
version = "7.2.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8f49c4d5f0abb602a93fb8736af2a4f4dd9512e36f7f570d66e65ff867ed3b9d"
dependencies = [
"zstd-sys",
]
[[package]]
name = "zstd-sys"
version = "2.0.16+zstd.1.5.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "91e19ebc2adc8f83e43039e79776e3fda8ca919132d68a1fed6a5faca2683748"
dependencies = [
"cc",
"pkg-config",
]

View File

@@ -33,10 +33,14 @@ sha2 = "0.10"
hex = "0.4"
uuid = { version = "1.0", features = ["v4"] }
# Security
subtle = "2.5"
aes-gcm = "0.10"
base64 = "0.22"
# Security
subtle = "2.5"
aes-gcm = "0.10"
base64 = "0.22"
# Text processing
jieba-rs = "0.8.1"
ferrous-opencc = { version = "0.3.1", features = ["s2t-conversion", "t2s-conversion"] }
# Cache
moka = { version = "0.12", features = ["future"] }
@@ -50,7 +54,7 @@ qdrant-client = "1.7"
reqwest = { version = "0.12", features = ["json"] }
# HTTP Server
axum = "0.7"
axum = { version = "0.7", features = ["multipart"] }
tower = "0.4"
# API Documentation
@@ -98,3 +102,11 @@ path = "src/playground.rs"
[[bin]]
name = "fix_chunks"
path = "src/bin/fix_chunks.rs"
[[bin]]
name = "migrate_chinese_text"
path = "src/bin/migrate_chinese_text.rs"
[[bin]]
name = "test_bm25_simple"
path = "src/bin/test_bm25_simple.rs"

View File

@@ -78,6 +78,15 @@ pub static SERVER_PORT: Lazy<u16> = Lazy::new(|| {
pub static REDIS_KEY_PREFIX: Lazy<String> =
Lazy::new(|| env::var("MOMENTRY_REDIS_PREFIX").unwrap_or_else(|_| "momentry:".to_string()));
pub static DATABASE_SCHEMA: Lazy<String> =
Lazy::new(|| env::var("DATABASE_SCHEMA").unwrap_or_else(|_| "public".to_string()));
pub static MONGODB_DATABASE: Lazy<String> =
Lazy::new(|| env::var("MONGODB_DATABASE").unwrap_or_else(|_| "momentry".to_string()));
pub static QDRANT_COLLECTION: Lazy<String> =
Lazy::new(|| env::var("QDRANT_COLLECTION").unwrap_or_else(|_| "momentry_rule1".to_string()));
pub mod processor {
use super::*;

View File

@@ -1,6 +1,8 @@
use anyhow::Result;
use async_trait::async_trait;
pub mod schema;
use crate::core::chunk::Chunk;
#[derive(Debug, Clone)]

File diff suppressed because it is too large Load Diff

30
src/core/db/schema.rs Normal file
View File

@@ -0,0 +1,30 @@
use crate::core::config::DATABASE_SCHEMA;
use once_cell::sync::Lazy;
pub static SCHEMA_PREFIX: Lazy<String> = Lazy::new(|| {
let schema = DATABASE_SCHEMA.as_str();
if schema == "public" {
String::new()
} else {
format!("{}.", schema)
}
});
pub fn table_name(table: &str) -> String {
let prefix = SCHEMA_PREFIX.as_str();
if prefix.is_empty() {
table.to_string()
} else {
format!("{}{}", prefix, table)
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_table_name_public() {
assert_eq!(table_name("videos"), "videos");
}
}