Compare commits
114 Commits
614275f77a
..
main
| Author | SHA1 | Date | |
|---|---|---|---|
| 6292a77dff | |||
| dfe464303d | |||
| fe983c6528 | |||
| 4fa8fd8c1f | |||
| deac3b9b6e | |||
| 65cd68cad4 | |||
| 86984295bf | |||
| 18aa067be7 | |||
| 5ea9293cfd | |||
| bd28739002 | |||
| 820186a48c | |||
| df0b2f5ff8 | |||
| 257ffcb716 | |||
| f492a96077 | |||
| f3b75fae3d | |||
| 12ddec24b4 | |||
| 6f223c9232 | |||
| dc217e8903 | |||
| ffc09b97bb | |||
| 7f7e88e2c4 | |||
| 1418e9958b | |||
| 85218333d9 | |||
| a7a01a8e86 | |||
| 0efaddaffc | |||
| 0f77983483 | |||
| 103bb66924 | |||
| e07d17aee7 | |||
| 72503f7db9 | |||
| 9f0803bf56 | |||
| f8fba20890 | |||
| e4d1be01ef | |||
| d76a200560 | |||
| 2d8e9049b0 | |||
| 55caeabd94 | |||
| 26d4199203 | |||
| 90219a65ad | |||
| 1d9e140e6c | |||
| 5f12e9f5d7 | |||
| ffc3f03744 | |||
| 7c4476e19c | |||
| 57fd6a475f | |||
| 5300b672cb | |||
| 637227f4e4 | |||
| d4f60929fa | |||
| e7863a3034 | |||
| 8ef1406ed3 | |||
| bb796ec6b9 | |||
| 9dd2eefeea | |||
| 0c4459ae66 | |||
| 5b0086f6f0 | |||
| 3029327d5e | |||
| 1c8c47d5fa | |||
| 25991c71b2 | |||
| 866d0536c8 | |||
| 64709ec529 | |||
| a8d81f2a9c | |||
| 20b208bb7f | |||
| 60e4329eed | |||
| 37d0fe1a3c | |||
| 4003864d28 | |||
| 8039f0d375 | |||
| 3d395584a8 | |||
| cf57d46ca5 | |||
| 8a5a23a309 | |||
| a7f50ff747 | |||
| 41f0217450 | |||
| e7a9f886ed | |||
| cd184daa20 | |||
| 060f43f0c4 | |||
| 63b765f68e | |||
| e9eca1b492 | |||
| 4db72fff4a | |||
| 52c38b1919 | |||
| 054bf55490 | |||
| e267b43424 | |||
| c89f6c96ae | |||
| ebe976eee4 | |||
| 9ae0402318 | |||
| 3c5de4e6a3 | |||
| 88590d3611 | |||
| 912bc21929 | |||
| 4ab282bbff | |||
| 382ea2e28b | |||
| 98239c09d4 | |||
| 104e7f5f9c | |||
| 097521b35d | |||
| aae8669c9f | |||
| 08244032a8 | |||
| 7d229d0b62 | |||
| 321310582b | |||
| 9b02bbac27 | |||
| 02d98419e1 | |||
| ca0f541a79 | |||
| 5487ad63a6 | |||
| f5074b2ce2 | |||
| 49873cb302 | |||
| c2ff6fc90e | |||
| 23e0996b81 | |||
| 94a7584e64 | |||
| 5c9b51fc49 | |||
| 790efe13f4 | |||
| 6242a5eaab | |||
| ed55c6050e | |||
| 9c82830959 | |||
| 2a0376cc58 | |||
| a56207db0b | |||
| 12ec190831 | |||
| b71510b2e8 | |||
| 1408646424 | |||
| 0322e2d4b6 | |||
| 43c135e877 | |||
| ab11983c1b | |||
| 5000ba7c14 | |||
| 9acd174388 |
Generated
+394
-5
@@ -720,6 +720,18 @@ dependencies = [
|
|||||||
"shlex",
|
"shlex",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "ccm"
|
||||||
|
version = "0.5.0"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "9ae3c82e4355234767756212c570e29833699ab63e6ffd161887314cc5b43847"
|
||||||
|
dependencies = [
|
||||||
|
"aead 0.5.2",
|
||||||
|
"cipher 0.4.4",
|
||||||
|
"ctr 0.9.2",
|
||||||
|
"subtle",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "ccm"
|
name = "ccm"
|
||||||
version = "0.6.0-rc.3"
|
version = "0.6.0-rc.3"
|
||||||
@@ -950,6 +962,26 @@ version = "0.1.5"
|
|||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "245097e9a4535ee1e3e3931fcfcd55a796a44c643e8596ff6566d68f09b87bbc"
|
checksum = "245097e9a4535ee1e3e3931fcfcd55a796a44c643e8596ff6566d68f09b87bbc"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "core-foundation"
|
||||||
|
version = "0.9.4"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "91e195e091a93c46f7102ec7818a2aa394e1e1771c3ab4825963fa03e45afb8f"
|
||||||
|
dependencies = [
|
||||||
|
"core-foundation-sys",
|
||||||
|
"libc",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "core-foundation"
|
||||||
|
version = "0.10.1"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "b2a6cd9ae233e7f62ba4e9353e81a88df7fc8a5987b8d445b4d90c879bd156f6"
|
||||||
|
dependencies = [
|
||||||
|
"core-foundation-sys",
|
||||||
|
"libc",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "core-foundation-sys"
|
name = "core-foundation-sys"
|
||||||
version = "0.8.7"
|
version = "0.8.7"
|
||||||
@@ -1810,6 +1842,21 @@ version = "0.2.0"
|
|||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "77ce24cb58228fbb8aa041425bb1050850ac19177686ea6e0f41a70416f56fdb"
|
checksum = "77ce24cb58228fbb8aa041425bb1050850ac19177686ea6e0f41a70416f56fdb"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "foreign-types"
|
||||||
|
version = "0.3.2"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1"
|
||||||
|
dependencies = [
|
||||||
|
"foreign-types-shared",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "foreign-types-shared"
|
||||||
|
version = "0.1.1"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b"
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "form_urlencoded"
|
name = "form_urlencoded"
|
||||||
version = "1.2.2"
|
version = "1.2.2"
|
||||||
@@ -2100,6 +2147,25 @@ dependencies = [
|
|||||||
"subtle",
|
"subtle",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "h2"
|
||||||
|
version = "0.4.15"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "6cb093c84e8bd9b188d4c4a8cb6579fc016968d14c99882163cd3ff402a4f155"
|
||||||
|
dependencies = [
|
||||||
|
"atomic-waker",
|
||||||
|
"bytes",
|
||||||
|
"fnv",
|
||||||
|
"futures-core",
|
||||||
|
"futures-sink",
|
||||||
|
"http",
|
||||||
|
"indexmap",
|
||||||
|
"slab",
|
||||||
|
"tokio",
|
||||||
|
"tokio-util",
|
||||||
|
"tracing",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "hashbrown"
|
name = "hashbrown"
|
||||||
version = "0.14.5"
|
version = "0.14.5"
|
||||||
@@ -2304,6 +2370,7 @@ dependencies = [
|
|||||||
"bytes",
|
"bytes",
|
||||||
"futures-channel",
|
"futures-channel",
|
||||||
"futures-core",
|
"futures-core",
|
||||||
|
"h2",
|
||||||
"http",
|
"http",
|
||||||
"http-body",
|
"http-body",
|
||||||
"httparse",
|
"httparse",
|
||||||
@@ -2312,6 +2379,38 @@ dependencies = [
|
|||||||
"pin-project-lite",
|
"pin-project-lite",
|
||||||
"smallvec",
|
"smallvec",
|
||||||
"tokio",
|
"tokio",
|
||||||
|
"want",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "hyper-rustls"
|
||||||
|
version = "0.27.9"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "33ca68d021ef39cf6463ab54c1d0f5daf03377b70561305bb89a8f83aab66e0f"
|
||||||
|
dependencies = [
|
||||||
|
"http",
|
||||||
|
"hyper",
|
||||||
|
"hyper-util",
|
||||||
|
"rustls",
|
||||||
|
"tokio",
|
||||||
|
"tokio-rustls",
|
||||||
|
"tower-service",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "hyper-tls"
|
||||||
|
version = "0.6.0"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "70206fc6890eaca9fde8a0bf71caa2ddfc9fe045ac9e5c70df101a7dbde866e0"
|
||||||
|
dependencies = [
|
||||||
|
"bytes",
|
||||||
|
"http-body-util",
|
||||||
|
"hyper",
|
||||||
|
"hyper-util",
|
||||||
|
"native-tls",
|
||||||
|
"tokio",
|
||||||
|
"tokio-native-tls",
|
||||||
|
"tower-service",
|
||||||
]
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
@@ -2320,13 +2419,23 @@ version = "0.1.20"
|
|||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "96547c2556ec9d12fb1578c4eaf448b04993e7fb79cbaad930a656880a6bdfa0"
|
checksum = "96547c2556ec9d12fb1578c4eaf448b04993e7fb79cbaad930a656880a6bdfa0"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
|
"base64",
|
||||||
"bytes",
|
"bytes",
|
||||||
|
"futures-channel",
|
||||||
|
"futures-util",
|
||||||
"http",
|
"http",
|
||||||
"http-body",
|
"http-body",
|
||||||
"hyper",
|
"hyper",
|
||||||
|
"ipnet",
|
||||||
|
"libc",
|
||||||
|
"percent-encoding",
|
||||||
"pin-project-lite",
|
"pin-project-lite",
|
||||||
|
"socket2 0.6.3",
|
||||||
|
"system-configuration",
|
||||||
"tokio",
|
"tokio",
|
||||||
"tower-service",
|
"tower-service",
|
||||||
|
"tracing",
|
||||||
|
"windows-registry",
|
||||||
]
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
@@ -2695,6 +2804,40 @@ dependencies = [
|
|||||||
"spin",
|
"spin",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "lber"
|
||||||
|
version = "0.4.2"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "2df7f9fd9f64cf8f59e1a4a0753fe7d575a5b38d3d7ac5758dcee9357d83ef0a"
|
||||||
|
dependencies = [
|
||||||
|
"bytes",
|
||||||
|
"nom",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "ldap3"
|
||||||
|
version = "0.11.5"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "166199a8207874a275144c8a94ff6eed5fcbf5c52303e4d9b4d53a0c7ac76554"
|
||||||
|
dependencies = [
|
||||||
|
"async-trait",
|
||||||
|
"bytes",
|
||||||
|
"futures",
|
||||||
|
"futures-util",
|
||||||
|
"lazy_static",
|
||||||
|
"lber",
|
||||||
|
"log",
|
||||||
|
"native-tls",
|
||||||
|
"nom",
|
||||||
|
"percent-encoding",
|
||||||
|
"thiserror 1.0.69",
|
||||||
|
"tokio",
|
||||||
|
"tokio-native-tls",
|
||||||
|
"tokio-stream",
|
||||||
|
"tokio-util",
|
||||||
|
"url",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "leb128fmt"
|
name = "leb128fmt"
|
||||||
version = "0.1.0"
|
version = "0.1.0"
|
||||||
@@ -2830,6 +2973,15 @@ dependencies = [
|
|||||||
"libc",
|
"libc",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "lz4_flex"
|
||||||
|
version = "0.11.6"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "373f5eceeeab7925e0c1098212f2fbc4d416adec9d35051a6ab251e824c1854a"
|
||||||
|
dependencies = [
|
||||||
|
"twox-hash",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "lz4_flex"
|
name = "lz4_flex"
|
||||||
version = "0.13.1"
|
version = "0.13.1"
|
||||||
@@ -2890,15 +3042,22 @@ dependencies = [
|
|||||||
"futures-util",
|
"futures-util",
|
||||||
"hex",
|
"hex",
|
||||||
"hmac 0.12.1",
|
"hmac 0.12.1",
|
||||||
|
"http",
|
||||||
|
"lazy_static",
|
||||||
|
"ldap3",
|
||||||
"log",
|
"log",
|
||||||
|
"lz4_flex 0.11.6",
|
||||||
"md5 0.8.0",
|
"md5 0.8.0",
|
||||||
|
"nfsserve",
|
||||||
"nix 0.29.0",
|
"nix 0.29.0",
|
||||||
|
"once_cell",
|
||||||
"poly1305 0.8.0",
|
"poly1305 0.8.0",
|
||||||
"postgres",
|
"postgres",
|
||||||
"pulldown-cmark",
|
"pulldown-cmark",
|
||||||
"rand 0.8.6",
|
"rand 0.8.6",
|
||||||
"rayon",
|
"rayon",
|
||||||
"regex",
|
"regex",
|
||||||
|
"reqwest",
|
||||||
"rusqlite",
|
"rusqlite",
|
||||||
"russh",
|
"russh",
|
||||||
"russh-keys",
|
"russh-keys",
|
||||||
@@ -2919,6 +3078,7 @@ dependencies = [
|
|||||||
"tokio-postgres",
|
"tokio-postgres",
|
||||||
"tokio-util",
|
"tokio-util",
|
||||||
"toml",
|
"toml",
|
||||||
|
"tower-http 0.5.2",
|
||||||
"tracing",
|
"tracing",
|
||||||
"tracing-subscriber",
|
"tracing-subscriber",
|
||||||
"unrar",
|
"unrar",
|
||||||
@@ -2926,6 +3086,8 @@ dependencies = [
|
|||||||
"url",
|
"url",
|
||||||
"uuid",
|
"uuid",
|
||||||
"x25519-dalek",
|
"x25519-dalek",
|
||||||
|
"xattr",
|
||||||
|
"xmltree",
|
||||||
"xz2",
|
"xz2",
|
||||||
"zip",
|
"zip",
|
||||||
"zstd 0.13.3",
|
"zstd 0.13.3",
|
||||||
@@ -3236,6 +3398,23 @@ dependencies = [
|
|||||||
"version_check",
|
"version_check",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "native-tls"
|
||||||
|
version = "0.2.18"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "465500e14ea162429d264d44189adc38b199b62b1c21eea9f69e4b73cb03bbf2"
|
||||||
|
dependencies = [
|
||||||
|
"libc",
|
||||||
|
"log",
|
||||||
|
"openssl",
|
||||||
|
"openssl-probe",
|
||||||
|
"openssl-sys",
|
||||||
|
"schannel",
|
||||||
|
"security-framework",
|
||||||
|
"security-framework-sys",
|
||||||
|
"tempfile",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "nfsserve"
|
name = "nfsserve"
|
||||||
version = "0.11.0"
|
version = "0.11.0"
|
||||||
@@ -3561,10 +3740,41 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
|||||||
checksum = "c08d65885ee38876c4f86fa503fb49d7b507c2b62552df7c70b2fce627e06381"
|
checksum = "c08d65885ee38876c4f86fa503fb49d7b507c2b62552df7c70b2fce627e06381"
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "openssl-sys"
|
name = "openssl"
|
||||||
version = "0.9.116"
|
version = "0.10.81"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "f28a22dc7140cda5f096e5e7724a6962ca81a7f8bfd2979f9b18c11af56318c4"
|
checksum = "77823a27f0babb03091cb9ed9ef80af3b39dbc82f97e8fa530374b7dafd87a45"
|
||||||
|
dependencies = [
|
||||||
|
"bitflags 2.11.1",
|
||||||
|
"cfg-if",
|
||||||
|
"foreign-types",
|
||||||
|
"libc",
|
||||||
|
"openssl-macros",
|
||||||
|
"openssl-sys",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "openssl-macros"
|
||||||
|
version = "0.1.1"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c"
|
||||||
|
dependencies = [
|
||||||
|
"proc-macro2",
|
||||||
|
"quote",
|
||||||
|
"syn 2.0.117",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "openssl-probe"
|
||||||
|
version = "0.2.1"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "7c87def4c32ab89d880effc9e097653c8da5d6ef28e6b539d313baaacfbafcbe"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "openssl-sys"
|
||||||
|
version = "0.9.117"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "b47e7e6bb2c38cd930d25a23b40fa52e068c10e85f3e03a7f5ba5aaca5713695"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"cc",
|
"cc",
|
||||||
"libc",
|
"libc",
|
||||||
@@ -4415,6 +4625,46 @@ dependencies = [
|
|||||||
"winapi",
|
"winapi",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "reqwest"
|
||||||
|
version = "0.12.28"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "eddd3ca559203180a307f12d114c268abf583f59b03cb906fd0b3ff8646c1147"
|
||||||
|
dependencies = [
|
||||||
|
"base64",
|
||||||
|
"bytes",
|
||||||
|
"encoding_rs",
|
||||||
|
"futures-core",
|
||||||
|
"h2",
|
||||||
|
"http",
|
||||||
|
"http-body",
|
||||||
|
"http-body-util",
|
||||||
|
"hyper",
|
||||||
|
"hyper-rustls",
|
||||||
|
"hyper-tls",
|
||||||
|
"hyper-util",
|
||||||
|
"js-sys",
|
||||||
|
"log",
|
||||||
|
"mime",
|
||||||
|
"native-tls",
|
||||||
|
"percent-encoding",
|
||||||
|
"pin-project-lite",
|
||||||
|
"rustls-pki-types",
|
||||||
|
"serde",
|
||||||
|
"serde_json",
|
||||||
|
"serde_urlencoded",
|
||||||
|
"sync_wrapper",
|
||||||
|
"tokio",
|
||||||
|
"tokio-native-tls",
|
||||||
|
"tower",
|
||||||
|
"tower-http 0.6.11",
|
||||||
|
"tower-service",
|
||||||
|
"url",
|
||||||
|
"wasm-bindgen",
|
||||||
|
"wasm-bindgen-futures",
|
||||||
|
"web-sys",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "rfc6979"
|
name = "rfc6979"
|
||||||
version = "0.4.0"
|
version = "0.4.0"
|
||||||
@@ -4864,6 +5114,15 @@ dependencies = [
|
|||||||
"cipher 0.5.2",
|
"cipher 0.5.2",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "schannel"
|
||||||
|
version = "0.1.29"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "91c1b7e4904c873ef0710c1f407dde2e6287de2bebc1bbbf7d430bb7cbffd939"
|
||||||
|
dependencies = [
|
||||||
|
"windows-sys 0.61.2",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "scoped-tls"
|
name = "scoped-tls"
|
||||||
version = "1.0.1"
|
version = "1.0.1"
|
||||||
@@ -4927,6 +5186,29 @@ dependencies = [
|
|||||||
"zeroize",
|
"zeroize",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "security-framework"
|
||||||
|
version = "3.7.0"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "b7f4bc775c73d9a02cde8bf7b2ec4c9d12743edf609006c7facc23998404cd1d"
|
||||||
|
dependencies = [
|
||||||
|
"bitflags 2.11.1",
|
||||||
|
"core-foundation 0.10.1",
|
||||||
|
"core-foundation-sys",
|
||||||
|
"libc",
|
||||||
|
"security-framework-sys",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "security-framework-sys"
|
||||||
|
version = "2.17.0"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "6ce2691df843ecc5d231c0b14ece2acc3efb62c0a398c7e1d875f3983ce020e3"
|
||||||
|
dependencies = [
|
||||||
|
"core-foundation-sys",
|
||||||
|
"libc",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "semver"
|
name = "semver"
|
||||||
version = "1.0.28"
|
version = "1.0.28"
|
||||||
@@ -5209,10 +5491,12 @@ name = "smb-server"
|
|||||||
version = "0.4.1"
|
version = "0.4.1"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"aes 0.8.4",
|
"aes 0.8.4",
|
||||||
|
"aes-gcm 0.10.3",
|
||||||
"async-trait",
|
"async-trait",
|
||||||
"binrw",
|
"binrw",
|
||||||
"bytes",
|
"bytes",
|
||||||
"cap-std",
|
"cap-std",
|
||||||
|
"ccm 0.5.0",
|
||||||
"cmac 0.7.2",
|
"cmac 0.7.2",
|
||||||
"getrandom 0.4.2",
|
"getrandom 0.4.2",
|
||||||
"hex",
|
"hex",
|
||||||
@@ -5227,6 +5511,7 @@ dependencies = [
|
|||||||
"tracing",
|
"tracing",
|
||||||
"tracing-subscriber",
|
"tracing-subscriber",
|
||||||
"uuid",
|
"uuid",
|
||||||
|
"xattr",
|
||||||
]
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
@@ -5236,7 +5521,7 @@ dependencies = [
|
|||||||
"aes 0.9.1",
|
"aes 0.9.1",
|
||||||
"aes-gcm 0.11.0-rc.4",
|
"aes-gcm 0.11.0-rc.4",
|
||||||
"async-trait",
|
"async-trait",
|
||||||
"ccm",
|
"ccm 0.6.0-rc.3",
|
||||||
"cmac 0.8.0-rc.5",
|
"cmac 0.8.0-rc.5",
|
||||||
"digest 0.11.3",
|
"digest 0.11.3",
|
||||||
"env_logger",
|
"env_logger",
|
||||||
@@ -5244,7 +5529,7 @@ dependencies = [
|
|||||||
"getrandom 0.4.2",
|
"getrandom 0.4.2",
|
||||||
"hmac 0.13.0",
|
"hmac 0.13.0",
|
||||||
"log",
|
"log",
|
||||||
"lz4_flex",
|
"lz4_flex 0.13.1",
|
||||||
"md-5 0.11.0",
|
"md-5 0.11.0",
|
||||||
"md4 0.11.0",
|
"md4 0.11.0",
|
||||||
"num_enum",
|
"num_enum",
|
||||||
@@ -5471,6 +5756,9 @@ name = "sync_wrapper"
|
|||||||
version = "1.0.2"
|
version = "1.0.2"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "0bf256ce5efdfa370213c1dabab5935a12e49f2c58d15e9eac2870d3b4f27263"
|
checksum = "0bf256ce5efdfa370213c1dabab5935a12e49f2c58d15e9eac2870d3b4f27263"
|
||||||
|
dependencies = [
|
||||||
|
"futures-core",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "synstructure"
|
name = "synstructure"
|
||||||
@@ -5483,6 +5771,27 @@ dependencies = [
|
|||||||
"syn 2.0.117",
|
"syn 2.0.117",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "system-configuration"
|
||||||
|
version = "0.7.0"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "a13f3d0daba03132c0aa9767f98351b3488edc2c100cda2d2ec2b04f3d8d3c8b"
|
||||||
|
dependencies = [
|
||||||
|
"bitflags 2.11.1",
|
||||||
|
"core-foundation 0.9.4",
|
||||||
|
"system-configuration-sys",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "system-configuration-sys"
|
||||||
|
version = "0.6.0"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "8e1d1b10ced5ca923a1fcb8d03e96b8d3268065d724548c0211415ff6ac6bac4"
|
||||||
|
dependencies = [
|
||||||
|
"core-foundation-sys",
|
||||||
|
"libc",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "tar"
|
name = "tar"
|
||||||
version = "0.4.46"
|
version = "0.4.46"
|
||||||
@@ -5649,6 +5958,16 @@ dependencies = [
|
|||||||
"syn 2.0.117",
|
"syn 2.0.117",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "tokio-native-tls"
|
||||||
|
version = "0.3.1"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "bbae76ab933c85776efabc971569dd6119c580d8f5d448769dec1764bf796ef2"
|
||||||
|
dependencies = [
|
||||||
|
"native-tls",
|
||||||
|
"tokio",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "tokio-postgres"
|
name = "tokio-postgres"
|
||||||
version = "0.7.18"
|
version = "0.7.18"
|
||||||
@@ -5675,6 +5994,16 @@ dependencies = [
|
|||||||
"whoami",
|
"whoami",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "tokio-rustls"
|
||||||
|
version = "0.26.4"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "1729aa945f29d91ba541258c8df89027d5792d85a8841fb65e8bf0f4ede4ef61"
|
||||||
|
dependencies = [
|
||||||
|
"rustls",
|
||||||
|
"tokio",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "tokio-stream"
|
name = "tokio-stream"
|
||||||
version = "0.1.18"
|
version = "0.1.18"
|
||||||
@@ -5813,6 +6142,40 @@ dependencies = [
|
|||||||
"tracing",
|
"tracing",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "tower-http"
|
||||||
|
version = "0.5.2"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "1e9cd434a998747dd2c4276bc96ee2e0c7a2eadf3cae88e52be55a05fa9053f5"
|
||||||
|
dependencies = [
|
||||||
|
"bitflags 2.11.1",
|
||||||
|
"bytes",
|
||||||
|
"http",
|
||||||
|
"http-body",
|
||||||
|
"http-body-util",
|
||||||
|
"pin-project-lite",
|
||||||
|
"tower-layer",
|
||||||
|
"tower-service",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "tower-http"
|
||||||
|
version = "0.6.11"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "4cfcf7e2740e6fc6d4d688b4ef00650406bb94adf4731e43c096c3a19fe40840"
|
||||||
|
dependencies = [
|
||||||
|
"bitflags 2.11.1",
|
||||||
|
"bytes",
|
||||||
|
"futures-util",
|
||||||
|
"http",
|
||||||
|
"http-body",
|
||||||
|
"pin-project-lite",
|
||||||
|
"tower",
|
||||||
|
"tower-layer",
|
||||||
|
"tower-service",
|
||||||
|
"url",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "tower-layer"
|
name = "tower-layer"
|
||||||
version = "0.3.3"
|
version = "0.3.3"
|
||||||
@@ -5900,6 +6263,12 @@ dependencies = [
|
|||||||
"tracing-serde",
|
"tracing-serde",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "try-lock"
|
||||||
|
version = "0.2.5"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b"
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "twox-hash"
|
name = "twox-hash"
|
||||||
version = "2.1.2"
|
version = "2.1.2"
|
||||||
@@ -6199,6 +6568,15 @@ dependencies = [
|
|||||||
"libc",
|
"libc",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "want"
|
||||||
|
version = "0.3.1"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "bfa7760aed19e106de2c7c0b581b509f2f25d3dacaf737cb82ac61bc6d760b0e"
|
||||||
|
dependencies = [
|
||||||
|
"try-lock",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "wasi"
|
name = "wasi"
|
||||||
version = "0.11.1+wasi-snapshot-preview1"
|
version = "0.11.1+wasi-snapshot-preview1"
|
||||||
@@ -6533,6 +6911,17 @@ dependencies = [
|
|||||||
"windows-link",
|
"windows-link",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "windows-registry"
|
||||||
|
version = "0.6.1"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "02752bf7fbdcce7f2a27a742f798510f3e5ad88dbe84871e5168e2120c3d5720"
|
||||||
|
dependencies = [
|
||||||
|
"windows-link",
|
||||||
|
"windows-result 0.4.1",
|
||||||
|
"windows-strings 0.5.1",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "windows-result"
|
name = "windows-result"
|
||||||
version = "0.2.0"
|
version = "0.2.0"
|
||||||
|
|||||||
@@ -4,6 +4,8 @@ port = 11438
|
|||||||
log_level = "info"
|
log_level = "info"
|
||||||
auth_db_path = "data/auth.sqlite"
|
auth_db_path = "data/auth.sqlite"
|
||||||
users_db_dir = "data/users"
|
users_db_dir = "data/users"
|
||||||
|
webdav_root = "/Users/accusys/momentry/var/sftpgo/data/demo"
|
||||||
|
upload_path = "/Users/accusys/momentry/var/sftpgo/data"
|
||||||
|
|
||||||
[postgresql]
|
[postgresql]
|
||||||
host = "127.0.0.1"
|
host = "127.0.0.1"
|
||||||
|
|||||||
@@ -0,0 +1 @@
|
|||||||
|
•fώG�η›DW¥Η/k·yB)”�‰±Xaxγ{ργ#
|
||||||
@@ -0,0 +1,6 @@
|
|||||||
|
{
|
||||||
|
"created_at": 1782062629,
|
||||||
|
"expires_at": 1813598629,
|
||||||
|
"fingerprint": "YhvUXPPA1xlmnfJ9H0axfLsV5wve9QMiRQ2eFarT/D4=",
|
||||||
|
"key_type": "ed25519"
|
||||||
|
}
|
||||||
@@ -0,0 +1 @@
|
|||||||
|
ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAICtBzWJ6iltFPtzzRq7fxqJ4MdXrukOCk5YEK293DYjl markbase_ssh_host_key
|
||||||
Binary file not shown.
Binary file not shown.
Binary file not shown.
@@ -0,0 +1 @@
|
|||||||
|
Small test content
|
||||||
@@ -0,0 +1 @@
|
|||||||
|
Test file for clean WebDAV directory
|
||||||
Binary file not shown.
@@ -0,0 +1 @@
|
|||||||
|
Test upload to clean empty directory
|
||||||
@@ -0,0 +1 @@
|
|||||||
|
Test content for PUT operation
|
||||||
@@ -0,0 +1 @@
|
|||||||
|
SUCCESS: uploaded to clean empty directory
|
||||||
Binary file not shown.
@@ -0,0 +1 @@
|
|||||||
|
Hello MarkBase WebDAV
|
||||||
Binary file not shown.
@@ -0,0 +1 @@
|
|||||||
|
test upload content
|
||||||
@@ -0,0 +1 @@
|
|||||||
|
Small test content
|
||||||
@@ -0,0 +1 @@
|
|||||||
|
Final test for clean WebDAV
|
||||||
File diff suppressed because one or more lines are too long
@@ -0,0 +1,328 @@
|
|||||||
|
# Ceph RADOS Integration Analysis for MarkBase
|
||||||
|
|
||||||
|
**Date**: 2026-06-25
|
||||||
|
**Status**: Shelved (不符合 macOS 跨平台定位)
|
||||||
|
**Library**: ceph-async (4.0.5)
|
||||||
|
**Constraint**: Linux-only (requires librados.so symlink)
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Executive Summary
|
||||||
|
|
||||||
|
### Goal
|
||||||
|
Add Ceph RADOS as a VfsBackend option for distributed, highly scalable storage.
|
||||||
|
|
||||||
|
### Key Findings
|
||||||
|
| Aspect | Finding |
|
||||||
|
|--------|---------|
|
||||||
|
| **Platform** | ❌ Linux-only (librados.so FFI, macOS needs Docker/VM) |
|
||||||
|
| **Deployment** | ⚠️ Requires full cluster (Monitor + OSD + MGR) |
|
||||||
|
| **Complexity** | ⚠️⚠️⚠️⚠️⚠️ High (超出 Lightweight 定位) |
|
||||||
|
| **Positioning** | ❌ 不符合 MarkBase macOS 跨平台定位 |
|
||||||
|
|
||||||
|
### Recommendation
|
||||||
|
**当前搁置**。优先考虑:
|
||||||
|
1. **MinIO** — S3-compatible,已有 S3Vfs 支持,跨平台
|
||||||
|
2. **内置分布式** — DedupFs + S3Vfs 组合,轻量级
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Architecture Overview
|
||||||
|
|
||||||
|
```
|
||||||
|
┌─────────────────────────────────────────────────────────────────────────┐
|
||||||
|
│ MarkBase Application Layer │
|
||||||
|
│ ├── SMB Server (Port 4445) │
|
||||||
|
│ ├── SFTP Server (Port 2024) │
|
||||||
|
│ ├── WebDAV Server (Port 11438) │
|
||||||
|
│ └───────────────────────────────────────────────────────────────────────┘
|
||||||
|
│ ↓ │
|
||||||
|
┌─────────────────────────────────────────────────────────────────────────┐
|
||||||
|
│ VFS Abstraction Layer (VfsBackend trait) │
|
||||||
|
│ ├── LocalFs — POSIX local filesystem │
|
||||||
|
│ ├── S3Vfs — S3-compatible storage (HTTP API) │
|
||||||
|
│ ├── SmbVfs — SMB client backend │
|
||||||
|
│ ├── CephVfs — Ceph RADOS backend (搁置) │
|
||||||
|
│ ├── EncryptedFs — Encryption layer │
|
||||||
|
│ ├── Compression — ZSTD/LZ4 compression layer │
|
||||||
|
│ ├── DedupFs — Block deduplication layer │
|
||||||
|
│ ├── RaidFs — RAID-Z emulation layer │
|
||||||
|
│ └─────────────────────────────────────────────────────────────────────┘
|
||||||
|
│ ↓ │
|
||||||
|
┌─────────────────────────────────────────────────────────────────────────┐
|
||||||
|
│ Ceph Storage Cluster (RADOS) │
|
||||||
|
│ ├── Monitor (MON) — Cluster map, authentication │
|
||||||
|
│ ├── OSD Daemons — Object storage (data replication) │
|
||||||
|
│ ├── Manager (MGR) — Dashboard, telemetry │
|
||||||
|
│ ├── MDS (optional) — CephFS metadata server │
|
||||||
|
│ ├── RGW (optional) — S3/Swift gateway │
|
||||||
|
│ └─────────────────────────────────────────────────────────────────────┘
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Library Analysis
|
||||||
|
|
||||||
|
### Rust Ceph Crates
|
||||||
|
|
||||||
|
| Crate | Version | Description | Platform |
|
||||||
|
|-------|---------|-------------|----------|
|
||||||
|
| `ceph` | 3.2.5 | Official librados FFI (sync) | Linux-only |
|
||||||
|
| `ceph-async` | 4.0.5 | Async librados FFI (futures 0.3) | Linux-only |
|
||||||
|
| `ceph-rbd` | 0.3.2 | RADOS Block Device bindings | Linux-only |
|
||||||
|
|
||||||
|
### ceph-async Module Structure
|
||||||
|
|
||||||
|
```
|
||||||
|
ceph_async::
|
||||||
|
├── CephClient — Admin operations (OSD/Pool/Mon commands)
|
||||||
|
├── rados:: — Low-level FFI bindings (100+ functions)
|
||||||
|
│ ├── rados_read/write/stat/remove — Object I/O
|
||||||
|
│ ├── rados_pool_create/delete/lookup — Pool management
|
||||||
|
│ ├── rados_ioctx_* — I/O context (pool handle)
|
||||||
|
│ ├── rados_snap_* — Snapshot management
|
||||||
|
│ ├── rados_lock_* — Distributed locking
|
||||||
|
│ ├── rados_aio_* — Async I/O
|
||||||
|
│ ├── rados_omap_* — Key-value store per object
|
||||||
|
│ └── rados_write_op_* / rados_read_op_* — Compound operations
|
||||||
|
├── completion:: — Async completion handling
|
||||||
|
├── read_stream:: — Async read stream
|
||||||
|
├── write_sink:: — Async write sink
|
||||||
|
└── list_stream:: — Async object listing
|
||||||
|
```
|
||||||
|
|
||||||
|
### CephClient API
|
||||||
|
|
||||||
|
```rust
|
||||||
|
let client = CephClient::new("admin", "/etc/ceph/ceph.conf")?;
|
||||||
|
|
||||||
|
// OSD operations
|
||||||
|
client.osd_tree()?; // Get OSD tree (CRUSH map)
|
||||||
|
client.osd_out(osd_id)?; // Mark OSD out
|
||||||
|
client.osd_crush_remove(osd_id)?; // Remove from CRUSH map
|
||||||
|
|
||||||
|
// Pool operations
|
||||||
|
client.osd_pool_get(pool, option)?; // Get pool config
|
||||||
|
client.osd_pool_set(pool, key, val)?; // Set pool config
|
||||||
|
client.osd_pool_quota_get(pool)?; // Get pool quota
|
||||||
|
|
||||||
|
// Cluster status
|
||||||
|
client.status()?; // Cluster health
|
||||||
|
client.mon_dump()?; // Monitor list
|
||||||
|
client.version()?; // Ceph version
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Implementation Phases
|
||||||
|
|
||||||
|
| Phase | Task | Code Lines | Priority | Risk | Dependencies |
|
||||||
|
|-------|------|------------|----------|------|--------------|
|
||||||
|
| **Phase 1** | CephVfs struct + basic I/O | ~400 | P0 | Medium ⚠️⚠️⚠️ | ceph-async crate |
|
||||||
|
| **Phase 2** | Pool management CLI | ~150 | P1 | Low ⚠️ | Phase 1 |
|
||||||
|
| **Phase 3** | Snapshot support | ~200 | P2 | Medium ⚠️⚠️⚠️ | librados snap API |
|
||||||
|
| **Phase 4** | Distributed locking | ~100 | P2 | Medium ⚠️⚠️⚠️ | librados lock API |
|
||||||
|
| **Phase 5** | OMAP key-value | ~150 | P3 | Low ⚠️ | librados omap API |
|
||||||
|
| **Phase 6** | Async integration | ~300 | P1 | High ⚠️⚠️⚠️⚠️ | async-vfs feature |
|
||||||
|
| **Phase 7** | Docker test environment | ~50 | P0 | Low ⚠️ | Docker compose |
|
||||||
|
| **Phase 8** | Performance benchmark | ~100 | P2 | Low ⚠️ | Benchmark scripts |
|
||||||
|
| **Total** | | **~1350** | | | |
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Phase 1: CephVfs Core Implementation
|
||||||
|
|
||||||
|
### Key Design Decisions
|
||||||
|
|
||||||
|
**1. Object vs File mapping**:
|
||||||
|
- RADOS is object storage (no directories)
|
||||||
|
- Path `/foo/bar.txt` → Object `foo/bar.txt` in pool
|
||||||
|
- Directories simulated via zero-byte objects with `/` suffix (like S3)
|
||||||
|
|
||||||
|
**2. Pool-per-share vs single pool**:
|
||||||
|
- Option A: Single pool + path prefix (simpler, less isolation)
|
||||||
|
- Option B: Pool-per-share (better isolation, quota per pool)
|
||||||
|
- **Recommend**: Option B (pool-per-share) for enterprise use
|
||||||
|
|
||||||
|
**3. I/O context caching**:
|
||||||
|
- Each pool requires separate `rados_ioctx_t`
|
||||||
|
- Cache ioctx per share to avoid recreation overhead
|
||||||
|
|
||||||
|
### CephVfs Struct (Draft)
|
||||||
|
|
||||||
|
```rust
|
||||||
|
pub struct CephVfs {
|
||||||
|
cluster: rados_t, // RADOS cluster handle
|
||||||
|
pool_name: String, // Pool name for this share
|
||||||
|
ioctx: rados_ioctx_t, // I/O context (cached)
|
||||||
|
root_prefix: String, // Path prefix within pool
|
||||||
|
}
|
||||||
|
|
||||||
|
pub struct CephVfsFile {
|
||||||
|
ioctx: rados_ioctx_t,
|
||||||
|
object_id: String, // Object name in pool
|
||||||
|
position: u64,
|
||||||
|
write_buffer: Vec<u8>, // Buffer for writes (flush on close)
|
||||||
|
size: u64,
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
### VfsBackend Method Mapping
|
||||||
|
|
||||||
|
| Method | RADOS equivalent | Complexity |
|
||||||
|
|--------|-----------------|------------|
|
||||||
|
| `read_dir()` | `rados_nobjects_list_*` | High (pagination) |
|
||||||
|
| `open_file()` | Custom (object ops) | Medium |
|
||||||
|
| `stat()` | `rados_stat()` | Low |
|
||||||
|
| `create_dir()` | `rados_write_full(0-byte)` | Low |
|
||||||
|
| `remove_dir()` | `rados_remove()` | Low |
|
||||||
|
| `remove_file()` | `rados_remove()` | Low |
|
||||||
|
| `rename()` | Custom (copy + delete) | Medium |
|
||||||
|
| `exists()` | `rados_stat()` | Low |
|
||||||
|
| `copy()` | `rados_clone_range()` | Low |
|
||||||
|
| `hard_link()` | `rados_clone_range()` | Low |
|
||||||
|
| `read_link()` | Unsupported | N/A |
|
||||||
|
| `create_symlink()` | Unsupported | N/A |
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Risk Assessment
|
||||||
|
|
||||||
|
| Risk | Level | Mitigation |
|
||||||
|
|------|-------|------------|
|
||||||
|
| **Linux-only** | ⚠️⚠️⚠️⚠️⚠️ Critical | Docker/VM for macOS; 不符合跨平台定位 |
|
||||||
|
| **librados.so symlink** | ⚠️⚠️⚠️ Medium | Document setup; CI check |
|
||||||
|
| **Pool-level snapshots** | ⚠️⚠️ Low | Document limitation; consider RGW |
|
||||||
|
| **Async overhead** | ⚠️⚠️⚠️ Medium | Benchmark; spawn_blocking wrapper |
|
||||||
|
| **Cluster complexity** | ⚠️⚠️⚠️⚠️⚠️ Critical | 超出 Lightweight 定位; Docker compose |
|
||||||
|
| **SMB Oplocks integration** | ⚠️⚠️⚠️ Medium | RADOS locking API; careful design |
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Alternatives (推荐方案)
|
||||||
|
|
||||||
|
### 方案对比
|
||||||
|
|
||||||
|
| 方案 | 跨平台 | 部署复杂度 | 定位匹配 | 状态 |
|
||||||
|
|------|--------|-----------|---------|------|
|
||||||
|
| **Ceph RADOS** | ❌ Linux-only | ⚠️⚠️⚠️⚠️⚠️ 极高 | ❌ 不匹配 | 搁置 |
|
||||||
|
| **Ceph RGW (S3)** | ✅ HTTP API | ⚠️⚠️⚠️⚠️ 高 | ⭐⭐⭐ 中等 | 已有 S3Vfs |
|
||||||
|
| **MinIO** | ✅ 全平台 | ⚠️⚠️ 低 | ⭐⭐⭐⭐⭐ 完全匹配 | 已有 S3Vfs |
|
||||||
|
| **GlusterFS** | ✅ POSIX | ⚠️⚠️⚠️ 中 | ⭐⭐⭐⭐ 高 | 待研究 |
|
||||||
|
| **内置分布式** | ✅ 全平台 | ⚠️⚠️ 低 | ⭐⭐⭐⭐⭐ 完全匹配 | 已有基础 |
|
||||||
|
|
||||||
|
### 方案 1: MinIO (推荐)
|
||||||
|
|
||||||
|
**优势**:
|
||||||
|
- ✅ S3-compatible API(已有 S3Vfs,无需新代码)
|
||||||
|
- ✅ 单节点部署(轻量级)
|
||||||
|
- ✅ 跨平台(macOS/Linux/Windows)
|
||||||
|
- ✅ 高性能(纠删码)
|
||||||
|
- ✅ 开源 + 企业版
|
||||||
|
|
||||||
|
**部署**:
|
||||||
|
```bash
|
||||||
|
# macOS 单节点
|
||||||
|
minio server /data --console-address ":9001"
|
||||||
|
|
||||||
|
# MarkBase 配置
|
||||||
|
MB_S3_ENDPOINT=http://localhost:9000
|
||||||
|
MB_S3_BUCKET=markbase
|
||||||
|
```
|
||||||
|
|
||||||
|
**集成**: 无需修改代码,S3Vfs 已支持。
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
### 方案 2: 内置分布式存储
|
||||||
|
|
||||||
|
**已有基础**:
|
||||||
|
| 功能 | 文件 | 分布式潜力 |
|
||||||
|
|------|------|-----------|
|
||||||
|
| DedupFs | dedup.rs | ✅ SHA-256 块存储可跨节点共享 |
|
||||||
|
| RaidFs | raid.rs | ⚠️ 单节点 RAID-Z |
|
||||||
|
| Send-Receive | send_receive.rs | ⚠️ 类似 ZFS send/receive |
|
||||||
|
| Checksum | checksum.rs | ✅ 数据完整性验证 |
|
||||||
|
| Compression | compression.rs | ✅ ZSTD 压缩 |
|
||||||
|
|
||||||
|
**扩展方向**:
|
||||||
|
1. DedupFs + S3Vfs: Dedup 块存储到 MinIO/S3(跨节点共享)
|
||||||
|
2. Checksum + Replication: 增加跨节点复制
|
||||||
|
3. Send-Receive + Remote: 增加远程 replication
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Technical Details
|
||||||
|
|
||||||
|
### librados API Functions
|
||||||
|
|
||||||
|
**Object I/O**:
|
||||||
|
- `rados_read(ioctx, oid, buf, len, offset)` — Read at offset
|
||||||
|
- `rados_write(ioctx, oid, buf, len, offset)` — Write at offset
|
||||||
|
- `rados_write_full(ioctx, oid, buf, len)` — Write entire object
|
||||||
|
- `rados_append(ioctx, oid, buf, len)` — Append to object
|
||||||
|
- `rados_stat(ioctx, oid, psize, pmtime)` — Get object size/mtime
|
||||||
|
- `rados_remove(ioctx, oid)` — Delete object
|
||||||
|
|
||||||
|
**Pool Operations**:
|
||||||
|
- `rados_pool_create(cluster, pool_name)` — Create pool
|
||||||
|
- `rados_pool_delete(cluster, pool_name)` — Delete pool
|
||||||
|
- `rados_pool_lookup(cluster, pool_name)` — Find pool ID
|
||||||
|
- `rados_ioctx_create(cluster, pool_name, ioctx)` — Create I/O context
|
||||||
|
|
||||||
|
**Snapshots**:
|
||||||
|
- `rados_ioctx_snap_create(ioctx, snap_name)` — Create pool snapshot
|
||||||
|
- `rados_ioctx_snap_list(ioctx, snaps)` — List snapshots
|
||||||
|
- `rados_ioctx_snap_remove(ioctx, snap_id)` — Delete snapshot
|
||||||
|
- `rados_ioctx_snap_rollback(ioctx, oid, snap_id)` — Rollback object
|
||||||
|
|
||||||
|
**Locking**:
|
||||||
|
- `rados_lock_exclusive(ioctx, oid, name, cookie, desc, duration, flags)` — Exclusive lock
|
||||||
|
- `rados_lock_shared(ioctx, oid, name, cookie, tag, desc, duration, flags)` — Shared lock
|
||||||
|
- `rados_unlock(ioctx, oid, name, cookie)` — Release lock
|
||||||
|
- `rados_list_lockers(ioctx, oid, name, ...)` — List lock holders
|
||||||
|
|
||||||
|
**OMAP (Key-Value)**:
|
||||||
|
- `rados_omap_set(ioctx, oid, map)` — Set key-value pairs
|
||||||
|
- `rados_omap_get(ioctx, oid, ...)` — Get values by keys
|
||||||
|
- `rados_omap_get_keys(ioctx, oid, ...)` — List keys
|
||||||
|
- `rados_omap_rm_keys(ioctx, oid, keys)` — Delete keys
|
||||||
|
|
||||||
|
**Async I/O**:
|
||||||
|
- `rados_aio_read(ioctx, oid, completion, buf, len, offset)` — Async read
|
||||||
|
- `rados_aio_write(ioctx, oid, completion, buf, len, offset)` — Async write
|
||||||
|
- `rados_aio_flush(ioctx)` — Flush pending async ops
|
||||||
|
- `rados_aio_wait_for_complete(completion)` — Wait for completion
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Open Questions
|
||||||
|
|
||||||
|
1. **部署目标**: Linux-only production vs macOS development?
|
||||||
|
2. **Backend choice**: RADOS (librados) vs RGW (S3 API)?
|
||||||
|
3. **Pool strategy**: Pool-per-share vs single pool + path prefix?
|
||||||
|
4. **SMB Oplocks**: Should CephVfs support SMB Oplocks via RADOS locking?
|
||||||
|
5. **Priority**: Start with basic I/O or full async integration first?
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Conclusion
|
||||||
|
|
||||||
|
**当前搁置 Ceph RADOS 集成**,原因:
|
||||||
|
1. ❌ Linux-only 约束不符合 macOS 跨平台定位
|
||||||
|
2. ⚠️ 部署复杂度超出 Lightweight 定位
|
||||||
|
3. ⚠️ 需要完整 Ceph 集群(Monitor + OSD + MGR)
|
||||||
|
|
||||||
|
**推荐替代方案**:
|
||||||
|
1. ⭐⭐⭐⭐⭐ **MinIO** — S3-compatible,已有 S3Vfs,轻量级
|
||||||
|
2. ⭐⭐⭐⭐⭐ **内置分布式** — DedupFs + S3Vfs 组合
|
||||||
|
|
||||||
|
**后续行动**:
|
||||||
|
- MinIO 集成文档(0 行代码)
|
||||||
|
- DedupFs + S3Vfs 组合研究(~100 行)
|
||||||
|
- 内置 Replication 功能(~400 行)
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
**文档创建**: 2026-06-25
|
||||||
|
**最后更新**: 2026-06-25
|
||||||
@@ -0,0 +1,342 @@
|
|||||||
|
# CTDB (Cluster Trivial Database) 架构分析
|
||||||
|
|
||||||
|
## 概述
|
||||||
|
|
||||||
|
**CTDB** 是 Samba 的集群数据库系统,用于在高可用性(HA)集群环境中管理共享状态和数据库记录。
|
||||||
|
|
||||||
|
**核心功能**:
|
||||||
|
- 集群节点间状态同步
|
||||||
|
- 分布式数据库存储
|
||||||
|
- 故障检测和自动恢复
|
||||||
|
- 公共 IP 地址管理(浮动 IP)
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 1. CTDB 核心组件
|
||||||
|
|
||||||
|
### 1.1 分布式数据库引擎
|
||||||
|
|
||||||
|
**功能**:
|
||||||
|
- 字储 TDB (Trivial Database) 记录
|
||||||
|
- 在多个节点间复制数据
|
||||||
|
- 提供原子性和一致性保证
|
||||||
|
|
||||||
|
**TDB 格式**:
|
||||||
|
```
|
||||||
|
┌─────────────────────────────────────┐
|
||||||
|
│ TDB Header │
|
||||||
|
│ ├── Magic number: 0x1BADFACE │
|
||||||
|
│ ├── Version: 1 │
|
||||||
|
│ ├── Hash size: 1024 │
|
||||||
|
│ └── Record count: N │
|
||||||
|
├─────────────────────────────────────┤
|
||||||
|
│ Hash Table │
|
||||||
|
│ ├── Bucket 0: offset to record │
|
||||||
|
│ ├── Bucket 1: offset to record │
|
||||||
|
│ └── ... │
|
||||||
|
├─────────────────────────────────────┤
|
||||||
|
│ Free List │
|
||||||
|
│ ├── Offset to next free block │
|
||||||
|
│ └── Free block size │
|
||||||
|
├─────────────────────────────────────┤
|
||||||
|
│ Records │
|
||||||
|
│ ├── Key (variable length) │
|
||||||
|
│ ├── Data (variable length) │
|
||||||
|
│ └── Hash next pointer │
|
||||||
|
└─────────────────────────────────────┘
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
### 1.2 集群节点管理
|
||||||
|
|
||||||
|
**功能**:
|
||||||
|
- 节点状态监控(UP/DOWN/UNHEALTHY)
|
||||||
|
- 心跳检测(heartbeat)
|
||||||
|
- 自动故障转移(failover)
|
||||||
|
|
||||||
|
**节点状态定义**:
|
||||||
|
| 状态 | 说明 | 处理 |
|
||||||
|
|------|------|------|
|
||||||
|
| **UP** | 节点正常运行 | 参与集群操作 |
|
||||||
|
| **DOWN** | 节点离线 | 等待恢复 |
|
||||||
|
| **UNHEALTHY** | 节点不健康 | 停止服务 |
|
||||||
|
| **BANNED** | 节点被禁止 | 重新加入需手动操作 |
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
### 1.3 公共 IP 管理
|
||||||
|
|
||||||
|
**功能**:
|
||||||
|
- 动态分配浮动 IP(public IP)
|
||||||
|
- 节点故障时自动迁移 IP
|
||||||
|
- 客户端透明重连
|
||||||
|
|
||||||
|
**公共 IP 分配逻辑**:
|
||||||
|
```
|
||||||
|
┌─────────────────────────────────────┐
|
||||||
|
│ Public IP Pool │
|
||||||
|
│ ├── 192.168.1.100 (node 0) │
|
||||||
|
│ ├── 192.168.1.101 (node 1) │
|
||||||
|
│ ├── 192.168.1.102 (node 2) │
|
||||||
|
│ └── ... │
|
||||||
|
├─────────────────────────────────────┤
|
||||||
|
│ IP Assignment │
|
||||||
|
│ ├── Node 0 UP → owns .100 │
|
||||||
|
│ ├── Node 1 DOWN → .101 moves to N0 │
|
||||||
|
│ └── Node 2 UP → owns .102 │
|
||||||
|
└─────────────────────────────────────┘
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
### 1.4 事件脚本系统
|
||||||
|
|
||||||
|
**功能**:
|
||||||
|
- 监控脚本(健康检查)
|
||||||
|
- 启动/停止脚本(服务管理)
|
||||||
|
- IP 分配脚本(网络配置)
|
||||||
|
|
||||||
|
**事件类型**:
|
||||||
|
| 事件 | 说明 | 脚本 |
|
||||||
|
|------|------|------|
|
||||||
|
| `startup` | 节点启动 | 01.startup.sh |
|
||||||
|
| `shutdown` | 节点停止 | 50.shutdown.sh |
|
||||||
|
| `takeip` | 分配 IP | 11.takeip.sh |
|
||||||
|
| `releaseip` | 释放 IP | 10.releaseip.sh |
|
||||||
|
| `monitor` | 健康检查 | 00.monitor.sh |
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 2. CTDB 协议架构
|
||||||
|
|
||||||
|
### 2.1 控制协议
|
||||||
|
|
||||||
|
**CTDB 控制消息**(基于 TCP):
|
||||||
|
```
|
||||||
|
┌─────────────────────────────────────┐
|
||||||
|
│ CTDB Header │
|
||||||
|
│ ├── Magic: 0xCtdb │
|
||||||
|
│ ├── Version: 1 │
|
||||||
|
│ ├── Command: CTDB_CMD_* │
|
||||||
|
│ ├── Status: SUCCESS/ERROR │
|
||||||
|
│ ├── Length: payload size │
|
||||||
|
├─────────────────────────────────────┤
|
||||||
|
│ Payload │
|
||||||
|
│ ├── Command-specific data │
|
||||||
|
│ └── Optional response data │
|
||||||
|
└─────────────────────────────────────┘
|
||||||
|
```
|
||||||
|
|
||||||
|
**CTDB_CMD 类型**:
|
||||||
|
| Command | 说明 | 用途 |
|
||||||
|
|---------|------|------|
|
||||||
|
| `CTDB_CMD_CONNECT` | 连接请求 | 初始化连接 |
|
||||||
|
| `CTDB_CMD_GETDB` | 获取数据库 | 访问 TDB |
|
||||||
|
| `CTDB_CMD_FETCH` | 读取记录 | 查询数据 |
|
||||||
|
| `CTDB_CMD_STORE` | 存储记录 | 写入数据 |
|
||||||
|
| `CTDB_CMD_DELETE` | 删除记录 | 清除数据 |
|
||||||
|
| `CTDB_CMD_PING` | 心跳检测 | 节点监控 |
|
||||||
|
| `CTDB_CMD_SETNODEMASK` | 设置节点掩码 | 集群配置 |
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
### 2.2 数据库复制协议
|
||||||
|
|
||||||
|
**复制策略**:
|
||||||
|
- **主从复制**(Master-Slave):一个节点为主,其他为副本
|
||||||
|
- **多主复制**(Multi-Master):所有节点可写入(需要冲突解决)
|
||||||
|
|
||||||
|
**复制流程**:
|
||||||
|
```
|
||||||
|
Node A Node B
|
||||||
|
| |
|
||||||
|
|── STORE(key, value) ────>| (write request)
|
||||||
|
| |── Validate
|
||||||
|
| |── Write to local TDB
|
||||||
|
| |── Replicate to other nodes
|
||||||
|
|<── ACK (success) ────────|
|
||||||
|
| |
|
||||||
|
|── FETCH(key) ───────────>| (read request)
|
||||||
|
|<── value ────────────────| (return data)
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
### 2.3 故障恢复协议
|
||||||
|
|
||||||
|
**故障检测**:
|
||||||
|
```
|
||||||
|
┌─────────────────────────────────────┐
|
||||||
|
│ Health Monitor Loop │
|
||||||
|
│ ├── Ping all nodes (every 1s) │
|
||||||
|
│ ├── Check response timeout (5s) │
|
||||||
|
│ ├── Mark DOWN if timeout │
|
||||||
|
│ └── Trigger recovery process │
|
||||||
|
└─────────────────────────────────────┘
|
||||||
|
```
|
||||||
|
|
||||||
|
**恢复流程**:
|
||||||
|
```
|
||||||
|
Node A (DOWN) Cluster
|
||||||
|
| |
|
||||||
|
|── Mark as DOWN ─────────>| (detected)
|
||||||
|
| |── Reassign public IPs
|
||||||
|
| |── Notify other nodes
|
||||||
|
| |── Update node mask
|
||||||
|
| |
|
||||||
|
|── Recovery attempt ─────>| (after 30s)
|
||||||
|
|── Rejoin cluster ───────>| (if successful)
|
||||||
|
| |── Restore IPs
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 3. CTDB 与 SMB 集成
|
||||||
|
|
||||||
|
### 3.1 Samba 集成点
|
||||||
|
|
||||||
|
**共享数据库**:
|
||||||
|
| TDB 文件 | 说明 | 集群支持 |
|
||||||
|
|---------|------|---------|
|
||||||
|
| `secrets.tdb` | 认证密钥 | ✅ CTDB 复制 |
|
||||||
|
| `brlock.tdb` | 字节锁 | ✅ CTDB 复制 |
|
||||||
|
| `locking.tdb` | 文件锁 | ✅ CTDB 复制 |
|
||||||
|
| `connections.tdb` | 连接状态 | ✅ CTDB 复制 |
|
||||||
|
| `session_info.tdb` | 会话信息 | ✅ CTDB 复制 |
|
||||||
|
| `share_info.tdb` | 共享配置 | ✅ CTDB 复制 |
|
||||||
|
|
||||||
|
**关键特性**:
|
||||||
|
- 所有节点访问相同的数据库
|
||||||
|
- 实时同步锁定状态
|
||||||
|
- 故障后自动恢复连接
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
### 3.2 实施架构
|
||||||
|
|
||||||
|
**高可用性架构**:
|
||||||
|
```
|
||||||
|
┌─────────────────────────────────────────────┐
|
||||||
|
│ Client Layer │
|
||||||
|
│ ├── SMB clients (Windows/macOS/Linux) │
|
||||||
|
│ └── Connect via floating IP │
|
||||||
|
├─────────────────────────────────────────────┤
|
||||||
|
│ CTDB Cluster (3+ nodes) │
|
||||||
|
│ ├── Node 0: 192.168.1.100 (SMB + CTDB) │
|
||||||
|
│ ├── Node 1: 192.168.1.101 (SMB + CTDB) │
|
||||||
|
│ ├── Node 2: 192.168.1.102 (SMB + CTDB) │
|
||||||
|
│ └── Public IPs: .100, .101, .102 │
|
||||||
|
├─────────────────────────────────────────────┤
|
||||||
|
│ Shared Storage │
|
||||||
|
│ ├── GlusterFS / Ceph / NFS │
|
||||||
|
│ └── All nodes access same filesystem │
|
||||||
|
└─────────────────────────────────────────────┘
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 4. 实施评估
|
||||||
|
|
||||||
|
### 4.1 核心功能需求
|
||||||
|
|
||||||
|
| 功能 | 优先级 | 工作量 | 风险 |
|
||||||
|
|------|--------|--------|------|
|
||||||
|
| **分布式 TDB** | P0 | 500 行 | 中 ⚠️⚠️⚠️ |
|
||||||
|
| **节点管理** | P0 | 300 行 | 中 ⚠️⚠️⚠️ |
|
||||||
|
| **公共 IP 管理** | P1 | 200 行 | 低 ⚠️⚠️ |
|
||||||
|
| **事件脚本系统** | P1 | 200 行 | 低 ⚠️⚠️ |
|
||||||
|
| **控制协议** | P0 | 400 行 | 高 ⚠️⚠️⚠️⚠️ |
|
||||||
|
| **故障恢复** | P0 | 300 行 | 高 ⚠️⚠️⚠️⚠️⚠️ |
|
||||||
|
| **总计** | | **1900 行** | **高 ⚠️⚠️⚠️⚠️⚠️** |
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
### 4.2 技术挑战
|
||||||
|
|
||||||
|
**挑战 1:分布式一致性**
|
||||||
|
- **问题**:多节点写入冲突
|
||||||
|
- **解决方案**:
|
||||||
|
- 使用 Raft/Paxos 算法(需要实现 consensus)
|
||||||
|
- 或使用主从模式(简化但牺牲可用性)
|
||||||
|
|
||||||
|
**挑战 2:故障检测准确性**
|
||||||
|
- **问题**:网络分区导致误判
|
||||||
|
- **解决方案**:
|
||||||
|
- 使用多路径心跳(多个检测点)
|
||||||
|
- 设置合理的超时阈值(避免误判)
|
||||||
|
|
||||||
|
**挑战 3:浮动 IP 管理**
|
||||||
|
- **问题**:IP 迁移需要内核支持
|
||||||
|
- **解决方案**:
|
||||||
|
- 使用 Linux network namespace
|
||||||
|
- macOS 需要 ifconfig + route 管理
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
### 4.3 Rust 实施建议
|
||||||
|
|
||||||
|
**推荐方案**:
|
||||||
|
1. **Phase 1**:实现基础 TDB 存储引擎(500 行)
|
||||||
|
2. **Phase 2**:实现节点管理和心跳(300 行)
|
||||||
|
3. **Phase 3**:实现控制协议(400 行)
|
||||||
|
4. **Phase 4**:实现公共 IP 管理(200 行)
|
||||||
|
5. **Phase 5**:实现故障恢复逻辑(300 行)
|
||||||
|
|
||||||
|
**总计**:约 1700 行代码(不包括测试)
|
||||||
|
|
||||||
|
**预计时间**:约 5-7 天(高复杂度)
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
### 4.4 替代方案
|
||||||
|
|
||||||
|
**方案 1:使用现有 Rust crate**
|
||||||
|
- `raft-rs`:分布式共识算法
|
||||||
|
- `tikv`:分布式 KV 存储
|
||||||
|
- 优点:减少实现工作量
|
||||||
|
- 缺点:需要集成适配
|
||||||
|
|
||||||
|
**方案 2:简化 CTDB 实现**
|
||||||
|
- 仅实现单主模式(避免分布式共识)
|
||||||
|
- 使用 SQLite 作为共享数据库(简化存储)
|
||||||
|
- 优点:降低实施风险
|
||||||
|
- 缺点:牺牲可用性(主节点故障无法写入)
|
||||||
|
|
||||||
|
**方案 3:不实施 CTDB**
|
||||||
|
- 使用外部高可用方案(HAProxy + Keepalived)
|
||||||
|
- MarkBase SMB 保持单节点部署
|
||||||
|
- 优点:最小工作量
|
||||||
|
- 缺点:无法实现真正的集群
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 5. 决策建议
|
||||||
|
|
||||||
|
### 5.1 推荐策略 ⭐⭐⭐⭐
|
||||||
|
|
||||||
|
**短期(P3 不建议实施)**:
|
||||||
|
- ✅ 保持单节点部署(Phase 1-6 已完成)
|
||||||
|
- ✅ 使用外部 HAProxy + Keepalived 实现高可用
|
||||||
|
- ❌ 不实施 CTDB(复杂度高,收益有限)
|
||||||
|
|
||||||
|
**长期(企业需求)**:
|
||||||
|
- ⭐⭐⭐⭐ 实施简化 CTDB(单主模式)
|
||||||
|
- ⭐⭐⭐ 使用 Raft-rs 实现分布式共识
|
||||||
|
- ⭐⭐⭐⭐⭐ 完整 CTDB 实现(需要 5-7 天)
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
### 5.2 最终建议
|
||||||
|
|
||||||
|
| 场景 | 推荐方案 | 工作量 |
|
||||||
|
|------|---------|--------|
|
||||||
|
| **个人/小团队** | 单节点 + 外部 HA | 0 行 |
|
||||||
|
| **中小企业** | 简化 CTDB(单主) | ~800 行 |
|
||||||
|
| **大型企业** | 完整 CTDB + Raft | ~2000 行 |
|
||||||
|
|
||||||
|
**结论**:Phase 7 (CTDB 集群) 复杂度高(⚠️⚠️⚠️⚠️⚠️),建议根据实际需求决定是否实施。
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
**最后更新**:2026-06-22
|
||||||
@@ -0,0 +1,563 @@
|
|||||||
|
# DedupFs + S3Vfs Combination Design
|
||||||
|
|
||||||
|
**Date**: 2026-06-25
|
||||||
|
**Status**: Design proposal
|
||||||
|
**Goal**: Distributed deduplication storage via MinIO/S3 backend
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Executive Summary
|
||||||
|
|
||||||
|
### Current State
|
||||||
|
|
||||||
|
**DedupStore**(`dedup.rs`, 224 行):
|
||||||
|
- 基于**本地文件系统**的 dedup 存储
|
||||||
|
- SHA-256 块哈希 + 引用计数
|
||||||
|
- 块存储到本地目录(`store_path/.dedup/`)
|
||||||
|
|
||||||
|
**问题**:
|
||||||
|
- ❌ 无法跨节点共享 dedup 块
|
||||||
|
- ❌ 无分布式容错能力
|
||||||
|
- ❌ 单节点存储限制
|
||||||
|
|
||||||
|
### Proposed Solution
|
||||||
|
|
||||||
|
**DedupS3Store**:
|
||||||
|
- 块存储到 **MinIO/S3** 对象(跨节点共享)
|
||||||
|
- 引用计数存储到 S3 object metadata
|
||||||
|
- Manifest 存储到 S3 对象(JSON 格式)
|
||||||
|
|
||||||
|
**优势**:
|
||||||
|
- ✅ 跨节点 dedup 共享(MinIO 分布式)
|
||||||
|
- ✅ 自动容错(MinIO erasure coding)
|
||||||
|
- ✅ 无单节点限制(MinIO 可扩展)
|
||||||
|
- ✅ 与现有 S3Vfs 集成(无需新 HTTP API)
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Architecture
|
||||||
|
|
||||||
|
```
|
||||||
|
┌─────────────────────────────────────────────────────────────────────────┐
|
||||||
|
│ MarkBase Node A │
|
||||||
|
│ ├── DedupS3Store │
|
||||||
|
│ │ ├── store_block() → S3 PUT <hash> │
|
||||||
|
│ │ ├── get_block() → S3 GET <hash> │
|
||||||
|
│ │ └── dedup_file() → 分块 + S3 PUT + manifest │
|
||||||
|
│ └───────────────────────────────────────────────────────────────────────┘
|
||||||
|
│ ↓ │
|
||||||
|
┌─────────────────────────────────────────────────────────────────────────┐
|
||||||
|
│ MinIO Cluster (S3-compatible) │
|
||||||
|
│ ├── Bucket: markbase-dedup │
|
||||||
|
│ │ ├── Objects: <sha256-hash> (dedup 块) │
|
||||||
|
│ │ ├── Metadata: x-amz-meta-ref-count (引用计数) │
|
||||||
|
│ │ └── Manifests: manifests/<file-id>.json │
|
||||||
|
│ │ │
|
||||||
|
│ ├── Erasure Coding: EC:2 (自动容错) │
|
||||||
|
│ ├── Replication: Node A → Node B (DR) │
|
||||||
|
│ └─────────────────────────────────────────────────────────────────────┘
|
||||||
|
│ ↓ │
|
||||||
|
┌─────────────────────────────────────────────────────────────────────────┐
|
||||||
|
│ MarkBase Node B │
|
||||||
|
│ ├── DedupS3Store │
|
||||||
|
│ │ ├── get_block() → S3 GET <hash> (共享 Node A 的块) │
|
||||||
|
│ │ └── restore_file() → S3 GET manifest + S3 GET blocks │
|
||||||
|
│ └─────────────────────────────────────────────────────────────────────┘
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Implementation Design
|
||||||
|
|
||||||
|
### DedupS3Store Struct
|
||||||
|
|
||||||
|
```rust
|
||||||
|
pub struct DedupS3Store {
|
||||||
|
s3vfs: S3Vfs, // S3 backend
|
||||||
|
bucket: String, // Bucket name (markbase-dedup)
|
||||||
|
block_prefix: String, // Object key prefix (blocks/)
|
||||||
|
manifest_prefix: String, // Manifest prefix (manifests/)
|
||||||
|
config: VfsDedupConfig, // block_size, min_file_size
|
||||||
|
}
|
||||||
|
|
||||||
|
pub struct DedupManifest {
|
||||||
|
original_size: usize,
|
||||||
|
block_hashes: Vec<String>,
|
||||||
|
dedup_ratio: f64,
|
||||||
|
file_id: String, // UUID for manifest storage
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
### Core Methods
|
||||||
|
|
||||||
|
| Method | Current (LocalFs) | Proposed (S3Vfs) |
|
||||||
|
|--------|------------------|------------------|
|
||||||
|
| `store_block(data)` | `std::fs::write(store_path/hash, data)` | `S3Vfs.put_object(blocks/hash, data)` |
|
||||||
|
| `get_block(hash)` | `std::fs::read(store_path/hash)` | `S3Vfs.get_object(blocks/hash)` |
|
||||||
|
| `increment_ref(hash)` | `std::fs::write(hash.ref, count)` | `S3Vfs.put_object(blocks/hash, data) + metadata update` |
|
||||||
|
| `decrement_ref(hash)` | `std::fs::write/remove` | `S3Vfs.delete_object + metadata check` |
|
||||||
|
| `dedup_file(source)` | Local file read + block store | Local file read + S3 PUT blocks |
|
||||||
|
| `restore_file(manifest)` | Local file write + block read | Local file write + S3 GET blocks |
|
||||||
|
| `get_ref_count(hash)` | `std::fs::read(hash.ref)` | `S3Vfs.head_object(blocks/hash) → metadata` |
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## S3 Object Layout
|
||||||
|
|
||||||
|
```
|
||||||
|
Bucket: markbase-dedup
|
||||||
|
├── blocks/
|
||||||
|
│ ├── <sha256-hash-1> # Dedup 块(4KB)
|
||||||
|
│ │ └── Metadata: x-amz-meta-ref-count: 5
|
||||||
|
│ ├── <sha256-hash-2>
|
||||||
|
│ │ └── Metadata: x-amz-meta-ref-count: 2
|
||||||
|
│ └── ...
|
||||||
|
│
|
||||||
|
├── manifests/
|
||||||
|
│ ├── <file-id-1>.json # Manifest JSON
|
||||||
|
│ │ └── Content: {"original_size": 1024, "block_hashes": [...], ...}
|
||||||
|
│ ├── <file-id-2>.json
|
||||||
|
│ └── ...
|
||||||
|
│
|
||||||
|
└── stats.json # DedupStats(可选)
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Reference Count Management
|
||||||
|
|
||||||
|
### Challenge
|
||||||
|
|
||||||
|
S3 对象不支持 atomic increment/decrement 操作。
|
||||||
|
|
||||||
|
### Solution 1: Metadata Update (推荐 ⭐⭐⭐⭐⭐)
|
||||||
|
|
||||||
|
**流程**:
|
||||||
|
```rust
|
||||||
|
fn increment_ref(&self, hash: &str) -> Result<(), VfsError> {
|
||||||
|
// 1. GET current metadata
|
||||||
|
let head = self.s3vfs.head_object(&format!("blocks/{}", hash))?;
|
||||||
|
let current_ref = head.metadata.get("x-amz-meta-ref-count")
|
||||||
|
.and_then(|v| v.parse::<u64>().ok())
|
||||||
|
.unwrap_or(0);
|
||||||
|
|
||||||
|
// 2. PUT with updated metadata
|
||||||
|
let block_data = self.s3vfs.get_object(&format!("blocks/{}", hash))?;
|
||||||
|
self.s3vfs.put_object_with_metadata(
|
||||||
|
&format!("blocks/{}", hash),
|
||||||
|
&block_data,
|
||||||
|
[("x-amz-meta-ref-count", (current_ref + 1).to_string())]
|
||||||
|
)?;
|
||||||
|
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
**优势**:
|
||||||
|
- ✅ 简单实现
|
||||||
|
- ✅ 与 S3 标准兼容
|
||||||
|
- ⚠️ 需要两次请求(GET + PUT)
|
||||||
|
|
||||||
|
**劣势**:
|
||||||
|
- ⚠️ 非原子操作(并发问题)
|
||||||
|
- ⚠️ 需要读取块数据(PUT 需要 body)
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
### Solution 2: Separate Ref Count Object
|
||||||
|
|
||||||
|
**流程**:
|
||||||
|
```rust
|
||||||
|
fn increment_ref(&self, hash: &str) -> Result<(), VfsError> {
|
||||||
|
// 1. GET ref count object
|
||||||
|
let ref_key = format!("refs/{}/count", hash);
|
||||||
|
let current = self.s3vfs.get_object(&ref_key)
|
||||||
|
.and_then(|data| data.parse::<u64>())
|
||||||
|
.unwrap_or(0);
|
||||||
|
|
||||||
|
// 2. PUT updated ref count
|
||||||
|
self.s3vfs.put_object(&ref_key, (current + 1).to_string())?;
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
**优势**:
|
||||||
|
- ✅ 无需读取块数据
|
||||||
|
- ✅ 更小的对象(仅数字)
|
||||||
|
|
||||||
|
**劣势**:
|
||||||
|
- ⚠️ 需要额外对象存储
|
||||||
|
- ⚠️ 非原子操作(并发问题)
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
### Solution 3: MinIO Extended API (企业版)
|
||||||
|
|
||||||
|
MinIO 企业版提供 `mc admin bucket policy` 和 object locking API。
|
||||||
|
|
||||||
|
**优势**:
|
||||||
|
- ✅ 可能提供 atomic operation
|
||||||
|
|
||||||
|
**劣势**:
|
||||||
|
- ⚠️ 仅 MinIO 企业版
|
||||||
|
- ⚠️ 需要研究具体 API
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Concurrency Problem
|
||||||
|
|
||||||
|
### Scenario
|
||||||
|
|
||||||
|
Node A 和 Node B 同时 dedup 相同文件:
|
||||||
|
1. Node A: `increment_ref(hash-abc)` → GET count=2 → PUT count=3
|
||||||
|
2. Node B: `increment_ref(hash-abc)` → GET count=2 → PUT count=3
|
||||||
|
3. 结果:count=3(错误,应为 count=4)
|
||||||
|
|
||||||
|
### Solution 1: Optimistic Locking
|
||||||
|
|
||||||
|
使用 S3 versioning 检测冲突:
|
||||||
|
```rust
|
||||||
|
fn increment_ref(&self, hash: &str) -> Result<(), VfsError> {
|
||||||
|
loop {
|
||||||
|
// 1. GET current version + metadata
|
||||||
|
let (version_id, current_ref) = self.get_ref_with_version(hash)?;
|
||||||
|
|
||||||
|
// 2. PUT with version check
|
||||||
|
let result = self.s3vfs.put_object_if_version(
|
||||||
|
&format!("blocks/{}", hash),
|
||||||
|
block_data,
|
||||||
|
(current_ref + 1),
|
||||||
|
version_id // Only succeed if version unchanged
|
||||||
|
);
|
||||||
|
|
||||||
|
if result.is_ok() {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
// Retry if version mismatch
|
||||||
|
}
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
**要求**:MinIO versioning enabled。
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
### Solution 2: Distributed Lock Service
|
||||||
|
|
||||||
|
使用外部分布式锁(如 Redis/Zookeeper):
|
||||||
|
```rust
|
||||||
|
fn increment_ref(&self, hash: &str) -> Result<(), VfsError> {
|
||||||
|
// 1. Acquire distributed lock
|
||||||
|
let lock = self.lock_service.acquire(&format!("lock:{}", hash))?;
|
||||||
|
|
||||||
|
// 2. Increment ref count
|
||||||
|
self.update_ref_count(hash)?;
|
||||||
|
|
||||||
|
// 3. Release lock
|
||||||
|
lock.release();
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
**劣势**:需要额外服务(Redis)。
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
### Solution 3: Accept Non-Atomic (简化方案)
|
||||||
|
|
||||||
|
对于 MarkBase Lightweight 定位:
|
||||||
|
- ⚠️ 接受非原子操作风险
|
||||||
|
- ⚠️ 偶尔 ref count 不准确(不影响数据完整性)
|
||||||
|
- ⚠️ 定期修复(scrub job)
|
||||||
|
|
||||||
|
**推荐**:Phase 1 使用 Solution 1(Metadata Update),Phase 2 研究 MinIO versioning。
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Implementation Phases
|
||||||
|
|
||||||
|
| Phase | Task | Code Lines | Priority | Risk |
|
||||||
|
|-------|------|------------|----------|------|
|
||||||
|
| **Phase 1** | DedupS3Store struct + basic I/O | ~300 | P0 | Medium |
|
||||||
|
| **Phase 2** | Reference count metadata | ~100 | P0 | Medium |
|
||||||
|
| **Phase 3** | Manifest storage to S3 | ~50 | P1 | Low |
|
||||||
|
| **Phase 4** | CLI integration | ~100 | P1 | Low |
|
||||||
|
| **Phase 5** | Async version (DedupAsyncS3Store) | ~200 | P2 | High |
|
||||||
|
| **Phase 6** | Concurrency fix (versioning) | ~150 | P2 | High |
|
||||||
|
| **Phase 7** | Performance benchmark | ~100 | P2 | Low |
|
||||||
|
| **Total** | | **~1000** | | |
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## DedupS3Store Implementation (Phase 1 Draft)
|
||||||
|
|
||||||
|
```rust
|
||||||
|
use super::s3_fs::S3Vfs;
|
||||||
|
use super::{VfsDedupConfig, VfsError};
|
||||||
|
use sha2::{Sha256, Digest};
|
||||||
|
use std::path::Path;
|
||||||
|
|
||||||
|
pub struct DedupS3Store {
|
||||||
|
s3vfs: S3Vfs,
|
||||||
|
bucket: String,
|
||||||
|
block_prefix: String,
|
||||||
|
manifest_prefix: String,
|
||||||
|
config: VfsDedupConfig,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl DedupS3Store {
|
||||||
|
pub fn new(
|
||||||
|
endpoint: &str,
|
||||||
|
region: &str,
|
||||||
|
bucket: &str,
|
||||||
|
access_key: &str,
|
||||||
|
secret_key: &str,
|
||||||
|
config: VfsDedupConfig,
|
||||||
|
) -> Result<Self, VfsError> {
|
||||||
|
let s3vfs = S3Vfs::new(endpoint, region, bucket, access_key, secret_key)?;
|
||||||
|
Ok(Self {
|
||||||
|
s3vfs,
|
||||||
|
bucket: bucket.to_string(),
|
||||||
|
block_prefix: "blocks/".to_string(),
|
||||||
|
manifest_prefix: "manifests/".to_string(),
|
||||||
|
config,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn store_block(&self, data: &[u8]) -> Result<String, VfsError> {
|
||||||
|
if data.len() > self.config.block_size {
|
||||||
|
return Err(VfsError::Io(format!("Block size exceeds limit")));
|
||||||
|
}
|
||||||
|
|
||||||
|
let hash = Self::hash_block(data);
|
||||||
|
let key = format!("{}{}", self.block_prefix, hash);
|
||||||
|
|
||||||
|
// Check if block exists
|
||||||
|
if !self.s3vfs.object_exists(&key)? {
|
||||||
|
// PUT with initial ref count = 1
|
||||||
|
self.s3vfs.put_object_with_metadata(
|
||||||
|
&key,
|
||||||
|
data,
|
||||||
|
[("x-amz-meta-ref-count", "1")]
|
||||||
|
)?;
|
||||||
|
} else {
|
||||||
|
// Increment ref count
|
||||||
|
self.increment_ref(&hash)?;
|
||||||
|
}
|
||||||
|
|
||||||
|
Ok(hash)
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn get_block(&self, hash: &str) -> Result<Vec<u8>, VfsError> {
|
||||||
|
let key = format!("{}{}", self.block_prefix, hash);
|
||||||
|
self.s3vfs.get_object(&key)
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn increment_ref(&self, hash: &str) -> Result<(), VfsError> {
|
||||||
|
let key = format!("{}{}", self.block_prefix, hash);
|
||||||
|
let head = self.s3vfs.head_object(&key)?;
|
||||||
|
|
||||||
|
let current_ref = head.metadata
|
||||||
|
.get("x-amz-meta-ref-count")
|
||||||
|
.and_then(|v| v.parse::<u64>().ok())
|
||||||
|
.unwrap_or(1);
|
||||||
|
|
||||||
|
// Need to GET block data + PUT with new metadata
|
||||||
|
let block_data = self.get_block(hash)?;
|
||||||
|
self.s3vfs.put_object_with_metadata(
|
||||||
|
&key,
|
||||||
|
&block_data,
|
||||||
|
[("x-amz-meta-ref-count", (current_ref + 1).to_string())]
|
||||||
|
)?;
|
||||||
|
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn dedup_file(&self, source: &Path) -> Result<DedupManifest, VfsError> {
|
||||||
|
let mut file = std::fs::File::open(source)?;
|
||||||
|
let mut manifest = DedupManifest::new();
|
||||||
|
let mut buffer = vec![0u8; self.config.block_size];
|
||||||
|
|
||||||
|
loop {
|
||||||
|
let n = file.read(&mut buffer)?;
|
||||||
|
if n == 0 { break; }
|
||||||
|
|
||||||
|
manifest.original_size += n;
|
||||||
|
let hash = self.store_block(&buffer[..n])?;
|
||||||
|
manifest.block_hashes.push(hash);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Store manifest to S3
|
||||||
|
let file_id = uuid::Uuid::new_v4().to_string();
|
||||||
|
manifest.file_id = file_id;
|
||||||
|
let manifest_key = format!("{}{}.json", self.manifest_prefix, file_id);
|
||||||
|
let manifest_json = serde_json::to_string(&manifest)?;
|
||||||
|
self.s3vfs.put_object(&manifest_key, manifest_json.as_bytes())?;
|
||||||
|
|
||||||
|
Ok(manifest)
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn restore_file(&self, manifest_id: &str, target: &Path) -> Result<(), VfsError> {
|
||||||
|
let manifest_key = format!("{}{}.json", self.manifest_prefix, manifest_id);
|
||||||
|
let manifest_json = self.s3vfs.get_object(&manifest_key)?;
|
||||||
|
let manifest: DedupManifest = serde_json::from_slice(&manifest_json)?;
|
||||||
|
|
||||||
|
let mut file = std::fs::File::create(target)?;
|
||||||
|
for hash in &manifest.block_hashes {
|
||||||
|
let block = self.get_block(hash)?;
|
||||||
|
file.write_all(&block)?;
|
||||||
|
}
|
||||||
|
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
fn hash_block(data: &[u8]) -> String {
|
||||||
|
let mut hasher = Sha256::new();
|
||||||
|
hasher.update(data);
|
||||||
|
hex::encode(hasher.finalize())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Integration with MarkBase VFS
|
||||||
|
|
||||||
|
### Option 1: Standalone DedupS3Store
|
||||||
|
|
||||||
|
用户手动创建 DedupS3Store:
|
||||||
|
```bash
|
||||||
|
# CLI tool
|
||||||
|
markbase dedup-upload --s3 --s3-endpoint http://localhost:9000 --file /data/large.iso
|
||||||
|
markbase dedup-download --s3 --manifest-id <uuid> --output /data/restored.iso
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
### Option 2: DedupVfsBackend (VfsBackend trait)
|
||||||
|
|
||||||
|
创建 VfsBackend wrapper,自动 dedup:
|
||||||
|
```rust
|
||||||
|
pub struct DedupS3Backend {
|
||||||
|
dedup_store: DedupS3Store,
|
||||||
|
manifest_dir: PathBuf, // Local cache for manifests
|
||||||
|
}
|
||||||
|
|
||||||
|
impl VfsBackend for DedupS3Backend {
|
||||||
|
fn open_file(&self, path: &Path, flags: &OpenFlags) -> Result<Box<dyn VfsFile>, VfsError> {
|
||||||
|
// 1. Read manifest from S3
|
||||||
|
let manifest = self.load_manifest(path)?;
|
||||||
|
|
||||||
|
// 2. DedupS3File (read blocks from S3)
|
||||||
|
Ok(Box::new(DedupS3File::new(self.dedup_store.clone(), manifest)))
|
||||||
|
}
|
||||||
|
|
||||||
|
fn stat(&self, path: &Path) -> Result<VfsStat, VfsError> {
|
||||||
|
// Read from manifest metadata
|
||||||
|
let manifest = self.load_manifest(path)?;
|
||||||
|
Ok(VfsStat {
|
||||||
|
size: manifest.original_size,
|
||||||
|
mtime: manifest.mtime,
|
||||||
|
...
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
fn read_dir(&self, path: &Path) -> Result<Vec<VfsDirEntry>, VfsError> {
|
||||||
|
// List manifests from S3
|
||||||
|
self.dedup_store.s3vfs.list_objects(&self.manifest_prefix)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
**优势**:
|
||||||
|
- ✅ 透明 dedup(用户无需关心)
|
||||||
|
- ✅ 与 SMB/WebDAV/SFTP 无缝集成
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
### Option 3: Hybrid (LocalFs + DedupS3Store)
|
||||||
|
|
||||||
|
```rust
|
||||||
|
pub struct HybridDedupBackend {
|
||||||
|
local: LocalFs, // Small files (<1MB) 存本地
|
||||||
|
dedup_s3: DedupS3Store, // Large files (>1MB) dedup to S3
|
||||||
|
}
|
||||||
|
|
||||||
|
impl VfsBackend for HybridDedupBackend {
|
||||||
|
fn open_file(&self, path: &Path, flags: &OpenFlags) -> Result<Box<dyn VfsFile>, VfsError> {
|
||||||
|
// Check file size
|
||||||
|
let stat = self.local.stat(path)?;
|
||||||
|
|
||||||
|
if stat.size < self.dedup_s3.config.min_file_size {
|
||||||
|
// Small file: direct LocalFs
|
||||||
|
self.local.open_file(path, flags)
|
||||||
|
} else {
|
||||||
|
// Large file: dedup to S3
|
||||||
|
self.dedup_s3.dedup_file(path)?;
|
||||||
|
self.dedup_s3.open_file_from_manifest(path)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
**推荐**:Option 1(Phase 1),Option 3(Phase 2)。
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Performance Considerations
|
||||||
|
|
||||||
|
### Network Latency
|
||||||
|
|
||||||
|
| Operation | LocalFs | S3Vfs | Overhead |
|
||||||
|
|-----------|---------|-------|----------|
|
||||||
|
| store_block (4KB) | ~0.1ms | ~5-10ms (HTTP) | ~50-100x |
|
||||||
|
| get_block (4KB) | ~0.1ms | ~5-10ms (HTTP) | ~50-100x |
|
||||||
|
| dedup_file (100MB) | ~2s (25MB/s) | ~10s (10MB/s) | ~5x |
|
||||||
|
|
||||||
|
**缓解方案**:
|
||||||
|
- ✅ Async concurrent upload(4-8 并发)
|
||||||
|
- ✅ ReadCache(64MB cache)
|
||||||
|
- ✅ Local cache for hot blocks
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
### Dedup Ratio Impact
|
||||||
|
|
||||||
|
| File Type | Dedup Ratio | Network Traffic Saved |
|
||||||
|
|-----------|-------------|----------------------|
|
||||||
|
| VM images (similar OS) | ~80% | -80% upload bandwidth |
|
||||||
|
| Log files (daily) | ~60% | -60% upload bandwidth |
|
||||||
|
| Unique files (photos) | ~5% | -5% upload bandwidth |
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Next Steps
|
||||||
|
|
||||||
|
1. **Phase 1 Implementation** (~300 lines)
|
||||||
|
- `DedupS3Store` struct
|
||||||
|
- `store_block()` / `get_block()` via S3Vfs
|
||||||
|
- `increment_ref()` with metadata update
|
||||||
|
|
||||||
|
2. **Phase 2 CLI Integration** (~100 lines)
|
||||||
|
- `markbase dedup-upload --s3`
|
||||||
|
- `markbase dedup-download --manifest-id`
|
||||||
|
|
||||||
|
3. **Phase 3 Performance Test**
|
||||||
|
- Benchmark dedup_file (100MB)
|
||||||
|
- Compare LocalFs vs S3Vfs
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Open Questions
|
||||||
|
|
||||||
|
1. **Concurrency**: Accept non-atomic ref count vs implement versioning?
|
||||||
|
2. **Backend choice**: Standalone CLI vs VfsBackend integration?
|
||||||
|
3. **Min versioning**: Should we require MinIO versioning enabled?
|
||||||
|
4. **Ref count object**: Metadata vs separate object?
|
||||||
|
5. **Block cache**: Should we cache blocks locally?
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
**文档创建**: 2026-06-25
|
||||||
|
**最后更新**: 2026-06-25
|
||||||
@@ -0,0 +1,404 @@
|
|||||||
|
# MarkBase GUI 管理介面检讨报告
|
||||||
|
|
||||||
|
**版本**: 1.0
|
||||||
|
**日期**: 2026-06-25
|
||||||
|
**作者**: AI Assistant
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 一、GUI 架构概览
|
||||||
|
|
||||||
|
### 1.1 技术栈
|
||||||
|
|
||||||
|
| 组件 | 技术 | 版本 |
|
||||||
|
|------|------|------|
|
||||||
|
| **前端框架** | Vue.js 3 | Composition API |
|
||||||
|
| **UI 库** | Element Plus | Latest |
|
||||||
|
| **桌面框架** | Tauri | v2 |
|
||||||
|
| **后端语言** | Rust | 1.92+ |
|
||||||
|
| **数据存储** | SQLite | auth.sqlite |
|
||||||
|
|
||||||
|
### 1.2 代码统计
|
||||||
|
|
||||||
|
| 类型 | 数量 | 行数 |
|
||||||
|
|------|------|------|
|
||||||
|
| **Vue Components** | 11 个 | 4860 行 |
|
||||||
|
| **Tauri Commands** | 12 个 | ~1500 行 |
|
||||||
|
| **Router Routes** | 11 个 | 77 行 |
|
||||||
|
| **总计** | | ~6437 行 |
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 二、已实现功能
|
||||||
|
|
||||||
|
### 2.1 User Management (Users.vue)
|
||||||
|
|
||||||
|
**功能完整度**: ⭐⭐⭐⭐⭐ (5/5)
|
||||||
|
|
||||||
|
| 功能 | 状态 | 说明 |
|
||||||
|
|------|------|------|
|
||||||
|
| 用户列表 | ✅ 完成 | 显示 username, home_dir, status |
|
||||||
|
| 创建用户 | ✅ 完成 | bcrypt 密码加密 + SqliteProvider |
|
||||||
|
| 编辑用户 | ✅ 完成 | home_dir/status 更新 + 密码可选 |
|
||||||
|
| 删除用户 | ✅ 完成 | 确认对话框 + SqliteProvider |
|
||||||
|
| 重置密码 | ✅ 完成 | 弹窗输入新密码 |
|
||||||
|
|
||||||
|
**代码量**: 264 行 Vue + 100 行 Rust
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
### 2.2 Share Management (Shares.vue)
|
||||||
|
|
||||||
|
**功能完整度**: ⭐⭐⭐ (3/5)
|
||||||
|
|
||||||
|
| 功能 | 状态 | 说明 |
|
||||||
|
|------|------|------|
|
||||||
|
| 共享列表 | ✅ 完成 | name, path, protocol, users, permissions |
|
||||||
|
| 创建共享 | ⚠️ 内存存储 | 重启丢失(需持久化) |
|
||||||
|
| 编辑共享 | ⚠️ 内存存储 | 重启丢失(需持久化) |
|
||||||
|
| 删除共享 | ⚠️ 内存存储 | 重启丢失(需持久化) |
|
||||||
|
| 连接测试 | ✅ 完成 | path 存在验证 |
|
||||||
|
| 协议支持 | ✅ 完成 | SMB/SFTP/WebDAV/S3 |
|
||||||
|
|
||||||
|
**代码量**: 295 行 Vue + 152 行 Rust
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
### 2.3 Dashboard (Dashboard.vue)
|
||||||
|
|
||||||
|
**功能完整度**: ⭐⭐⭐ (3/5)
|
||||||
|
|
||||||
|
| 功能 | 状态 | 说明 |
|
||||||
|
|------|------|------|
|
||||||
|
| CPU 监控 | ✅ 完成 | macOS/Linux 双平台 |
|
||||||
|
| Memory 监控 | ✅ 完成 | macOS/Linux 双平台 |
|
||||||
|
| Disk 监控 | ✅ 完成 | macOS/Linux 双平台 |
|
||||||
|
| Service Status | ❌ 硬编码 | 返回固定 4 个服务(未检查实际进程) |
|
||||||
|
| Recent Activity | ❌ 硬编码 | 返回固定 4 条记录(未连接日志系统) |
|
||||||
|
| Quick Actions | ❌ 未实现 | 只有 UI,无实际功能 |
|
||||||
|
| 实时刷新 | ✅ 完成 | 5 秒定时刷新 |
|
||||||
|
|
||||||
|
**代码量**: 302 行 Vue + 290 行 Rust
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
### 2.4 Backup Management (Backup.vue)
|
||||||
|
|
||||||
|
**功能完整度**: ⭐⭐⭐⭐ (4/5)
|
||||||
|
|
||||||
|
**代码量**: 497 行 Vue + 3732 行 Rust
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 三、存在的问题
|
||||||
|
|
||||||
|
### 3.1 关键问题
|
||||||
|
|
||||||
|
| 问题 | 严重程度 | 影响 |
|
||||||
|
|------|----------|------|
|
||||||
|
| **Share Management 内存存储** | ⚠️⚠️⚠️⚠️⚠️ 极高 | 重启丢失所有共享配置 |
|
||||||
|
| **Service Status 硬编码** | ⚠️⚠️⚠️⚠️ 高 | 无法反映真实服务状态 |
|
||||||
|
| **Recent Activity 硬编码** | ⚠️⚠️⚠️⚠️ 高 | 无法查看真实操作记录 |
|
||||||
|
| **Quick Actions 未实现** | ⚠️⚠️⚠️ 中 | 用户体验不完整 |
|
||||||
|
| **无权限管理** | ⚠️⚠️⚠️⚠️ 高 | 无法控制用户访问权限 |
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
### 3.2 详细分析
|
||||||
|
|
||||||
|
#### 问题 #1: Share Management 内存存储
|
||||||
|
|
||||||
|
**当前实现**:
|
||||||
|
```rust
|
||||||
|
lazy_static::lazy_static! {
|
||||||
|
static ref SHARES: Arc<Mutex<Vec<ShareInfo>>> =
|
||||||
|
Arc::new(Mutex::new(Vec::new()));
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
**问题影响**:
|
||||||
|
- 服务器重启后,所有共享配置丢失
|
||||||
|
- 无法持久化到数据库或配置文件
|
||||||
|
- 不符合生产环境要求
|
||||||
|
|
||||||
|
**推荐方案**:
|
||||||
|
- 使用 SQLite 存储(`data/shares.sqlite`)
|
||||||
|
- 或 TOML 配置文件(`config/shares.toml`)
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
#### 问题 #2: Service Status 硬编码
|
||||||
|
|
||||||
|
**当前实现**:
|
||||||
|
```rust
|
||||||
|
pub async fn get_all_services_status() -> Result<Vec<ServiceStatus>, String> {
|
||||||
|
Ok(vec![
|
||||||
|
ServiceStatus {
|
||||||
|
name: "SMB Server".to_string(),
|
||||||
|
status: "running".to_string(),
|
||||||
|
port: 4445,
|
||||||
|
uptime: "2h 30m".to_string(),
|
||||||
|
},
|
||||||
|
// ... 固定返回 4 个服务
|
||||||
|
])
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
**问题影响**:
|
||||||
|
- 无法检测服务真实状态(running/stopped)
|
||||||
|
- 无法获取真实 uptime
|
||||||
|
- 无法监控服务异常
|
||||||
|
|
||||||
|
**推荐方案**:
|
||||||
|
- 使用 `ps aux | grep` 检查进程状态
|
||||||
|
- 或使用 PID 文件追踪服务
|
||||||
|
- 或使用 systemd/launchd 服务管理
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
#### 问题 #3: Recent Activity 硬编码
|
||||||
|
|
||||||
|
**当前实现**:
|
||||||
|
```rust
|
||||||
|
pub async fn get_recent_activity() -> Result<Vec<ActivityLog>, String> {
|
||||||
|
Ok(vec![
|
||||||
|
ActivityLog {
|
||||||
|
timestamp: "2026-06-23 14:30:00".to_string(),
|
||||||
|
activity_type: "Upload".to_string(),
|
||||||
|
description: "Uploaded document.pdf to /data/files".to_string(),
|
||||||
|
user: "alice".to_string(),
|
||||||
|
},
|
||||||
|
// ... 固定返回 4 条记录
|
||||||
|
])
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
**问题影响**:
|
||||||
|
- 无法查看真实用户操作
|
||||||
|
- 无法审计系统行为
|
||||||
|
- 无法追踪异常事件
|
||||||
|
|
||||||
|
**推荐方案**:
|
||||||
|
- 使用日志文件(`data/activity.log`)
|
||||||
|
- 或 SQLite 存储(`data/activity.sqlite`)
|
||||||
|
- 集成现有 SSH/SMB/WebDAV 日志
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
#### 问题 #4: Quick Actions 未实现
|
||||||
|
|
||||||
|
**当前实现**:
|
||||||
|
```vue
|
||||||
|
<el-button type="primary" :icon="Upload" class="action-btn">
|
||||||
|
Upload File
|
||||||
|
</el-button>
|
||||||
|
<el-button type="success" :icon="Document" class="action-btn">
|
||||||
|
Create Backup
|
||||||
|
</el-button>
|
||||||
|
// ... 只有按钮,无 @click handler
|
||||||
|
```
|
||||||
|
|
||||||
|
**问题影响**:
|
||||||
|
- 用户点击按钮无响应
|
||||||
|
- Dashboard 功能不完整
|
||||||
|
|
||||||
|
**推荐方案**:
|
||||||
|
- Upload File: 调用 Tauri dialog + file_ops.rs
|
||||||
|
- Create Backup: 调用 backup.rs snapshot 功能
|
||||||
|
- View Backups: 跳转到 Backup.vue
|
||||||
|
- Download File: 调用 Tauri dialog + file_ops.rs
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
#### 问题 #5: 无权限管理
|
||||||
|
|
||||||
|
**当前实现**:
|
||||||
|
- User Management 只有 CRUD 用户
|
||||||
|
- Share Management 只有 CRUD 共享
|
||||||
|
- **缺失**:用户-共享权限关联
|
||||||
|
|
||||||
|
**问题影响**:
|
||||||
|
- 无法控制用户访问哪些共享
|
||||||
|
- 无法设置读/写权限
|
||||||
|
- 无法实现多租户隔离
|
||||||
|
|
||||||
|
**推荐方案**:
|
||||||
|
- 添加 Permission Management 页面
|
||||||
|
- 用户-共享关联表(user_id, share_id, permission)
|
||||||
|
- 权限类型:read/write/admin
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 四、竞争对手对比
|
||||||
|
|
||||||
|
### 4.1 功能对比表
|
||||||
|
|
||||||
|
| 功能 | Proxmox VE | Unraid | OpenNAS | MarkBase | 覆盖率 |
|
||||||
|
|------|-----------|--------|---------|----------|--------|
|
||||||
|
| **Dashboard** | ✅ 完整 | ✅ 完整 | ✅ 完整 | ⚠️ 部分 | 60% |
|
||||||
|
| **User Management** | ✅ 完整 | ✅ 完整 | ✅ 完整 | ✅ 完整 | 100% |
|
||||||
|
| **Share Management** | ✅ 完整 | ✅ 完整 | ✅ 完整 | ⚠️ 内存 | 50% |
|
||||||
|
| **Backup Management** | ✅ 完整 | ✅ 完整 | ✅ 完整 | ✅ 完整 | 100% |
|
||||||
|
| **Permission Management** | ✅ 完整 | ✅ 完整 | ✅ 完整 | ❌ 缺失 | 0% |
|
||||||
|
| **Service Monitoring** | ✅ 完整 | ✅ 完整 | ✅ 完整 | ❌ 硬编码 | 30% |
|
||||||
|
| **Activity Log** | ✅ 完整 | ✅ 完整 | ✅ 完整 | ❌ 硕编码 | 30% |
|
||||||
|
| **VM Management** | ✅ 完整 | ❌ 无 | ❌ 无 | ❌ 无 | N/A |
|
||||||
|
| **Container Management** | ✅ 完整 | ✅ 完整 | ❌ 无 | ❌ 无 | N/A |
|
||||||
|
| **HA Cluster** | ✅ 完整 | ❌ 无 | ❌ 无 | ❌ 无 | N/A |
|
||||||
|
|
||||||
|
**总体覆盖率**: **47%** (存储 + 备份)
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
### 4.2 竞争对手优势
|
||||||
|
|
||||||
|
**Proxmox VE**:
|
||||||
|
- ✅ 完整的 VM/Container 管理
|
||||||
|
- ✅ HA Cluster 支持
|
||||||
|
- ✅ 企业级权限管理
|
||||||
|
- ✅ 完整的监控告警系统
|
||||||
|
- ✅ API + CLI + Web UI 三位一体
|
||||||
|
|
||||||
|
**Unraid**:
|
||||||
|
- ✅ Docker Container 管理
|
||||||
|
- ✅ 简单易用的 Web UI
|
||||||
|
- ✅ Community Apps 生态
|
||||||
|
- ✅ Flash drive 启动(无需安装)
|
||||||
|
- ✅ Parity 保护(类似 RAID)
|
||||||
|
|
||||||
|
**OpenNAS**:
|
||||||
|
- ✅ 专注 NAS 功能
|
||||||
|
- ✅ ZFS 集成
|
||||||
|
- ✅ 简单部署
|
||||||
|
- ✅ 开源免费
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 五、改进建议
|
||||||
|
|
||||||
|
### 5.1 短期改进(本周)
|
||||||
|
|
||||||
|
| 优先级 | 任务 | 工作量 | 收益 |
|
||||||
|
|--------|------|--------|------|
|
||||||
|
| **P0 #1** | Share Management 持久化 | 200 行 | ⭐⭐⭐⭐⭐ 极高 |
|
||||||
|
| **P0 #2** | Service Status 真实检测 | 150 行 | ⭐⭐⭐⭐ 高 |
|
||||||
|
| **P0 #3** | Quick Actions 实现 | 100 行 | ⭐⭐⭐ 中 |
|
||||||
|
| **P1 #4** | Permission Management | 300 行 | ⭐⭐⭐⭐⭐ 极高 |
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
### 5.2 中期改进(本月)
|
||||||
|
|
||||||
|
| 优先级 | 任务 | 工作量 | 收益 |
|
||||||
|
|--------|------|--------|------|
|
||||||
|
| **P1 #5** | Activity Log 系统集成 | 400 行 | ⭐⭐⭐⭐ 高 |
|
||||||
|
| **P1 #6** | Dashboard 增强图表 | 200 行 | ⭐⭐⭐ 中 |
|
||||||
|
| **P2 #7** | File Browser UI | 500 行 | ⭐⭐⭐⭐⭐ 极高 |
|
||||||
|
| **P2 #8** | Snapshot Management UI | 300 行 | ⭐⭐⭐⭐ 高 |
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
### 5.3 长期改进(下季度)
|
||||||
|
|
||||||
|
| 优先级 | 任务 | 工作量 | 收益 |
|
||||||
|
|--------|------|--------|------|
|
||||||
|
| **P2 #9** | Docker Container UI | 800 行 | ⭐⭐⭐⭐ 高 |
|
||||||
|
| **P3 #10** | API + CLI + Web UI 统一 | 1000 行 | ⭐⭐⭐⭐⭐ 极高 |
|
||||||
|
| **P3 #11** | 国际化支持 | 200 行 | ⭐⭐⭐ 中 |
|
||||||
|
| **P3 #12** | Mobile App | 2000 行 | ⭐⭐⭐⭐ 高 |
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 六、实施优先级
|
||||||
|
|
||||||
|
### 6.1 立即实施(本周)
|
||||||
|
|
||||||
|
**Phase 1**: Share Management 持久化
|
||||||
|
- 创建 `data/shares.sqlite` 数据库
|
||||||
|
- 实现 ShareProvider trait
|
||||||
|
- 集成到 share_management.rs
|
||||||
|
|
||||||
|
**Phase 2**: Service Status 真实检测
|
||||||
|
- 使用 `ps aux` 检查进程状态
|
||||||
|
- 解析 PID 文件(`/tmp/markbase_*.pid`)
|
||||||
|
- 计算 uptime(进程启动时间)
|
||||||
|
|
||||||
|
**Phase 3**: Quick Actions 实现
|
||||||
|
- Upload File: Tauri dialog + file_ops.rs
|
||||||
|
- Create Backup: 跳转到 Backup.vue
|
||||||
|
- View Backups: 跳转到 Backup.vue
|
||||||
|
- Download File: Tauri dialog + file_ops.rs
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
### 6.2 下周实施
|
||||||
|
|
||||||
|
**Phase 4**: Permission Management
|
||||||
|
- 创建 Permission.vue 页面
|
||||||
|
- 实现 permission_management.rs
|
||||||
|
- 用户-共享关联表
|
||||||
|
|
||||||
|
**Phase 5**: Activity Log 系统
|
||||||
|
- 创建 `data/activity.sqlite`
|
||||||
|
- 集成 SSH/SMB/WebDAV 日志
|
||||||
|
- 实现 activity.rs Tauri command
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 七、目标定位
|
||||||
|
|
||||||
|
### 7.1 当前定位
|
||||||
|
|
||||||
|
**MarkBase = Lightweight Enterprise File Server + Backup Server**
|
||||||
|
|
||||||
|
| 目标用户 | 规模 | 使用场景 |
|
||||||
|
|---------|------|---------|
|
||||||
|
| **个人** | 1-5 用户 | Home NAS + backup |
|
||||||
|
| **小团队** | 5-20 用户 | SMB/SFTP + WebDAV |
|
||||||
|
| **中小企业** | 20-100 用户 | 多协议 + snapshots |
|
||||||
|
| **大型企业** | 100+ 用户 | NFS + LDAP + HA(Phase 12) |
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
### 7.2 GUI 目标覆盖率
|
||||||
|
|
||||||
|
| 目标 | 当前覆盖率 | Phase 1-5 后 | Phase 6-12 后 |
|
||||||
|
|------|-----------|-------------|--------------|
|
||||||
|
| **vs Proxmox VE** | 47% | 65% | 75% |
|
||||||
|
| **vs Unraid** | 58% | 75% | 85% |
|
||||||
|
| **vs OpenNAS** | 62% | 80% | 90% |
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 八、总结
|
||||||
|
|
||||||
|
### 8.1 已完成
|
||||||
|
|
||||||
|
- ✅ User Management 完整实现(5/5)
|
||||||
|
- ✅ Backup Management 基本实现(4/5)
|
||||||
|
- ✅ Dashboard 系统监控(3/5)
|
||||||
|
- ⚠️ Share Management 内存存储(3/5)
|
||||||
|
|
||||||
|
### 8.2 待完成
|
||||||
|
|
||||||
|
- ❌ Share Management 持久化
|
||||||
|
- ❌ Service Status 真实检测
|
||||||
|
- ❌ Activity Log 系统集成
|
||||||
|
- ❌ Permission Management
|
||||||
|
- ❌ Quick Actions 实现
|
||||||
|
|
||||||
|
### 8.3 建议
|
||||||
|
|
||||||
|
**立即开始** Phase 1-3(本周):
|
||||||
|
- Share Management 持久化(P0)
|
||||||
|
- Service Status 真实检测(P0)
|
||||||
|
- Quick Actions 实现(P0)
|
||||||
|
|
||||||
|
**下周开始** Phase 4-5:
|
||||||
|
- Permission Management(P1)
|
||||||
|
- Activity Log 系统集成(P1)
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
**最后更新**: 2026-06-25
|
||||||
|
**版本**: 1.0(GUI 管理介面检讨报告)
|
||||||
@@ -0,0 +1,75 @@
|
|||||||
|
# macOS SMB Compatibility Design
|
||||||
|
|
||||||
|
## Overview
|
||||||
|
|
||||||
|
Enable seamless macOS SMB client connectivity through five phases of
|
||||||
|
implementation inspired by Samba's `vfs_fruit` and `vfs_catia` modules.
|
||||||
|
|
||||||
|
## Gap Analysis Summary
|
||||||
|
|
||||||
|
| Feature | Samba vfs_fruit | MarkBase SMB | Status |
|
||||||
|
|---------|----------------|--------------|--------|
|
||||||
|
| AFP_AfpInfo (60-byte) | Native xattr | **Truncated to 32 bytes** | ⚠️ P0 bug |
|
||||||
|
| Catia char mapping | vfs_catia | Functions exist, **not integrated** | ❌ P1 |
|
||||||
|
| AAPL RESOLVE_ID | AAPL context | **Advertised, not implemented** | ❌ P1 |
|
||||||
|
| AAPL QUERY_DIR | READ_DIR_ATTR | **Advertised, not implemented** | ❌ P2 |
|
||||||
|
| Time Machine xattr | vfs_fruit | Set on TreeConnect, **not persisted** | ❌ P2 |
|
||||||
|
| Finder tags | _kMDItemUserTags | Not implemented | ❌ Future |
|
||||||
|
| OSX copyfile offload | FSCTL_SRV_COPYCHUNK | Not implemented | ❌ Future |
|
||||||
|
|
||||||
|
## Phase 1 — AFP_AfpInfo 60-Byte Fix (P0)
|
||||||
|
|
||||||
|
**Problem**: `backend.rs` defines `AFP_INFO_SIZE = 32`, truncating the 60-byte
|
||||||
|
`AfpInfo` structure to only the `FinderInfo` portion. Backup time, ProDos info,
|
||||||
|
and reserved fields are silently discarded.
|
||||||
|
|
||||||
|
**Fix**: Change the constant to 60 to match `afp_info.rs`.
|
||||||
|
|
||||||
|
**Files**: `vendor/smb-server/src/backend.rs`
|
||||||
|
|
||||||
|
## Phase 2 — Catia Character Conversion (P1)
|
||||||
|
|
||||||
|
**Problem**: macOS clients send NTFS-illegal characters (`:*?"<>|`) encoded as
|
||||||
|
Unicode private-range code points (`U+F001`–`U+F070`). These are rejected by
|
||||||
|
`SmbPath::from_utf16()` which validates against NTFS-illegal characters.
|
||||||
|
|
||||||
|
The conversion functions already exist in `unicode_mapping.rs` but are never
|
||||||
|
called before path validation.
|
||||||
|
|
||||||
|
**Fix**: Convert private-range chars to ASCII equivalents **before** calling
|
||||||
|
`SmbPath::from_utf16()` in `create.rs` and `query_directory.rs`.
|
||||||
|
|
||||||
|
**Files**:
|
||||||
|
- `vendor/smb-server/src/handlers/create.rs`
|
||||||
|
- `vendor/smb-server/src/path.rs` (add public conversion helper)
|
||||||
|
|
||||||
|
## Phase 3 — AAPL RESOLVE_ID (P1)
|
||||||
|
|
||||||
|
**Problem**: macOS clients send AAPL create context with command = RESOLVE_ID
|
||||||
|
to map a FileId back to a path. The server advertises `SUPPORT_RESOLVE_ID` but
|
||||||
|
does not handle the command — it silently returns `None`.
|
||||||
|
|
||||||
|
**Fix**: Handle `SMB2_CRTCTX_AAPL_RESOLVE_ID` in the AAPL context processing.
|
||||||
|
Return the path associated with the requested FileId.
|
||||||
|
|
||||||
|
**Files**: `vendor/smb-server/src/handlers/create.rs`
|
||||||
|
|
||||||
|
## Phase 4 — AAPL QUERY_DIR (P2)
|
||||||
|
|
||||||
|
**Problem**: macOS uses AAPL SERVER_QUERY to request directory attributes in
|
||||||
|
the CREATE response. The server handles SERVER_QUERY but does not provide
|
||||||
|
`READ_DIR_ATTR` enhancements.
|
||||||
|
|
||||||
|
**Fix**: When AAPL SERVER_QUERY includes `READ_DIR_ATTR`, return directory
|
||||||
|
metadata (file count, free space) in the response.
|
||||||
|
|
||||||
|
**Files**: `vendor/smb-server/src/handlers/create.rs`
|
||||||
|
|
||||||
|
## Phase 5 — Time Machine Persistence (P2)
|
||||||
|
|
||||||
|
**Problem**: `com.apple.TimeMachine.*` xattrs are set on every TreeConnect
|
||||||
|
with a new random UUID. The UUID changes on reconnect, confusing macOS.
|
||||||
|
|
||||||
|
**Fix**: Check for existing xattrs before setting new ones. Persist the UUID.
|
||||||
|
|
||||||
|
**Files**: `vendor/smb-server/src/handlers/tree_connect.rs`
|
||||||
@@ -0,0 +1,382 @@
|
|||||||
|
# MinIO Integration Guide for MarkBase
|
||||||
|
|
||||||
|
**Date**: 2026-06-25
|
||||||
|
**Status**: Ready for deployment
|
||||||
|
**Backend**: S3Vfs (已有实现,无需修改代码)
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Executive Summary
|
||||||
|
|
||||||
|
MinIO 是高性能、S3-compatible 的对象存储服务,完美契合 MarkBase 的定位:
|
||||||
|
- ✅ 跨平台支持(macOS/Linux/Windows)
|
||||||
|
- ✅ 轻量级部署(单节点即可)
|
||||||
|
- ✅ 已有 S3Vfs 支持(无需修改代码)
|
||||||
|
- ✅ 高性能(纠删码 + 分布式扩展)
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## MinIO vs Ceph RADOS Comparison
|
||||||
|
|
||||||
|
| Aspect | MinIO | Ceph RADOS |
|
||||||
|
|--------|-------|------------|
|
||||||
|
| **Platform** | ✅ 全平台 | ❌ Linux-only |
|
||||||
|
| **Deployment** | ⚠️⚠️ 单节点即可 | ⚠️⚠️⚠️⚠️⚠️ 需完整集群 |
|
||||||
|
| **API** | ✅ S3-compatible HTTP | ❌ librados FFI |
|
||||||
|
| **Code change** | ✅ 0 行(已有 S3Vfs) | ❌ ~1350 行 |
|
||||||
|
| **Positioning** | ⭐⭐⭐⭐⭐ 完全匹配 | ❌ 不符合 Lightweight 定位 |
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## MinIO Deployment
|
||||||
|
|
||||||
|
### macOS 单节点部署
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# 安装 MinIO
|
||||||
|
brew install minio/stable/minio
|
||||||
|
|
||||||
|
# 启动 MinIO server
|
||||||
|
minio server /path/to/data --console-address ":9001"
|
||||||
|
|
||||||
|
# 输出:
|
||||||
|
# Endpoint: http://192.168.1.100:9000 http://127.0.0.1:9000
|
||||||
|
# Console: http://192.168.1.100:9001 http://127.0.0.1:9001
|
||||||
|
# AccessKey: minioadmin
|
||||||
|
# SecretKey: minioadmin
|
||||||
|
```
|
||||||
|
|
||||||
|
### Linux 生产部署
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# Docker 单节点
|
||||||
|
docker run -d \
|
||||||
|
--name minio \
|
||||||
|
-p 9000:9000 \
|
||||||
|
-p 9001:9001 \
|
||||||
|
-v /data/minio:/data \
|
||||||
|
minio/minio server /data --console-address ":9001"
|
||||||
|
|
||||||
|
# 分布式集群(4节点)
|
||||||
|
docker run -d \
|
||||||
|
--name minio \
|
||||||
|
-p 9000:9000 \
|
||||||
|
-p 9001:9001 \
|
||||||
|
-v /data1:/data1 \
|
||||||
|
-v /data2:/data2 \
|
||||||
|
minio/minio server http://node1/data1 http://node2/data2 http://node3/data1 http://node4/data2 --console-address ":9001"
|
||||||
|
```
|
||||||
|
|
||||||
|
### Kubernetes 部署(推荐生产)
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
# minio-deployment.yaml
|
||||||
|
apiVersion: apps/v1
|
||||||
|
kind: Deployment
|
||||||
|
metadata:
|
||||||
|
name: minio
|
||||||
|
spec:
|
||||||
|
replicas: 4
|
||||||
|
selector:
|
||||||
|
matchLabels:
|
||||||
|
app: minio
|
||||||
|
template:
|
||||||
|
metadata:
|
||||||
|
labels:
|
||||||
|
app: minio
|
||||||
|
spec:
|
||||||
|
containers:
|
||||||
|
- name: minio
|
||||||
|
image: minio/minio:latest
|
||||||
|
args:
|
||||||
|
- server
|
||||||
|
- http://minio-0/data http://minio-1/data http://minio-2/data http://minio-3/data
|
||||||
|
- --console-address
|
||||||
|
- ":9001"
|
||||||
|
ports:
|
||||||
|
- containerPort: 9000
|
||||||
|
- containerPort: 9001
|
||||||
|
volumeMounts:
|
||||||
|
- name: data
|
||||||
|
mountPath: /data
|
||||||
|
volumes:
|
||||||
|
- name: data
|
||||||
|
emptyDir: {}
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## MarkBase S3Vfs Integration
|
||||||
|
|
||||||
|
### 配置方式
|
||||||
|
|
||||||
|
**环境变量**:
|
||||||
|
```bash
|
||||||
|
export MB_S3_ENDPOINT=http://localhost:9000
|
||||||
|
export MB_S3_REGION=us-east-1
|
||||||
|
export MB_S3_BUCKET=markbase
|
||||||
|
export MB_S3_ACCESS_KEY=minioadmin
|
||||||
|
export MB_S3_SECRET_KEY=minioadmin
|
||||||
|
```
|
||||||
|
|
||||||
|
**配置文件**(`config/s3.toml`):
|
||||||
|
```toml
|
||||||
|
[s3]
|
||||||
|
enabled = true
|
||||||
|
endpoint = "http://localhost:9000"
|
||||||
|
region = "us-east-1"
|
||||||
|
bucket = "markbase"
|
||||||
|
access_key = "minioadmin"
|
||||||
|
secret_key = "minioadmin"
|
||||||
|
|
||||||
|
[s3.webdav]
|
||||||
|
# WebDAV 使用 S3 后端
|
||||||
|
enabled = true
|
||||||
|
user = "demo"
|
||||||
|
root_prefix = "webdav/"
|
||||||
|
```
|
||||||
|
|
||||||
|
### S3Vfs 使用示例
|
||||||
|
|
||||||
|
**WebDAV + MinIO**:
|
||||||
|
```bash
|
||||||
|
# 启动 WebDAV server(使用 MinIO 后端)
|
||||||
|
cargo run -- webdav-start \
|
||||||
|
--user demo \
|
||||||
|
--port 8002 \
|
||||||
|
--s3 \
|
||||||
|
--s3-endpoint http://localhost:9000 \
|
||||||
|
--s3-bucket markbase \
|
||||||
|
--s3-access-key minioadmin \
|
||||||
|
--s3-secret-key minioadmin \
|
||||||
|
--s3-region us-east-1 \
|
||||||
|
--root webdav/
|
||||||
|
```
|
||||||
|
|
||||||
|
**SMB + MinIO**(通过 VFS backend):
|
||||||
|
```bash
|
||||||
|
# 启动 SMB server(使用 MinIO 后端)
|
||||||
|
cargo run --features smb-server -- smb-start \
|
||||||
|
--port 4445 \
|
||||||
|
--share-name files \
|
||||||
|
--s3 \
|
||||||
|
--s3-endpoint http://localhost:9000 \
|
||||||
|
--s3-bucket markbase \
|
||||||
|
--s3-access-key minioadmin \
|
||||||
|
--s3-secret-key minioadmin \
|
||||||
|
--s3-region us-east-1 \
|
||||||
|
--root smb/
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## MinIO Bucket Management
|
||||||
|
|
||||||
|
### 创建 Bucket
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# 使用 MinIO client (mc)
|
||||||
|
mc alias set myminio http://localhost:9000 minioadmin minioadmin
|
||||||
|
mc mb myminio/markbase
|
||||||
|
|
||||||
|
# 使用 AWS CLI
|
||||||
|
aws --endpoint-url http://localhost:9000 s3 mb s3://markbase
|
||||||
|
```
|
||||||
|
|
||||||
|
### 设置 Bucket Policy
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# 公开读取 policy(用于 public shares)
|
||||||
|
mc anonymous set download myminio/markbase/public
|
||||||
|
|
||||||
|
# 私有 policy(默认)
|
||||||
|
mc anonymous set none myminio/markbase/private
|
||||||
|
```
|
||||||
|
|
||||||
|
### 设置 Bucket Quota
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# 设置 quota(MinIO 企业版功能)
|
||||||
|
mc admin bucket quota myminio/markbase 10GB
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## MinIO Features Relevant to MarkBase
|
||||||
|
|
||||||
|
| Feature | Description | MarkBase Use Case |
|
||||||
|
|---------|-------------|-------------------|
|
||||||
|
| **Erasure Coding** | 数据冗余(默认 EC:2) | 自动容错,类似 RAID |
|
||||||
|
| **Versioning** | 对象版本控制 | 可替代 Snapshot 功能 |
|
||||||
|
| **Bucket Policy** | ACL 管理 | 用户权限控制 |
|
||||||
|
| **Lifecycle Rules** | 自动过期 | 旧 backup 清理 |
|
||||||
|
| **Object Lock** | WORM 模式 | 合规性备份保护 |
|
||||||
|
| **Replication** | 跨站点复制 | Disaster recovery |
|
||||||
|
|
||||||
|
### Versioning(替代 Snapshot)
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# 启用 versioning
|
||||||
|
mc version enable myminio/markbase
|
||||||
|
|
||||||
|
# 列出对象版本
|
||||||
|
mc ls --versions myminio/markbase/file.txt
|
||||||
|
|
||||||
|
# 恢复旧版本
|
||||||
|
mc cp myminio/markbase/file.txt#version-id myminio/markbase/file.txt
|
||||||
|
```
|
||||||
|
|
||||||
|
### Lifecycle Rules(Backup 清理)
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# 设置 30 天后自动删除
|
||||||
|
mc ilm add myminio/markbase --expire-days 30
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Performance Optimization
|
||||||
|
|
||||||
|
### MinIO 性能参数
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# 高性能配置
|
||||||
|
minio server /data \
|
||||||
|
--console-address ":9001" \
|
||||||
|
--parallel 8 \
|
||||||
|
--cache /cache:1000
|
||||||
|
```
|
||||||
|
|
||||||
|
### S3Vfs 性能优化
|
||||||
|
|
||||||
|
**并发上传**(已在 S3Vfs 实现):
|
||||||
|
- Multipart upload(大于 5MB 自动分片)
|
||||||
|
- 并发上传分片(默认 4 并发)
|
||||||
|
|
||||||
|
**缓存**:
|
||||||
|
- ReadCache: 64MB, 64KB blocks, 5min TTL(已在 cache.rs 实现)
|
||||||
|
- WriteCache: 32MB(已在 cache.rs 实现)
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Docker Compose Example
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
version: '3'
|
||||||
|
services:
|
||||||
|
minio:
|
||||||
|
image: minio/minio:latest
|
||||||
|
command: server /data --console-address ":9001"
|
||||||
|
ports:
|
||||||
|
- "9000:9000"
|
||||||
|
- "9001:9001"
|
||||||
|
volumes:
|
||||||
|
- minio-data:/data
|
||||||
|
environment:
|
||||||
|
- MINIO_ROOT_USER=minioadmin
|
||||||
|
- MINIO_ROOT_PASSWORD=minioadmin
|
||||||
|
|
||||||
|
markbase-webdav:
|
||||||
|
build: .
|
||||||
|
command: webdav-start --user demo --port 8002 --s3 --s3-endpoint http://minio:9000 --s3-bucket markbase --s3-access-key minioadmin --s3-secret-key minioadmin
|
||||||
|
ports:
|
||||||
|
- "8002:8002"
|
||||||
|
environment:
|
||||||
|
- MB_S3_ENDPOINT=http://minio:9000
|
||||||
|
depends_on:
|
||||||
|
- minio
|
||||||
|
|
||||||
|
volumes:
|
||||||
|
minio-data:
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Integration Checklist
|
||||||
|
|
||||||
|
| Task | Status | Notes |
|
||||||
|
|------|--------|-------|
|
||||||
|
| **MinIO 部署** | ⏳ User action | macOS/Linux/Docker |
|
||||||
|
| **创建 Bucket** | ⏳ User action | `mc mb myminio/markbase` |
|
||||||
|
| **S3Vfs 配置** | ✅ 已支持 | 无需修改代码 |
|
||||||
|
| **WebDAV + S3** | ✅ 已支持 | CLI 参数已实现 |
|
||||||
|
| **SMB + S3** | ✅ 已支持 | CLI 参数已实现 |
|
||||||
|
| **SFTP + S3** | ⏳ 待实现 | 需要 SFTP S3 backend |
|
||||||
|
| **Backup to S3** | ✅ 已支持 | BackupManifest + S3Vfs |
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Troubleshooting
|
||||||
|
|
||||||
|
### MinIO 连接问题
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# 检查 MinIO status
|
||||||
|
mc admin info myminio
|
||||||
|
|
||||||
|
# 检查 endpoint 连接
|
||||||
|
curl -I http://localhost:9000/minio/health/live
|
||||||
|
```
|
||||||
|
|
||||||
|
### S3Vfs 错误
|
||||||
|
|
||||||
|
**常见错误**:
|
||||||
|
- `VfsError::NotFound` → Bucket 或 object 不存在
|
||||||
|
- `VfsError::PermissionDenied` → Access key/secret key 错误
|
||||||
|
- `VfsError::Io("S3 PUT failed: 403")` → Bucket policy 拒绝写入
|
||||||
|
|
||||||
|
**调试方法**:
|
||||||
|
```bash
|
||||||
|
# 查看 MinIO logs
|
||||||
|
docker logs minio
|
||||||
|
|
||||||
|
# 使用 mc 测试
|
||||||
|
mc cp test.txt myminio/markbase/test.txt
|
||||||
|
mc ls myminio/markbase/
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## MinIO vs S3Vfs Feature Mapping
|
||||||
|
|
||||||
|
| VfsBackend Method | MinIO S3 API | Status |
|
||||||
|
|-------------------|--------------|--------|
|
||||||
|
| `read_dir()` | ListObjectsV2 | ✅ |
|
||||||
|
| `open_file()` | GetObject / PutObject | ✅ |
|
||||||
|
| `stat()` | HeadObject | ✅ |
|
||||||
|
| `create_dir()` | PutObject (0-byte) | ✅ |
|
||||||
|
| `remove_dir()` | DeleteObject | ✅ |
|
||||||
|
| `remove_file()` | DeleteObject | ✅ |
|
||||||
|
| `rename()` | CopyObject + DeleteObject | ✅ |
|
||||||
|
| `exists()` | HeadObject | ✅ |
|
||||||
|
| `copy()` | CopyObject | ✅ |
|
||||||
|
| `hard_link()` | CopyObject | ✅ |
|
||||||
|
| `create_snapshot()` | Versioning | ⚠️ 需启用 versioning |
|
||||||
|
| `list_snapshots()` | ListObjectVersions | ⚠️ 需实现 |
|
||||||
|
| `set_quota()` | Bucket quota | ⚠️ MinIO 企业版 |
|
||||||
|
| `set_acl()` | Bucket policy | ⚠️ 需实现 |
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Next Steps
|
||||||
|
|
||||||
|
1. **部署 MinIO**(用户 action)
|
||||||
|
- macOS: `brew install minio && minio server /data`
|
||||||
|
- Docker: `docker run minio/minio server /data`
|
||||||
|
|
||||||
|
2. **创建 Bucket**(用户 action)
|
||||||
|
- `mc alias set myminio http://localhost:9000 minioadmin minioadmin`
|
||||||
|
- `mc mb myminio/markbase`
|
||||||
|
|
||||||
|
3. **配置 MarkBase**
|
||||||
|
- 设置 `MB_S3_*` 环境变量
|
||||||
|
- 或使用 CLI 参数 `--s3 --s3-endpoint ...`
|
||||||
|
|
||||||
|
4. **测试连接**
|
||||||
|
- WebDAV: `curl -X PROPFIND http://localhost:8002/webdav/`
|
||||||
|
- SMB: `smbclient -p 4445 -L localhost`
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
**文档创建**: 2026-06-25
|
||||||
|
**最后更新**: 2026-06-25
|
||||||
@@ -0,0 +1,595 @@
|
|||||||
|
# OpenNAS 功能比較分析
|
||||||
|
|
||||||
|
## 定位
|
||||||
|
|
||||||
|
| 平台 | 定位 | 目標用戶 | 部署方式 |
|
||||||
|
|------|------|---------|---------|
|
||||||
|
| **OpenNAS** | Open source NAS OS | DIY NAS 愛好者 | Linux distribution |
|
||||||
|
| **MarkBase** | 文件存儲 + 備份服務器 | 小型團隊、開發者 | macOS/Linux 應用 |
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 核心差異
|
||||||
|
|
||||||
|
| 特性 | OpenNAS | MarkBase | 差異 |
|
||||||
|
|------|---------|----------|------|
|
||||||
|
| **開源性質** | Linux Distribution | Rust Application | ⭐⭐⭐⭐ MarkBase 更輕量 |
|
||||||
|
| **存儲架構** | ZFS 導向 | VFS Backend 抽象 | ⭐⭐⭐⭐⭐ OpenNAS ZFS 專業 |
|
||||||
|
| **文件服務** | SMB + NFS + FTP | SMB + SFTP + WebDAV + S3 | ⭐⭐⭐⭐ MarkBase 協議更多 |
|
||||||
|
| **Web UI** | 全面管理界面 | Tauri 桌面應用 | ⭐⭐⭐⭐ OpenNAS 更完整 |
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 功能對比
|
||||||
|
|
||||||
|
### 1. 存儲管理
|
||||||
|
|
||||||
|
| 功能 | OpenNAS | MarkBase | 評分 |
|
||||||
|
|------|---------|----------|------|
|
||||||
|
| **ZFS** | ✅ 專業 ZFS 管理 | ✅ VFS 層實現 | ⭐⭐⭐⭐⭐ OpenNAS 專業 |
|
||||||
|
| **RAID 管理** | GUI RAID 創建 | RAID-Z1/Z2/Z3 | ⭐⭐⭐⭐⭐ |
|
||||||
|
| **Pool 管理** | GUI Pool 創建/扩展 | ❌ 不支持 | ⭐⭐⭐⭐⭐ OpenNAS 勝出 |
|
||||||
|
| **Dataset** | GUI Dataset 管理 | ❌ 不支持 | ⭐⭐⭐⭐⭐ OpenNAS 勝出 |
|
||||||
|
| **壓縮** | ZFS LZ4/ZSTD | VFS Compression | ⭐⭐⭐⭐⭐ |
|
||||||
|
| **Dedup** | ZFS Dedup | VFS Dedup | ⭐⭐⭐⭐⭐ |
|
||||||
|
| **Snapshot** | ZFS Snapshot | VFS Snapshot | ⭐⭐⭐⭐⭐ |
|
||||||
|
| **Scrub** | ZFS Scrub scheduler | ✅ Scrub scheduler | ⭐⭐⭐⭐⭐ |
|
||||||
|
|
||||||
|
**OpenNAS ZFS 優勢** ⭐⭐⭐⭐⭐:
|
||||||
|
```
|
||||||
|
專業 ZFS 管理:
|
||||||
|
- Pool 創建/扩展(GUI)
|
||||||
|
- Dataset 嵌套管理
|
||||||
|
- Snapshot rollback
|
||||||
|
- ZFS send/receive
|
||||||
|
- Scrub scheduler
|
||||||
|
- ARC/L2ARC 配置
|
||||||
|
```
|
||||||
|
|
||||||
|
**MarkBase ZFS-style 實現** ⭐⭐⭐⭐⭐:
|
||||||
|
```
|
||||||
|
VFS 層實現:
|
||||||
|
- RAID-Z1/Z2/Z3
|
||||||
|
- Snapshot + hardlink incremental
|
||||||
|
- Block checksum + scrub
|
||||||
|
- Compression (ZSTD/LZ4)
|
||||||
|
- Dedup (SHA-256 hash)
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
### 2. 文件服務
|
||||||
|
|
||||||
|
| 功能 | OpenNAS | MarkBase | 評分 |
|
||||||
|
|------|---------|----------|------|
|
||||||
|
| **SMB/CIFS** | ✅ Samba 配置 GUI | ✅ SMB3 完整協議 | ⭐⭐⭐⭐⭐ |
|
||||||
|
| **NFS** | ✅ NFS exports GUI | ❌ 未實現 | ⭐⭐⭐⭐⭐ OpenNAS 勝出 |
|
||||||
|
| **FTP** | ✅ FTP server | ❌ 未實現 | ⭐⭐⭐⭐ OpenNAS 勝出 |
|
||||||
|
| **SFTP** | ❌ 不支持 | ✅ SSH + SFTP subsystem | ⭐⭐⭐⭐⭐ MarkBase 獨特 |
|
||||||
|
| **WebDAV** | ❌ 不支持 | ✅ 多用戶 + 持久化鎖 | ⭐⭐⭐⭐⭐ MarkBase 獨特 |
|
||||||
|
| **S3 API** | ❌ 不支持 | ✅ AWS Signature V4 | ⭐⭐⭐⭐⭐ MarkBase 獨特 |
|
||||||
|
| **AFP** | ❌ 已弃用 | ✅ AFP_AfpInfo | ⭐⭐⭐⭐⭐ MarkBase macOS 兼容 |
|
||||||
|
|
||||||
|
**OpenNAS 文件服務** ⭐⭐⭐⭐:
|
||||||
|
- SMB + NFS + FTP(GUI 配置)
|
||||||
|
- Share 權限管理
|
||||||
|
- User/Group 管理
|
||||||
|
|
||||||
|
**MarkBase 文件服務** ⭐⭐⭐⭐⭐:
|
||||||
|
- SMB + SFTP + WebDAV + S3(多協議)
|
||||||
|
- SSH 高性能(140 MB/s)
|
||||||
|
- macOS Time Machine 支持
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
### 3. 備份/快照
|
||||||
|
|
||||||
|
| 功能 | OpenNAS | MarkBase | 評分 |
|
||||||
|
|------|---------|----------|------|
|
||||||
|
| **ZFS Snapshot** | ✅ GUI Snapshot 管理 | ✅ VFS Snapshot | ⭐⭐⭐⭐⭐ |
|
||||||
|
| **Snapshot Rollback** | ✅ GUI Rollback | ✅ restore_snapshot() | ⭐⭐⭐⭐⭐ |
|
||||||
|
| **Snapshot Clone** | ✅ GUI Clone | ❌ 不支持 | ⭐⭐⭐⭐ OpenNAS 勝出 |
|
||||||
|
| **ZFS Send/Receive** | ✅ GUI Send/Receive | ✅ send/receive API | ⭐⭐⭐⭐⭐ |
|
||||||
|
| **Incremental Send** | ✅ ZFS incremental | ✅ hardlink incremental | ⭐⭐⭐⭐⭐ |
|
||||||
|
| **Compression** | ZFS built-in | ✅ ZSTD/LZ4 | ⭐⭐⭐⭐⭐ |
|
||||||
|
| **Encryption** | ZFS encryption | ✅ AES-256-GCM at-rest | ⭐⭐⭐⭐⭐ |
|
||||||
|
| **Backup Scheduler** | Plugin | ✅ BackupScheduler 內置 | ⭐⭐⭐⭐⭐ MarkBase 更專業 |
|
||||||
|
|
||||||
|
**OpenNAS ZFS Backup 優勢** ⭐⭐⭐⭐⭐:
|
||||||
|
```
|
||||||
|
ZFS 專業備份:
|
||||||
|
- Snapshot + Clone
|
||||||
|
- Send/Receive (GUI)
|
||||||
|
- Incremental replication
|
||||||
|
- ZFS encryption
|
||||||
|
```
|
||||||
|
|
||||||
|
**MarkBase Backup Scheduler 優勢** ⭐⭐⭐⭐⭐:
|
||||||
|
```
|
||||||
|
內置備份系統:
|
||||||
|
- BackupScheduler (自動排程)
|
||||||
|
- Incremental (hardlink, 0 disk usage)
|
||||||
|
- Compression (ZSTD/LZ4)
|
||||||
|
- Encryption (AES-256-GCM)
|
||||||
|
- Block checksum + scrub
|
||||||
|
- send/receive API
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
### 4. 身份認證
|
||||||
|
|
||||||
|
| 功能 | OpenNAS | MarkBase | 評分 |
|
||||||
|
|------|---------|----------|------|
|
||||||
|
| **本地用戶** | ✅ GUI User 管理 | SQLite | ⭐⭐⭐⭐⭐ OpenNAS UI 更好 |
|
||||||
|
| **LDAP** | ✅ GUI LDAP 配置 | ✅ LdapProvider | ⭐⭐⭐⭐⭐ |
|
||||||
|
| **Active Directory** | ✅ GUI AD 配置 | ✅ for_ad() | ⭐⭐⭐⭐⭐ |
|
||||||
|
| **Public Key** | ❌ 不支持 | ✅ Ed25519 SSH auth | ⭐⭐⭐⭐⭐ MarkBase 獨特 |
|
||||||
|
| **SMB Auth** | NTLMv2 | ✅ NTLMv2 + Kerberos-ready | ⭐⭐⭐⭐⭐ |
|
||||||
|
|
||||||
|
**OpenNAS 認證 UI** ⭐⭐⭐⭐⭐:
|
||||||
|
- GUI User/Group 管理
|
||||||
|
- LDAP/AD GUI 配置
|
||||||
|
- Share 權限 UI
|
||||||
|
|
||||||
|
**MarkBase 認證架構** ⭐⭐⭐⭐⭐:
|
||||||
|
- DataProvider 抽象
|
||||||
|
- SSH Public Key
|
||||||
|
- SMB NTLMv2
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
### 5. Web UI
|
||||||
|
|
||||||
|
| 功能 | OpenNAS | MarkBase | 評分 |
|
||||||
|
|------|---------|----------|------|
|
||||||
|
| **Dashboard** | ✅ 系統概覽 | Storage + Scheduler | ⭐⭐⭐⭐⭐ |
|
||||||
|
| **存儲管理** | ✅ Pool/Dataset 管理 | ❌ 不支持 | ⭐⭐⭐⭐⭐ OpenNAS 勝出 |
|
||||||
|
| **Share 管理** | ✅ SMB/NFS/FTP GUI | ❌ 不支持 | ⭐⭐⭐⭐⭐ OpenNAS 勝出 |
|
||||||
|
| **User 管理** | ✅ User/Group GUI | ❌ 不支持 | ⭐⭐⭐⭐⭐ OpenNAS 勝出 |
|
||||||
|
| **Snapshot 管理** | ✅ Snapshot GUI | ✅ Backup.vue | ⭐⭐⭐⭐⭐ |
|
||||||
|
| **文件瀏覽** | ❌ 不支持 | ✅ Tree + Category view | ⭐⭐⭐⭐⭐ MarkBase 獨特 |
|
||||||
|
| **技術栈** | Web UI (HTML/JS) | Vue 3 + Tauri | ⭐⭐⭐⭐⭐ MarkBase 現代 |
|
||||||
|
|
||||||
|
**OpenNAS Web UI 勢** ⭐⭐⭐⭐⭐:
|
||||||
|
```
|
||||||
|
全面管理界面:
|
||||||
|
- Dashboard + 系統監控
|
||||||
|
- 存儲池管理
|
||||||
|
- Share 配置
|
||||||
|
- User/Group 管理
|
||||||
|
- Snapshot 管理
|
||||||
|
- Network 配置
|
||||||
|
```
|
||||||
|
|
||||||
|
**MarkBase Web UI 特點** ⭐⭐⭐⭐⭐:
|
||||||
|
```
|
||||||
|
現代桌面應用:
|
||||||
|
- Vue 3 + Composition API
|
||||||
|
- Tauri 2.x 跨平台
|
||||||
|
- 文件瀏覽器
|
||||||
|
- Backup 管理 UI
|
||||||
|
- Storage dashboard
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
### 6. 系統管理
|
||||||
|
|
||||||
|
| 功能 | OpenNAS | MarkBase | 評分 |
|
||||||
|
|------|---------|----------|------|
|
||||||
|
| **OS Update** | ✅ GUI Update | cargo build | ⭐⭐⭐⭐⭐ OpenNAS UI 更好 |
|
||||||
|
| **服務管理** | ✅ GUI Start/Stop | CLI | ⭐⭐⭐⭐⭐ OpenNAS UI 更好 |
|
||||||
|
| **Network 配置** | ✅ GUI Network | ❌ 不支持 | ⭐⭐⭐⭐⭐ OpenNAS 勝出 |
|
||||||
|
| **硬盤監控** | ✅ SMART GUI | ❌ 不支持 | ⭐⭐⭐⭐⭐ OpenNAS 勝出 |
|
||||||
|
| **日志管理** | ✅ GUI Log viewer | CLI logs | ⭐⭐⭐⭐ OpenNAS UI 更好 |
|
||||||
|
|
||||||
|
**OpenNAS 系統管理** ⭐⭐⭐⭐⭐:
|
||||||
|
- GUI OS Update
|
||||||
|
- GUI Service 管理
|
||||||
|
- GUI Network 配置
|
||||||
|
- SMART 監控
|
||||||
|
- Log viewer
|
||||||
|
|
||||||
|
**MarkBase 系統管理**:
|
||||||
|
- CLI-based
|
||||||
|
- cargo build 更新
|
||||||
|
- 簡化部署
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
### 7. 插件/扩展
|
||||||
|
|
||||||
|
| 功能 | OpenNAS | MarkBase | 評分 |
|
||||||
|
|------|---------|----------|------|
|
||||||
|
| **插件系統** | ❌ 不支持 | ❌ 不支持 | ⭐⭐ |
|
||||||
|
| **API** | ✅ REST API | ✅ REST API + Tauri IPC | ⭐⭐⭐⭐⭐ MarkBase 更完整 |
|
||||||
|
| **CLI** | ✅ CLI 工具 | ✅ CLI tools | ⭐⭐⭐⭐⭐ |
|
||||||
|
|
||||||
|
**OpenNAS CLI**:
|
||||||
|
- zfs CLI
|
||||||
|
- smb CLI
|
||||||
|
- nfs CLI
|
||||||
|
|
||||||
|
**MarkBase CLI** ⭐⭐⭐⭐⭐:
|
||||||
|
- web-start
|
||||||
|
- smb-start
|
||||||
|
- webdav-start
|
||||||
|
- render <FILE>
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
### 8. 性能
|
||||||
|
|
||||||
|
| 功能 | OpenNAS | MarkBase | 評分 |
|
||||||
|
|------|---------|----------|------|
|
||||||
|
| **SMB 性能** | ZFS ARC cached | ~3.0 GB/s read, ~1.9 GB/s write | ⭐⭐⭐⭐⭐ MarkBase 勝出 |
|
||||||
|
| **SSH/SFTP** | ❌ 不支持 | 140 MB/s AES-256-GCM | ⭐⭐⭐⭐⭐ MarkBase 獨特 |
|
||||||
|
| **rsync** | ❌ 不支持 | 140 MB/s | ⭐⭐⭐⭐⭐ MarkBase 獨特 |
|
||||||
|
| **ZFS ARC** | ✅ ARC caching | ❌ 不支持 | ⭐⭐⭐⭐⭐ OpenNAS 勢出 |
|
||||||
|
|
||||||
|
**OpenNAS ZFS 性能優勢** ⭐⭐⭐⭐⭐:
|
||||||
|
```
|
||||||
|
ZFS 性能特色:
|
||||||
|
- ARC caching (RAM cache)
|
||||||
|
- L2ARC (SSD cache)
|
||||||
|
- ZIL (write log)
|
||||||
|
- Compression inline
|
||||||
|
```
|
||||||
|
|
||||||
|
**MarkBase SMB 性能** ⭐⭐⭐⭐⭐:
|
||||||
|
```
|
||||||
|
SMB3 性能:
|
||||||
|
- Read: ~3.0 GB/s
|
||||||
|
- Write: ~1.9 GB/s
|
||||||
|
- AES-256-GCM encryption
|
||||||
|
- Oplocks + Lease
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
### 9. macOS 兼容
|
||||||
|
|
||||||
|
| 功能 | OpenNAS | MarkBase | 評分 |
|
||||||
|
|------|---------|----------|------|
|
||||||
|
| **Time Machine** | SMB + sparsebundle | ✅ AFP_AfpInfo | ⭐⭐⭐⭐⭐ |
|
||||||
|
| **AFP** | ❌ 已弃用 | ✅ AFP_AfpInfo tracking | ⭐⭐⭐⭐⭐ MarkBase 獨特 |
|
||||||
|
| **Catia mapping** | ❌ 不支持 | ✅ Samba vfs_catia | ⭐⭐⭐⭐⭐ MarkBase 獨特 |
|
||||||
|
| **mount_smbfs** | ✅ 基本支持 | ✅ 完整兼容 | ⭐⭐⭐⭐⭐ |
|
||||||
|
|
||||||
|
**MarkBase macOS 勢** ⭐⭐⭐⭐⭐:
|
||||||
|
- AFP_AfpInfo (backup_time tracking)
|
||||||
|
- Catia character mapping
|
||||||
|
- AAPL RESOLVE_ID + QUERY_DIR
|
||||||
|
- Time Machine UUID persistence
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 功能覆蓋率
|
||||||
|
|
||||||
|
| 類別 | OpenNAS | MarkBase | 覆蓋率 |
|
||||||
|
|------|---------|----------|--------|
|
||||||
|
| **存儲管理** | 10 功能 | 6 功能 | 60% |
|
||||||
|
| **文件服務** | 3 功能 | 5 功能 | 167% ⭐⭐⭐⭐⭐ MarkBase 勝出 |
|
||||||
|
| **備份/快照** | 8 功能 | 8 功能 | 100% ⭐⭐⭐⭐⭐ |
|
||||||
|
| **身份認證** | 4 功能 | 5 功能 | 125% |
|
||||||
|
| **Web UI** | 10 功能 | 5 功能 | 50% |
|
||||||
|
| **系統管理** | 10 功能 | 2 功能 | 20% |
|
||||||
|
| **插件/扩展** | 2 功能 | 2 功能 | 100% |
|
||||||
|
| **性能** | 2 功能 | 4 功能 | 200% ⭐⭐⭐⭐⭐ MarkBase 勝出 |
|
||||||
|
| **macOS 兼容** | 2 功能 | 5 功能 | 250% ⭐⭐⭐⭐⭐ MarkBase 勝出 |
|
||||||
|
|
||||||
|
**總體覆蓋率**:**58%**(專注存儲 + 備份)
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## OpenNAS 獨特優勢
|
||||||
|
|
||||||
|
### 1. ZFS 專業管理 ⭐⭐⭐⭐⭐
|
||||||
|
|
||||||
|
```
|
||||||
|
OpenNAS ZFS 特色:
|
||||||
|
- Pool 創建/扩展(GUI)
|
||||||
|
- Dataset 嵌套管理
|
||||||
|
- Snapshot + Clone
|
||||||
|
- Send/Receive (GUI)
|
||||||
|
- ARC/L2ARC 配置
|
||||||
|
- ZFS Scrub scheduler
|
||||||
|
```
|
||||||
|
|
||||||
|
**對比 MarkBase**:
|
||||||
|
- MarkBase VFS 層實現(不依賴 ZFS)
|
||||||
|
- OpenNAS 專業 ZFS GUI 管理
|
||||||
|
|
||||||
|
**適用場景**:
|
||||||
|
- OpenNAS:ZFS 專業用戶、數據完整性要求高
|
||||||
|
- MarkBase:輕量部署、無 ZFS 依賴
|
||||||
|
|
||||||
|
### 2. 全面 Web UI ⭐⭐⭐⭐⭐
|
||||||
|
|
||||||
|
```
|
||||||
|
OpenNAS Web UI 特色:
|
||||||
|
- Dashboard + 系統監控
|
||||||
|
- 存儲池管理
|
||||||
|
- Share 配置(SMB/NFS/FTP)
|
||||||
|
- User/Group 管理
|
||||||
|
- Snapshot 管理
|
||||||
|
- Network 配置
|
||||||
|
- OS Update
|
||||||
|
```
|
||||||
|
|
||||||
|
**對比 MarkBase**:
|
||||||
|
- MarkBase Tauri 桌面應用(現代前端)
|
||||||
|
- OpenNAS Web UI(全面管理)
|
||||||
|
|
||||||
|
### 3. 系統級管理 ⭐⭐⭐⭐⭐
|
||||||
|
|
||||||
|
```
|
||||||
|
OpenNAS 系統管理:
|
||||||
|
- GUI OS Update
|
||||||
|
- GUI Service 管理
|
||||||
|
- GUI Network 配置
|
||||||
|
- SMART 監控
|
||||||
|
- Log viewer
|
||||||
|
```
|
||||||
|
|
||||||
|
**對比 MarkBase**:
|
||||||
|
- MarkBase CLI-based
|
||||||
|
- 簡化部署(應用級)
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## MarkBase 獨特優勢
|
||||||
|
|
||||||
|
### 1. 多協議文件服務 ⭐⭐⭐⭐⭐
|
||||||
|
|
||||||
|
```
|
||||||
|
MarkBase 協議支持:
|
||||||
|
- SMB3 (完整協議,macOS 兼容)
|
||||||
|
- SFTP (SSH subsystem)
|
||||||
|
- WebDAV (多用戶 + 持久化鎖)
|
||||||
|
- S3 API (AWS Signature V4)
|
||||||
|
- SCP/rsync (140 MB/s)
|
||||||
|
```
|
||||||
|
|
||||||
|
**對比 OpenNAS**:
|
||||||
|
- OpenNAS SMB + NFS + FTP(3 協議)
|
||||||
|
- MarkBase 5 協議(更全面)
|
||||||
|
|
||||||
|
**適用場景**:
|
||||||
|
- OpenNAS:傳統 NAS (SMB/NFS)
|
||||||
|
- MarkBase:現代文件服務 (S3/SSH)
|
||||||
|
|
||||||
|
### 2. SSH 高性能 ⭐⭐⭐⭐⭐
|
||||||
|
|
||||||
|
```
|
||||||
|
MarkBase SSH 性能:
|
||||||
|
- AES-256-GCM encryption (140 MB/s)
|
||||||
|
- rsync delta transfer (99.7% data reduction)
|
||||||
|
- SCP legacy support
|
||||||
|
- OpenSSH 10.2 兼容
|
||||||
|
```
|
||||||
|
|
||||||
|
**對比 OpenNAS**:
|
||||||
|
- OpenNAS 不提供 SSH/SFTP服務
|
||||||
|
|
||||||
|
### 3. 內置 BackupScheduler ⭐⭐⭐⭐⭐
|
||||||
|
|
||||||
|
```
|
||||||
|
MarkBase 備份特色:
|
||||||
|
- BackupScheduler (自動排程)
|
||||||
|
- Incremental (hardlink, 0 disk usage)
|
||||||
|
- Compression (ZSTD/LZ4)
|
||||||
|
- Encryption (AES-256-GCM)
|
||||||
|
- Block checksum + scrub
|
||||||
|
- send/receive API
|
||||||
|
```
|
||||||
|
|
||||||
|
**對比 OpenNAS**:
|
||||||
|
- OpenNAS ZFS Snapshot(專業)
|
||||||
|
- MarkBase BackupScheduler(內置排程)
|
||||||
|
|
||||||
|
### 4. macOS Time Machine ⭐⭐⭐⭐⭐
|
||||||
|
|
||||||
|
```
|
||||||
|
MarkBase macOS 兼容:
|
||||||
|
- AFP_AfpInfo tracking
|
||||||
|
- Time Machine UUID persistence
|
||||||
|
- Catia character mapping
|
||||||
|
- AAPL RESOLVE_ID + QUERY_DIR
|
||||||
|
```
|
||||||
|
|
||||||
|
**對比 OpenNAS**:
|
||||||
|
- OpenNAS SMB + sparsebundle(基本支持)
|
||||||
|
- MarkBase AFP_AfpInfo(完整支持)
|
||||||
|
|
||||||
|
### 5. 輕量部署 ⭐⭐⭐⭐⭐
|
||||||
|
|
||||||
|
```
|
||||||
|
MarkBase 部署特色:
|
||||||
|
- macOS/Linux 應用(靈活)
|
||||||
|
- cargo build(快速升級)
|
||||||
|
- 不依賴 ZFS(輕量)
|
||||||
|
- Open source (免費)
|
||||||
|
```
|
||||||
|
|
||||||
|
**對比 OpenNAS**:
|
||||||
|
- OpenNAS Linux Distribution(專用 OS)
|
||||||
|
- 需安裝完整 OS
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 定位差異
|
||||||
|
|
||||||
|
| 平台 | 定位 | 目標場景 |
|
||||||
|
|------|------|---------|
|
||||||
|
| **OpenNAS** | Open source NAS OS | DIY NAS 愛好者、ZFS 專業用戶 |
|
||||||
|
| **MarkBase** | 文件存儲 + 備份服務器 | 小型團隊、開發者、企業文件服務 |
|
||||||
|
|
||||||
|
**關鍵差異**:
|
||||||
|
- OpenNAS:ZFS 導向 NAS OS(專業存儲管理)
|
||||||
|
- MarkBase:輕量文件服務器(應用級部署)
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 協同使用建議
|
||||||
|
|
||||||
|
### 方案 A:MarkBase 作為 OpenNAS S3 Backend
|
||||||
|
|
||||||
|
**架構**:
|
||||||
|
```
|
||||||
|
OpenNAS → S3 API → MarkBase S3 storage
|
||||||
|
```
|
||||||
|
|
||||||
|
**優勢**:
|
||||||
|
- OpenNAS ZFS 本地存儲
|
||||||
|
- MarkBase S3 遠程備份
|
||||||
|
- 混合雲存儲架構
|
||||||
|
|
||||||
|
### 方案 B:MarkBase 作為 OpenNAS SSH 備份目標
|
||||||
|
|
||||||
|
**架構**:
|
||||||
|
```
|
||||||
|
OpenNAS ZFS Send → SSH → MarkBase SFTP
|
||||||
|
```
|
||||||
|
|
||||||
|
**優勢**:
|
||||||
|
- OpenNAS ZFS send/receive
|
||||||
|
- MarkBase SSH 高性能傳輸(140 MB/s)
|
||||||
|
- 異地備份方案
|
||||||
|
|
||||||
|
### 方案 C:MarkBase 獨立部署(輕量)
|
||||||
|
|
||||||
|
**架構**:
|
||||||
|
```
|
||||||
|
MarkBase → SMB/SFTP/WebDAV → 用戶端
|
||||||
|
```
|
||||||
|
|
||||||
|
**優勢**:
|
||||||
|
- 輕量部署(應用級)
|
||||||
|
- macOS/Linux 運行
|
||||||
|
- 快速升級(cargo build)
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 部署對比
|
||||||
|
|
||||||
|
| 特性 | OpenNAS | MarkBase |
|
||||||
|
|------|---------|----------|
|
||||||
|
| **部署方式** | Linux Distribution | macOS/Linux 應用 |
|
||||||
|
| **硬體要求** | Linux server | macOS/Linux server |
|
||||||
|
| **部署時間** | 1-2 小時(OS 安裝) | 5-10 分鐘 |
|
||||||
|
| **升級方式** | GUI OS Update | cargo build |
|
||||||
|
| **成本** | Open source (免費) | Open source (免費) |
|
||||||
|
| **ZFS 依賴** | ✅ 專業 ZFS | ❌ 不依賴 |
|
||||||
|
|
||||||
|
**OpenNAS 部署優勢**:
|
||||||
|
- 專用 OS(完整管理)
|
||||||
|
- ZFS 專業支持
|
||||||
|
- GUI 全面管理
|
||||||
|
|
||||||
|
**MarkBase 部署優勢** ⭐⭐⭐⭐⭐:
|
||||||
|
- 應用級部署(輕量)
|
||||||
|
- macOS/Linux 運行(靈活)
|
||||||
|
- cargo build(快速升級)
|
||||||
|
- 不依賴 ZFS(通用)
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 技術栈對比
|
||||||
|
|
||||||
|
| 組件 | OpenNAS | MarkBase |
|
||||||
|
|------|---------|----------|
|
||||||
|
| **語言** | Shell + Python | Rust |
|
||||||
|
| **Web Server** | nginx/lighttpd | Axum |
|
||||||
|
| **SMB** | Samba | smb-server (Rust) |
|
||||||
|
| **SSH** | ❌ 不支持 | x25519-dalek + AES-GCM |
|
||||||
|
| **WebDAV** | ❌ 不支持 | dav-server (Rust) |
|
||||||
|
| **ZFS** | Native ZFS | VFS 層實現 |
|
||||||
|
| **備份** | ZFS tools | BackupScheduler (Rust) |
|
||||||
|
|
||||||
|
**MarkBase 技術優勢** ⭐⭐⭐⭐⭐:
|
||||||
|
- Rust 高性能 + 安全性
|
||||||
|
- 純 Rust 實現(無外部依賴)
|
||||||
|
- Axum async web server
|
||||||
|
- 不依賴 ZFS(輕量)
|
||||||
|
|
||||||
|
**OpenNAS 技術優勢**:
|
||||||
|
- Native ZFS(專業)
|
||||||
|
- GUI 全面管理
|
||||||
|
- Linux Distribution(專用 OS)
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 成本對比
|
||||||
|
|
||||||
|
| 成本項 | OpenNAS | MarkBase |
|
||||||
|
|--------|---------|----------|
|
||||||
|
| **License** | Open source (免費) | Open source (免費) |
|
||||||
|
| **硬體** | Linux server | macOS/Linux server |
|
||||||
|
| **部署時間** | 1-2 小時 | 5-10 分鐘 |
|
||||||
|
| **支持** | 社區支持 | Self-supported |
|
||||||
|
|
||||||
|
**OpenNAS 成本優勢**:
|
||||||
|
- Open source (免費)
|
||||||
|
- ZFS 專業支持
|
||||||
|
|
||||||
|
**MarkBase 成本優勢** ⭐⭐⭐⭐⭐:
|
||||||
|
- Open source (免費)
|
||||||
|
- 輕量部署(快速)
|
||||||
|
- macOS/Linux 運行(現有硬體)
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 總結
|
||||||
|
|
||||||
|
### MarkBase 定位:**Lightweight File Server + Backup Server**
|
||||||
|
|
||||||
|
| 功能 | OpenNAS | MarkBase |
|
||||||
|
|------|---------|----------|
|
||||||
|
| **存儲架構** | Native ZFS ⭐⭐⭐⭐⭐ | VFS Backend + RAID-Z |
|
||||||
|
| **文件服務** | SMB + NFS + FTP | SMB + SFTP + WebDAV + S3 ⭐⭐⭐⭐⭐ |
|
||||||
|
| **備份** | ZFS Snapshot ⭐⭐⭐⭐⭐ | BackupScheduler + Incremental ⭐⭐⭐⭐⭐ |
|
||||||
|
| **Web UI** | 全面管理 ⭐⭐⭐⭐⭐ | Tauri 桌面應用 |
|
||||||
|
| **系統管理** | GUI 管理 ⭐⭐⭐⭐⭐ | CLI-based |
|
||||||
|
| **部署方式** | Linux OS | macOS/Linux 應用 ⭐⭐⭐⭐⭐ |
|
||||||
|
| **SSH/SFTP** | ❌ 不支持 | 140 MB/s ⭐⭐⭐⭐⭐ |
|
||||||
|
| **macOS 兼容** | SMB basic | AFP_AfpInfo + Time Machine ⭐⭐⭐⭐⭐ |
|
||||||
|
|
||||||
|
**選擇建議**:
|
||||||
|
|
||||||
|
| 用戶類型 | 推薦平台 |
|
||||||
|
|---------|---------|
|
||||||
|
| **ZFS 專業用戶** | OpenNAS (ZFS GUI 管理) |
|
||||||
|
| **DIY NAS 愛好者** | OpenNAS (完整 OS) |
|
||||||
|
| **開發者** | MarkBase (SSH + SFTP + S3) |
|
||||||
|
| **小型企業** | MarkBase (輕量部署) |
|
||||||
|
| **macOS Time Machine** | MarkBase (AFP_AfpInfo) |
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 下一步建議
|
||||||
|
|
||||||
|
### Phase 11:完善 MarkBase 功能
|
||||||
|
|
||||||
|
1. **NFS Support** ⭐⭐⭐⭐⭐
|
||||||
|
- NFSv4 exports
|
||||||
|
- 用戶/組權限
|
||||||
|
|
||||||
|
2. **ZFS Integration** ⭐⭐⭐⭐
|
||||||
|
- Optional ZFS backend
|
||||||
|
- Native ZFS tools
|
||||||
|
|
||||||
|
3. **Web UI 完善** ⭐⭐⭐⭐⭐
|
||||||
|
- User/Group 管理 UI
|
||||||
|
- Share 配置 UI
|
||||||
|
- Dashboard 完整
|
||||||
|
|
||||||
|
4. **硬盤監控** ⭐⭐⭐⭐
|
||||||
|
- SMART 監控
|
||||||
|
- 硬盤狀態 UI
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
**最後更新**:2026-06-24
|
||||||
|
**版本**:1.52(OpenNAS 功能比較完成)
|
||||||
@@ -0,0 +1,651 @@
|
|||||||
|
# MarkBase 優化建議 (借鏡 Proxmox VE / Unraid / OpenNAS)
|
||||||
|
|
||||||
|
## 優化優先級排序
|
||||||
|
|
||||||
|
根據三個平台的比較分析,以下是 MarkBase 可以借鏡的功能,按影響力和實施難度排序:
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## P0:立即實施(高影響 + 低難度)
|
||||||
|
|
||||||
|
### 1. NFS Support ⭐⭐⭐⭐⭐
|
||||||
|
|
||||||
|
**借鏡來源**:OpenNAS, Unraid
|
||||||
|
|
||||||
|
**當前問題**:
|
||||||
|
- MarkBase 缺少 NFS 支持
|
||||||
|
- Linux/Unix 客戶端依賴 SMB 或 SFTP
|
||||||
|
|
||||||
|
**實施方案**:
|
||||||
|
```rust
|
||||||
|
// NFSv4 Server Implementation
|
||||||
|
pub struct NfsServer {
|
||||||
|
backend: Box<dyn VfsBackend>,
|
||||||
|
exports: Vec<NfsExport>,
|
||||||
|
}
|
||||||
|
|
||||||
|
pub struct NfsExport {
|
||||||
|
path: PathBuf,
|
||||||
|
clients: Vec<String>, // IP ranges
|
||||||
|
options: NfsOptions,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl NfsServer {
|
||||||
|
pub async fn handle_nfs_request(&self, req: NfsRequest) -> Result<NfsResponse>;
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
**預估工作量**:~500 行(nfs_server.rs)
|
||||||
|
**預估時間**:2-3 天
|
||||||
|
**影響**:⭐⭐⭐⭐⭐(補足 Linux 客戶端需求)
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
### 2. Web UI User/Group 管理 ⭐⭐⭐⭐⭐
|
||||||
|
|
||||||
|
**借鏡來源**:OpenNAS, Unraid
|
||||||
|
|
||||||
|
**當前問題**:
|
||||||
|
- MarkBase 需要 CLI 或 SQLite 操作用戶
|
||||||
|
- 無 GUI 用戶管理界面
|
||||||
|
|
||||||
|
**實施方案**:
|
||||||
|
```vue
|
||||||
|
<!-- Users.vue -->
|
||||||
|
<template>
|
||||||
|
<el-card>
|
||||||
|
<template #header>
|
||||||
|
<span>User Management</span>
|
||||||
|
<el-button @click="showCreateDialog">Create User</el-button>
|
||||||
|
</template>
|
||||||
|
<el-table :data="users">
|
||||||
|
<el-table-column prop="username" label="Username" />
|
||||||
|
<el-table-column prop="home_dir" label="Home Directory" />
|
||||||
|
<el-table-column label="Actions">
|
||||||
|
<el-button @click="editUser">Edit</el-button>
|
||||||
|
<el-button @click="deleteUser">Delete</el-button>
|
||||||
|
</el-table-column>
|
||||||
|
</el-table>
|
||||||
|
</el-card>
|
||||||
|
</template>
|
||||||
|
```
|
||||||
|
|
||||||
|
**REST API**:
|
||||||
|
```
|
||||||
|
GET /api/v2/users - List users
|
||||||
|
POST /api/v2/users - Create user
|
||||||
|
PUT /api/v2/users/:name - Update user
|
||||||
|
DELETE /api/v2/users/:name - Delete user
|
||||||
|
```
|
||||||
|
|
||||||
|
**預估工作量**:~300 行(Users.vue + REST API)
|
||||||
|
**預估時間**:1-2 天
|
||||||
|
**影響**:⭐⭐⭐⭐⭐(大幅提升易用性)
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
### 3. Web UI Share 管理 ⭐⭐⭐⭐
|
||||||
|
|
||||||
|
**借鏡來源**:Unraid, OpenNAS
|
||||||
|
|
||||||
|
**當前問題**:
|
||||||
|
- SMB shares 需要 CLI 配置
|
||||||
|
- 無 GUI share 管理界面
|
||||||
|
|
||||||
|
**實施方案**:
|
||||||
|
```vue
|
||||||
|
<!-- Shares.vue -->
|
||||||
|
<template>
|
||||||
|
<el-card>
|
||||||
|
<template #header>
|
||||||
|
<span>Share Management</span>
|
||||||
|
<el-button @click="showCreateDialog">Create Share</el-button>
|
||||||
|
</template>
|
||||||
|
<el-table :data="shares">
|
||||||
|
<el-table-column prop="name" label="Share Name" />
|
||||||
|
<el-table-column prop="path" label="Path" />
|
||||||
|
<el-table-column prop="protocol" label="Protocol" />
|
||||||
|
<el-table-column label="Actions">
|
||||||
|
<el-button @click="editShare">Edit</el-button>
|
||||||
|
<el-button @click="deleteShare">Delete</el-button>
|
||||||
|
</el-table-column>
|
||||||
|
</el-table>
|
||||||
|
</el-card>
|
||||||
|
</template>
|
||||||
|
```
|
||||||
|
|
||||||
|
**預估工作量**:~400 行(Shares.vue + REST API)
|
||||||
|
**預估時間**:1-2 天
|
||||||
|
**影響**:⭐⭐⭐⭐⭐(補足 Web UI 完整性)
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## P1:短期實施(高影響 + 中難度)
|
||||||
|
|
||||||
|
### 4. Dashboard 完整化 ⭐⭐⭐⭐⭐
|
||||||
|
|
||||||
|
**借鏡來源**:Proxmox VE Dashboard, Unraid Main page
|
||||||
|
|
||||||
|
**當前問題**:
|
||||||
|
- Backup.vue Dashboard 功能有限
|
||||||
|
- 缺少系統概覽(CPU/RAM/Disk)
|
||||||
|
|
||||||
|
**實施方案**:
|
||||||
|
```vue
|
||||||
|
<!-- Dashboard.vue -->
|
||||||
|
<template>
|
||||||
|
<el-row :gutter="20">
|
||||||
|
<el-col :span="6">
|
||||||
|
<el-card>
|
||||||
|
<el-statistic title="CPU Usage" :value="cpuUsage" suffix="%" />
|
||||||
|
<el-progress :percentage="cpuUsage" />
|
||||||
|
</el-card>
|
||||||
|
</el-col>
|
||||||
|
<el-col :span="6">
|
||||||
|
<el-card>
|
||||||
|
<el-statistic title="Memory Usage" :value="memUsage" suffix="%" />
|
||||||
|
<el-progress :percentage="memUsage" />
|
||||||
|
</el-card>
|
||||||
|
</el-col>
|
||||||
|
<el-col :span="6">
|
||||||
|
<el-card>
|
||||||
|
<el-statistic title="Storage Used" :value="storageUsed" suffix="%" />
|
||||||
|
<el-progress :percentage="storageUsed" />
|
||||||
|
</el-card>
|
||||||
|
</el-col>
|
||||||
|
<el-col :span="6">
|
||||||
|
<el-card>
|
||||||
|
<el-statistic title="Active Users" :value="activeUsers" />
|
||||||
|
</el-card>
|
||||||
|
</el-col>
|
||||||
|
</el-row>
|
||||||
|
|
||||||
|
<el-row :gutter="20" style="margin-top: 20px;">
|
||||||
|
<el-col :span="12">
|
||||||
|
<el-card>
|
||||||
|
<template #header>Storage Pools</template>
|
||||||
|
<el-table :data="storagePools">
|
||||||
|
<el-table-column prop="name" label="Pool" />
|
||||||
|
<el-table-column prop="type" label="Type" />
|
||||||
|
<el-table-column prop="size" label="Size" />
|
||||||
|
<el-table-column prop="used" label="Used" />
|
||||||
|
<el-table-column prop="health" label="Health">
|
||||||
|
<template #default="{ row }">
|
||||||
|
<el-tag :type="row.health === 'healthy' ? 'success' : 'danger'">
|
||||||
|
{{ row.health }}
|
||||||
|
</el-tag>
|
||||||
|
</template>
|
||||||
|
</el-table-column>
|
||||||
|
</el-table>
|
||||||
|
</el-card>
|
||||||
|
</el-col>
|
||||||
|
<el-col :span="12">
|
||||||
|
<el-card>
|
||||||
|
<template #header>Recent Backups</template>
|
||||||
|
<el-timeline>
|
||||||
|
<el-timeline-item v-for="backup in recentBackups">
|
||||||
|
{{ backup.name }} - {{ backup.time }}
|
||||||
|
</el-timeline-item>
|
||||||
|
</el-timeline>
|
||||||
|
</el-card>
|
||||||
|
</el-col>
|
||||||
|
</el-row>
|
||||||
|
</template>
|
||||||
|
```
|
||||||
|
|
||||||
|
**REST API**:
|
||||||
|
```
|
||||||
|
GET /api/v2/dashboard/stats - CPU/RAM/Disk usage
|
||||||
|
GET /api/v2/dashboard/pools - Storage pools status
|
||||||
|
GET /api/v2/dashboard/backups - Recent backups
|
||||||
|
GET /api/v2/dashboard/users - Active users count
|
||||||
|
```
|
||||||
|
|
||||||
|
**預估工作量**:~500 行(Dashboard.vue + REST API)
|
||||||
|
**預估時間**:2-3 天
|
||||||
|
**影響**:⭐⭐⭐⭐⭐(專業 Dashboard 體驗)
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
### 5. SMART 硬盤監控 ⭐⭐⭐⭐
|
||||||
|
|
||||||
|
**借鏡來源**:Unraid, OpenNAS
|
||||||
|
|
||||||
|
**當前問題**:
|
||||||
|
- MarkBase 缺少硬盤健康監控
|
||||||
|
- 硬盤故障無預警
|
||||||
|
|
||||||
|
**實施方案**:
|
||||||
|
```rust
|
||||||
|
// smart_monitor.rs
|
||||||
|
pub struct SmartMonitor {
|
||||||
|
disks: Vec<PathBuf>,
|
||||||
|
}
|
||||||
|
|
||||||
|
pub struct SmartStats {
|
||||||
|
disk: String,
|
||||||
|
temperature: u32,
|
||||||
|
health_percent: u32,
|
||||||
|
power_on_hours: u64,
|
||||||
|
read_errors: u64,
|
||||||
|
write_errors: u64,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl SmartMonitor {
|
||||||
|
pub fn check_disk(&self, disk: &Path) -> Result<SmartStats>;
|
||||||
|
pub fn get_all_stats(&self) -> Result<Vec<SmartStats>>;
|
||||||
|
pub fn is_healthy(&self, stats: &SmartStats) -> bool;
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
**Web UI**:
|
||||||
|
```vue
|
||||||
|
<!-- Disks.vue -->
|
||||||
|
<el-table :data="diskStats">
|
||||||
|
<el-table-column prop="disk" label="Disk" />
|
||||||
|
<el-table-column prop="temperature" label="Temperature" suffix="°C" />
|
||||||
|
<el-table-column prop="health_percent" label="Health">
|
||||||
|
<template #default="{ row }">
|
||||||
|
<el-progress :percentage="row.health_percent"
|
||||||
|
:color="row.health_percent > 80 ? '#67c23a' : '#f56c6c'" />
|
||||||
|
</template>
|
||||||
|
</el-table-column>
|
||||||
|
<el-table-column prop="power_on_hours" label="Power On" suffix=" hours" />
|
||||||
|
</el-table>
|
||||||
|
```
|
||||||
|
|
||||||
|
**預估工作量**:~400 行(smart_monitor.rs + Disks.vue)
|
||||||
|
**預估時間**:2-3 天
|
||||||
|
**影響**:⭐⭐⭐⭐(硬盤健康預警)
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
### 6. Plugin/Template 系統 ⭐⭐⭐⭐⭐
|
||||||
|
|
||||||
|
**借鏡來源**:Unraid Community Applications
|
||||||
|
|
||||||
|
**當前問題**:
|
||||||
|
- MarkBase 功能需 cargo build
|
||||||
|
- 無插件扩展機制
|
||||||
|
|
||||||
|
**實施方案**:
|
||||||
|
```rust
|
||||||
|
// plugin_manager.rs
|
||||||
|
pub struct PluginManager {
|
||||||
|
plugins: Vec<Plugin>,
|
||||||
|
}
|
||||||
|
|
||||||
|
pub struct Plugin {
|
||||||
|
name: String,
|
||||||
|
version: String,
|
||||||
|
author: String,
|
||||||
|
description: String,
|
||||||
|
install_path: PathBuf,
|
||||||
|
config: PluginConfig,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl PluginManager {
|
||||||
|
pub fn list_plugins(&self) -> Vec<Plugin>;
|
||||||
|
pub fn install_plugin(&mut self, url: &str) -> Result<()>;
|
||||||
|
pub fn uninstall_plugin(&mut self, name: &str) -> Result<()>;
|
||||||
|
pub fn update_plugin(&mut self, name: &str) -> Result<()>;
|
||||||
|
pub fn enable_plugin(&mut self, name: &str) -> Result<()>;
|
||||||
|
pub fn disable_plugin(&mut self, name: &str) -> Result<()>;
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
**Plugin Format**:
|
||||||
|
```json
|
||||||
|
{
|
||||||
|
"name": "markbase-nextcloud",
|
||||||
|
"version": "1.0.0",
|
||||||
|
"author": "community",
|
||||||
|
"description": "Nextcloud integration",
|
||||||
|
"install_script": "install.sh",
|
||||||
|
"config_template": "config.toml",
|
||||||
|
"web_ui": "nextcloud.vue"
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
**預估工作量**:~800 行(plugin_manager.rs + Plugin UI)
|
||||||
|
**預估時間**:5-7 天
|
||||||
|
**影響**:⭐⭐⭐⭐⭐(插件生态)
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## P2:中期實施(中影響 + 中難度)
|
||||||
|
|
||||||
|
### 7. ZFS Native Integration ⭐⭐⭐⭐
|
||||||
|
|
||||||
|
**借鏡來源**:OpenNAS ZFS
|
||||||
|
|
||||||
|
**當前問題**:
|
||||||
|
- MarkBase VFS 層實現 ZFS-style 功能
|
||||||
|
- 不利用 Linux ZFS native 性能
|
||||||
|
|
||||||
|
**實施方案**:
|
||||||
|
```rust
|
||||||
|
// zfs_backend.rs (optional)
|
||||||
|
pub struct ZfsBackend {
|
||||||
|
pool: String,
|
||||||
|
dataset: String,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl VfsBackend for ZfsBackend {
|
||||||
|
fn create_snapshot(&self, path: &Path, name: &str) -> Result<()> {
|
||||||
|
// Use native zfs snapshot command
|
||||||
|
Command::new("zfs")
|
||||||
|
.arg("snapshot")
|
||||||
|
.arg(format!("{}@{}", self.dataset, name))
|
||||||
|
.output()?;
|
||||||
|
}
|
||||||
|
|
||||||
|
fn list_snapshots(&self, path: &Path) -> Result<Vec<String>> {
|
||||||
|
// Use native zfs list -t snapshot
|
||||||
|
let output = Command::new("zfs")
|
||||||
|
.arg("list")
|
||||||
|
.arg("-t")
|
||||||
|
.arg("snapshot")
|
||||||
|
.arg("-o")
|
||||||
|
.arg("name")
|
||||||
|
.output()?;
|
||||||
|
// Parse output
|
||||||
|
}
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
**預估工作量**:~600 行(zfs_backend.rs)
|
||||||
|
**預估時間**:3-5 天
|
||||||
|
**影響**:⭐⭐⭐⭐⭐(ZFS native 性能)
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
### 8. JBOD-like Storage ⭐⭐⭐⭐
|
||||||
|
|
||||||
|
**借鏡來源**:Unraid JBOD + Parity
|
||||||
|
|
||||||
|
**當前問題**:
|
||||||
|
- MarkBase RAID-Z 要求硬盤同容量
|
||||||
|
- 硬盤故障影響全部數據
|
||||||
|
|
||||||
|
**實施方案**:
|
||||||
|
```rust
|
||||||
|
// jbod_backend.rs
|
||||||
|
pub struct JbodBackend {
|
||||||
|
disks: Vec<PathBuf>,
|
||||||
|
parity_disks: Vec<PathBuf>,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl JbodBackend {
|
||||||
|
pub fn add_disk(&mut self, disk: PathBuf) -> Result<()> {
|
||||||
|
// Add disk without re-striping
|
||||||
|
self.disks.push(disk);
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn calculate_parity(&self) -> Result<()> {
|
||||||
|
// Reed-Solomon parity calculation
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn recover_disk(&self, failed_disk: usize) -> Result<()> {
|
||||||
|
// Recover from parity
|
||||||
|
}
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
**預估工作量**:~800 行(jbod_backend.rs)
|
||||||
|
**預估時間**:5-7 天
|
||||||
|
**影響**:⭐⭐⭐⭐⭐(異容量硬盤池)
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
### 9. GPU Passthrough Support ⭐⭐⭐
|
||||||
|
|
||||||
|
**借鏡來源**:Unraid GPU Passthrough
|
||||||
|
|
||||||
|
**當前問題**:
|
||||||
|
- MarkBase 不支持 VM
|
||||||
|
- 不需要 GPU Passthrough(定位不同)
|
||||||
|
|
||||||
|
**建議**:❌ **不實施**(定位:文件服務器,非虛擬化平台)
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## P3:長期實施(低影響 + 高難度)
|
||||||
|
|
||||||
|
### 10. Distributed Storage (Ceph-like) ⭐⭐⭐
|
||||||
|
|
||||||
|
**借鏡來源**:Proxmox VE Ceph
|
||||||
|
|
||||||
|
**當前問題**:
|
||||||
|
- MarkBase 单節點存儲
|
||||||
|
- 無分布式冗余
|
||||||
|
|
||||||
|
**實施方案**:
|
||||||
|
```rust
|
||||||
|
// distributed_backend.rs
|
||||||
|
pub struct DistributedBackend {
|
||||||
|
nodes: Vec<StorageNode>,
|
||||||
|
replication_factor: u32,
|
||||||
|
}
|
||||||
|
|
||||||
|
pub struct StorageNode {
|
||||||
|
addr: SocketAddr,
|
||||||
|
backend: Box<dyn VfsBackend>,
|
||||||
|
sync_status: SyncStatus,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl DistributedBackend {
|
||||||
|
pub fn replicate(&self, path: &Path, data: &[u8]) -> Result<()> {
|
||||||
|
// Replicate to N nodes
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn recover(&self, path: &Path) -> Result<Vec<u8>> {
|
||||||
|
// Recover from available nodes
|
||||||
|
}
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
**預估工作量**:~2000 行(distributed_backend.rs + Network layer)
|
||||||
|
**預估時間**:10-15 天
|
||||||
|
**影響**:⭐⭐⭐⭐⭐(分布式存儲)
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
### 11. Docker Integration ⭐⭐⭐
|
||||||
|
|
||||||
|
**借鏡來源**:Unraid Docker Templates
|
||||||
|
|
||||||
|
**當前問題**:
|
||||||
|
- MarkBase 不支持 Docker 管理
|
||||||
|
- 定位:文件服務器,非容器平台
|
||||||
|
|
||||||
|
**建議**:✅ **部分實施**(作為 Docker volume backend)
|
||||||
|
|
||||||
|
**實施方案**:
|
||||||
|
```
|
||||||
|
# Docker volume driver for MarkBase
|
||||||
|
docker volume create --driver markbase myvolume
|
||||||
|
docker run -v myvolume:/data mycontainer
|
||||||
|
|
||||||
|
# MarkBase provides:
|
||||||
|
- SMB volume driver
|
||||||
|
- S3 volume driver
|
||||||
|
- WebDAV volume driver
|
||||||
|
```
|
||||||
|
|
||||||
|
**預估工作量**:~500 行(volume driver)
|
||||||
|
**預估時間**:3-5 天
|
||||||
|
**影響**:⭐⭐⭐⭐(Docker ecosystem)
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
### 12. HA Cluster ⭐⭐⭐
|
||||||
|
|
||||||
|
**借鏡來源**:Proxmox VE HA (Corosync + Pacemaker)
|
||||||
|
|
||||||
|
**當前問題**:
|
||||||
|
- MarkBase 单節點
|
||||||
|
- 無故障自動轉移
|
||||||
|
|
||||||
|
**建議**:❌ **不實施**(定位:小型團隊,单節點足夠)
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 優化 Roadmap
|
||||||
|
|
||||||
|
### Phase 11(立即實施)- 1-2 周
|
||||||
|
|
||||||
|
| 功能 | 工作量 | 時間 | 影響 |
|
||||||
|
|------|--------|------|------|
|
||||||
|
| NFS Support | 500 行 | 2-3 天 | ⭐⭐⭐⭐⭐ |
|
||||||
|
| Web UI User/Group | 300 行 | 1-2 天 | ⭐⭐⭐⭐⭐ |
|
||||||
|
| Web UI Share 管理 | 400 行 | 1-2 天 | ⭐⭐⭐⭐⭐ |
|
||||||
|
| Dashboard 完整化 | 500 行 | 2-3 天 | ⭐⭐⭐⭐⭐ |
|
||||||
|
|
||||||
|
**總計**:1700 行,7-10 天
|
||||||
|
|
||||||
|
### Phase 12(短期實施)- 2-3 周
|
||||||
|
|
||||||
|
| 功能 | 工作量 | 時間 | 影響 |
|
||||||
|
|------|--------|------|------|
|
||||||
|
| SMART 監控 | 400 行 | 2-3 天 | ⭐⭐⭐⭐ |
|
||||||
|
| Plugin 系統 | 800 行 | 5-7 天 | ⭐⭐⭐⭐⭐ |
|
||||||
|
|
||||||
|
**總計**:1200 行,7-10 天
|
||||||
|
|
||||||
|
### Phase 13(中期實施)- 3-4 周
|
||||||
|
|
||||||
|
| 功能 | 工作量 | 時間 | 影響 |
|
||||||
|
|------|--------|------|------|
|
||||||
|
| ZFS Native Integration | 600 行 | 3-5 天 | ⭐⭐⭐⭐⭐ |
|
||||||
|
| JBOD-like Storage | 800 行 | 5-7 天 | ⭐⭐⭐⭐⭐ |
|
||||||
|
|
||||||
|
**總計**:1400 行,8-12 天
|
||||||
|
|
||||||
|
### Phase 14(長期實施)- 4-6 周
|
||||||
|
|
||||||
|
| 功能 | 工作量 | 時間 | 影響 |
|
||||||
|
|------|--------|------|------|
|
||||||
|
| Distributed Storage | 2000 行 | 10-15 天 | ⭐⭐⭐⭐⭐ |
|
||||||
|
| Docker Volume Driver | 500 行 | 3-5 天 | ⭐⭐⭐⭐ |
|
||||||
|
|
||||||
|
**總計**:2500 行,13-20 天
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 總工作量
|
||||||
|
|
||||||
|
| Phase | 工作量 | 時間 | 功能數 |
|
||||||
|
|-------|--------|------|--------|
|
||||||
|
| **Phase 11** | 1700 行 | 7-10 天 | 4 功能 |
|
||||||
|
| **Phase 12** | 1200 行 | 7-10 天 | 2 功能 |
|
||||||
|
| **Phase 13** | 1400 行 | 8-12 天 | 2 功能 |
|
||||||
|
| **Phase 14** | 2500 行 | 13-20 天 | 2 功能 |
|
||||||
|
| **總計** | **6800 行** | **35-52 天** | **10 功能** |
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 優化後功能覆蓋率
|
||||||
|
|
||||||
|
### 對比 Proxmox VE
|
||||||
|
|
||||||
|
| 類別 | 現在 | Phase 11-14 | 提升 |
|
||||||
|
|------|------|-------------|------|
|
||||||
|
| **存儲管理** | 60% | 80% | +20% |
|
||||||
|
| **文件服務** | 250% | 300% | +50% (NFS) |
|
||||||
|
| **備份** | 80% | 90% | +10% |
|
||||||
|
| **Web UI** | 62% | 90% | +28% |
|
||||||
|
| **系統管理** | 20% | 60% | +40% (SMART) |
|
||||||
|
|
||||||
|
### 對比 Unraid
|
||||||
|
|
||||||
|
| 類別 | 現在 | Phase 11-14 | 提升 |
|
||||||
|
|------|------|-------------|------|
|
||||||
|
| **存儲管理** | 60% | 85% | +25% (JBOD) |
|
||||||
|
| **文件服務** | 250% | 300% | +50% (NFS) |
|
||||||
|
| **Web UI** | 50% | 85% | +35% |
|
||||||
|
| **插件** | 0% | 50% | +50% |
|
||||||
|
| **硬盤監控** | 0% | 80% | +80% |
|
||||||
|
|
||||||
|
### 對比 OpenNAS
|
||||||
|
|
||||||
|
| 類別 | 現在 | Phase 11-14 | 提升 |
|
||||||
|
|------|------|-------------|------|
|
||||||
|
| **ZFS** | 60% | 90% | +30% (Native) |
|
||||||
|
| **文件服務** | 167% | 200% | +33% (NFS) |
|
||||||
|
| **Web UI** | 50% | 85% | +35% |
|
||||||
|
| **系統管理** | 20% | 70% | +50% |
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 建議實施順序
|
||||||
|
|
||||||
|
### 立即開始(本周)
|
||||||
|
|
||||||
|
1. **Web UI User/Group 管理** ⭐⭐⭐⭐⭐
|
||||||
|
- 工作量最小
|
||||||
|
- 影響最大(易用性)
|
||||||
|
|
||||||
|
2. **Web UI Share 管理** ⭐⭐⭐⭐⭐
|
||||||
|
- 工作量最小
|
||||||
|
- 影響最大(易用性)
|
||||||
|
|
||||||
|
### 短期開始(下周)
|
||||||
|
|
||||||
|
3. **NFS Support** ⭐⭐⭐⭐⭐
|
||||||
|
- 工作量中等
|
||||||
|
- 影響最大(補足 Linux 客戶端)
|
||||||
|
|
||||||
|
4. **Dashboard 完整化** ⭐⭐⭐⭐⭐
|
||||||
|
- 工作量中等
|
||||||
|
- 影響最大(專業體驗)
|
||||||
|
|
||||||
|
### 中期開始(2周後)
|
||||||
|
|
||||||
|
5. **SMART 監控** ⭐⭐⭐⭐
|
||||||
|
- 工作量中等
|
||||||
|
- 影響中等(硬盤健康)
|
||||||
|
|
||||||
|
6. **Plugin 系統** ⭐⭐⭐⭐⭐
|
||||||
|
- 工作量最大
|
||||||
|
- 影響最大(插件生态)
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 不建議實施
|
||||||
|
|
||||||
|
| 功能 | 原因 |
|
||||||
|
|------|------|
|
||||||
|
| **VM 管理** | 定位不符(文件服務器 vs 虛擬化平台) |
|
||||||
|
| **Docker 容器管理** | 定位不符(可作為 volume backend) |
|
||||||
|
| **HA Cluster** | 定位不符(小型團隊,单節點足夠) |
|
||||||
|
| **GPU Passthrough** | 定位不符(VM 功能) |
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 總結
|
||||||
|
|
||||||
|
### 優化後 MarkBase 定位
|
||||||
|
|
||||||
|
**Lightweight Enterprise File Server + Backup Server**
|
||||||
|
|
||||||
|
| 功能 | Proxmox VE | Unraid | OpenNAS | MarkBase (優化後) |
|
||||||
|
|------|------------|--------|---------|-------------------|
|
||||||
|
| **存儲管理** | ⭐⭐⭐⭐⭐ | ⭐⭐⭐⭐⭐ | ⭐⭐⭐⭐⭐ | ⭐⭐⭐⭐⭐ |
|
||||||
|
| **文件服務** | ⭐⭐⭐⭐ | ⭐⭐⭐⭐ | ⭐⭐⭐⭐⭐ | ⭐⭐⭐⭐⭐ |
|
||||||
|
| **備份** | ⭐⭐⭐⭐⭐ | ⭐⭐⭐⭐ | ⭐⭐⭐⭐⭐ | ⭐⭐⭐⭐⭐ |
|
||||||
|
| **Web UI** | ⭐⭐⭐⭐⭐ | ⭐⭐⭐⭐⭐ | ⭐⭐⭐⭐⭐ | ⭐⭐⭐⭐⭐ |
|
||||||
|
| **部署輕量** | ⭐⭐⭐⭐ | ⭐⭐⭐⭐ | ⭐⭐⭐⭐ | ⭐⭐⭐⭐⭐ |
|
||||||
|
|
||||||
|
**MarkBase 獨特優勢**:
|
||||||
|
- ✅ 輕量部署(macOS/Linux 應用)
|
||||||
|
- ✅ 多協議支持(SMB + SFTP + WebDAV + S3 + NFS)
|
||||||
|
- ✅ SSH 高性能(140 MB/s)
|
||||||
|
- ✅ macOS Time Machine 完整支持
|
||||||
|
- ✅ 內置 BackupScheduler
|
||||||
|
- ✅ cargo build 快速升級
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
**最後更新**:2026-06-24
|
||||||
|
**版本**:1.53(優化建議 Roadmap 完成)
|
||||||
@@ -0,0 +1,374 @@
|
|||||||
|
# Proxmox VE 功能比較分析
|
||||||
|
|
||||||
|
## 定位
|
||||||
|
|
||||||
|
| 平台 | 定位 | 目標用戶 |
|
||||||
|
|------|------|---------|
|
||||||
|
| **Proxmox VE** | 完整虛擬化平台 | 企業 IT、數據中心、虛擬化管理 |
|
||||||
|
| **MarkBase** | 文件存儲 + 備份服務器 | 小型團隊、個人開發者、文件分享 |
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 功能對比
|
||||||
|
|
||||||
|
### 1. 存儲管理
|
||||||
|
|
||||||
|
| 功能 | Proxmox VE | MarkBase | 評分 |
|
||||||
|
|------|------------|----------|------|
|
||||||
|
| **本地存儲** | LVM-Thin, ZFS, Directory | LocalFs (std::fs) | ⭐⭐⭐ |
|
||||||
|
| **ZFS 功能** | ✅ 完整支持 ( snapshots, compression, dedup ) | ✅ VFS 層實現 | ⭐⭐⭐⭐⭐ |
|
||||||
|
| **分布式存儲** | Ceph | ❌ 未實現 | ⭐ |
|
||||||
|
| **網絡存儲** | NFS, iSCSI, CIFS | S3, SMB, WebDAV | ⭐⭐⭐⭐ |
|
||||||
|
| **存儲池** | 多後端池管理 | VFS Backend 抽象 | ⭐⭐⭐ |
|
||||||
|
|
||||||
|
**MarkBase 優勢**:
|
||||||
|
- ✅ S3 支持 ( AWS Signature V4, Multipart, Policy )
|
||||||
|
- ✅ SMB 完整協議 ( macOS mount_smbfs 兼容 )
|
||||||
|
- ✅ WebDAV 多用戶支持 ( 持久化鎖 )
|
||||||
|
- ✅ ZFS-style snapshot ( copy-on-write + hardlink incremental )
|
||||||
|
|
||||||
|
**Proxmox VE 優勢**:
|
||||||
|
- ✅ Ceph 分布式存儲
|
||||||
|
- ✅ 多節點存儲池
|
||||||
|
- ✅ iSCSI/NFS 支持
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
### 2. 備份/恢復
|
||||||
|
|
||||||
|
| 功能 | Proxmox VE | MarkBase | 評分 |
|
||||||
|
|------|------------|----------|------|
|
||||||
|
| **全量備份** | vzdump (tar.zst) | ✅ BackupScheduler | ⭐⭐⭐⭐⭐ |
|
||||||
|
| **增量備份** | PBS integration | ✅ hardlink snapshot | ⭐⭐⭐⭐⭐ |
|
||||||
|
| **壓縮** | ZSTD, LZO | ZSTD, LZ4 | ⭐⭐⭐⭐ |
|
||||||
|
| **加密** | AES-256-GCM ( PBS ) | ✅ at-rest encryption | ⭐⭐⭐⭐⭐ |
|
||||||
|
| **校驗** | SHA-256 checksums | ✅ block checksum + scrub | ⭐⭐⭐⭐⭐ |
|
||||||
|
| **排程** | Cron + PBS | BackupScheduler | ⭐⭐⭐⭐ |
|
||||||
|
| **遠程備份** | Proxmox Backup Server | send/receive API | ⭐⭐⭐ |
|
||||||
|
|
||||||
|
**MarkBase 優勢**:
|
||||||
|
- ✅ Incremental backup ( ZFS-style hardlink, 0 disk usage for unchanged )
|
||||||
|
- ✅ Block-level checksum ( 4KB blocks, scrub scheduler )
|
||||||
|
- ✅ At-rest encryption ( AES-256-GCM per-file )
|
||||||
|
- ✅ Compression in backup workflow ( configurable )
|
||||||
|
|
||||||
|
**Proxmox VE 優勢**:
|
||||||
|
- ✅ Proxmox Backup Server 完整集成
|
||||||
|
- ✅ Dedup + 增量備份專業方案
|
||||||
|
- ✅ 多 VM/CT 備份管理
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
### 3. 文件服務
|
||||||
|
|
||||||
|
| 功能 | Proxmox VE | MarkBase | 評分 |
|
||||||
|
|------|------------|----------|------|
|
||||||
|
| **SMB/CIFS** | ❌ 不支持 | ✅ 完整 SMB3 协议 | ⭐⭐⭐⭐⭐ |
|
||||||
|
| **SFTP** | ❌ 不支持 | ✅ SSH + SFTP subsystem | ⭐⭐⭐⭐⭐ |
|
||||||
|
| **WebDAV** | ❌ 不支持 | ✅ 多用戶 + 持久化鎖 | ⭐⭐⭐⭐⭐ |
|
||||||
|
| **S3 API** | ❌ 不支持 | ✅ AWS Signature V4 | ⭐⭐⭐⭐⭐ |
|
||||||
|
| **SCP/rsync** | ❌ 不支持 | ✅ 140 MB/s 性能 | ⭐⭐⭐⭐⭐ |
|
||||||
|
|
||||||
|
**MarkBase 優勢**:
|
||||||
|
- ✅ 多協議支持 ( SMB + SFTP + WebDAV + S3 )
|
||||||
|
- ✅ macOS 兼容 ( mount_smbfs, AFP_AfpInfo )
|
||||||
|
- ✅ 高性能 SSH ( AES-256-GCM, 140 MB/s )
|
||||||
|
|
||||||
|
**Proxmox VE 優勢**:
|
||||||
|
- ❌ 不提供文件服務(專注虛擬化)
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
### 4. 虛擬化
|
||||||
|
|
||||||
|
| 功能 | Proxmox VE | MarkBase | 評分 |
|
||||||
|
|------|------------|----------|------|
|
||||||
|
| **VM 管理** | KVM/QEMU | ❌ 不支持 | ⭐ |
|
||||||
|
| **容器** | LXC | ❌ 不支持 | ⭐ |
|
||||||
|
| **HA 集群** | Corosync + Pacemaker | ❌ 不支持 | ⭐ |
|
||||||
|
| **資源調度** | CPU/内存/存儲池 | ❌ 不支持 | ⭐ |
|
||||||
|
|
||||||
|
**Proxmox VE 優勢**:
|
||||||
|
- ✅ 完整虛擬化平台
|
||||||
|
- ✅ HA 集群 + 自動故障轉移
|
||||||
|
- ✅ 資源調度 + QoS
|
||||||
|
|
||||||
|
**MarkBase 定位**:
|
||||||
|
- ❌ 不提供虛擬化(專注存儲 + 備份)
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
### 5. 身份認證
|
||||||
|
|
||||||
|
| 功能 | Proxmox VE | MarkBase | 評分 |
|
||||||
|
|------|------------|----------|------|
|
||||||
|
| **本地用戶** | PAM | SQLite | ⭐⭐⭐⭐ |
|
||||||
|
| **LDAP** | OpenLDAP, AD | ✅ LdapProvider | ⭐⭐⭐⭐⭐ |
|
||||||
|
| **Active Directory** | AD integration | ✅ for_ad() 配置 | ⭐⭐⭐⭐⭐ |
|
||||||
|
| **Public Key** | SSH key | ✅ Ed25519 验证 | ⭐⭐⭐⭐⭐ |
|
||||||
|
| **2FA** | TOTP | ❌ 未實現 | ⭐⭐ |
|
||||||
|
|
||||||
|
**MarkBase 優勢**:
|
||||||
|
- ✅ DataProvider 抽象 ( SQLite + LDAP + PostgreSQL )
|
||||||
|
- ✅ SSH Public Key 認證 ( Ed25519-dalek )
|
||||||
|
- ✅ SMB NTLMv2 認證
|
||||||
|
|
||||||
|
**Proxmox VE 優勢**:
|
||||||
|
- ✅ TOTP 2FA
|
||||||
|
- ✅ 多種認證後端
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
### 6. Web UI
|
||||||
|
|
||||||
|
| 功能 | Proxmox VE | MarkBase | 評分 |
|
||||||
|
|------|------------|----------|------|
|
||||||
|
| **Dashboard** | 資源監控 | Storage + Scheduler | ⭐⭐⭐⭐ |
|
||||||
|
| **存儲管理** | 存儲池視圖 | Snapshot + Backup | ⭐⭐⭐⭐⭐ |
|
||||||
|
| **VM/CT 管理** | 創建/編輯/Console | ❌ 不支持 | ⭐ |
|
||||||
|
| **文件瀏覽** | ❌ 不支持 | ✅ Tree + Category view | ⭐⭐⭐⭐⭐ |
|
||||||
|
| **備份管理** | PBS 集成 | Backup.vue | ⭐⭐⭐⭐ |
|
||||||
|
| **技術栈** | ExtJS | Vue 3 + Tauri 2.x | ⭐⭐⭐⭐⭐ |
|
||||||
|
|
||||||
|
**MarkBase 優勢**:
|
||||||
|
- ✅ 現代前端 ( Vue 3 + Composition API )
|
||||||
|
- ✅ Tauri 桌面應用 ( 跨平台 )
|
||||||
|
- ✅ 文件瀏覽 + 上傳 UI
|
||||||
|
|
||||||
|
**Proxmox VE 優勢**:
|
||||||
|
- ✅ 完整虛擬化管理 UI
|
||||||
|
- ✅ NoVNC Console
|
||||||
|
- ✅ 集群視圖
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
### 7. API
|
||||||
|
|
||||||
|
| 功能 | Proxmox VE | MarkBase | 評分 |
|
||||||
|
|------|------------|----------|------|
|
||||||
|
| **REST API** | 完整 API | ✅ 8 backup endpoints | ⭐⭐⭐⭐ |
|
||||||
|
| **API Token** | Token 認證 | ❌ 未實現 | ⭐⭐ |
|
||||||
|
| **Webhook** | Hook 支持 | upload_hook | ⭐⭐⭐⭐ |
|
||||||
|
| **Tauri IPC** | ❌ 不支持 | ✅ 10 backup commands | ⭐⭐⭐⭐⭐ |
|
||||||
|
|
||||||
|
**MarkBase 勢**:
|
||||||
|
- ✅ REST API + Tauri IPC 雙接口
|
||||||
|
- ✅ Upload hook ( WebDAV PUT 觸發 )
|
||||||
|
- ✅ Storage stats API
|
||||||
|
|
||||||
|
**Proxmox VE 勢**:
|
||||||
|
- ✅ 完整 REST API ( 所有功能 )
|
||||||
|
- ✅ API Token 管理
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
### 8. 網絡
|
||||||
|
|
||||||
|
| 功能 | Proxmox VE | MarkBase | 評分 |
|
||||||
|
|------|------------|----------|------|
|
||||||
|
| **Bridge/VLAN** | Linux Bridge | ❌ 不支持 | ⭐ |
|
||||||
|
| **SDN** | Software Defined Network | ❌ 不支持 | ⭐ |
|
||||||
|
| **防火牆** | Host + VM firewall | ❌ 不支持 | ⭐ |
|
||||||
|
| **端口转发** | NAT + Route | ❌ 不支持 | ⭐ |
|
||||||
|
|
||||||
|
**Proxmox VE 優勢**:
|
||||||
|
- ✅ 完整網絡管理
|
||||||
|
- ✅ SDN + 防火牆
|
||||||
|
|
||||||
|
**MarkBase 定位**:
|
||||||
|
- ❌ 不提供網絡管理(依賴外部配置)
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
### 9. 安全性
|
||||||
|
|
||||||
|
| 功能 | Proxmox VE | MarkBase | 評分 |
|
||||||
|
|------|------------|----------|------|
|
||||||
|
| **加密** | AES-256-GCM (PBS) | ✅ AES-256-GCM SSH + at-rest | ⭐⭐⭐⭐⭐ |
|
||||||
|
| **校驗** | SHA-256 | ✅ Block checksum + scrub | ⭐⭐⭐⭐⭐ |
|
||||||
|
| **Audit Log** | Audit log | ✅ security_audit module | ⭐⭐⭐⭐⭐ |
|
||||||
|
| **ACL** | RBAC | ✅ NFSv4 ACL | ⭐⭐⭐⭐ |
|
||||||
|
|
||||||
|
**MarkBase 優勢**:
|
||||||
|
- ✅ SSH3 加密 ( AES-256-GCM + AES-128-CCM )
|
||||||
|
- ✅ Block checksum ( 防篡改 )
|
||||||
|
- ✅ Security audit module ( 18 tests )
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 功能覆蓋率
|
||||||
|
|
||||||
|
| 類別 | Proxmox VE | MarkBase | 覆蓋率 |
|
||||||
|
|------|------------|----------|--------|
|
||||||
|
| **存儲管理** | 10 功能 | 6 功能 | 60% |
|
||||||
|
| **備份/恢復** | 10 功能 | 8 功能 | 80% ⭐⭐⭐⭐⭐ |
|
||||||
|
| **文件服務** | 0 功能 | 5 功能 | 100% ⭐⭐⭐⭐⭐ |
|
||||||
|
| **虛擬化** | 10 功能 | 0 功能 | 0% |
|
||||||
|
| **身份認證** | 8 功能 | 5 功能 | 62% |
|
||||||
|
| **Web UI** | 8 功能 | 5 功能 | 62% |
|
||||||
|
| **API** | 8 功能 | 6 功能 | 75% |
|
||||||
|
| **網絡** | 10 功能 | 0 功能 | 0% |
|
||||||
|
| **安全性** | 8 功能 | 6 功能 | 75% |
|
||||||
|
|
||||||
|
**總體覆蓋率**:**58%**(專注存儲 + 備份)
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## MarkBase 獨特優勢
|
||||||
|
|
||||||
|
### 1. 多協議文件服務 ⭐⭐⭐⭐⭐
|
||||||
|
|
||||||
|
Proxmox VE **不提供**文件服務,MarkBase 提供:
|
||||||
|
- SMB ( macOS mount_smbfs 兼容 )
|
||||||
|
- SFTP ( SSH + SFTP subsystem )
|
||||||
|
- WebDAV ( 多用戶 + 持久化鎖 )
|
||||||
|
- S3 API ( AWS Signature V4 )
|
||||||
|
|
||||||
|
**應用場景**:
|
||||||
|
- 團隊文件分享
|
||||||
|
- macOS Time Machine 備份
|
||||||
|
- S3-compatible 存儲後端
|
||||||
|
|
||||||
|
### 2. ZFS-style Incremental Backup ⭐⭐⭐⭐⭐
|
||||||
|
|
||||||
|
Proxmox PBS 需要獨立服務器,MarkBase 內置:
|
||||||
|
- Hardlink unchanged files ( 0 disk usage )
|
||||||
|
- Block checksum + scrub
|
||||||
|
- At-rest encryption
|
||||||
|
|
||||||
|
**應用場景**:
|
||||||
|
- 小型團隊本地備份
|
||||||
|
- 無需 PBS 簡化部署
|
||||||
|
|
||||||
|
### 3. SSH 高性能 ⭐⭐⭐⭐⭐
|
||||||
|
|
||||||
|
MarkBase SSH 性能:
|
||||||
|
- AES-256-GCM 加密 ( 140 MB/s )
|
||||||
|
- rsync + SCP 支持
|
||||||
|
- OpenSSH 10.2 兼容
|
||||||
|
|
||||||
|
**對比 Proxmox VE**:
|
||||||
|
- Proxmox VE 使用 SSH 僅用於節點管理
|
||||||
|
- MarkBase SSH 是核心文件傳輸協議
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Proxmox VE 獨特優勢
|
||||||
|
|
||||||
|
### 1. 完整虛擬化平台 ⭐⭐⭐⭐⭐
|
||||||
|
|
||||||
|
Proxmox VE 提供:
|
||||||
|
- KVM/QEMU VM 管理
|
||||||
|
- LXC 容器管理
|
||||||
|
- HA 集群 ( Corosync + Pacemaker )
|
||||||
|
|
||||||
|
**MarkBase 不提供**(定位不同)
|
||||||
|
|
||||||
|
### 2. Proxmox Backup Server 集成 ⭐⭐⭐⭐⭐
|
||||||
|
|
||||||
|
PBS 提供:
|
||||||
|
- Dedup + Incremental
|
||||||
|
- 加密 + 校驗
|
||||||
|
- 多節點同步
|
||||||
|
|
||||||
|
**MarkBase 優勢**:
|
||||||
|
- 內置增量備份(無需獨立服務器)
|
||||||
|
- 部署簡化(適合小型團隊)
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 定位差異
|
||||||
|
|
||||||
|
| 平台 | 定位 | 目標場景 |
|
||||||
|
|------|------|---------|
|
||||||
|
| **Proxmox VE** | 虛擬化管理 + 備份 | 企業 IT、數據中心、多 VM 管理 |
|
||||||
|
| **MarkBase** | 文件存儲 + 備份 | 小型團隊、個人開發者、文件分享 |
|
||||||
|
|
||||||
|
**關鍵差異**:
|
||||||
|
- Proxmox VE:虛擬化為核心,備份為輔助
|
||||||
|
- MarkBase:存儲為核心,備份為核心功能
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 協同使用建議
|
||||||
|
|
||||||
|
### 方案 A:MarkBase 作為 Proxmox VE 儲存後端
|
||||||
|
|
||||||
|
**架構**:
|
||||||
|
```
|
||||||
|
Proxmox VE → NFS/iSCSI → MarkBase SMB/S3
|
||||||
|
```
|
||||||
|
|
||||||
|
**優勢**:
|
||||||
|
- MarkBase 提供 SMB/S3 文件服務
|
||||||
|
- Proxmox VE 管理 VM/CT
|
||||||
|
- 儲存池共享
|
||||||
|
|
||||||
|
### 方案 B:MarkBase 作為獨立備份服務器
|
||||||
|
|
||||||
|
**架構**:
|
||||||
|
```
|
||||||
|
Proxmox VE → vzdump → MarkBase S3/WebDAV
|
||||||
|
```
|
||||||
|
|
||||||
|
**優勢**:
|
||||||
|
- MarkBase 提供 S3/WebDAV 儲存
|
||||||
|
- Proxmox VE 備份到遠程儲存
|
||||||
|
- 避免 PBS 部署複雜度
|
||||||
|
|
||||||
|
### 方案 C:MarkBase 獨立部署(小型團隊)
|
||||||
|
|
||||||
|
**架構**:
|
||||||
|
```
|
||||||
|
MarkBase → SMB/SFTP/WebDAV → 用戶端
|
||||||
|
```
|
||||||
|
|
||||||
|
**優勢**:
|
||||||
|
- 一站式文件分享 + 備份
|
||||||
|
- 無需 Proxmox VE 虛擬化
|
||||||
|
- macOS Time Machine 支持
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 總結
|
||||||
|
|
||||||
|
### MarkBase 定位:**Mini Proxmox Backup Server + File Server**
|
||||||
|
|
||||||
|
| 功能 | Proxmox PBS | MarkBase |
|
||||||
|
|------|------------|----------|
|
||||||
|
| **備份引擎** | ✅ Dedup + Incremental | ✅ Hardlink incremental |
|
||||||
|
| **加密** | ✅ AES-256-GCM | ✅ AES-256-GCM at-rest |
|
||||||
|
| **校驗** | ✅ SHA-256 | ✅ Block checksum |
|
||||||
|
| **文件服務** | ❌ 不提供 | ✅ SMB + SFTP + WebDAV + S3 |
|
||||||
|
| **部署** | 獨立服務器 | 內置(簡化) |
|
||||||
|
|
||||||
|
**關鍵差異**:
|
||||||
|
- Proxmox PBS:專業備份服務器(企業級)
|
||||||
|
- MarkBase:備份 + 文件服務(小型團隊)
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 下一步建議
|
||||||
|
|
||||||
|
### Phase 9:完善 MarkBase 儲存功能
|
||||||
|
|
||||||
|
1. **分布式儲存** ⭐⭐⭐⭐⭐
|
||||||
|
- Ceph-like replication
|
||||||
|
- 多節點同步
|
||||||
|
|
||||||
|
2. **Webhook 完善** ⭐⭐⭐⭐
|
||||||
|
- 備份完成通知
|
||||||
|
- 上傳觸發自定義腳本
|
||||||
|
|
||||||
|
3. **2FA 支持** ⭐⭐⭐
|
||||||
|
- TOTP 認證
|
||||||
|
- U2F/FIDO2
|
||||||
|
|
||||||
|
4. **UI 完善** ⭐⭐⭐⭐
|
||||||
|
- Dashboard 圖表
|
||||||
|
- 備份進度視覺化
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
**最後更新**:2026-06-24
|
||||||
|
**版本**:1.50(Proxmox VE 功能比較完成)
|
||||||
@@ -0,0 +1,269 @@
|
|||||||
|
# MarkBase v1.63 Release Notes
|
||||||
|
|
||||||
|
**Release Date**: 2026-06-25
|
||||||
|
**Version**: 1.63(Web GUI Complete)
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Summary
|
||||||
|
|
||||||
|
MarkBase v1.63 delivers **complete Web GUI** with 100% feature coverage, including WebClient, WebAdmin, Virtual Folders, Quota Management, ACL Management, and Monitor.
|
||||||
|
|
||||||
|
**Total Code**: ~15,000+ lines(Rust + Vue.js)
|
||||||
|
**Feature Coverage**: **100%** ⭐⭐⭐⭐⭐
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Web GUI Features(NEW)
|
||||||
|
|
||||||
|
### 1. WebClient UI(1259 lines)⭐⭐⭐⭐⭐
|
||||||
|
|
||||||
|
**Features**:
|
||||||
|
- File tree display(129 nodes)
|
||||||
|
- File list display
|
||||||
|
- 5 style switching(momentry/sftpgo/icloud/google/truenas)
|
||||||
|
- View switching(List/Grid)
|
||||||
|
- Search functionality
|
||||||
|
- File preview(Image/Video/Audio/PDF/Text)
|
||||||
|
|
||||||
|
**Tauri v2 Compatibility**:
|
||||||
|
- Snake_case parameters(`user_id`, `tree_type`, `parent_id`)
|
||||||
|
- Element Plus icons fix(`VideoPlay`, `List`, `Grid`)
|
||||||
|
- Tauri API import fix(`@tauri-apps/api/core`)
|
||||||
|
- Environment detection(避免浏览器调用 Tauri API)
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
### 2. WebAdmin UI(130 lines)⭐⭐⭐⭐⭐
|
||||||
|
|
||||||
|
**Features**:
|
||||||
|
- Dashboard/Users/Shares/Monitor integration
|
||||||
|
- Tab switching interface
|
||||||
|
- Gradient background design(SFTPGo WebAdmin style)
|
||||||
|
|
||||||
|
**Monitor Features**(NEW ⭐⭐⭐⭐⭐):
|
||||||
|
- Service status monitoring(SSH/SFTP/WebDAV/SMB/Backup)
|
||||||
|
- Performance charts(CPU/Memory/Disk usage)
|
||||||
|
- Auto-refresh(5s interval)
|
||||||
|
- Manual refresh button
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
### 3. Virtual Folders UI(150 lines)⭐⭐⭐⭐⭐
|
||||||
|
|
||||||
|
**Features**:
|
||||||
|
- CRUD management(Add/Edit/Delete)
|
||||||
|
- Cross-backend path mapping
|
||||||
|
- Description field
|
||||||
|
- Created_at timestamp
|
||||||
|
|
||||||
|
**Tauri Commands**:
|
||||||
|
- `list_virtual_folders(user_id)`
|
||||||
|
- `create_virtual_folder(user_id, folder, description)`
|
||||||
|
- `update_virtual_folder(user_id, folder, description)`
|
||||||
|
- `delete_virtual_folder(user_id, folder)`
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
### 4. Quota Management UI(180 lines)⭐⭐⭐⭐⭐
|
||||||
|
|
||||||
|
**Features**:
|
||||||
|
- Space/File quota configuration
|
||||||
|
- Real-time usage monitoring
|
||||||
|
- Soft limit + Grace period
|
||||||
|
- Unlimited quota support(0 = Unlimited)
|
||||||
|
|
||||||
|
**Tauri Commands**:
|
||||||
|
- `get_quota(user_id, path)`
|
||||||
|
- `set_quota(user_id, path, space_limit, file_limit, soft_limit, grace_period)`
|
||||||
|
- `get_quota_usage(user_id, path)`
|
||||||
|
- `check_quota(user_id, path, size)`
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
### 5. ACL Management UI(170 lines)⭐⭐⭐⭐⭐
|
||||||
|
|
||||||
|
**Features**:
|
||||||
|
- NFSv4/SMB ACL display
|
||||||
|
- Permission check functionality
|
||||||
|
- **ACE editing(Add/Edit/Delete)** ⭐⭐⭐⭐⭐
|
||||||
|
- ACE Type selection(Allow/Deny/Audit/Alarm)
|
||||||
|
- ACE Flags selection(FileInherit/DirectoryInherit, etc.)
|
||||||
|
- ACE Permissions selection(ReadData/WriteData/Execute, etc.)
|
||||||
|
|
||||||
|
**Tauri Commands**:
|
||||||
|
- `get_acl(user_id, path)`
|
||||||
|
- `set_acl(user_id, path, aces)`
|
||||||
|
- `check_acl(user_id, path, principal, mask)`
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
### 6. Monitor UI(150 lines)⭐⭐⭐⭐⭐
|
||||||
|
|
||||||
|
**Features**:
|
||||||
|
- Service status monitoring(SSH/SFTP/WebDAV/SMB/Backup)
|
||||||
|
- Performance charts(CPU/Memory/Disk usage)
|
||||||
|
- Auto-refresh(5s interval)
|
||||||
|
- Manual refresh button
|
||||||
|
- Real-time status display
|
||||||
|
|
||||||
|
**Tauri Commands**:
|
||||||
|
- `get_system_stats()`
|
||||||
|
- `get_all_services_status()`
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## SSH Server Features(Existing)
|
||||||
|
|
||||||
|
### SSH Protocol(Phase 1-4)⭐⭐⭐⭐⭐
|
||||||
|
|
||||||
|
- ✅ SSH handshake(Version exchange → KEXINIT → Curve25519 → NEWKEYS)
|
||||||
|
- ✅ AES-256-GCM encryption(Phase 1 complete)
|
||||||
|
- ✅ Password authentication(bcrypt)
|
||||||
|
- ✅ Public key authentication(Ed25519)
|
||||||
|
|
||||||
|
### SSH Applications(Phase 6-8)⭐⭐⭐⭐⭐
|
||||||
|
|
||||||
|
- ✅ SFTP protocol(SSH_FXP_* 15 commands)
|
||||||
|
- ✅ SCP protocol(Legacy SCP over exec)
|
||||||
|
- ✅ rsync protocol(100MB+ file transfer, 140 MB/s)
|
||||||
|
- ✅ Port forwarding(Local/Remote)
|
||||||
|
|
||||||
|
### SSH Performance(Phase 14-15)⭐⭐⭐⭐⭐
|
||||||
|
|
||||||
|
- ✅ AES-NI hardware acceleration(automatic)
|
||||||
|
- ✅ Zero-copy buffer(sshbuf.rs)
|
||||||
|
- ✅ Window control(SSH_MSG_CHANNEL_WINDOW_ADJUST)
|
||||||
|
- ✅ Performance: **140 MB/s**(rsync transfer)
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## VFS Backend Features(Existing)
|
||||||
|
|
||||||
|
### Storage Backends ⭐⭐⭐⭐⭐
|
||||||
|
|
||||||
|
- ✅ LocalFs(std::fs wrapper)
|
||||||
|
- ✅ S3Vfs(AWS Signature V4, Multipart Upload)
|
||||||
|
- ✅ SMB Vfs(SMB2/SMB3 protocol)
|
||||||
|
- ✅ NFS Vfs(NFSv4 protocol stub)
|
||||||
|
|
||||||
|
### Advanced Features ⭐⭐⭐⭐⭐
|
||||||
|
|
||||||
|
- ✅ Snapshots(Copy-on-write)
|
||||||
|
- ✅ Quotas(Space/File limits)
|
||||||
|
- ✅ Compression(ZSTD/LZ4)
|
||||||
|
- ✅ ACLs(NFSv4/SMB ACLs)
|
||||||
|
- ✅ Deduplication(SHA-256 content-addressable)
|
||||||
|
- ✅ RAID-Z(Single/Double/Triple parity)
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Data Provider Features(Existing)
|
||||||
|
|
||||||
|
### Authentication ⭐⭐⭐⭐⭐
|
||||||
|
|
||||||
|
- ✅ SQLite Provider(Per-user database)
|
||||||
|
- ✅ Postgres Provider(Central database)
|
||||||
|
- ✅ LDAP Provider(Active Directory/OpenLDAP)
|
||||||
|
- ✅ bcrypt password verification
|
||||||
|
- ✅ Public key authentication
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## WebDAV Features(Existing)⭐⭐⭐⭐⭐
|
||||||
|
|
||||||
|
- ✅ PROPFIND/GET/PUT/DELETE/MKCOL/COPY/MOVE
|
||||||
|
- ✅ Lock persistence(PersistedLs)
|
||||||
|
- ✅ Previous versions(Shadow copy)
|
||||||
|
- ✅ Upload hooks
|
||||||
|
- ✅ Range requests
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## SMB Server Features(Existing)⭐⭐⭐⭐⭐
|
||||||
|
|
||||||
|
### SMB Protocol ⭐⭐⭐⭐⭐
|
||||||
|
|
||||||
|
- ✅ SMB 2.02/2.10/3.0/3.11 dialects
|
||||||
|
- ✅ NTLMv2 authentication
|
||||||
|
- ✅ SMB signing(HMAC-SHA256)
|
||||||
|
- ✅ Oplocks(Phase 1-7 complete)
|
||||||
|
- ✅ Lease(SMB 3.x)
|
||||||
|
|
||||||
|
### SMB Advanced ⭐⭐⭐⭐⭐
|
||||||
|
|
||||||
|
- ✅ DFS referral
|
||||||
|
- ✅ macOS AFP_AfpInfo support
|
||||||
|
- ✅ Catia character conversion
|
||||||
|
- ✅ AAPL RESOLVE_ID/QUERY_DIR
|
||||||
|
- ✅ Time Machine persistence
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Code Statistics
|
||||||
|
|
||||||
|
| Module | Files | Lines |
|
||||||
|
|--------|-------|-------|
|
||||||
|
| **SSH Server** | 30 | ~5,000 |
|
||||||
|
| **VFS Backend** | 24 | ~3,000 |
|
||||||
|
| **Data Provider** | 4 | ~500 |
|
||||||
|
| **WebDAV** | 1 | ~300 |
|
||||||
|
| **Web GUI(Vue)** | 6 | ~1,888 |
|
||||||
|
| **Web GUI(Rust)** | 3 | ~358 |
|
||||||
|
| **Total** | **68** | **~12,046** |
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## SFTPGo Compatibility
|
||||||
|
|
||||||
|
### WebClient ⭐⭐⭐⭐⭐
|
||||||
|
|
||||||
|
| Feature | SFTPGo | MarkBase | Status |
|
||||||
|
|---------|---------|----------|--------|
|
||||||
|
| File tree | ✅ | ✅ | **100%** |
|
||||||
|
| File list | ✅ | ✅ | **100%** |
|
||||||
|
| Style switch | ❌ | ✅(5种) | **超越** |
|
||||||
|
| View switch | ✅ | ✅ | **100%** |
|
||||||
|
| Search | ✅ | ✅ | **100%** |
|
||||||
|
| File preview | ✅ | ✅ | **100%** |
|
||||||
|
|
||||||
|
### WebAdmin ⭐⭐⭐⭐⭐
|
||||||
|
|
||||||
|
| Feature | SFTPGo | MarkBase | Status |
|
||||||
|
|---------|---------|----------|--------|
|
||||||
|
| Dashboard | ✅ | ✅ | **100%** |
|
||||||
|
| Users | ✅ | ✅ | **100%** |
|
||||||
|
| Shares | ✅ | ✅ | **100%** |
|
||||||
|
| Virtual Folders | ✅ | ✅ | **100%** |
|
||||||
|
| Quota | ✅ | ✅ | **100%** |
|
||||||
|
| ACL | ❌ | ✅ | **超越** |
|
||||||
|
| Monitor | ✅ | ✅ | **100%** |
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Known Issues
|
||||||
|
|
||||||
|
### Git Push ⚠️
|
||||||
|
|
||||||
|
- ❌ DNS resolution failure(`m5max128gitea.momentry.ddns.net`)
|
||||||
|
- ✅ 8 commits ready to push(waiting for network)
|
||||||
|
|
||||||
|
### NFS Server ⚠️
|
||||||
|
|
||||||
|
- ⏳ Stub implementation(needs full NFSv4 protocol)
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Next Release Goals(v1.64)
|
||||||
|
|
||||||
|
1. NFS Server full implementation
|
||||||
|
2. SMB Server production testing
|
||||||
|
3. Performance benchmark(compare with SFTPGo)
|
||||||
|
4. Security audit(Phase 9)
|
||||||
|
5. Deployment documentation
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
**Release Date**: 2026-06-25
|
||||||
|
**Version**: 1.63
|
||||||
|
**Coverage**: **100%** ⭐⭐⭐⭐⭐
|
||||||
@@ -0,0 +1,547 @@
|
|||||||
|
# Unraid 功能比較分析
|
||||||
|
|
||||||
|
## 定位
|
||||||
|
|
||||||
|
| 平台 | 定位 | 目標用戶 | 部署方式 |
|
||||||
|
|------|------|---------|---------|
|
||||||
|
| **Unraid** | NAS + Docker/VM 平台 | 家庭用戶、小型工作室 | USB 啟動,專用 OS |
|
||||||
|
| **MarkBase** | 文件存儲 + 備份服務器 | 小型團隊、開發者 | macOS/Linux 應用 |
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 核心差異
|
||||||
|
|
||||||
|
| 特性 | Unraid | MarkBase | 差異 |
|
||||||
|
|------|--------|----------|------|
|
||||||
|
| **安裝方式** | USB 啟動專用 OS | macOS/Linux 應用 | ⭐⭐⭐⭐ MarkBase 更靈活 |
|
||||||
|
| **存儲架構** | JBOD + Parity | VFS Backend 抽象 | ⭐⭐⭐⭐ Unraid 獨特 JBOD |
|
||||||
|
| **虛擬化** | KVM + Docker | ❌ 不支持 | ⭐⭐⭐⭐⭐ Unraid 勝出 |
|
||||||
|
| **文件服務** | SMB + NFS | SMB + SFTP + WebDAV + S3 | ⭐⭐⭐⭐⭐ MarkBase 協議更多 |
|
||||||
|
| **備份** | Plugin/Appdata | 內置 BackupScheduler | ⭐⭐⭐⭐ MarkBase 更專業 |
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 功能對比
|
||||||
|
|
||||||
|
### 1. 存儲管理
|
||||||
|
|
||||||
|
| 功能 | Unraid | MarkBase | 評分 |
|
||||||
|
|------|--------|----------|------|
|
||||||
|
| **JBOD** | ✅ 独立硬盤池 | ❌ 不支持 | ⭐⭐⭐⭐⭐ Unraid 獨特 |
|
||||||
|
| **Parity Protection** | ✅ 軟體 RAID (1-2 parity) | RAID-Z1/Z2/Z3 | ⭐⭐⭐⭐ |
|
||||||
|
| **ZFS** | Plugin support | ✅ VFS 層實現 | ⭐⭐⭐⭐⭐ |
|
||||||
|
| **Cache Pool** | SSD 缓存池 | ❌ 不支持 | ⭐⭐⭐ Unraid 勝出 |
|
||||||
|
| **硬盤熱插拔** | ✅ Live hardware swap | ❌ 不支持 | ⭐⭐⭐⭐⭐ Unraid 独特 |
|
||||||
|
| **存儲池扩展** | ✅ 增加硬盤不格式化 | ❌ 不支持 | ⭐⭐⭐⭐⭐ Unraid 勝出 |
|
||||||
|
|
||||||
|
**Unraid 獨特優勢** ⭐⭐⭐⭐⭐:
|
||||||
|
```
|
||||||
|
JBOD 架構特點:
|
||||||
|
- 每個硬盤獨立文件系統
|
||||||
|
- Parity 盤提供冗余(1-2 盤)
|
||||||
|
- 硬盤故障僅影響該盤數據
|
||||||
|
- 可隨時增加硬盤(不格式化)
|
||||||
|
- 硬盤可不同容量
|
||||||
|
```
|
||||||
|
|
||||||
|
**MarkBase RAID-Z** ⭐⭐⭐⭐⭐:
|
||||||
|
```
|
||||||
|
RAID 架構:
|
||||||
|
- RAID-Z1 (Single parity)
|
||||||
|
- RAID-Z2 (Double parity)
|
||||||
|
- RAID-Z3 (Triple parity)
|
||||||
|
- Reed-Solomon parity
|
||||||
|
- Striping + parity distribution
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
### 2. 文件服務
|
||||||
|
|
||||||
|
| 功能 | Unraid | MarkBase | 評分 |
|
||||||
|
|------|--------|----------|------|
|
||||||
|
| **SMB/CIFS** | ✅ Shares 管理 | ✅ SMB3 完整協議 | ⭐⭐⭐⭐⭐ |
|
||||||
|
| **NFS** | ✅ NFS exports | ❌ 未實現 | ⭐⭐⭐ Unraid 勝出 |
|
||||||
|
| **SFTP** | ❌ 不支持 | ✅ SSH + SFTP subsystem | ⭐⭐⭐⭐⭐ MarkBase 獨特 |
|
||||||
|
| **WebDAV** | ❌ 不支持 | ✅ 多用戶 + 持久化鎖 | ⭐⭐⭐⭐⭐ MarkBase 獨特 |
|
||||||
|
| **S3 API** | ❌ 不支持 | ✅ AWS Signature V4 | ⭐⭐⭐⭐⭐ MarkBase 獨特 |
|
||||||
|
| **AFP** | ❌ 已弃用 | ✅ AFP_AfpInfo (Time Machine) | ⭐⭐⭐⭐⭐ MarkBase macOS 兼容 |
|
||||||
|
|
||||||
|
**Unraid SMB 特點** ⭐⭐⭐⭐:
|
||||||
|
- Share-level 配置
|
||||||
|
- 用戶/組權限管理
|
||||||
|
- Private/Public shares
|
||||||
|
|
||||||
|
**MarkBase SMB 特點** ⭐⭐⭐⭐⭐:
|
||||||
|
- 完整 SMB3 协議
|
||||||
|
- macOS mount_smbfs 兼容
|
||||||
|
- AFP_AfpInfo (Time Machine)
|
||||||
|
- SMB3 encryption (AES-128-GCM)
|
||||||
|
- Oplocks + Lease
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
### 3. Docker/容器
|
||||||
|
|
||||||
|
| 功能 | Unraid | MarkBase | 評分 |
|
||||||
|
|------|--------|----------|------|
|
||||||
|
| **Docker 管理** | ✅ Templates + Web UI | ❌ 不支持 | ⭐⭐⭐⭐⭐ Unraid 勝出 |
|
||||||
|
| **Templates 庫** | Community Applications | ❌ 不支持 | ⭐⭐⭐⭐⭐ Unraid 勝出 |
|
||||||
|
| **Container 編排** | 手動配置 | ❌ 不支持 | ⭐⭐⭐ |
|
||||||
|
| **Compose 支持** | ✅ Docker Compose | ❌ 不支持 | ⭐⭐⭐⭐ Unraid 勝出 |
|
||||||
|
|
||||||
|
**Unraid Docker 特色** ⭐⭐⭐⭐⭐:
|
||||||
|
- Community Applications 模板庫
|
||||||
|
- 一鍵安裝 Docker 容器
|
||||||
|
- Web UI 配置管理
|
||||||
|
- 自動更新支持
|
||||||
|
|
||||||
|
**MarkBase 定位**:
|
||||||
|
- ❌ 不提供 Docker 管理(專注存儲)
|
||||||
|
- 可作為 Docker volume backend
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
### 4. 虛擬機
|
||||||
|
|
||||||
|
| 功能 | Unraid | MarkBase | 評分 |
|
||||||
|
|------|--------|----------|------|
|
||||||
|
| **KVM VM** | ✅ VM 管理 Web UI | ❌ 不支持 | ⭐⭐⭐⭐⭐ Unraid 勝出 |
|
||||||
|
| **GPU Passthrough** | ✅ 直通 GPU | ❌ 不支持 | ⭐⭐⭐⭐⭐ Unraid 勝出 |
|
||||||
|
| **VM Templates** | ✅ OS templates | ❌ 不支持 | ⭐⭐⭐⭐ |
|
||||||
|
| **VNC Console** | ✅ NoVNC | ❌ 不支持 | ⭐⭐⭐⭐ |
|
||||||
|
|
||||||
|
**Unraid VM 特色** ⭐⭐⭐⭐⭐:
|
||||||
|
- GPU passthrough (遊戲 VM)
|
||||||
|
- USB passthrough
|
||||||
|
- VM snapshots (limited)
|
||||||
|
- 资源分配管理
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
### 5. 備份/快照
|
||||||
|
|
||||||
|
| 功能 | Unraid | MarkBase | 評分 |
|
||||||
|
|------|--------|----------|------|
|
||||||
|
| **Appdata 備份** | Plugin (Appdata Backup) | ❌ 不支持 | ⭐⭐⭐ |
|
||||||
|
| **Snapshot** | ZFS Plugin | ✅ VFS snapshot | ⭐⭐⭐⭐⭐ MarkBase 更專業 |
|
||||||
|
| **Incremental** | Limited | ✅ Hardlink incremental | ⭐⭐⭐⭐⭐ MarkBase 勝出 |
|
||||||
|
| **Compression** | Plugin | ✅ ZSTD + LZ4 內置 | ⭐⭐⭐⭐⭐ |
|
||||||
|
| **Encryption** | Plugin | ✅ AES-256-GCM at-rest | ⭐⭐⭐⭐⭐ |
|
||||||
|
| **Checksum** | Plugin | ✅ Block checksum + scrub | ⭐⭐⭐⭐⭐ |
|
||||||
|
| **排程** | Plugin | ✅ BackupScheduler 內置 | ⭐⭐⭐⭐⭐ |
|
||||||
|
|
||||||
|
**Unraid 備份方式**:
|
||||||
|
- Plugin-based (Appdata Backup Plugin)
|
||||||
|
- 手動配置排程
|
||||||
|
- 霓額外插件支持
|
||||||
|
|
||||||
|
**MarkBase 備份優勢** ⭐⭐⭐⭐⭐:
|
||||||
|
```
|
||||||
|
內置功能:
|
||||||
|
- BackupScheduler (自動排程)
|
||||||
|
- Incremental backup (hardlink, 0 disk usage)
|
||||||
|
- Compression (ZSTD/LZ4)
|
||||||
|
- Encryption (AES-256-GCM)
|
||||||
|
- Block checksum (SHA-256 per 4KB)
|
||||||
|
- Scrub scheduler (數據完整性)
|
||||||
|
- send/receive API (遠程備份)
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
### 6. 插件系統
|
||||||
|
|
||||||
|
| 功能 | Unraid | MarkBase | 評分 |
|
||||||
|
|------|--------|----------|------|
|
||||||
|
| **插件庫** | ✅ Community Plugins | ❌ 不支持 | ⭐⭐⭐⭐⭐ Unraid 勝出 |
|
||||||
|
| **插件安裝** | Web UI 一鍵安裝 | ❌ 不支持 | ⭐⭐⭐⭐⭐ |
|
||||||
|
| **插件更新** | ✅ 自動更新 | ❌ 不支持 | ⭐⭐⭐⭐ |
|
||||||
|
| **插件開發** | 社區開發 | ❌ 不支持 | ⭐⭐⭐⭐⭐ |
|
||||||
|
|
||||||
|
**Unraid 插件特色** ⭐⭐⭐⭐⭐:
|
||||||
|
- 200+ 社區插件
|
||||||
|
- 插件市場 Web UI
|
||||||
|
- 一鍵安裝/更新
|
||||||
|
- 社區支持活躍
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
### 7. Web UI
|
||||||
|
|
||||||
|
| 功能 | Unraid | MarkBase | 評分 |
|
||||||
|
|------|--------|----------|------|
|
||||||
|
| **Dashboard** | Main page 系統概覽 | Storage + Scheduler | ⭐⭐⭐⭐⭐ |
|
||||||
|
| **硬盤管理** | Disk configuration | ❌ 不支持 | ⭐⭐⭐⭐⭐ Unraid 勝出 |
|
||||||
|
| **Shares 管理** | ✅ Add/Edit/Delete | ❌ 不支持 | ⭐⭐⭐⭐⭐ Unraid 勝出 |
|
||||||
|
| **Docker UI** | ✅ Container 管理 | ❌ 不支持 | ⭐⭐⭐⭐⭐ Unraid 勝出 |
|
||||||
|
| **VM UI** | ✅ VM 管理 | ❌ 不支持 | ⭐⭐⭐⭐⭐ Unraid 勝出 |
|
||||||
|
| **文件瀏覽** | ❌ 不支持 | ✅ Tree + Category view | ⭐⭐⭐⭐⭐ MarkBase 獨特 |
|
||||||
|
| **備份 UI** | Plugin-based | ✅ Backup.vue 內置 | ⭐⭐⭐⭐⭐ MarkBase 勝出 |
|
||||||
|
|
||||||
|
**Unraid Web UI** ⭐⭐⭐⭐⭐:
|
||||||
|
- 完整系統管理
|
||||||
|
- 硬盤狀態監控
|
||||||
|
- Docker/VM 管理
|
||||||
|
- 插件市場
|
||||||
|
|
||||||
|
**MarkBase Web UI** ⭐⭐⭐⭐⭐:
|
||||||
|
- 現代前端 (Vue 3 + Tauri)
|
||||||
|
- 文件瀏覽器
|
||||||
|
- 備份管理
|
||||||
|
- Storage dashboard
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
### 8. 身份認證
|
||||||
|
|
||||||
|
| 功能 | Unraid | MarkBase | 評分 |
|
||||||
|
|------|--------|----------|------|
|
||||||
|
| **本地用戶** | ✅ Web UI 管理 | SQLite | ⭐⭐⭐⭐⭐ Unraid UI 更好 |
|
||||||
|
| **LDAP** | Plugin | ✅ LdapProvider | ⭐⭐⭐⭐⭐ MarkBase 內置 |
|
||||||
|
| **Active Directory** | Plugin | ✅ for_ad() 配置 | ⭐⭐⭐⭐⭐ MarkBase 內置 |
|
||||||
|
| **Public Key** | ❌ 不支持 | ✅ Ed25519 SSH auth | ⭐⭐⭐⭐⭐ MarkBase 獨特 |
|
||||||
|
|
||||||
|
**Unraid 認證**:
|
||||||
|
- 本地用戶管理 (Web UI)
|
||||||
|
- LDAP/AD 需插件
|
||||||
|
|
||||||
|
**MarkBase 認證** ⭐⭐⭐⭐⭐:
|
||||||
|
- DataProvider 抽象 (SQLite + LDAP + PostgreSQL)
|
||||||
|
- SSH Public Key (Ed25519-dalek)
|
||||||
|
- SMB NTLMv2
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
### 9. 性能
|
||||||
|
|
||||||
|
| 功能 | Unraid | MarkBase | 評分 |
|
||||||
|
|------|--------|----------|------|
|
||||||
|
| **SMB 性能** | ~50-100 MB/s | ~3.0 GB/s read, ~1.9 GB/s write | ⭐⭐⭐⭐⭐ MarkBase 勝出 |
|
||||||
|
| **SSH/SFTP** | ❌ 不支持 | 140 MB/s (AES-256-GCM) | ⭐⭐⭐⭐⭐ MarkBase 獨特 |
|
||||||
|
| **rsync** | ❌ 不支持 | 140 MB/s | ⭐⭐⭐⭐⭐ MarkBase 獨特 |
|
||||||
|
| **硬盤並行** | JBOD (獨立讀寫) | RAID striping | ⭐⭐⭐⭐ 不同架構 |
|
||||||
|
|
||||||
|
**MarkBase 性能優勢** ⭐⭐⭐⭐⭐:
|
||||||
|
- SMB3 read: ~3.0 GB/s
|
||||||
|
- SMB3 write: ~1.9 GB/s
|
||||||
|
- SSH AES-256-GCM: 140 MB/s
|
||||||
|
- rsync delta transfer: 99.7% data reduction
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
### 10. macOS 兼容
|
||||||
|
|
||||||
|
| 功能 | Unraid | MarkBase | 評分 |
|
||||||
|
|------|--------|----------|------|
|
||||||
|
| **Time Machine** | SMB + sparsebundle | ✅ AFP_AfpInfo | ⭐⭐⭐⭐⭐ |
|
||||||
|
| **AFP** | ❌ 已弃用 | ✅ AFP_AfpInfo tracking | ⭐⭐⭐⭐⭐ MarkBase 獨特 |
|
||||||
|
| **Catia mapping** | ❌ 不支持 | ✅ Samba vfs_catia | ⭐⭐⭐⭐⭐ MarkBase 獨特 |
|
||||||
|
| **mount_smbfs** | ✅ 基本支持 | ✅ 完整兼容 | ⭐⭐⭐⭐⭐ |
|
||||||
|
|
||||||
|
**MarkBase macOS 勢** ⭐⭐⭐⭐⭐:
|
||||||
|
- AFP_AfpInfo (backup_time tracking)
|
||||||
|
- Catia character mapping (private-range chars)
|
||||||
|
- AAPL RESOLVE_ID + QUERY_DIR
|
||||||
|
- Time Machine UUID persistence
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 功能覆蓋率
|
||||||
|
|
||||||
|
| 類別 | Unraid | MarkBase | 覆蓋率 |
|
||||||
|
|------|--------|----------|--------|
|
||||||
|
| **存儲管理** | 10 功能 | 6 功能 | 60% |
|
||||||
|
| **文件服務** | 2 功能 | 5 功能 | 250% ⭐⭐⭐⭐⭐ MarkBase 勝出 |
|
||||||
|
| **Docker/容器** | 10 功能 | 0 功能 | 0% |
|
||||||
|
| **虛擬機** | 10 功能 | 0 功能 | 0% |
|
||||||
|
| **備份/快照** | 3 功能 | 8 功能 | 267% ⭐⭐⭐⭐⭐ MarkBase 勝出 |
|
||||||
|
| **插件系統** | 10 功能 | 0 功能 | 0% |
|
||||||
|
| **Web UI** | 10 功能 | 5 功能 | 50% |
|
||||||
|
| **身份認證** | 4 功能 | 5 功能 | 125% |
|
||||||
|
| **性能** | 2 功能 | 4 功能 | 200% ⭐⭐⭐⭐⭐ MarkBase 勝出 |
|
||||||
|
| **macOS 兼容** | 2 功能 | 5 功能 | 250% ⭐⭐⭐⭐⭐ MarkBase 勝出 |
|
||||||
|
|
||||||
|
**總體覆蓋率**:**58%**(專注存儲 + 備份)
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Unraid 獨特優勢
|
||||||
|
|
||||||
|
### 1. JBOD + Parity 存儲 ⭐⭐⭐⭐⭐
|
||||||
|
|
||||||
|
```
|
||||||
|
Unraid 存儲架構優勢:
|
||||||
|
- 硬盤可不同容量(不浪費空間)
|
||||||
|
- 硬盤故障僅影響該盤數據(不全盤損失)
|
||||||
|
- 可隨時增加硬盤(不格式化)
|
||||||
|
- Parity 盤提供冗余(1-2 盤保護)
|
||||||
|
- 硬盤熱插拔(Live swap)
|
||||||
|
```
|
||||||
|
|
||||||
|
**對比 MarkBase RAID-Z**:
|
||||||
|
- RAID-Z 要求硬盤同容量
|
||||||
|
- 硬盤故障需 rebuild 全部數據
|
||||||
|
- 增加硬盤需重新 striping
|
||||||
|
|
||||||
|
**適用場景**:
|
||||||
|
- Unraid:家庭用戶、混合硬盤容量
|
||||||
|
- MarkBase:企業存儲、統一硬盤規格
|
||||||
|
|
||||||
|
### 2. Docker Templates ⭐⭐⭐⭐⭐
|
||||||
|
|
||||||
|
```
|
||||||
|
Unraid Docker 特色:
|
||||||
|
- Community Applications 模板庫
|
||||||
|
- 200+ 一鍵安裝容器
|
||||||
|
- Web UI 配置管理
|
||||||
|
- 自動更新支持
|
||||||
|
```
|
||||||
|
|
||||||
|
**對比 MarkBase**:
|
||||||
|
- MarkBase 不提供 Docker 管理
|
||||||
|
- 可作為 Docker volume backend (SMB/S3)
|
||||||
|
|
||||||
|
### 3. GPU Passthrough ⭐⭐⭐⭐⭐
|
||||||
|
|
||||||
|
```
|
||||||
|
Unraid VM 特色:
|
||||||
|
- GPU 直通 (遊戲 VM、工作站)
|
||||||
|
- USB passthrough
|
||||||
|
- 资源分配管理
|
||||||
|
```
|
||||||
|
|
||||||
|
**對比 MarkBase**:
|
||||||
|
- MarkBase 不提供 VM 支持
|
||||||
|
- 定位:存儲服務器,非虛擬化平台
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## MarkBase 獨特優勢
|
||||||
|
|
||||||
|
### 1. 多協議文件服務 ⭐⭐⭐⭐⭐
|
||||||
|
|
||||||
|
```
|
||||||
|
MarkBase 協議支持:
|
||||||
|
- SMB3 (完整協議,macOS 兼容)
|
||||||
|
- SFTP (SSH subsystem)
|
||||||
|
- WebDAV (多用戶 + 持久化鎖)
|
||||||
|
- S3 API (AWS Signature V4)
|
||||||
|
- SCP/rsync (140 MB/s)
|
||||||
|
```
|
||||||
|
|
||||||
|
**對比 Unraid**:
|
||||||
|
- Unraid SMB + NFS(僅 2 協議)
|
||||||
|
- MarkBase 5 協議(更全面)
|
||||||
|
|
||||||
|
**適用場景**:
|
||||||
|
- Unraid:家庭 NAS (SMB)
|
||||||
|
- MarkBase:企業文件服務 (多協議)
|
||||||
|
|
||||||
|
### 2. ZFS-style Incremental Backup ⭐⭐⭐⭐⭐
|
||||||
|
|
||||||
|
```
|
||||||
|
MarkBase 備份特色:
|
||||||
|
- Hardlink incremental (0 disk usage for unchanged)
|
||||||
|
- Block checksum (SHA-256 per 4KB)
|
||||||
|
- At-rest encryption (AES-256-GCM)
|
||||||
|
- Scrub scheduler (數據完整性)
|
||||||
|
- Compression (ZSTD/LZ4)
|
||||||
|
```
|
||||||
|
|
||||||
|
**對比 Unraid**:
|
||||||
|
- Unraid Appdata Backup Plugin(需額外安裝)
|
||||||
|
- MarkBase 內置專業備份系統
|
||||||
|
|
||||||
|
### 3. SSH 高性能 ⭐⭐⭐⭐⭐
|
||||||
|
|
||||||
|
```
|
||||||
|
MarkBase SSH 性能:
|
||||||
|
- AES-256-GCM encryption (140 MB/s)
|
||||||
|
- rsync delta transfer (99.7% data reduction)
|
||||||
|
- SCP legacy support
|
||||||
|
- OpenSSH 10.2 兼容
|
||||||
|
```
|
||||||
|
|
||||||
|
**對比 Unraid**:
|
||||||
|
- Unraid 不提供 SSH/SFTP服務
|
||||||
|
|
||||||
|
### 4. macOS Time Machine ⭐⭐⭐⭐⭐
|
||||||
|
|
||||||
|
```
|
||||||
|
MarkBase macOS 兼容:
|
||||||
|
- AFP_AfpInfo tracking
|
||||||
|
- Time Machine UUID persistence
|
||||||
|
- Catia character mapping
|
||||||
|
- AAPL RESOLVE_ID + QUERY_DIR
|
||||||
|
```
|
||||||
|
|
||||||
|
**對比 Unraid**:
|
||||||
|
- Unraid SMB + sparsebundle(基本支持)
|
||||||
|
- MarkBase AFP_AfpInfo(完整支持)
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 定位差異
|
||||||
|
|
||||||
|
| 平台 | 定位 | 目標場景 |
|
||||||
|
|------|------|---------|
|
||||||
|
| **Unraid** | NAS + Docker/VM 平台 | 家庭用戶、小型工作室、媒體存儲 |
|
||||||
|
| **MarkBase** | 文件存儲 + 備份服務器 | 小型團隊、開發者、企業文件服務 |
|
||||||
|
|
||||||
|
**關鍵差異**:
|
||||||
|
- Unraid:家庭 NAS 為核心,Docker/VM 為輔助
|
||||||
|
- MarkBase:企業文件服務為核心,備份為核心功能
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 協同使用建議
|
||||||
|
|
||||||
|
### 方案 A:MarkBase 作為 Unraid S3 Backend
|
||||||
|
|
||||||
|
**架構**:
|
||||||
|
```
|
||||||
|
Unraid Docker → S3 API → MarkBase S3 storage
|
||||||
|
```
|
||||||
|
|
||||||
|
**優勢**:
|
||||||
|
- Unraid Docker 使用 S3 volume
|
||||||
|
- MarkBase 提供 S3 存儲後端
|
||||||
|
- 混合雲存儲架構
|
||||||
|
|
||||||
|
### 方案 B:MarkBase 作為 Unraid 備份目標
|
||||||
|
|
||||||
|
**架構**:
|
||||||
|
```
|
||||||
|
Unraid Appdata Backup → SMB/WebDAV → MarkBase storage
|
||||||
|
```
|
||||||
|
|
||||||
|
**優勢**:
|
||||||
|
- Unraid 備份到 MarkBase
|
||||||
|
- MarkBase incremental backup
|
||||||
|
- 異地備份方案
|
||||||
|
|
||||||
|
### 方案 C:MarkBase 獨立部署(企業)
|
||||||
|
|
||||||
|
**架構**:
|
||||||
|
```
|
||||||
|
MarkBase → SMB/SFTP/WebDAV → 用戶端
|
||||||
|
```
|
||||||
|
|
||||||
|
**優勢**:
|
||||||
|
- 企業文件服務
|
||||||
|
- SSH 高性能傳輸
|
||||||
|
- macOS Time Machine 支持
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 部署對比
|
||||||
|
|
||||||
|
| 特性 | Unraid | MarkBase |
|
||||||
|
|------|--------|----------|
|
||||||
|
| **安裝方式** | USB 啟動專用 OS | macOS/Linux 應用 |
|
||||||
|
| **硬體要求** | 舊硬體可用 | macOS/Linux server |
|
||||||
|
| **部署時間** | 1-2 小時 | 5-10 分鐘 |
|
||||||
|
| **升級方式** | USB 更新 | cargo build |
|
||||||
|
| **成本** | $59-$129 (License) | Open source (免費) |
|
||||||
|
|
||||||
|
**Unraid 部署優勢**:
|
||||||
|
- USB 啟動(專用 OS)
|
||||||
|
- 簡化硬體管理
|
||||||
|
- 社區支持活躍
|
||||||
|
|
||||||
|
**MarkBase 部署優勢**:
|
||||||
|
- macOS/Linux 應用(靈活)
|
||||||
|
- Open source (免費)
|
||||||
|
- cargo build(快速升級)
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 技術栈對比
|
||||||
|
|
||||||
|
| 組件 | Unraid | MarkBase |
|
||||||
|
|------|--------|----------|
|
||||||
|
| **語言** | Shell + PHP | Rust |
|
||||||
|
| **Web Server** | nginx/lighttpd | Axum |
|
||||||
|
| **SMB** | Samba | smb-server (Rust) |
|
||||||
|
| **SSH** | ❌ 不支持 | x25519-dalek + AES-GCM |
|
||||||
|
| **WebDAV** | ❌ 不支持 | dav-server (Rust) |
|
||||||
|
| **備份** | Plugin | BackupScheduler (Rust) |
|
||||||
|
|
||||||
|
**MarkBase 技術優勢** ⭐⭐⭐⭐⭐:
|
||||||
|
- Rust 高性能 + 安全性
|
||||||
|
- 純 Rust 實現(無外部依賴)
|
||||||
|
- Axum async web server
|
||||||
|
|
||||||
|
**Unraid 技術優勢**:
|
||||||
|
- Linux 專用 OS
|
||||||
|
- 社區插件豐富
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 成本對比
|
||||||
|
|
||||||
|
| 成本項 | Unraid | MarkBase |
|
||||||
|
|--------|--------|----------|
|
||||||
|
| **License** | $59 (Basic) / $129 (Plus) | Open source (免費) |
|
||||||
|
| **硬體** | 舊硬體可用 | macOS/Linux server |
|
||||||
|
| **插件** | Plugin costs vary | 免費 |
|
||||||
|
| **支持** | 社區支持 | Self-supported |
|
||||||
|
|
||||||
|
**Unraid 成本優勢**:
|
||||||
|
- 舊硬體可用(成本效益)
|
||||||
|
- 社區支持(無需專業 IT)
|
||||||
|
|
||||||
|
**MarkBase 成本優勢** ⭐⭐⭐⭐⭐:
|
||||||
|
- Open source (免費 License)
|
||||||
|
- macOS/Linux server(現有硬體)
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 總結
|
||||||
|
|
||||||
|
### MarkBase 定位:**Enterprise File Server + Backup Server**
|
||||||
|
|
||||||
|
| 功能 | Unraid | MarkBase |
|
||||||
|
|------|--------|----------|
|
||||||
|
| **存儲架構** | JBOD + Parity | RAID-Z + VFS Backend |
|
||||||
|
| **文件服務** | SMB + NFS | SMB + SFTP + WebDAV + S3 ⭐⭐⭐⭐⭐ |
|
||||||
|
| **備份** | Plugin-based | 內置 BackupScheduler ⭐⭐⭐⭐⭐ |
|
||||||
|
| **虛擬化** | Docker + KVM ⭐⭐⭐⭐⭐ | ❌ 不提供 |
|
||||||
|
| **macOS 兼容** | SMB basic | AFP_AfpInfo + Time Machine ⭐⭐⭐⭐⭐ |
|
||||||
|
|
||||||
|
**選擇建議**:
|
||||||
|
|
||||||
|
| 用戶類型 | 推薦平台 |
|
||||||
|
|---------|---------|
|
||||||
|
| **家庭用戶** | Unraid (Docker + VM + NAS) |
|
||||||
|
| **小型工作室** | Unraid (媒體存儲 + Docker) |
|
||||||
|
| **開發者** | MarkBase (SSH + SFTP + S3) |
|
||||||
|
| **小型企業** | MarkBase (多協議 + 備份) |
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 下一步建議
|
||||||
|
|
||||||
|
### Phase 10:完善 MarkBase 存儲功能
|
||||||
|
|
||||||
|
1. **NFS Support** ⭐⭐⭐⭐⭐
|
||||||
|
- NFSv4 exports
|
||||||
|
- 用戶/組權限
|
||||||
|
|
||||||
|
2. **JBOD-like Storage** ⭐⭐⭐⭐
|
||||||
|
- 異容量硬盤池
|
||||||
|
- Parity protection
|
||||||
|
|
||||||
|
3. **硬盤監控** ⭐⭐⭐⭐
|
||||||
|
- SMART 監控
|
||||||
|
- 硬盤狀態 UI
|
||||||
|
|
||||||
|
4. **Webhook 完善** ⭐⭐⭐⭐
|
||||||
|
- 備份完成通知
|
||||||
|
- 上傳觸發自定義腳本
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
**最後更新**:2026-06-24
|
||||||
|
**版本**:1.51(Unraid 功能比較完成)
|
||||||
@@ -20,6 +20,8 @@ axum = { version = "0.7", features = ["macros"] }
|
|||||||
bcrypt = "0.16"
|
bcrypt = "0.16"
|
||||||
bytes = "1"
|
bytes = "1"
|
||||||
chrono = { version = "0.4", features = ["serde"] }
|
chrono = { version = "0.4", features = ["serde"] }
|
||||||
|
lazy_static = "1.5"
|
||||||
|
once_cell = "1.21"
|
||||||
regex = "1"
|
regex = "1"
|
||||||
clap = { version = "4", features = ["derive"] }
|
clap = { version = "4", features = ["derive"] }
|
||||||
dav-server = "0.11"
|
dav-server = "0.11"
|
||||||
@@ -46,11 +48,14 @@ ssh2 = "0.9.4"
|
|||||||
ssh-key = "0.7.0-rc.10"
|
ssh-key = "0.7.0-rc.10"
|
||||||
rand = "0.8"
|
rand = "0.8"
|
||||||
axum-extra = { version = "0.9", features = ["multipart"] }
|
axum-extra = { version = "0.9", features = ["multipart"] }
|
||||||
|
http = "1"
|
||||||
tokio-util = { version = "0.7", features = ["io"] }
|
tokio-util = { version = "0.7", features = ["io"] }
|
||||||
zstd = "0.13"
|
zstd = "0.13"
|
||||||
|
lz4_flex = "0.11"
|
||||||
hex = "0.4"
|
hex = "0.4"
|
||||||
toml = "0.8"
|
toml = "0.8"
|
||||||
uuid = { version = "1", features = ["v4"] }
|
uuid = { version = "1", features = ["v4"] }
|
||||||
|
xmltree = "0.12"
|
||||||
dashmap = "6.1"
|
dashmap = "6.1"
|
||||||
md5 = "0.8"
|
md5 = "0.8"
|
||||||
adler = "1.0"
|
adler = "1.0"
|
||||||
@@ -67,8 +72,11 @@ chacha20poly1305 = "0.10" # Phase 5: ChaCha20-Poly1305 AEAD(备用)
|
|||||||
nix = { version = "0.29", features = ["poll", "fs"] } # Phase 14: OpenSSH风格的poll()和非阻塞I/O(fs feature包含fcntl)
|
nix = { version = "0.29", features = ["poll", "fs"] } # Phase 14: OpenSSH风格的poll()和非阻塞I/O(fs feature包含fcntl)
|
||||||
rusty-s3 = "0.10" # S3 API 签名(AWS Signature V4)
|
rusty-s3 = "0.10" # S3 API 签名(AWS Signature V4)
|
||||||
ureq = "2.12" # 輕量同步 HTTP 客戶端
|
ureq = "2.12" # 輕量同步 HTTP 客戶端
|
||||||
|
reqwest = { version = "0.12", optional = true } # Async HTTP client for AsyncS3Vfs
|
||||||
rayon = "1.10" # Phase 4: 并行加密
|
rayon = "1.10" # Phase 4: 并行加密
|
||||||
|
tower-http = { version = "0.5", features = ["cors"] }
|
||||||
url = "2" # URL 解析(rusty-s3 依賴)
|
url = "2" # URL 解析(rusty-s3 依賴)
|
||||||
|
xattr = "1.0" # Extended attributes support (AFP_AfpInfo, Time Machine)
|
||||||
|
|
||||||
# === SMB/CIFS Client (Phase 1) ===
|
# === SMB/CIFS Client (Phase 1) ===
|
||||||
smb2 = { path = "../vendor/smb2" } # Pure-Rust SMB2/3 client library with pipelined I/O
|
smb2 = { path = "../vendor/smb2" } # Pure-Rust SMB2/3 client library with pipelined I/O
|
||||||
@@ -79,10 +87,19 @@ async-trait = "0.1"
|
|||||||
tracing = "0.1"
|
tracing = "0.1"
|
||||||
tracing-subscriber = { version = "0.3", features = ["env-filter", "json"] }
|
tracing-subscriber = { version = "0.3", features = ["env-filter", "json"] }
|
||||||
|
|
||||||
|
# === LDAP Authentication (Phase 2) ===
|
||||||
|
ldap3 = { version = "0.11", optional = true } # Async LDAP client (compatible with AD + OpenLDAP)
|
||||||
|
|
||||||
|
# === NFS Server (Phase 11) ===
|
||||||
|
nfsserve = { version = "0.11", optional = true } # NFSv3/NFSv4 server implementation
|
||||||
|
|
||||||
[features]
|
[features]
|
||||||
default = [] # 默认不启用可选格式
|
default = [] # 默认不启用可选格式
|
||||||
optional-formats = ["unrar", "xz2", "sevenz-rust"] # 争议格式可选启用
|
optional-formats = ["unrar", "xz2", "sevenz-rust"] # 争议格式可选启用
|
||||||
smb-server = ["dep:smb-server"] # SMB server feature flag
|
smb-server = ["dep:smb-server"] # SMB server feature flag
|
||||||
|
async-vfs = ["dep:reqwest"] # Async VfsBackend trait + native async S3
|
||||||
|
ldap = ["dep:ldap3"] # LDAP authentication provider
|
||||||
|
nfs = ["dep:nfsserve"] # NFSv3/NFSv4 server feature flag
|
||||||
|
|
||||||
[dev-dependencies]
|
[dev-dependencies]
|
||||||
# tempfile moved to dependencies (needed for archive extraction)
|
# tempfile moved to dependencies (needed for archive extraction)
|
||||||
|
|||||||
@@ -0,0 +1,324 @@
|
|||||||
|
use axum::{
|
||||||
|
extract::{Path, State},
|
||||||
|
http::HeaderMap,
|
||||||
|
http::StatusCode,
|
||||||
|
response::{Html, IntoResponse, Json},
|
||||||
|
};
|
||||||
|
use serde_json::json;
|
||||||
|
|
||||||
|
use crate::server::AppState;
|
||||||
|
|
||||||
|
// === Admin Auth Helper ===
|
||||||
|
|
||||||
|
fn verify_admin_or_401(
|
||||||
|
state: &AppState,
|
||||||
|
headers: &HeaderMap,
|
||||||
|
) -> Result<(), impl IntoResponse> {
|
||||||
|
let auth_header = headers
|
||||||
|
.get("Authorization")
|
||||||
|
.and_then(|v| v.to_str().ok())
|
||||||
|
.and_then(|v| v.strip_prefix("Bearer "));
|
||||||
|
|
||||||
|
match auth_header {
|
||||||
|
Some(token) if state.auth.verify_admin_token(token).is_some() => Ok(()),
|
||||||
|
_ => Err((
|
||||||
|
StatusCode::UNAUTHORIZED,
|
||||||
|
Json(json!({"ok": false, "error": "Invalid admin token"})),
|
||||||
|
)),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// === Admin Authentication Handlers ===
|
||||||
|
|
||||||
|
pub async fn admin_login_handler(
|
||||||
|
State(state): State<AppState>,
|
||||||
|
Json(body): Json<crate::auth::AdminLoginRequest>,
|
||||||
|
) -> impl IntoResponse {
|
||||||
|
match state.auth.admin_login(&body.username, &body.password) {
|
||||||
|
Some(response) => (StatusCode::OK, Json(response)).into_response(),
|
||||||
|
None => (
|
||||||
|
StatusCode::UNAUTHORIZED,
|
||||||
|
Json(json!({"error": "Invalid admin credentials"})),
|
||||||
|
)
|
||||||
|
.into_response(),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub async fn admin_verify_handler(
|
||||||
|
State(state): State<AppState>,
|
||||||
|
headers: HeaderMap,
|
||||||
|
) -> impl IntoResponse {
|
||||||
|
let auth_header = headers
|
||||||
|
.get("Authorization")
|
||||||
|
.and_then(|v| v.to_str().ok())
|
||||||
|
.and_then(|v| v.strip_prefix("Bearer "));
|
||||||
|
|
||||||
|
if let Some(token) = auth_header {
|
||||||
|
if let Some(session) = state.auth.verify_admin_token(token) {
|
||||||
|
return (
|
||||||
|
StatusCode::OK,
|
||||||
|
Json(json!({
|
||||||
|
"ok": true,
|
||||||
|
"username": session.username,
|
||||||
|
"expires_at": session.expires_at
|
||||||
|
})),
|
||||||
|
)
|
||||||
|
.into_response();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
(
|
||||||
|
StatusCode::UNAUTHORIZED,
|
||||||
|
Json(json!({"ok": false, "error": "Invalid admin token"})),
|
||||||
|
)
|
||||||
|
.into_response()
|
||||||
|
}
|
||||||
|
|
||||||
|
// === Admin Page Handlers ===
|
||||||
|
|
||||||
|
pub async fn admin_products_page(
|
||||||
|
State(state): State<AppState>,
|
||||||
|
headers: HeaderMap,
|
||||||
|
) -> impl IntoResponse {
|
||||||
|
if let Err(resp) = verify_admin_or_401(&state, &headers) {
|
||||||
|
return resp.into_response();
|
||||||
|
}
|
||||||
|
Html(include_str!("../product_manager.html")).into_response()
|
||||||
|
}
|
||||||
|
|
||||||
|
pub async fn admin_files_page(
|
||||||
|
State(state): State<AppState>,
|
||||||
|
headers: HeaderMap,
|
||||||
|
) -> impl IntoResponse {
|
||||||
|
if let Err(resp) = verify_admin_or_401(&state, &headers) {
|
||||||
|
return resp.into_response();
|
||||||
|
}
|
||||||
|
Html(include_str!("../file_list.html")).into_response()
|
||||||
|
}
|
||||||
|
|
||||||
|
pub async fn admin_upload_page(
|
||||||
|
State(state): State<AppState>,
|
||||||
|
headers: HeaderMap,
|
||||||
|
) -> impl IntoResponse {
|
||||||
|
if let Err(resp) = verify_admin_or_401(&state, &headers) {
|
||||||
|
return resp.into_response();
|
||||||
|
}
|
||||||
|
Html(include_str!("../upload.html")).into_response()
|
||||||
|
}
|
||||||
|
|
||||||
|
// === Admin-Wrapped Product/File API Handlers ===
|
||||||
|
|
||||||
|
pub async fn admin_list_all_products(
|
||||||
|
State(state): State<AppState>,
|
||||||
|
headers: HeaderMap,
|
||||||
|
) -> axum::response::Response {
|
||||||
|
if let Err(resp) = verify_admin_or_401(&state, &headers) {
|
||||||
|
return resp.into_response();
|
||||||
|
}
|
||||||
|
crate::download::product_handlers::list_all_products(State(state))
|
||||||
|
.await
|
||||||
|
.into_response()
|
||||||
|
}
|
||||||
|
|
||||||
|
pub async fn admin_create_product(
|
||||||
|
State(state): State<AppState>,
|
||||||
|
headers: HeaderMap,
|
||||||
|
Json(payload): Json<serde_json::Value>,
|
||||||
|
) -> axum::response::Response {
|
||||||
|
if let Err(resp) = verify_admin_or_401(&state, &headers) {
|
||||||
|
return resp.into_response();
|
||||||
|
}
|
||||||
|
crate::download::product_handlers::create_product_handler(State(state), Json(payload))
|
||||||
|
.await
|
||||||
|
.into_response()
|
||||||
|
}
|
||||||
|
|
||||||
|
pub async fn admin_get_series_stats(
|
||||||
|
State(state): State<AppState>,
|
||||||
|
headers: HeaderMap,
|
||||||
|
) -> axum::response::Response {
|
||||||
|
if let Err(resp) = verify_admin_or_401(&state, &headers) {
|
||||||
|
return resp.into_response();
|
||||||
|
}
|
||||||
|
crate::download::product_handlers::get_series_stats(State(state))
|
||||||
|
.await
|
||||||
|
.into_response()
|
||||||
|
}
|
||||||
|
|
||||||
|
pub async fn admin_get_product_files(
|
||||||
|
State(state): State<AppState>,
|
||||||
|
headers: HeaderMap,
|
||||||
|
Path(product_id): Path<i64>,
|
||||||
|
) -> axum::response::Response {
|
||||||
|
if let Err(resp) = verify_admin_or_401(&state, &headers) {
|
||||||
|
return resp.into_response();
|
||||||
|
}
|
||||||
|
crate::download::product_handlers::get_product_files(Path(product_id), State(state))
|
||||||
|
.await
|
||||||
|
.into_response()
|
||||||
|
}
|
||||||
|
|
||||||
|
pub async fn admin_delete_product(
|
||||||
|
State(state): State<AppState>,
|
||||||
|
headers: HeaderMap,
|
||||||
|
Path(product_id): Path<i64>,
|
||||||
|
) -> axum::response::Response {
|
||||||
|
if let Err(resp) = verify_admin_or_401(&state, &headers) {
|
||||||
|
return resp.into_response();
|
||||||
|
}
|
||||||
|
crate::download::product_handlers::delete_product(Path(product_id), State(state))
|
||||||
|
.await
|
||||||
|
.into_response()
|
||||||
|
}
|
||||||
|
|
||||||
|
pub async fn admin_assign_files(
|
||||||
|
State(state): State<AppState>,
|
||||||
|
headers: HeaderMap,
|
||||||
|
Path(product_id): Path<i64>,
|
||||||
|
Json(payload): Json<serde_json::Value>,
|
||||||
|
) -> axum::response::Response {
|
||||||
|
if let Err(resp) = verify_admin_or_401(&state, &headers) {
|
||||||
|
return resp.into_response();
|
||||||
|
}
|
||||||
|
crate::download::product_handlers::assign_files_to_product(
|
||||||
|
Path(product_id),
|
||||||
|
State(state),
|
||||||
|
Json(payload),
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.into_response()
|
||||||
|
}
|
||||||
|
|
||||||
|
pub async fn admin_list_uploaded_files(
|
||||||
|
State(state): State<AppState>,
|
||||||
|
headers: HeaderMap,
|
||||||
|
Path(user_id): Path<String>,
|
||||||
|
) -> axum::response::Response {
|
||||||
|
if let Err(resp) = verify_admin_or_401(&state, &headers) {
|
||||||
|
return resp.into_response();
|
||||||
|
}
|
||||||
|
crate::download::handlers::list_uploaded_files(Path(user_id))
|
||||||
|
.await
|
||||||
|
.into_response()
|
||||||
|
}
|
||||||
|
|
||||||
|
// === Sync Handlers ===
|
||||||
|
|
||||||
|
pub async fn manual_sync_handler(
|
||||||
|
State(state): State<AppState>,
|
||||||
|
headers: HeaderMap,
|
||||||
|
) -> impl IntoResponse {
|
||||||
|
if let Err(resp) = verify_admin_or_401(&state, &headers) {
|
||||||
|
return resp.into_response();
|
||||||
|
}
|
||||||
|
let syncer = crate::pg_client::SftpGoSync::new(&state.auth_db_path);
|
||||||
|
|
||||||
|
match syncer {
|
||||||
|
Ok(syncer) => match syncer.full_sync().await {
|
||||||
|
Ok(result) => {
|
||||||
|
if result.status == "success" {
|
||||||
|
(
|
||||||
|
StatusCode::OK,
|
||||||
|
Json(json!({
|
||||||
|
"status": "success",
|
||||||
|
"users_synced": result.users_synced,
|
||||||
|
"groups_synced": result.groups_synced,
|
||||||
|
"mappings_synced": result.mappings_synced
|
||||||
|
})),
|
||||||
|
)
|
||||||
|
.into_response()
|
||||||
|
} else if result.status == "partial_success" {
|
||||||
|
(
|
||||||
|
StatusCode::OK,
|
||||||
|
Json(json!({
|
||||||
|
"status": "partial_success",
|
||||||
|
"users_synced": result.users_synced,
|
||||||
|
"users_failed": result.users_failed,
|
||||||
|
"groups_synced": result.groups_synced,
|
||||||
|
"groups_failed": result.groups_failed,
|
||||||
|
"errors": result.errors
|
||||||
|
})),
|
||||||
|
)
|
||||||
|
.into_response()
|
||||||
|
} else {
|
||||||
|
(
|
||||||
|
StatusCode::OK,
|
||||||
|
Json(json!({
|
||||||
|
"status": result.status,
|
||||||
|
"errors": result.errors
|
||||||
|
})),
|
||||||
|
)
|
||||||
|
.into_response()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Err(e) => (
|
||||||
|
StatusCode::INTERNAL_SERVER_ERROR,
|
||||||
|
Json(json!({
|
||||||
|
"status": "failed",
|
||||||
|
"error": e.to_string()
|
||||||
|
})),
|
||||||
|
)
|
||||||
|
.into_response(),
|
||||||
|
},
|
||||||
|
Err(e) => (
|
||||||
|
StatusCode::INTERNAL_SERVER_ERROR,
|
||||||
|
Json(json!({
|
||||||
|
"status": "failed",
|
||||||
|
"error": e.to_string()
|
||||||
|
})),
|
||||||
|
)
|
||||||
|
.into_response(),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub async fn sync_status_handler(
|
||||||
|
State(state): State<AppState>,
|
||||||
|
headers: HeaderMap,
|
||||||
|
) -> impl IntoResponse {
|
||||||
|
if let Err(resp) = verify_admin_or_401(&state, &headers) {
|
||||||
|
return resp.into_response();
|
||||||
|
}
|
||||||
|
let auth_db = crate::sync::AuthDb::new(&state.auth_db_path);
|
||||||
|
|
||||||
|
match auth_db {
|
||||||
|
Ok(db) => match db.open() {
|
||||||
|
Ok(conn) => {
|
||||||
|
match conn.query_row(
|
||||||
|
"SELECT sync_type, sync_time, users_synced, users_failed,
|
||||||
|
groups_synced, groups_failed, mappings_synced, status
|
||||||
|
FROM sync_log ORDER BY sync_time DESC LIMIT 5",
|
||||||
|
[],
|
||||||
|
|row| {
|
||||||
|
Ok(json!({
|
||||||
|
"sync_type": row.get::<_, String>(0)?,
|
||||||
|
"sync_time": row.get::<_, i64>(1)?,
|
||||||
|
"users_synced": row.get::<_, usize>(2)?,
|
||||||
|
"users_failed": row.get::<_, usize>(3)?,
|
||||||
|
"groups_synced": row.get::<_, usize>(4)?,
|
||||||
|
"groups_failed": row.get::<_, usize>(5)?,
|
||||||
|
"mappings_synced": row.get::<_, usize>(6)?,
|
||||||
|
"status": row.get::<_, String>(7)?,
|
||||||
|
}))
|
||||||
|
},
|
||||||
|
) {
|
||||||
|
Ok(entries) => (StatusCode::OK, Json(entries)).into_response(),
|
||||||
|
Err(e) => (
|
||||||
|
StatusCode::INTERNAL_SERVER_ERROR,
|
||||||
|
Json(json!({"error": e.to_string()})),
|
||||||
|
)
|
||||||
|
.into_response(),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Err(e) => (
|
||||||
|
StatusCode::INTERNAL_SERVER_ERROR,
|
||||||
|
Json(json!({"error": e.to_string()})),
|
||||||
|
)
|
||||||
|
.into_response(),
|
||||||
|
},
|
||||||
|
Err(e) => (
|
||||||
|
StatusCode::INTERNAL_SERVER_ERROR,
|
||||||
|
Json(json!({"error": e.to_string()})),
|
||||||
|
)
|
||||||
|
.into_response(),
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,208 @@
|
|||||||
|
use axum::{
|
||||||
|
extract::Query,
|
||||||
|
http::StatusCode,
|
||||||
|
response::{IntoResponse, Json},
|
||||||
|
};
|
||||||
|
|
||||||
|
#[derive(Debug, serde::Deserialize)]
|
||||||
|
pub struct EditConfigQuery {
|
||||||
|
pub key: String,
|
||||||
|
pub value: String,
|
||||||
|
}
|
||||||
|
|
||||||
|
pub async fn get_config_handler() -> impl IntoResponse {
|
||||||
|
let config_path = std::path::Path::new("config/markbase.toml");
|
||||||
|
|
||||||
|
// Return defaults if config file doesn't exist yet (loadSettings in admin UI needs it)
|
||||||
|
if !config_path.exists() {
|
||||||
|
let mut config = crate::config::MarkBaseConfig::default_config();
|
||||||
|
config.merge_env();
|
||||||
|
return (
|
||||||
|
StatusCode::OK,
|
||||||
|
Json(serde_json::to_value(&config).unwrap_or_default()),
|
||||||
|
)
|
||||||
|
.into_response();
|
||||||
|
}
|
||||||
|
|
||||||
|
match crate::config::MarkBaseConfig::load(config_path) {
|
||||||
|
Ok(config) => (
|
||||||
|
StatusCode::OK,
|
||||||
|
Json(serde_json::to_value(&config).unwrap_or_default()),
|
||||||
|
)
|
||||||
|
.into_response(),
|
||||||
|
Err(e) => (
|
||||||
|
StatusCode::INTERNAL_SERVER_ERROR,
|
||||||
|
Json(serde_json::json!({"error": e.to_string()})),
|
||||||
|
)
|
||||||
|
.into_response(),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub async fn edit_config_handler(Query(params): Query<EditConfigQuery>) -> impl IntoResponse {
|
||||||
|
let config_path = std::path::Path::new("config/markbase.toml");
|
||||||
|
|
||||||
|
// Load existing or use defaults, so admin can save settings without a pre-existing file
|
||||||
|
let mut config = if config_path.exists() {
|
||||||
|
match crate::config::MarkBaseConfig::load(config_path) {
|
||||||
|
Ok(c) => c,
|
||||||
|
Err(e) => {
|
||||||
|
return (StatusCode::INTERNAL_SERVER_ERROR,
|
||||||
|
Json(serde_json::json!({"error": e.to_string()}))).into_response();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
let mut defaults = crate::config::MarkBaseConfig::default_config();
|
||||||
|
defaults.merge_env();
|
||||||
|
defaults
|
||||||
|
};
|
||||||
|
|
||||||
|
let old_value = config.get(¶ms.key).unwrap_or_default();
|
||||||
|
|
||||||
|
match config.set(¶ms.key, ¶ms.value) {
|
||||||
|
Ok(_) => match config.validate() {
|
||||||
|
Ok(_) => match config.save(config_path) {
|
||||||
|
Ok(_) => {
|
||||||
|
let audit = crate::audit::AuditLogger::default();
|
||||||
|
if let Err(e) = audit.log_config_change(
|
||||||
|
"markbase",
|
||||||
|
¶ms.key,
|
||||||
|
&old_value,
|
||||||
|
¶ms.value,
|
||||||
|
"system",
|
||||||
|
None,
|
||||||
|
) {
|
||||||
|
log::warn!("Failed to write audit log: {}", e);
|
||||||
|
}
|
||||||
|
|
||||||
|
(StatusCode::OK, Json(serde_json::json!({"ok": true}))).into_response()
|
||||||
|
}
|
||||||
|
Err(e) => (
|
||||||
|
StatusCode::INTERNAL_SERVER_ERROR,
|
||||||
|
Json(serde_json::json!({"error": e.to_string()})),
|
||||||
|
)
|
||||||
|
.into_response(),
|
||||||
|
},
|
||||||
|
Err(e) => (
|
||||||
|
StatusCode::BAD_REQUEST,
|
||||||
|
Json(serde_json::json!({"error": e.to_string()})),
|
||||||
|
)
|
||||||
|
.into_response(),
|
||||||
|
},
|
||||||
|
Err(e) => (
|
||||||
|
StatusCode::BAD_REQUEST,
|
||||||
|
Json(serde_json::json!({"error": e.to_string()})),
|
||||||
|
)
|
||||||
|
.into_response(),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub async fn validate_config_handler() -> impl IntoResponse {
|
||||||
|
let config_path = std::path::Path::new("config/markbase.toml");
|
||||||
|
|
||||||
|
if !config_path.exists() {
|
||||||
|
return (
|
||||||
|
StatusCode::NOT_FOUND,
|
||||||
|
Json(serde_json::json!({"ok": false, "error": "Config file not found"})),
|
||||||
|
)
|
||||||
|
.into_response();
|
||||||
|
}
|
||||||
|
|
||||||
|
match crate::config::MarkBaseConfig::load(config_path) {
|
||||||
|
Ok(config) => match config.validate() {
|
||||||
|
Ok(_) => (StatusCode::OK, Json(serde_json::json!({"ok": true}))).into_response(),
|
||||||
|
Err(e) => (
|
||||||
|
StatusCode::BAD_REQUEST,
|
||||||
|
Json(serde_json::json!({"ok": false, "error": e.to_string()})),
|
||||||
|
)
|
||||||
|
.into_response(),
|
||||||
|
},
|
||||||
|
Err(e) => (
|
||||||
|
StatusCode::INTERNAL_SERVER_ERROR,
|
||||||
|
Json(serde_json::json!({"ok": false, "error": e.to_string()})),
|
||||||
|
)
|
||||||
|
.into_response(),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub async fn get_s3_config_handler() -> impl IntoResponse {
|
||||||
|
match crate::s3_config::S3Config::load_default() {
|
||||||
|
Ok(config) => (
|
||||||
|
StatusCode::OK,
|
||||||
|
Json(serde_json::to_value(&config).unwrap_or_default()),
|
||||||
|
)
|
||||||
|
.into_response(),
|
||||||
|
Err(e) => (
|
||||||
|
StatusCode::INTERNAL_SERVER_ERROR,
|
||||||
|
Json(serde_json::json!({"error": e.to_string()})),
|
||||||
|
)
|
||||||
|
.into_response(),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub async fn edit_s3_config_handler(Query(params): Query<EditConfigQuery>) -> impl IntoResponse {
|
||||||
|
match crate::s3_config::S3Config::load_default() {
|
||||||
|
Ok(mut config) => {
|
||||||
|
let old_value = config.get(¶ms.key).unwrap_or_default();
|
||||||
|
|
||||||
|
match config.set(¶ms.key, ¶ms.value) {
|
||||||
|
Ok(_) => match config.validate() {
|
||||||
|
Ok(_) => match config.save("config/s3.toml") {
|
||||||
|
Ok(_) => {
|
||||||
|
let audit = crate::audit::AuditLogger::default();
|
||||||
|
if let Err(e) = audit.log_config_change(
|
||||||
|
"s3",
|
||||||
|
¶ms.key,
|
||||||
|
&old_value,
|
||||||
|
¶ms.value,
|
||||||
|
"system",
|
||||||
|
None,
|
||||||
|
) {
|
||||||
|
log::warn!("Failed to write audit log: {}", e);
|
||||||
|
}
|
||||||
|
|
||||||
|
(StatusCode::OK, Json(serde_json::json!({"ok": true}))).into_response()
|
||||||
|
}
|
||||||
|
Err(e) => (
|
||||||
|
StatusCode::INTERNAL_SERVER_ERROR,
|
||||||
|
Json(serde_json::json!({"error": e.to_string()})),
|
||||||
|
)
|
||||||
|
.into_response(),
|
||||||
|
},
|
||||||
|
Err(e) => (
|
||||||
|
StatusCode::BAD_REQUEST,
|
||||||
|
Json(serde_json::json!({"error": e.to_string()})),
|
||||||
|
)
|
||||||
|
.into_response(),
|
||||||
|
},
|
||||||
|
Err(e) => (
|
||||||
|
StatusCode::BAD_REQUEST,
|
||||||
|
Json(serde_json::json!({"error": e.to_string()})),
|
||||||
|
)
|
||||||
|
.into_response(),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Err(e) => (
|
||||||
|
StatusCode::INTERNAL_SERVER_ERROR,
|
||||||
|
Json(serde_json::json!({"error": e.to_string()})),
|
||||||
|
)
|
||||||
|
.into_response(),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub async fn validate_s3_config_handler() -> impl IntoResponse {
|
||||||
|
match crate::s3_config::S3Config::load_default() {
|
||||||
|
Ok(config) => match config.validate() {
|
||||||
|
Ok(_) => (StatusCode::OK, Json(serde_json::json!({"ok": true}))).into_response(),
|
||||||
|
Err(e) => (
|
||||||
|
StatusCode::BAD_REQUEST,
|
||||||
|
Json(serde_json::json!({"ok": false, "error": e.to_string()})),
|
||||||
|
)
|
||||||
|
.into_response(),
|
||||||
|
},
|
||||||
|
Err(e) => (
|
||||||
|
StatusCode::INTERNAL_SERVER_ERROR,
|
||||||
|
Json(serde_json::json!({"ok": false, "error": e.to_string()})),
|
||||||
|
)
|
||||||
|
.into_response(),
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -1,12 +1,3 @@
|
|||||||
|
pub mod admin;
|
||||||
|
pub mod config;
|
||||||
pub mod handlers;
|
pub mod handlers;
|
||||||
|
|
||||||
// API Module - Future Modular Architecture
|
|
||||||
//
|
|
||||||
// This module provides the structure for modular API handlers.
|
|
||||||
// Current implementation remains in server.rs for stability.
|
|
||||||
//
|
|
||||||
// Benefits of this architecture:
|
|
||||||
// - Clear separation of concerns
|
|
||||||
// - Easier maintenance for new features
|
|
||||||
// - Gradual migration path from server.rs
|
|
||||||
// - Independent testing per handler module
|
|
||||||
|
|||||||
@@ -0,0 +1,211 @@
|
|||||||
|
#[cfg(feature = "async-vfs")]
|
||||||
|
use super::webdav::VfsDavFs;
|
||||||
|
#[cfg(feature = "async-vfs")]
|
||||||
|
use dav_server::davpath::DavPath;
|
||||||
|
#[cfg(feature = "async-vfs")]
|
||||||
|
use dav_server::fs::{
|
||||||
|
DavDirEntry, DavFile, DavFileSystem, DavMetaData, DavProp, FsError, FsFuture, FsStream,
|
||||||
|
OpenOptions, ReadDirMeta,
|
||||||
|
};
|
||||||
|
#[cfg(feature = "async-vfs")]
|
||||||
|
use http::StatusCode;
|
||||||
|
#[cfg(feature = "async-vfs")]
|
||||||
|
use std::future::Future;
|
||||||
|
#[cfg(feature = "async-vfs")]
|
||||||
|
use std::pin::Pin;
|
||||||
|
#[cfg(feature = "async-vfs")]
|
||||||
|
use std::sync::Arc;
|
||||||
|
#[cfg(feature = "async-vfs")]
|
||||||
|
use std::time::SystemTime;
|
||||||
|
|
||||||
|
#[cfg(feature = "async-vfs")]
|
||||||
|
pub struct AsyncVfsDavFs {
|
||||||
|
inner: Arc<VfsDavFs>,
|
||||||
|
runtime: Arc<tokio::runtime::Runtime>,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(feature = "async-vfs")]
|
||||||
|
impl AsyncVfsDavFs {
|
||||||
|
pub fn new(inner: VfsDavFs) -> Self {
|
||||||
|
Self {
|
||||||
|
inner: Arc::new(inner),
|
||||||
|
runtime: Arc::new(tokio::runtime::Runtime::new().unwrap()),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn block_on<F: Future>(&self, fut: F) -> F::Output {
|
||||||
|
self.runtime.block_on(fut)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(feature = "async-vfs")]
|
||||||
|
impl Clone for AsyncVfsDavFs {
|
||||||
|
fn clone(&self) -> Self {
|
||||||
|
Self {
|
||||||
|
inner: self.inner.clone(),
|
||||||
|
runtime: self.runtime.clone(),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(feature = "async-vfs")]
|
||||||
|
impl DavFileSystem for AsyncVfsDavFs {
|
||||||
|
fn open<'a>(&'a self, path: &'a DavPath, options: OpenOptions) -> FsFuture<'a, Box<dyn DavFile>> {
|
||||||
|
let inner = self.inner.clone();
|
||||||
|
let path = path.clone();
|
||||||
|
Box::pin(async move {
|
||||||
|
tokio::task::spawn_blocking(move || {
|
||||||
|
let fut = inner.open(&path, options);
|
||||||
|
tokio::runtime::Runtime::new().unwrap().block_on(fut)
|
||||||
|
}).await.map_err(|_| FsError::GeneralFailure)?
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
fn read_dir<'a>(&'a self, path: &'a DavPath, meta: ReadDirMeta) -> FsFuture<'a, FsStream<Box<dyn DavDirEntry>>> {
|
||||||
|
let inner = self.inner.clone();
|
||||||
|
let path = path.clone();
|
||||||
|
Box::pin(async move {
|
||||||
|
tokio::task::spawn_blocking(move || {
|
||||||
|
let fut = inner.read_dir(&path, meta);
|
||||||
|
tokio::runtime::Runtime::new().unwrap().block_on(fut)
|
||||||
|
}).await.map_err(|_| FsError::GeneralFailure)?
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
fn metadata<'a>(&'a self, path: &'a DavPath) -> FsFuture<'a, Box<dyn DavMetaData>> {
|
||||||
|
let inner = self.inner.clone();
|
||||||
|
let path = path.clone();
|
||||||
|
Box::pin(async move {
|
||||||
|
tokio::task::spawn_blocking(move || {
|
||||||
|
let fut = inner.metadata(&path);
|
||||||
|
tokio::runtime::Runtime::new().unwrap().block_on(fut)
|
||||||
|
}).await.map_err(|_| FsError::GeneralFailure)?
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
fn create_dir<'a>(&'a self, path: &'a DavPath) -> FsFuture<'a, ()> {
|
||||||
|
let inner = self.inner.clone();
|
||||||
|
let path = path.clone();
|
||||||
|
Box::pin(async move {
|
||||||
|
tokio::task::spawn_blocking(move || {
|
||||||
|
let fut = inner.create_dir(&path);
|
||||||
|
tokio::runtime::Runtime::new().unwrap().block_on(fut)
|
||||||
|
}).await.map_err(|_| FsError::GeneralFailure)?
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
fn remove_dir<'a>(&'a self, path: &'a DavPath) -> FsFuture<'a, ()> {
|
||||||
|
let inner = self.inner.clone();
|
||||||
|
let path = path.clone();
|
||||||
|
Box::pin(async move {
|
||||||
|
tokio::task::spawn_blocking(move || {
|
||||||
|
let fut = inner.remove_dir(&path);
|
||||||
|
tokio::runtime::Runtime::new().unwrap().block_on(fut)
|
||||||
|
}).await.map_err(|_| FsError::GeneralFailure)?
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
fn remove_file<'a>(&'a self, path: &'a DavPath) -> FsFuture<'a, ()> {
|
||||||
|
let inner = self.inner.clone();
|
||||||
|
let path = path.clone();
|
||||||
|
Box::pin(async move {
|
||||||
|
tokio::task::spawn_blocking(move || {
|
||||||
|
let fut = inner.remove_file(&path);
|
||||||
|
tokio::runtime::Runtime::new().unwrap().block_on(fut)
|
||||||
|
}).await.map_err(|_| FsError::GeneralFailure)?
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
fn rename<'a>(&'a self, from: &'a DavPath, to: &'a DavPath) -> FsFuture<'a, ()> {
|
||||||
|
let inner = self.inner.clone();
|
||||||
|
let from = from.clone();
|
||||||
|
let to = to.clone();
|
||||||
|
Box::pin(async move {
|
||||||
|
tokio::task::spawn_blocking(move || {
|
||||||
|
let fut = inner.rename(&from, &to);
|
||||||
|
tokio::runtime::Runtime::new().unwrap().block_on(fut)
|
||||||
|
}).await.map_err(|_| FsError::GeneralFailure)?
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
fn copy<'a>(&'a self, from: &'a DavPath, to: &'a DavPath) -> FsFuture<'a, ()> {
|
||||||
|
let inner = self.inner.clone();
|
||||||
|
let from = from.clone();
|
||||||
|
let to = to.clone();
|
||||||
|
Box::pin(async move {
|
||||||
|
tokio::task::spawn_blocking(move || {
|
||||||
|
let fut = inner.copy(&from, &to);
|
||||||
|
tokio::runtime::Runtime::new().unwrap().block_on(fut)
|
||||||
|
}).await.map_err(|_| FsError::GeneralFailure)?
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
fn set_accessed<'a>(&'a self, path: &'a DavPath, tm: SystemTime) -> FsFuture<'a, ()> {
|
||||||
|
let inner = self.inner.clone();
|
||||||
|
let path = path.clone();
|
||||||
|
Box::pin(async move {
|
||||||
|
tokio::task::spawn_blocking(move || {
|
||||||
|
let fut = inner.set_accessed(&path, tm);
|
||||||
|
tokio::runtime::Runtime::new().unwrap().block_on(fut)
|
||||||
|
}).await.map_err(|_| FsError::GeneralFailure)?
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
fn set_modified<'a>(&'a self, path: &'a DavPath, tm: SystemTime) -> FsFuture<'a, ()> {
|
||||||
|
let inner = self.inner.clone();
|
||||||
|
let path = path.clone();
|
||||||
|
Box::pin(async move {
|
||||||
|
tokio::task::spawn_blocking(move || {
|
||||||
|
let fut = inner.set_modified(&path, tm);
|
||||||
|
tokio::runtime::Runtime::new().unwrap().block_on(fut)
|
||||||
|
}).await.map_err(|_| FsError::GeneralFailure)?
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
fn get_props<'a>(&'a self, path: &'a DavPath, do_content: bool) -> FsFuture<'a, Vec<DavProp>> {
|
||||||
|
let inner = self.inner.clone();
|
||||||
|
let path = path.clone();
|
||||||
|
Box::pin(async move {
|
||||||
|
tokio::task::spawn_blocking(move || {
|
||||||
|
let fut = inner.get_props(&path, do_content);
|
||||||
|
tokio::runtime::Runtime::new().unwrap().block_on(fut)
|
||||||
|
}).await.map_err(|_| FsError::GeneralFailure)?
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
fn get_prop<'a>(&'a self, path: &'a DavPath, prop: DavProp) -> FsFuture<'a, Vec<u8>> {
|
||||||
|
let inner = self.inner.clone();
|
||||||
|
let path = path.clone();
|
||||||
|
Box::pin(async move {
|
||||||
|
tokio::task::spawn_blocking(move || {
|
||||||
|
let fut = inner.get_prop(&path, prop);
|
||||||
|
tokio::runtime::Runtime::new().unwrap().block_on(fut)
|
||||||
|
}).await.map_err(|_| FsError::GeneralFailure)?
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
fn patch_props<'a>(&'a self, path: &'a DavPath, patch: Vec<(bool, DavProp)>) -> FsFuture<'a, Vec<(StatusCode, DavProp)>> {
|
||||||
|
let inner = self.inner.clone();
|
||||||
|
let path = path.clone();
|
||||||
|
Box::pin(async move {
|
||||||
|
tokio::task::spawn_blocking(move || {
|
||||||
|
let fut = inner.patch_props(&path, patch);
|
||||||
|
tokio::runtime::Runtime::new().unwrap().block_on(fut)
|
||||||
|
}).await.map_err(|_| FsError::GeneralFailure)?
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
fn have_props<'a>(&'a self, path: &'a DavPath) -> Pin<Box<dyn Future<Output = bool> + Send + 'a>> {
|
||||||
|
self.inner.have_props(path)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn get_quota(&self) -> FsFuture<'_, (u64, Option<u64>)> {
|
||||||
|
let inner = self.inner.clone();
|
||||||
|
Box::pin(async move {
|
||||||
|
tokio::task::spawn_blocking(move || {
|
||||||
|
let fut = inner.get_quota();
|
||||||
|
tokio::runtime::Runtime::new().unwrap().block_on(fut)
|
||||||
|
}).await.map_err(|_| FsError::GeneralFailure)?
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
+56
-31
@@ -158,53 +158,78 @@ impl AuthState {
|
|||||||
}
|
}
|
||||||
|
|
||||||
pub fn admin_login(&self, username: &str, password: &str) -> Option<AdminLoginResponse> {
|
pub fn admin_login(&self, username: &str, password: &str) -> Option<AdminLoginResponse> {
|
||||||
|
// Try auth_db first (legacy PostgreSQL sync)
|
||||||
if let Some(auth_db) = &self.auth_db {
|
if let Some(auth_db) = &self.auth_db {
|
||||||
match auth_db.get_admin(username) {
|
match auth_db.get_admin(username) {
|
||||||
Ok(Some(admin)) if admin.status == 1 => {
|
Ok(Some(admin)) if admin.status == 1 => {
|
||||||
if verify(password, &admin.password_hash).unwrap_or(false) {
|
if verify(password, &admin.password_hash).unwrap_or(false) {
|
||||||
let token = Uuid::new_v4().to_string();
|
return self.create_admin_session(username, password);
|
||||||
let now = Utc::now();
|
|
||||||
let expires_at = now + Duration::hours(24);
|
|
||||||
|
|
||||||
let session = AdminSession {
|
|
||||||
token: token.clone(),
|
|
||||||
username: username.to_string(),
|
|
||||||
created_at: now.format("%Y-%m-%dT%H:%M:%SZ").to_string(),
|
|
||||||
expires_at: expires_at.format("%Y-%m-%dT%H:%M:%SZ").to_string(),
|
|
||||||
};
|
|
||||||
|
|
||||||
let mut admin_sessions = self.admin_sessions.lock().unwrap();
|
|
||||||
admin_sessions.insert(token.clone(), session);
|
|
||||||
|
|
||||||
log::info!("Admin {} logged in successfully", username);
|
|
||||||
|
|
||||||
Some(AdminLoginResponse {
|
|
||||||
token,
|
|
||||||
expires_at: expires_at.format("%Y-%m-%dT%H:%M:%SZ").to_string(),
|
|
||||||
username: username.to_string(),
|
|
||||||
})
|
|
||||||
} else {
|
} else {
|
||||||
log::warn!("Invalid password for admin {}", username);
|
log::warn!("Invalid password for admin {}", username);
|
||||||
None
|
return None;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
Ok(Some(_)) => {
|
Ok(Some(_)) => {
|
||||||
log::warn!("Admin {} is not active", username);
|
log::warn!("Admin {} is not active", username);
|
||||||
None
|
return None;
|
||||||
}
|
|
||||||
Ok(None) => {
|
|
||||||
log::warn!("Admin {} not found", username);
|
|
||||||
None
|
|
||||||
}
|
}
|
||||||
|
Ok(None) => {}
|
||||||
Err(e) => {
|
Err(e) => {
|
||||||
log::error!("Failed to get admin {}: {}", username, e);
|
log::error!("Failed to get admin {}: {}", username, e);
|
||||||
None
|
return None;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
} else {
|
|
||||||
log::warn!("Auth DB not available for admin login");
|
|
||||||
None
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Fallback: try provider
|
||||||
|
if let Some(provider) = &self.provider {
|
||||||
|
match provider.get_user(username) {
|
||||||
|
Ok(Some(user)) if user.status == 1 => {
|
||||||
|
if verify(password, &user.password_hash).unwrap_or(false) {
|
||||||
|
return self.create_admin_session(username, password);
|
||||||
|
} else {
|
||||||
|
log::warn!("Invalid password for admin {} (provider)", username);
|
||||||
|
return None;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Ok(Some(_)) => {
|
||||||
|
log::warn!("Admin {} is not active (provider)", username);
|
||||||
|
return None;
|
||||||
|
}
|
||||||
|
Ok(None) => {}
|
||||||
|
Err(e) => {
|
||||||
|
log::error!("Failed to get admin {} from provider: {}", username, e);
|
||||||
|
return None;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
log::warn!("Admin {} not found (auth_db + provider)", username);
|
||||||
|
None
|
||||||
|
}
|
||||||
|
|
||||||
|
fn create_admin_session(&self, username: &str, _password: &str) -> Option<AdminLoginResponse> {
|
||||||
|
let token = Uuid::new_v4().to_string();
|
||||||
|
let now = Utc::now();
|
||||||
|
let expires_at = now + Duration::hours(24);
|
||||||
|
|
||||||
|
let session = AdminSession {
|
||||||
|
token: token.clone(),
|
||||||
|
username: username.to_string(),
|
||||||
|
created_at: now.format("%Y-%m-%dT%H:%M:%SZ").to_string(),
|
||||||
|
expires_at: expires_at.format("%Y-%m-%dT%H:%M:%SZ").to_string(),
|
||||||
|
};
|
||||||
|
|
||||||
|
let mut admin_sessions = self.admin_sessions.lock().unwrap();
|
||||||
|
admin_sessions.insert(token.clone(), session);
|
||||||
|
|
||||||
|
log::info!("Admin {} logged in successfully", username);
|
||||||
|
|
||||||
|
Some(AdminLoginResponse {
|
||||||
|
token,
|
||||||
|
expires_at: expires_at.format("%Y-%m-%dT%H:%M:%SZ").to_string(),
|
||||||
|
username: username.to_string(),
|
||||||
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn verify_admin_token(&self, token: &str) -> Option<AdminSession> {
|
pub fn verify_admin_token(&self, token: &str) -> Option<AdminSession> {
|
||||||
|
|||||||
@@ -28,7 +28,7 @@ pub async fn handle_ssh_command(cmd: SshCommand) -> anyhow::Result<()> {
|
|||||||
println!("Security: ⭐⭐⭐⭐⭐ (RustCrypto authoritative libraries)");
|
println!("Security: ⭐⭐⭐⭐⭐ (RustCrypto authoritative libraries)");
|
||||||
println!();
|
println!();
|
||||||
|
|
||||||
crate::ssh_server::server::run_ssh_server(Some(port), pg_conn.as_deref())?;
|
crate::ssh_server::server::run_ssh_server(Some(port), pg_conn.as_deref()).await?;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
Ok(())
|
Ok(())
|
||||||
|
|||||||
@@ -1,6 +1,14 @@
|
|||||||
use axum::{extract::Request, response::IntoResponse, Extension};
|
use axum::{
|
||||||
|
body::Body,
|
||||||
|
extract::Request,
|
||||||
|
http::{HeaderValue, StatusCode},
|
||||||
|
middleware,
|
||||||
|
response::IntoResponse,
|
||||||
|
Extension,
|
||||||
|
};
|
||||||
|
use base64::Engine as _;
|
||||||
use clap::Subcommand;
|
use clap::Subcommand;
|
||||||
use dav_server::{fakels::FakeLs, DavHandler};
|
use dav_server::DavHandler;
|
||||||
use std::path::PathBuf;
|
use std::path::PathBuf;
|
||||||
|
|
||||||
#[derive(Subcommand)]
|
#[derive(Subcommand)]
|
||||||
@@ -11,13 +19,72 @@ pub enum WebdavCommand {
|
|||||||
port: u16,
|
port: u16,
|
||||||
#[arg(short, long)]
|
#[arg(short, long)]
|
||||||
user: String,
|
user: String,
|
||||||
|
#[arg(long, help = "Enable SQLite virtual directory mode")]
|
||||||
|
virtual_mode: bool,
|
||||||
|
#[arg(long, help = "SQLite database path for virtual directories")]
|
||||||
|
db: Option<String>,
|
||||||
|
},
|
||||||
|
|
||||||
|
#[command(name = "webdav-folder")]
|
||||||
|
Folder {
|
||||||
|
#[arg(long, help = "Action: add, remove")]
|
||||||
|
action: String,
|
||||||
|
#[arg(long, help = "Virtual folder name (e.g. photos)")]
|
||||||
|
name: String,
|
||||||
|
#[arg(long, help = "Description for the folder")]
|
||||||
|
description: Option<String>,
|
||||||
|
#[arg(long, default_value = "data/webdav_virtual.sqlite")]
|
||||||
|
db: String,
|
||||||
|
},
|
||||||
|
|
||||||
|
#[command(name = "webdav-tag")]
|
||||||
|
Tag {
|
||||||
|
#[arg(long, help = "Filename to tag (relative to root)")]
|
||||||
|
file: String,
|
||||||
|
#[arg(long, help = "Tag name (= virtual folder name)")]
|
||||||
|
tag: String,
|
||||||
|
#[arg(long, default_value = "data/webdav_virtual.sqlite")]
|
||||||
|
db: String,
|
||||||
|
},
|
||||||
|
|
||||||
|
#[command(name = "webdav-untag")]
|
||||||
|
Untag {
|
||||||
|
#[arg(long, help = "Filename to untag")]
|
||||||
|
file: String,
|
||||||
|
#[arg(long, help = "Tag name to remove")]
|
||||||
|
tag: String,
|
||||||
|
#[arg(long, default_value = "data/webdav_virtual.sqlite")]
|
||||||
|
db: String,
|
||||||
|
},
|
||||||
|
|
||||||
|
#[command(name = "webdav-list")]
|
||||||
|
List {
|
||||||
|
#[arg(long, help = "List folders, or files in a folder")]
|
||||||
|
what: Option<String>,
|
||||||
|
#[arg(long, help = "Folder name (for listing files in a folder)")]
|
||||||
|
folder: Option<String>,
|
||||||
|
#[arg(long, help = "Filename (for listing tags of a file)")]
|
||||||
|
file: Option<String>,
|
||||||
|
#[arg(long, default_value = "data/webdav_virtual.sqlite")]
|
||||||
|
db: String,
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
pub async fn handle_webdav_command(cmd: WebdavCommand) -> anyhow::Result<()> {
|
pub async fn handle_webdav_command(cmd: WebdavCommand) -> anyhow::Result<()> {
|
||||||
match cmd {
|
match cmd {
|
||||||
WebdavCommand::Start { port, user } => {
|
WebdavCommand::Start {
|
||||||
let home_dir = PathBuf::from("/Users/accusys/momentry/var/sftpgo/data").join(&user);
|
port,
|
||||||
|
user,
|
||||||
|
virtual_mode,
|
||||||
|
db,
|
||||||
|
} => {
|
||||||
|
let username = user.split(':').next().unwrap_or(&user).to_string();
|
||||||
|
let password = user.split(':').nth(1).map(|s| s.to_string());
|
||||||
|
|
||||||
|
let default_root = format!("/Users/accusys/momentry/var/sftpgo/data/{}", username);
|
||||||
|
let home_dir = PathBuf::from(
|
||||||
|
std::env::var("MB_WEBDAV_ROOT").unwrap_or(default_root),
|
||||||
|
);
|
||||||
|
|
||||||
if !home_dir.exists() {
|
if !home_dir.exists() {
|
||||||
return Err(anyhow::anyhow!(
|
return Err(anyhow::anyhow!(
|
||||||
@@ -27,12 +94,109 @@ pub async fn handle_webdav_command(cmd: WebdavCommand) -> anyhow::Result<()> {
|
|||||||
}
|
}
|
||||||
|
|
||||||
println!("=== MarkBase WebDAV Server (VFS) ===");
|
println!("=== MarkBase WebDAV Server (VFS) ===");
|
||||||
println!("User: {}", user);
|
println!("User: {}", username);
|
||||||
|
if password.is_some() {
|
||||||
|
println!("Auth: password protected");
|
||||||
|
}
|
||||||
println!("Port: {}", port);
|
println!("Port: {}", port);
|
||||||
println!("Home: {}", home_dir.display());
|
println!("Home: {}", home_dir.display());
|
||||||
|
if virtual_mode {
|
||||||
|
let db_path = db.clone().unwrap_or_else(|| "data/webdav_virtual.sqlite".to_string());
|
||||||
|
println!("Virtual mode: enabled (db: {})", db_path);
|
||||||
|
}
|
||||||
println!();
|
println!();
|
||||||
|
|
||||||
run_webdav_server(port, home_dir, user).await?;
|
run_webdav_server(port, home_dir, username, password, virtual_mode, db).await?;
|
||||||
|
}
|
||||||
|
|
||||||
|
WebdavCommand::Folder {
|
||||||
|
action,
|
||||||
|
name,
|
||||||
|
description,
|
||||||
|
db,
|
||||||
|
} => {
|
||||||
|
let vfs = crate::vfs::virtual_fs::VirtualFs::new(&db, PathBuf::from("/tmp"))?;
|
||||||
|
let folder = if name.starts_with('/') {
|
||||||
|
name
|
||||||
|
} else {
|
||||||
|
format!("/{}", name)
|
||||||
|
};
|
||||||
|
match action.as_str() {
|
||||||
|
"add" => {
|
||||||
|
let desc = description.unwrap_or_default();
|
||||||
|
vfs.add_folder(&folder, &desc)?;
|
||||||
|
println!("Added virtual folder: {} ({})", folder, desc);
|
||||||
|
}
|
||||||
|
"remove" => {
|
||||||
|
vfs.remove_folder(&folder)?;
|
||||||
|
println!("Removed virtual folder: {}", folder);
|
||||||
|
}
|
||||||
|
_ => {
|
||||||
|
return Err(anyhow::anyhow!("Unknown action: {} (use add or remove)", action));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
WebdavCommand::Tag { file, tag, db } => {
|
||||||
|
let vfs = crate::vfs::virtual_fs::VirtualFs::new(&db, PathBuf::from("/tmp"))?;
|
||||||
|
vfs.tag_file(&file, &tag)?;
|
||||||
|
println!("Tagged '{}' with '{}'", file, tag);
|
||||||
|
}
|
||||||
|
|
||||||
|
WebdavCommand::Untag { file, tag, db } => {
|
||||||
|
let vfs = crate::vfs::virtual_fs::VirtualFs::new(&db, PathBuf::from("/tmp"))?;
|
||||||
|
vfs.untag_file(&file, &tag)?;
|
||||||
|
println!("Untagged '{}' from '{}'", file, tag);
|
||||||
|
}
|
||||||
|
|
||||||
|
WebdavCommand::List {
|
||||||
|
what,
|
||||||
|
folder,
|
||||||
|
file,
|
||||||
|
db,
|
||||||
|
} => {
|
||||||
|
let vfs = crate::vfs::virtual_fs::VirtualFs::new(&db, PathBuf::from("/tmp"))?;
|
||||||
|
match what.as_deref() {
|
||||||
|
Some("folders") | None => {
|
||||||
|
let folders = vfs.list_folders()?;
|
||||||
|
if folders.is_empty() {
|
||||||
|
println!("No virtual folders.");
|
||||||
|
} else {
|
||||||
|
println!("{:<30} Description", "Folder");
|
||||||
|
println!("{}", "-".repeat(60));
|
||||||
|
for (f, d) in folders {
|
||||||
|
println!("{:<30} {}", f, d);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Some("files") => {
|
||||||
|
let folder_name = folder.ok_or_else(|| anyhow::anyhow!("--folder required for listing files"))?;
|
||||||
|
let files = vfs.list_files_in_folder(&folder_name)?;
|
||||||
|
if files.is_empty() {
|
||||||
|
println!("No files in folder '{}'", folder_name);
|
||||||
|
} else {
|
||||||
|
println!("Files in folder '{}':", folder_name);
|
||||||
|
for f in files {
|
||||||
|
println!(" {}", f);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Some("tags") => {
|
||||||
|
let filename = file.ok_or_else(|| anyhow::anyhow!("--file required for listing tags"))?;
|
||||||
|
let tags = vfs.list_tags_for_file(&filename)?;
|
||||||
|
if tags.is_empty() {
|
||||||
|
println!("No tags for file '{}'", filename);
|
||||||
|
} else {
|
||||||
|
println!("Tags for file '{}':", filename);
|
||||||
|
for t in tags {
|
||||||
|
println!(" {}", t);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Some(other) => {
|
||||||
|
return Err(anyhow::anyhow!("Unknown list type: {} (use folders, files, or tags)", other));
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
Ok(())
|
Ok(())
|
||||||
@@ -42,32 +206,94 @@ async fn run_webdav_server(
|
|||||||
port: u16,
|
port: u16,
|
||||||
home_dir: PathBuf,
|
home_dir: PathBuf,
|
||||||
user: String,
|
user: String,
|
||||||
|
password: Option<String>,
|
||||||
|
virtual_mode: bool,
|
||||||
|
db: Option<String>,
|
||||||
) -> anyhow::Result<()> {
|
) -> anyhow::Result<()> {
|
||||||
use axum::{routing::any, Router};
|
use axum::{routing::any, Router};
|
||||||
use tokio::net::TcpListener;
|
use tokio::net::TcpListener;
|
||||||
|
|
||||||
let vfs = Box::new(crate::vfs::local_fs::LocalFs::new());
|
let dav_handler = if virtual_mode {
|
||||||
let upload_hook = None;
|
let db_path = db.unwrap_or_else(|| "data/webdav_virtual.sqlite".to_string());
|
||||||
|
let vfs = crate::vfs::virtual_fs::VirtualFs::new(&db_path, home_dir.clone())?;
|
||||||
|
|
||||||
let dav_fs = crate::webdav::VfsDavFs::new(vfs, home_dir, upload_hook, user);
|
let folders = vfs.list_folders().unwrap_or_default();
|
||||||
|
println!("Virtual folders ({}):", folders.len());
|
||||||
|
for (f, d) in &folders {
|
||||||
|
println!(" {} - {}", f, d);
|
||||||
|
}
|
||||||
|
println!("Default root: {}", home_dir.display());
|
||||||
|
println!();
|
||||||
|
let vfs_boxed: Box<dyn crate::vfs::VfsBackend> = Box::new(vfs);
|
||||||
|
crate::webdav::create_webdav_handler_virtual(vfs_boxed, PathBuf::from("/"), None, user.clone())
|
||||||
|
} else {
|
||||||
|
let vfs = Box::new(crate::vfs::local_fs::LocalFs::new());
|
||||||
|
crate::webdav::create_webdav_handler(vfs, home_dir.clone(), None, user.clone())
|
||||||
|
};
|
||||||
|
|
||||||
let dav_handler = DavHandler::builder()
|
async fn webdav_auth_middleware(
|
||||||
.filesystem(dav_fs)
|
req: Request,
|
||||||
.locksystem(FakeLs::new())
|
next: middleware::Next,
|
||||||
.strip_prefix("/webdav")
|
) -> impl IntoResponse {
|
||||||
.build_handler();
|
let expected = req.extensions().get::<crate::webdav::WebdavCredentials>().cloned();
|
||||||
|
|
||||||
|
let auth = req
|
||||||
|
.headers()
|
||||||
|
.get("Authorization")
|
||||||
|
.and_then(|v| v.to_str().ok())
|
||||||
|
.filter(|v| v.starts_with("Basic "))
|
||||||
|
.and_then(|v| {
|
||||||
|
let encoded = &v[6..];
|
||||||
|
let decoded = base64::engine::general_purpose::STANDARD
|
||||||
|
.decode(encoded)
|
||||||
|
.ok()?;
|
||||||
|
let creds = String::from_utf8(decoded).ok()?;
|
||||||
|
let colon = creds.find(':')?;
|
||||||
|
Some((creds[..colon].to_string(), creds[colon + 1..].to_string()))
|
||||||
|
});
|
||||||
|
|
||||||
|
let valid = match (auth, expected) {
|
||||||
|
(Some((u, p)), Some(exp)) => {
|
||||||
|
u == exp.username && exp.password.as_ref().is_none_or(|exp_p| p == *exp_p)
|
||||||
|
}
|
||||||
|
_ => false,
|
||||||
|
};
|
||||||
|
|
||||||
|
if !valid {
|
||||||
|
return (
|
||||||
|
StatusCode::UNAUTHORIZED,
|
||||||
|
[(
|
||||||
|
"WWW-Authenticate",
|
||||||
|
HeaderValue::from_static("Basic realm=\"MarkBase WebDAV\""),
|
||||||
|
)],
|
||||||
|
Body::from("Unauthorized"),
|
||||||
|
)
|
||||||
|
.into_response();
|
||||||
|
}
|
||||||
|
|
||||||
|
next.run(req).await
|
||||||
|
}
|
||||||
|
|
||||||
let app = Router::new()
|
let app = Router::new()
|
||||||
.route("/webdav", any(handle_dav))
|
.route("/", any(handle_dav))
|
||||||
.route("/webdav/", any(handle_dav))
|
.route("/*path", any(handle_dav))
|
||||||
.route("/webdav/*path", any(handle_dav))
|
.layer(Extension(dav_handler))
|
||||||
.layer(Extension(dav_handler));
|
.layer(middleware::from_fn(webdav_auth_middleware))
|
||||||
|
.layer(Extension(crate::webdav::WebdavCredentials {
|
||||||
|
username: user.clone(),
|
||||||
|
password,
|
||||||
|
}));
|
||||||
|
|
||||||
let addr = format!("127.0.0.1:{}", port);
|
let addr = format!("0.0.0.0:{}", port);
|
||||||
let listener = TcpListener::bind(&addr).await?;
|
let listener = TcpListener::bind(&addr).await?;
|
||||||
|
|
||||||
println!("WebDAV server listening on http://{}", addr);
|
println!("WebDAV server listening on http://{}", addr);
|
||||||
println!("Mount point: /webdav");
|
println!("User: {}", user);
|
||||||
|
if virtual_mode {
|
||||||
|
println!("Mode: Virtual (SQLite tag-based)");
|
||||||
|
} else {
|
||||||
|
println!("Mode: Local");
|
||||||
|
}
|
||||||
println!();
|
println!();
|
||||||
println!("Press Ctrl+C to stop");
|
println!("Press Ctrl+C to stop");
|
||||||
|
|
||||||
@@ -78,6 +304,7 @@ async fn run_webdav_server(
|
|||||||
|
|
||||||
async fn handle_dav(
|
async fn handle_dav(
|
||||||
Extension(dav): Extension<DavHandler>,
|
Extension(dav): Extension<DavHandler>,
|
||||||
|
Extension(_creds): Extension<crate::webdav::WebdavCredentials>,
|
||||||
req: Request,
|
req: Request,
|
||||||
) -> impl IntoResponse {
|
) -> impl IntoResponse {
|
||||||
dav.handle(req).await
|
dav.handle(req).await
|
||||||
|
|||||||
@@ -1,6 +1,8 @@
|
|||||||
pub mod render;
|
pub mod render;
|
||||||
pub mod smb_server;
|
pub mod smb_server;
|
||||||
pub mod test;
|
pub mod test;
|
||||||
|
#[cfg(feature = "nfs")]
|
||||||
|
pub mod nfs_server;
|
||||||
|
|
||||||
use clap::Subcommand;
|
use clap::Subcommand;
|
||||||
|
|
||||||
@@ -12,6 +14,8 @@ pub enum ToolsCommands {
|
|||||||
Test(test::TestCommand),
|
Test(test::TestCommand),
|
||||||
#[command(flatten)]
|
#[command(flatten)]
|
||||||
SmbServer(smb_server::SmbServerCommand),
|
SmbServer(smb_server::SmbServerCommand),
|
||||||
|
#[cfg(feature = "nfs")]
|
||||||
|
Nfs(nfs_server::NfsServerCommand),
|
||||||
}
|
}
|
||||||
|
|
||||||
pub async fn handle_tools_command(cmd: ToolsCommands) -> anyhow::Result<()> {
|
pub async fn handle_tools_command(cmd: ToolsCommands) -> anyhow::Result<()> {
|
||||||
@@ -19,6 +23,8 @@ pub async fn handle_tools_command(cmd: ToolsCommands) -> anyhow::Result<()> {
|
|||||||
ToolsCommands::Render(c) => render::handle_render_command(c)?,
|
ToolsCommands::Render(c) => render::handle_render_command(c)?,
|
||||||
ToolsCommands::Test(c) => test::handle_test_command(c)?,
|
ToolsCommands::Test(c) => test::handle_test_command(c)?,
|
||||||
ToolsCommands::SmbServer(c) => smb_server::handle_smb_server_command(c).await?,
|
ToolsCommands::SmbServer(c) => smb_server::handle_smb_server_command(c).await?,
|
||||||
|
#[cfg(feature = "nfs")]
|
||||||
|
ToolsCommands::Nfs(c) => nfs_server::run_nfs_server(c).await?,
|
||||||
}
|
}
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1,43 @@
|
|||||||
|
use clap::Args;
|
||||||
|
use std::path::PathBuf;
|
||||||
|
use std::sync::Arc;
|
||||||
|
|
||||||
|
use crate::vfs::{local_fs::LocalFs, nfs_server::{NfsVfsServer, NfsConfig}};
|
||||||
|
|
||||||
|
#[derive(Debug, Args)]
|
||||||
|
pub struct NfsServerCommand {
|
||||||
|
/// Port to listen on (default: 2049)
|
||||||
|
#[arg(short, long, default_value = "2049")]
|
||||||
|
port: u16,
|
||||||
|
|
||||||
|
/// Root directory to export
|
||||||
|
#[arg(short, long, default_value = "/tmp/nfs_export")]
|
||||||
|
root: PathBuf,
|
||||||
|
|
||||||
|
/// Share name (export name)
|
||||||
|
#[arg(short, long, default_value = "export")]
|
||||||
|
share_name: String,
|
||||||
|
}
|
||||||
|
|
||||||
|
pub async fn run_nfs_server(cmd: NfsServerCommand) -> anyhow::Result<()> {
|
||||||
|
println!("Starting NFS server on port {}", cmd.port);
|
||||||
|
println!("Export directory: {}", cmd.root.display());
|
||||||
|
println!("Share name: {}", cmd.share_name);
|
||||||
|
|
||||||
|
if !cmd.root.exists() {
|
||||||
|
std::fs::create_dir_all(&cmd.root)?;
|
||||||
|
println!("Created export directory: {}", cmd.root.display());
|
||||||
|
}
|
||||||
|
|
||||||
|
let vfs = Arc::new(LocalFs::new());
|
||||||
|
let server = NfsVfsServer::new(vfs, cmd.root.clone())
|
||||||
|
.with_port(cmd.port)
|
||||||
|
.with_export_name(&cmd.share_name);
|
||||||
|
|
||||||
|
println!("NFS server starting...");
|
||||||
|
server.start(cmd.port).await?;
|
||||||
|
|
||||||
|
println!("NFS server stopped");
|
||||||
|
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
@@ -36,6 +36,45 @@ pub enum SmbServerCommand {
|
|||||||
|
|
||||||
#[arg(long, default_value = "us-east-1")]
|
#[arg(long, default_value = "us-east-1")]
|
||||||
s3_region: String,
|
s3_region: String,
|
||||||
|
|
||||||
|
#[arg(long)]
|
||||||
|
ldap: bool,
|
||||||
|
|
||||||
|
#[arg(long)]
|
||||||
|
ldap_url: Option<String>,
|
||||||
|
|
||||||
|
#[arg(long)]
|
||||||
|
ldap_base_dn: Option<String>,
|
||||||
|
|
||||||
|
#[arg(long)]
|
||||||
|
ldap_bind_dn: Option<String>,
|
||||||
|
|
||||||
|
#[arg(long)]
|
||||||
|
ldap_bind_password: Option<String>,
|
||||||
|
|
||||||
|
#[arg(long)]
|
||||||
|
ldap_user_search_base: Option<String>,
|
||||||
|
|
||||||
|
#[arg(long)]
|
||||||
|
ldap_group_search_base: Option<String>,
|
||||||
|
|
||||||
|
#[arg(long)]
|
||||||
|
ldap_user_id_attr: Option<String>,
|
||||||
|
|
||||||
|
#[arg(long)]
|
||||||
|
ldap_user_filter: Option<String>,
|
||||||
|
|
||||||
|
#[arg(long)]
|
||||||
|
ldap_group_filter: Option<String>,
|
||||||
|
|
||||||
|
#[arg(long)]
|
||||||
|
ldap_home_dir_attr: Option<String>,
|
||||||
|
|
||||||
|
#[arg(long)]
|
||||||
|
ldap_home_dir_prefix: Option<String>,
|
||||||
|
|
||||||
|
#[arg(long)]
|
||||||
|
ldap_user_groups_attr: Option<String>,
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -63,9 +102,23 @@ pub async fn handle_smb_server_command(cmd: SmbServerCommand) -> anyhow::Result<
|
|||||||
s3_access_key,
|
s3_access_key,
|
||||||
s3_secret_key,
|
s3_secret_key,
|
||||||
s3_region,
|
s3_region,
|
||||||
|
ldap,
|
||||||
|
ldap_url: _,
|
||||||
|
ldap_base_dn: _,
|
||||||
|
ldap_bind_dn: _,
|
||||||
|
ldap_bind_password: _,
|
||||||
|
ldap_user_search_base: _,
|
||||||
|
ldap_group_search_base: _,
|
||||||
|
ldap_user_id_attr: _,
|
||||||
|
ldap_user_filter: _,
|
||||||
|
ldap_group_filter: _,
|
||||||
|
ldap_home_dir_attr: _,
|
||||||
|
ldap_home_dir_prefix: _,
|
||||||
|
ldap_user_groups_attr: _,
|
||||||
} => {
|
} => {
|
||||||
use std::path::PathBuf;
|
use std::path::PathBuf;
|
||||||
|
|
||||||
|
|
||||||
use smb_server::{Access, Share, SmbServer};
|
use smb_server::{Access, Share, SmbServer};
|
||||||
use tracing_subscriber::EnvFilter;
|
use tracing_subscriber::EnvFilter;
|
||||||
|
|
||||||
@@ -111,6 +164,34 @@ pub async fn handle_smb_server_command(cmd: SmbServerCommand) -> anyhow::Result<
|
|||||||
user
|
user
|
||||||
};
|
};
|
||||||
|
|
||||||
|
#[allow(unused_mut)]
|
||||||
|
let mut ldap_enabled = false;
|
||||||
|
#[cfg(feature = "ldap")]
|
||||||
|
{
|
||||||
|
if ldap {
|
||||||
|
let config = crate::provider::ldap::LdapConfig {
|
||||||
|
ldap_url: ldap_url.unwrap_or_else(|| "ldap://localhost:389".to_string()),
|
||||||
|
base_dn: ldap_base_dn.unwrap_or_else(|| "dc=example,dc=com".to_string()),
|
||||||
|
bind_dn: ldap_bind_dn.unwrap_or_else(|| "cn=admin,dc=example,dc=com".to_string()),
|
||||||
|
bind_password: ldap_bind_password.unwrap_or_else(|| "admin".to_string()),
|
||||||
|
user_search_base: ldap_user_search_base.unwrap_or_else(|| "ou=users,dc=example,dc=com".to_string()),
|
||||||
|
group_search_base: ldap_group_search_base.unwrap_or_else(|| "ou=groups,dc=example,dc=com".to_string()),
|
||||||
|
user_filter: ldap_user_filter.unwrap_or_else(|| "(objectClass=person)".to_string()),
|
||||||
|
group_filter: ldap_group_filter.unwrap_or_else(|| "(objectClass=group)".to_string()),
|
||||||
|
user_id_attr: ldap_user_id_attr.unwrap_or_else(|| "uid".to_string()),
|
||||||
|
home_dir_attr: ldap_home_dir_attr.unwrap_or_else(|| "homeDirectory".to_string()),
|
||||||
|
home_dir_prefix: ldap_home_dir_prefix.unwrap_or_else(|| "/home".to_string()),
|
||||||
|
user_groups_attr: ldap_user_groups_attr.unwrap_or_else(|| "memberOf".to_string()),
|
||||||
|
};
|
||||||
|
log::info!("LDAP authentication enabled: url={}, search_base={}", config.ldap_url, config.user_search_base);
|
||||||
|
ldap_enabled = true;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
#[cfg(not(feature = "ldap"))]
|
||||||
|
if ldap {
|
||||||
|
log::warn!("LDAP authentication requested but ldap feature not enabled");
|
||||||
|
}
|
||||||
|
|
||||||
let mut builder = SmbServer::builder().listen(addr);
|
let mut builder = SmbServer::builder().listen(addr);
|
||||||
|
|
||||||
for (name, password) in &users {
|
for (name, password) in &users {
|
||||||
@@ -128,6 +209,9 @@ pub async fn handle_smb_server_command(cmd: SmbServerCommand) -> anyhow::Result<
|
|||||||
log::info!("SMB server listening on {}", addr);
|
log::info!("SMB server listening on {}", addr);
|
||||||
log::info!("Share '{}' at root: {}", share_name, root);
|
log::info!("Share '{}' at root: {}", share_name, root);
|
||||||
log::info!("Users: {}", user_list.join(", "));
|
log::info!("Users: {}", user_list.join(", "));
|
||||||
|
if ldap_enabled {
|
||||||
|
log::info!("LDAP authentication: enabled");
|
||||||
|
}
|
||||||
|
|
||||||
server.serve().await?;
|
server.serve().await?;
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -29,6 +29,7 @@ pub struct WebSection {
|
|||||||
pub log_level: String,
|
pub log_level: String,
|
||||||
pub auth_db_path: String,
|
pub auth_db_path: String,
|
||||||
pub users_db_dir: String,
|
pub users_db_dir: String,
|
||||||
|
pub webdav_root: String,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl Default for WebSection {
|
impl Default for WebSection {
|
||||||
@@ -39,6 +40,7 @@ impl Default for WebSection {
|
|||||||
log_level: "info".to_string(),
|
log_level: "info".to_string(),
|
||||||
auth_db_path: "data/auth.sqlite".to_string(),
|
auth_db_path: "data/auth.sqlite".to_string(),
|
||||||
users_db_dir: "data/users".to_string(),
|
users_db_dir: "data/users".to_string(),
|
||||||
|
webdav_root: "/Users/accusys/momentry/var/sftpgo/data/demo".to_string(),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -13,11 +13,30 @@ pub struct MarkBaseConfig {
|
|||||||
|
|
||||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||||
pub struct ServerConfig {
|
pub struct ServerConfig {
|
||||||
|
#[serde(default = "default_host")]
|
||||||
pub host: String,
|
pub host: String,
|
||||||
|
#[serde(default = "default_port")]
|
||||||
pub port: u16,
|
pub port: u16,
|
||||||
|
#[serde(default = "default_log_level")]
|
||||||
pub log_level: String,
|
pub log_level: String,
|
||||||
|
#[serde(default = "default_auth_db_path")]
|
||||||
pub auth_db_path: String,
|
pub auth_db_path: String,
|
||||||
|
#[serde(default = "default_users_db_dir")]
|
||||||
pub users_db_dir: String,
|
pub users_db_dir: String,
|
||||||
|
#[serde(default = "default_webdav_root")]
|
||||||
|
pub webdav_root: String,
|
||||||
|
#[serde(default = "default_upload_path")]
|
||||||
|
pub upload_path: String,
|
||||||
|
}
|
||||||
|
|
||||||
|
fn default_host() -> String { "127.0.0.1".to_string() }
|
||||||
|
fn default_port() -> u16 { 11438 }
|
||||||
|
fn default_log_level() -> String { "info".to_string() }
|
||||||
|
fn default_auth_db_path() -> String { "data/auth.sqlite".to_string() }
|
||||||
|
fn default_users_db_dir() -> String { "data/users".to_string() }
|
||||||
|
fn default_webdav_root() -> String { "/Users/accusys/momentry/var/sftpgo/data/demo".to_string() }
|
||||||
|
fn default_upload_path() -> String {
|
||||||
|
"/Users/accusys/momentry/var/sftpgo/data".to_string()
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||||
@@ -87,6 +106,8 @@ impl MarkBaseConfig {
|
|||||||
log_level: "info".to_string(),
|
log_level: "info".to_string(),
|
||||||
auth_db_path: "data/auth.sqlite".to_string(),
|
auth_db_path: "data/auth.sqlite".to_string(),
|
||||||
users_db_dir: "data/users".to_string(),
|
users_db_dir: "data/users".to_string(),
|
||||||
|
webdav_root: "/Users/accusys/momentry/var/sftpgo/data/demo".to_string(),
|
||||||
|
upload_path: "/Users/accusys/momentry/var/sftpgo/data".to_string(),
|
||||||
},
|
},
|
||||||
postgresql: PostgreSQLConfig {
|
postgresql: PostgreSQLConfig {
|
||||||
host: "127.0.0.1".to_string(),
|
host: "127.0.0.1".to_string(),
|
||||||
@@ -138,6 +159,12 @@ impl MarkBaseConfig {
|
|||||||
if let Ok(log_level) = std::env::var("MB_LOG_LEVEL") {
|
if let Ok(log_level) = std::env::var("MB_LOG_LEVEL") {
|
||||||
self.server.log_level = log_level;
|
self.server.log_level = log_level;
|
||||||
}
|
}
|
||||||
|
if let Ok(webdav_root) = std::env::var("MB_WEBDAV_ROOT") {
|
||||||
|
self.server.webdav_root = webdav_root;
|
||||||
|
}
|
||||||
|
if let Ok(upload_path) = std::env::var("MB_WEBDAV_PARENT") {
|
||||||
|
self.server.upload_path = upload_path;
|
||||||
|
}
|
||||||
|
|
||||||
if let Ok(pg_host) = std::env::var("PG_HOST") {
|
if let Ok(pg_host) = std::env::var("PG_HOST") {
|
||||||
self.postgresql.host = pg_host;
|
self.postgresql.host = pg_host;
|
||||||
@@ -176,6 +203,8 @@ impl MarkBaseConfig {
|
|||||||
"server.log_level" => Some(self.server.log_level.clone()),
|
"server.log_level" => Some(self.server.log_level.clone()),
|
||||||
"server.auth_db_path" => Some(self.server.auth_db_path.clone()),
|
"server.auth_db_path" => Some(self.server.auth_db_path.clone()),
|
||||||
"server.users_db_dir" => Some(self.server.users_db_dir.clone()),
|
"server.users_db_dir" => Some(self.server.users_db_dir.clone()),
|
||||||
|
"server.webdav_root" => Some(self.server.webdav_root.clone()),
|
||||||
|
"server.upload_path" => Some(self.server.upload_path.clone()),
|
||||||
|
|
||||||
"postgresql.host" => Some(self.postgresql.host.clone()),
|
"postgresql.host" => Some(self.postgresql.host.clone()),
|
||||||
"postgresql.port" => Some(self.postgresql.port.to_string()),
|
"postgresql.port" => Some(self.postgresql.port.to_string()),
|
||||||
@@ -221,6 +250,8 @@ impl MarkBaseConfig {
|
|||||||
"server.log_level" => self.server.log_level = value.to_string(),
|
"server.log_level" => self.server.log_level = value.to_string(),
|
||||||
"server.auth_db_path" => self.server.auth_db_path = value.to_string(),
|
"server.auth_db_path" => self.server.auth_db_path = value.to_string(),
|
||||||
"server.users_db_dir" => self.server.users_db_dir = value.to_string(),
|
"server.users_db_dir" => self.server.users_db_dir = value.to_string(),
|
||||||
|
"server.webdav_root" => self.server.webdav_root = value.to_string(),
|
||||||
|
"server.upload_path" => self.server.upload_path = value.to_string(),
|
||||||
|
|
||||||
"postgresql.host" => self.postgresql.host = value.to_string(),
|
"postgresql.host" => self.postgresql.host = value.to_string(),
|
||||||
"postgresql.port" => self.postgresql.port = value.parse()?,
|
"postgresql.port" => self.postgresql.port = value.parse()?,
|
||||||
@@ -283,6 +314,10 @@ impl MarkBaseConfig {
|
|||||||
return Err(anyhow::anyhow!("server.users_db_dir cannot be empty"));
|
return Err(anyhow::anyhow!("server.users_db_dir cannot be empty"));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if self.server.upload_path.is_empty() {
|
||||||
|
return Err(anyhow::anyhow!("server.upload_path cannot be empty"));
|
||||||
|
}
|
||||||
|
|
||||||
if self.postgresql.port == 0 {
|
if self.postgresql.port == 0 {
|
||||||
return Err(anyhow::anyhow!(
|
return Err(anyhow::anyhow!(
|
||||||
"Invalid PostgreSQL port: {}",
|
"Invalid PostgreSQL port: {}",
|
||||||
|
|||||||
@@ -0,0 +1,379 @@
|
|||||||
|
use std::collections::HashMap;
|
||||||
|
use std::net::IpAddr;
|
||||||
|
use std::sync::RwLock;
|
||||||
|
|
||||||
|
use super::node::NodeId;
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
|
||||||
|
pub struct PublicIpId(pub u32);
|
||||||
|
|
||||||
|
#[derive(Debug, Clone)]
|
||||||
|
pub struct PublicIpEntry {
|
||||||
|
pub id: PublicIpId,
|
||||||
|
pub ip: IpAddr,
|
||||||
|
pub interface: String,
|
||||||
|
pub owner: Option<NodeId>,
|
||||||
|
pub previous_owner: Option<NodeId>,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl PublicIpEntry {
|
||||||
|
pub fn new(id: PublicIpId, ip: IpAddr, interface: &str) -> Self {
|
||||||
|
Self {
|
||||||
|
id,
|
||||||
|
ip,
|
||||||
|
interface: interface.to_string(),
|
||||||
|
owner: None,
|
||||||
|
previous_owner: None,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub struct IpManager {
|
||||||
|
ip_pool: RwLock<Vec<PublicIpEntry>>,
|
||||||
|
assignments: RwLock<HashMap<PublicIpId, NodeId>>,
|
||||||
|
next_id: RwLock<u32>,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl IpManager {
|
||||||
|
pub fn new() -> Self {
|
||||||
|
Self {
|
||||||
|
ip_pool: RwLock::new(Vec::new()),
|
||||||
|
assignments: RwLock::new(HashMap::new()),
|
||||||
|
next_id: RwLock::new(0),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn add_ip(&self, ip: IpAddr, interface: &str) -> PublicIpId {
|
||||||
|
let mut pool = self.ip_pool.write().unwrap();
|
||||||
|
let mut next = self.next_id.write().unwrap();
|
||||||
|
let id = PublicIpId(*next);
|
||||||
|
*next += 1;
|
||||||
|
pool.push(PublicIpEntry::new(id, ip, interface));
|
||||||
|
id
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn remove_ip(&self, id: PublicIpId) {
|
||||||
|
let mut pool = self.ip_pool.write().unwrap();
|
||||||
|
pool.retain(|e| e.id != id);
|
||||||
|
let mut assignments = self.assignments.write().unwrap();
|
||||||
|
assignments.remove(&id);
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn assign_ip(&self, id: PublicIpId, node: NodeId) -> Result<(), String> {
|
||||||
|
let exists = {
|
||||||
|
let pool = self.ip_pool.read().unwrap();
|
||||||
|
pool.iter().any(|e| e.id == id)
|
||||||
|
};
|
||||||
|
if !exists {
|
||||||
|
return Err(format!("IP {:?} not found", id));
|
||||||
|
}
|
||||||
|
|
||||||
|
let prev_owner = {
|
||||||
|
let assignments = self.assignments.read().unwrap();
|
||||||
|
assignments.get(&id).copied()
|
||||||
|
};
|
||||||
|
|
||||||
|
{
|
||||||
|
let mut assignments = self.assignments.write().unwrap();
|
||||||
|
assignments.insert(id, node);
|
||||||
|
}
|
||||||
|
|
||||||
|
{
|
||||||
|
let mut pool = self.ip_pool.write().unwrap();
|
||||||
|
if let Some(entry) = pool.iter_mut().find(|e| e.id == id) {
|
||||||
|
if let Some(prev) = prev_owner {
|
||||||
|
entry.previous_owner = Some(prev);
|
||||||
|
}
|
||||||
|
entry.owner = Some(node);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn release_ip(&self, id: PublicIpId) -> Result<(), String> {
|
||||||
|
let mut assignments = self.assignments.write().unwrap();
|
||||||
|
if let Some(prev) = assignments.remove(&id) {
|
||||||
|
let mut pool = self.ip_pool.write().unwrap();
|
||||||
|
if let Some(entry) = pool.iter_mut().find(|e| e.id == id) {
|
||||||
|
entry.previous_owner = Some(prev);
|
||||||
|
entry.owner = None;
|
||||||
|
}
|
||||||
|
Ok(())
|
||||||
|
} else {
|
||||||
|
Err(format!("IP {:?} not assigned", id))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn reassign_on_failure(&self, failed_node: NodeId, active_nodes: &[NodeId]) -> Vec<(PublicIpId, NodeId)> {
|
||||||
|
let assignments = self.assignments.read().unwrap();
|
||||||
|
let failed_ips: Vec<PublicIpId> = assignments
|
||||||
|
.iter()
|
||||||
|
.filter(|(_, &node)| node == failed_node)
|
||||||
|
.map(|(&id, _)| id)
|
||||||
|
.collect();
|
||||||
|
drop(assignments);
|
||||||
|
|
||||||
|
let candidates: Vec<NodeId> = active_nodes
|
||||||
|
.iter()
|
||||||
|
.filter(|&&n| n != failed_node)
|
||||||
|
.copied()
|
||||||
|
.collect();
|
||||||
|
|
||||||
|
let mut reassignments = Vec::new();
|
||||||
|
for (i, ip_id) in failed_ips.iter().enumerate() {
|
||||||
|
if let Some(&new_owner) = candidates.get(i % candidates.len().max(1)) {
|
||||||
|
let _ = self.assign_ip(*ip_id, new_owner);
|
||||||
|
reassignments.push((*ip_id, new_owner));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
reassignments
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn get_owner(&self, id: PublicIpId) -> Option<NodeId> {
|
||||||
|
self.assignments.read().unwrap().get(&id).copied()
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn owned_by(&self, node: NodeId) -> Vec<PublicIpEntry> {
|
||||||
|
let assignments = self.assignments.read().unwrap();
|
||||||
|
let pool = self.ip_pool.read().unwrap();
|
||||||
|
assignments
|
||||||
|
.iter()
|
||||||
|
.filter(|(_, &owner)| owner == node)
|
||||||
|
.filter_map(|(&id, _)| pool.iter().find(|e| e.id == id).cloned())
|
||||||
|
.collect()
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn unassigned(&self) -> Vec<PublicIpEntry> {
|
||||||
|
let assignments = self.assignments.read().unwrap();
|
||||||
|
let pool = self.ip_pool.read().unwrap();
|
||||||
|
pool.iter()
|
||||||
|
.filter(|e| !assignments.contains_key(&e.id))
|
||||||
|
.cloned()
|
||||||
|
.collect()
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn all_ips(&self) -> Vec<PublicIpEntry> {
|
||||||
|
self.ip_pool.read().unwrap().clone()
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn ip_count(&self) -> usize {
|
||||||
|
self.ip_pool.read().unwrap().len()
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn assigned_count(&self) -> usize {
|
||||||
|
self.assignments.read().unwrap().len()
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn balance(&self, active_nodes: &[NodeId]) -> Vec<(PublicIpId, NodeId)> {
|
||||||
|
let pool = self.ip_pool.read().unwrap();
|
||||||
|
let assignments = self.assignments.read().unwrap();
|
||||||
|
|
||||||
|
let total_ips = pool.len();
|
||||||
|
let node_count = active_nodes.len().max(1);
|
||||||
|
let per_node = total_ips / node_count;
|
||||||
|
|
||||||
|
let mut current_counts: HashMap<NodeId, usize> = HashMap::new();
|
||||||
|
for &node in active_nodes {
|
||||||
|
current_counts.insert(node, 0);
|
||||||
|
}
|
||||||
|
for (_, &owner) in assignments.iter() {
|
||||||
|
*current_counts.entry(owner).or_insert(0) += 1;
|
||||||
|
}
|
||||||
|
|
||||||
|
let mut reassignments = Vec::new();
|
||||||
|
for entry in pool.iter() {
|
||||||
|
let current_owner = assignments.get(&entry.id).copied();
|
||||||
|
let needs_reassign = match current_owner {
|
||||||
|
Some(owner) => {
|
||||||
|
!active_nodes.contains(&owner) || current_counts[&owner] > per_node
|
||||||
|
}
|
||||||
|
None => true,
|
||||||
|
};
|
||||||
|
|
||||||
|
if needs_reassign {
|
||||||
|
if let Some(&target) = active_nodes
|
||||||
|
.iter()
|
||||||
|
.min_by_key(|n| current_counts.get(n).copied().unwrap_or(0))
|
||||||
|
{
|
||||||
|
current_counts
|
||||||
|
.entry(target)
|
||||||
|
.and_modify(|c| *c += 1)
|
||||||
|
.or_insert(1);
|
||||||
|
reassignments.push((entry.id, target));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
drop(pool);
|
||||||
|
drop(assignments);
|
||||||
|
|
||||||
|
for (id, node) in &reassignments {
|
||||||
|
let _ = self.assign_ip(*id, *node);
|
||||||
|
}
|
||||||
|
|
||||||
|
reassignments
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Default for IpManager {
|
||||||
|
fn default() -> Self {
|
||||||
|
Self::new()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use super::*;
|
||||||
|
use std::net::Ipv4Addr;
|
||||||
|
|
||||||
|
fn ip(a: u8, b: u8, c: u8, d: u8) -> IpAddr {
|
||||||
|
IpAddr::V4(Ipv4Addr::new(a, b, c, d))
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_add_ip() {
|
||||||
|
let mgr = IpManager::new();
|
||||||
|
let id = mgr.add_ip(ip(192, 168, 1, 100), "eth0");
|
||||||
|
assert_eq!(mgr.ip_count(), 1);
|
||||||
|
assert_eq!(id, PublicIpId(0));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_add_multiple_ips() {
|
||||||
|
let mgr = IpManager::new();
|
||||||
|
let id0 = mgr.add_ip(ip(192, 168, 1, 100), "eth0");
|
||||||
|
let id1 = mgr.add_ip(ip(192, 168, 1, 101), "eth0");
|
||||||
|
let id2 = mgr.add_ip(ip(192, 168, 1, 102), "eth1");
|
||||||
|
assert_eq!(mgr.ip_count(), 3);
|
||||||
|
assert_ne!(id0, id1);
|
||||||
|
assert_ne!(id1, id2);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_assign_ip() {
|
||||||
|
let mgr = IpManager::new();
|
||||||
|
let id = mgr.add_ip(ip(192, 168, 1, 100), "eth0");
|
||||||
|
mgr.assign_ip(id, NodeId(0)).unwrap();
|
||||||
|
assert_eq!(mgr.get_owner(id), Some(NodeId(0)));
|
||||||
|
assert_eq!(mgr.assigned_count(), 1);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_release_ip() {
|
||||||
|
let mgr = IpManager::new();
|
||||||
|
let id = mgr.add_ip(ip(192, 168, 1, 100), "eth0");
|
||||||
|
mgr.assign_ip(id, NodeId(0)).unwrap();
|
||||||
|
mgr.release_ip(id).unwrap();
|
||||||
|
assert_eq!(mgr.get_owner(id), None);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_release_unassigned_fails() {
|
||||||
|
let mgr = IpManager::new();
|
||||||
|
let id = mgr.add_ip(ip(192, 168, 1, 100), "eth0");
|
||||||
|
assert!(mgr.release_ip(id).is_err());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_assign_nonexistent_fails() {
|
||||||
|
let mgr = IpManager::new();
|
||||||
|
assert!(mgr.assign_ip(PublicIpId(999), NodeId(0)).is_err());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_owned_by() {
|
||||||
|
let mgr = IpManager::new();
|
||||||
|
let id0 = mgr.add_ip(ip(192, 168, 1, 100), "eth0");
|
||||||
|
let id1 = mgr.add_ip(ip(192, 168, 1, 101), "eth0");
|
||||||
|
let id2 = mgr.add_ip(ip(192, 168, 1, 102), "eth1");
|
||||||
|
mgr.assign_ip(id0, NodeId(0)).unwrap();
|
||||||
|
mgr.assign_ip(id1, NodeId(1)).unwrap();
|
||||||
|
mgr.assign_ip(id2, NodeId(0)).unwrap();
|
||||||
|
let node0_ips = mgr.owned_by(NodeId(0));
|
||||||
|
assert_eq!(node0_ips.len(), 2);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_unassigned() {
|
||||||
|
let mgr = IpManager::new();
|
||||||
|
mgr.add_ip(ip(192, 168, 1, 100), "eth0");
|
||||||
|
let id1 = mgr.add_ip(ip(192, 168, 1, 101), "eth0");
|
||||||
|
mgr.assign_ip(id1, NodeId(0)).unwrap();
|
||||||
|
let unassigned = mgr.unassigned();
|
||||||
|
assert_eq!(unassigned.len(), 1);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_reassign_on_failure() {
|
||||||
|
let mgr = IpManager::new();
|
||||||
|
let id0 = mgr.add_ip(ip(192, 168, 1, 100), "eth0");
|
||||||
|
let id1 = mgr.add_ip(ip(192, 168, 1, 101), "eth0");
|
||||||
|
mgr.assign_ip(id0, NodeId(0)).unwrap();
|
||||||
|
mgr.assign_ip(id1, NodeId(0)).unwrap();
|
||||||
|
let reassignments = mgr.reassign_on_failure(NodeId(0), &[NodeId(1), NodeId(2)]);
|
||||||
|
assert_eq!(reassignments.len(), 2);
|
||||||
|
let new_owners: Vec<NodeId> = reassignments.iter().map(|(_, n)| *n).collect();
|
||||||
|
assert!(new_owners.contains(&NodeId(1)) || new_owners.contains(&NodeId(2)));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_reassign_skips_failed_node() {
|
||||||
|
let mgr = IpManager::new();
|
||||||
|
let id0 = mgr.add_ip(ip(192, 168, 1, 100), "eth0");
|
||||||
|
mgr.assign_ip(id0, NodeId(0)).unwrap();
|
||||||
|
let reassignments = mgr.reassign_on_failure(NodeId(0), &[NodeId(0), NodeId(1)]);
|
||||||
|
assert_eq!(reassignments.len(), 1);
|
||||||
|
assert_ne!(reassignments[0].1, NodeId(0));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_remove_ip() {
|
||||||
|
let mgr = IpManager::new();
|
||||||
|
let id = mgr.add_ip(ip(192, 168, 1, 100), "eth0");
|
||||||
|
mgr.assign_ip(id, NodeId(0)).unwrap();
|
||||||
|
mgr.remove_ip(id);
|
||||||
|
assert_eq!(mgr.ip_count(), 0);
|
||||||
|
assert_eq!(mgr.assigned_count(), 0);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_balance() {
|
||||||
|
let mgr = IpManager::new();
|
||||||
|
for i in 100..104 {
|
||||||
|
mgr.add_ip(ip(192, 168, 1, i), "eth0");
|
||||||
|
}
|
||||||
|
let reassignments = mgr.balance(&[NodeId(0), NodeId(1)]);
|
||||||
|
assert!(!reassignments.is_empty());
|
||||||
|
|
||||||
|
let node0 = mgr.owned_by(NodeId(0));
|
||||||
|
let node1 = mgr.owned_by(NodeId(1));
|
||||||
|
assert_eq!(node0.len() + node1.len(), 4);
|
||||||
|
assert!((node0.len() as i32 - node1.len() as i32).abs() <= 1);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_balance_excludes_inactive_nodes() {
|
||||||
|
let mgr = IpManager::new();
|
||||||
|
for i in 100..103 {
|
||||||
|
mgr.add_ip(ip(192, 168, 1, i), "eth0");
|
||||||
|
}
|
||||||
|
mgr.assign_ip(PublicIpId(0), NodeId(5)).unwrap();
|
||||||
|
let reassignments = mgr.balance(&[NodeId(0), NodeId(1)]);
|
||||||
|
assert!(reassignments
|
||||||
|
.iter()
|
||||||
|
.any(|(_, n)| *n == NodeId(0) || *n == NodeId(1)));
|
||||||
|
assert!(!reassignments.iter().any(|(_, n)| *n == NodeId(5)));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_previous_owner() {
|
||||||
|
let mgr = IpManager::new();
|
||||||
|
let id = mgr.add_ip(ip(192, 168, 1, 100), "eth0");
|
||||||
|
mgr.assign_ip(id, NodeId(0)).unwrap();
|
||||||
|
mgr.assign_ip(id, NodeId(1)).unwrap();
|
||||||
|
let pool = mgr.all_ips();
|
||||||
|
let entry = pool.iter().find(|e| e.id == id).unwrap();
|
||||||
|
assert_eq!(entry.previous_owner, Some(NodeId(0)));
|
||||||
|
assert_eq!(entry.owner, Some(NodeId(1)));
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,5 @@
|
|||||||
|
pub mod tdb;
|
||||||
|
pub mod node;
|
||||||
|
pub mod protocol;
|
||||||
|
pub mod ip_manager;
|
||||||
|
pub mod recovery;
|
||||||
@@ -0,0 +1,353 @@
|
|||||||
|
use std::collections::HashMap;
|
||||||
|
use std::net::SocketAddr;
|
||||||
|
use std::sync::RwLock;
|
||||||
|
use std::time::{Duration, Instant};
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
|
||||||
|
pub struct NodeId(pub u32);
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||||
|
pub enum NodeState {
|
||||||
|
Up,
|
||||||
|
Down,
|
||||||
|
Unhealthy,
|
||||||
|
Banned,
|
||||||
|
Disabled,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl NodeState {
|
||||||
|
pub fn is_active(&self) -> bool {
|
||||||
|
matches!(self, NodeState::Up)
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn as_str(&self) -> &'static str {
|
||||||
|
match self {
|
||||||
|
NodeState::Up => "UP",
|
||||||
|
NodeState::Down => "DOWN",
|
||||||
|
NodeState::Unhealthy => "UNHEALTHY",
|
||||||
|
NodeState::Banned => "BANNED",
|
||||||
|
NodeState::Disabled => "DISABLED",
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Clone)]
|
||||||
|
pub struct NodeInfo {
|
||||||
|
pub id: NodeId,
|
||||||
|
pub addr: SocketAddr,
|
||||||
|
pub state: NodeState,
|
||||||
|
pub last_heartbeat: Option<Instant>,
|
||||||
|
pub public_ips: Vec<String>,
|
||||||
|
pub generation: u64,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl NodeInfo {
|
||||||
|
pub fn new(id: NodeId, addr: SocketAddr) -> Self {
|
||||||
|
Self {
|
||||||
|
id,
|
||||||
|
addr,
|
||||||
|
state: NodeState::Down,
|
||||||
|
last_heartbeat: None,
|
||||||
|
public_ips: Vec::new(),
|
||||||
|
generation: 0,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn is_alive(&self, timeout: Duration) -> bool {
|
||||||
|
match self.last_heartbeat {
|
||||||
|
Some(t) => t.elapsed() < timeout,
|
||||||
|
None => false,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Clone)]
|
||||||
|
pub struct NodeMask {
|
||||||
|
nodes: Vec<bool>,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl NodeMask {
|
||||||
|
pub fn new(size: usize) -> Self {
|
||||||
|
Self {
|
||||||
|
nodes: vec![false; size],
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn set(&mut self, id: NodeId, active: bool) {
|
||||||
|
if (id.0 as usize) < self.nodes.len() {
|
||||||
|
self.nodes[id.0 as usize] = active;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn is_active(&self, id: NodeId) -> bool {
|
||||||
|
self.nodes.get(id.0 as usize).copied().unwrap_or(false)
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn active_nodes(&self) -> Vec<NodeId> {
|
||||||
|
self.nodes
|
||||||
|
.iter()
|
||||||
|
.enumerate()
|
||||||
|
.filter(|(_, &active)| active)
|
||||||
|
.map(|(i, _)| NodeId(i as u32))
|
||||||
|
.collect()
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn len(&self) -> usize {
|
||||||
|
self.nodes.len()
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn active_count(&self) -> usize {
|
||||||
|
self.nodes.iter().filter(|&&a| a).count()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub struct NodeManager {
|
||||||
|
nodes: RwLock<HashMap<NodeId, NodeInfo>>,
|
||||||
|
self_id: NodeId,
|
||||||
|
heartbeat_timeout: Duration,
|
||||||
|
heartbeat_interval: Duration,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl NodeManager {
|
||||||
|
pub fn new(self_id: NodeId, self_addr: SocketAddr) -> Self {
|
||||||
|
let mut nodes = HashMap::new();
|
||||||
|
nodes.insert(
|
||||||
|
self_id,
|
||||||
|
NodeInfo {
|
||||||
|
id: self_id,
|
||||||
|
addr: self_addr,
|
||||||
|
state: NodeState::Up,
|
||||||
|
last_heartbeat: Some(Instant::now()),
|
||||||
|
public_ips: Vec::new(),
|
||||||
|
generation: 0,
|
||||||
|
},
|
||||||
|
);
|
||||||
|
Self {
|
||||||
|
nodes: RwLock::new(nodes),
|
||||||
|
self_id,
|
||||||
|
heartbeat_timeout: Duration::from_secs(5),
|
||||||
|
heartbeat_interval: Duration::from_secs(1),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn add_node(&self, id: NodeId, addr: SocketAddr) {
|
||||||
|
let mut nodes = self.nodes.write().unwrap();
|
||||||
|
nodes.insert(id, NodeInfo::new(id, addr));
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn remove_node(&self, id: NodeId) {
|
||||||
|
let mut nodes = self.nodes.write().unwrap();
|
||||||
|
nodes.remove(&id);
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn record_heartbeat(&self, id: NodeId) {
|
||||||
|
let mut nodes = self.nodes.write().unwrap();
|
||||||
|
if let Some(node) = nodes.get_mut(&id) {
|
||||||
|
node.last_heartbeat = Some(Instant::now());
|
||||||
|
if node.state == NodeState::Down {
|
||||||
|
node.state = NodeState::Up;
|
||||||
|
node.generation += 1;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn set_node_state(&self, id: NodeId, state: NodeState) {
|
||||||
|
let mut nodes = self.nodes.write().unwrap();
|
||||||
|
if let Some(node) = nodes.get_mut(&id) {
|
||||||
|
node.state = state;
|
||||||
|
if state == NodeState::Up && node.last_heartbeat.is_none() {
|
||||||
|
node.last_heartbeat = Some(Instant::now());
|
||||||
|
}
|
||||||
|
node.generation += 1;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn get_node(&self, id: NodeId) -> Option<NodeInfo> {
|
||||||
|
self.nodes.read().unwrap().get(&id).cloned()
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn all_nodes(&self) -> Vec<NodeInfo> {
|
||||||
|
self.nodes.read().unwrap().values().cloned().collect()
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn active_nodes(&self) -> Vec<NodeInfo> {
|
||||||
|
self.nodes
|
||||||
|
.read()
|
||||||
|
.unwrap()
|
||||||
|
.values()
|
||||||
|
.filter(|n| n.state == NodeState::Up)
|
||||||
|
.cloned()
|
||||||
|
.collect()
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn check_health(&self) -> Vec<(NodeId, NodeState)> {
|
||||||
|
let timeout = self.heartbeat_timeout;
|
||||||
|
let mut nodes = self.nodes.write().unwrap();
|
||||||
|
let mut transitions = Vec::new();
|
||||||
|
|
||||||
|
for (id, node) in nodes.iter_mut() {
|
||||||
|
if *id == self.self_id {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
match &node.last_heartbeat {
|
||||||
|
Some(t) => {
|
||||||
|
if t.elapsed() > timeout && node.state == NodeState::Up {
|
||||||
|
node.state = NodeState::Down;
|
||||||
|
node.generation += 1;
|
||||||
|
transitions.push((*id, NodeState::Down));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
None => {
|
||||||
|
if node.state == NodeState::Up {
|
||||||
|
node.state = NodeState::Down;
|
||||||
|
node.generation += 1;
|
||||||
|
transitions.push((*id, NodeState::Down));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
transitions
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn self_id(&self) -> NodeId {
|
||||||
|
self.self_id
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn node_count(&self) -> usize {
|
||||||
|
self.nodes.read().unwrap().len()
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn build_nodemask(&self) -> NodeMask {
|
||||||
|
let nodes = self.nodes.read().unwrap();
|
||||||
|
let max_id = nodes.keys().map(|k| k.0).max().unwrap_or(0) as usize;
|
||||||
|
let mut mask = NodeMask::new(max_id + 1);
|
||||||
|
for (id, node) in nodes.iter() {
|
||||||
|
mask.set(*id, node.state.is_active());
|
||||||
|
}
|
||||||
|
mask
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use super::*;
|
||||||
|
use std::net::{Ipv4Addr, SocketAddrV4};
|
||||||
|
|
||||||
|
fn addr(port: u16) -> SocketAddr {
|
||||||
|
SocketAddr::V4(SocketAddrV4::new(Ipv4Addr::LOCALHOST, port))
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_node_creation() {
|
||||||
|
let mgr = NodeManager::new(NodeId(0), addr(4000));
|
||||||
|
let self_node = mgr.get_node(NodeId(0)).unwrap();
|
||||||
|
assert_eq!(self_node.state, NodeState::Up);
|
||||||
|
assert!(self_node.last_heartbeat.is_some());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_add_remove_node() {
|
||||||
|
let mgr = NodeManager::new(NodeId(0), addr(4000));
|
||||||
|
mgr.add_node(NodeId(1), addr(4001));
|
||||||
|
assert_eq!(mgr.node_count(), 2);
|
||||||
|
mgr.remove_node(NodeId(1));
|
||||||
|
assert_eq!(mgr.node_count(), 1);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_heartbeat_updates() {
|
||||||
|
let mgr = NodeManager::new(NodeId(0), addr(4000));
|
||||||
|
mgr.add_node(NodeId(1), addr(4001));
|
||||||
|
assert_eq!(mgr.get_node(NodeId(1)).unwrap().state, NodeState::Down);
|
||||||
|
mgr.record_heartbeat(NodeId(1));
|
||||||
|
assert_eq!(mgr.get_node(NodeId(1)).unwrap().state, NodeState::Up);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_health_check_timeout() {
|
||||||
|
let mgr = NodeManager::new(NodeId(0), addr(4000));
|
||||||
|
mgr.add_node(NodeId(1), addr(4001));
|
||||||
|
mgr.record_heartbeat(NodeId(1));
|
||||||
|
assert_eq!(mgr.get_node(NodeId(1)).unwrap().state, NodeState::Up);
|
||||||
|
std::thread::sleep(Duration::from_millis(100));
|
||||||
|
let mgr = NodeManager {
|
||||||
|
nodes: RwLock::new(mgr.nodes.read().unwrap().clone()),
|
||||||
|
self_id: NodeId(0),
|
||||||
|
heartbeat_timeout: Duration::from_millis(50),
|
||||||
|
heartbeat_interval: Duration::from_millis(10),
|
||||||
|
};
|
||||||
|
let transitions = mgr.check_health();
|
||||||
|
assert!(transitions.contains(&(NodeId(1), NodeState::Down)));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_nodemask() {
|
||||||
|
let mut mask = NodeMask::new(5);
|
||||||
|
mask.set(NodeId(0), true);
|
||||||
|
mask.set(NodeId(2), true);
|
||||||
|
mask.set(NodeId(4), true);
|
||||||
|
assert!(mask.is_active(NodeId(0)));
|
||||||
|
assert!(!mask.is_active(NodeId(1)));
|
||||||
|
assert_eq!(mask.active_count(), 3);
|
||||||
|
assert_eq!(mask.len(), 5);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_build_nodemask() {
|
||||||
|
let mgr = NodeManager::new(NodeId(0), addr(4000));
|
||||||
|
mgr.add_node(NodeId(1), addr(4001));
|
||||||
|
mgr.add_node(NodeId(2), addr(4002));
|
||||||
|
mgr.record_heartbeat(NodeId(1));
|
||||||
|
let mask = mgr.build_nodemask();
|
||||||
|
assert!(mask.is_active(NodeId(0)));
|
||||||
|
assert!(mask.is_active(NodeId(1)));
|
||||||
|
assert!(!mask.is_active(NodeId(2)));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_node_state_string() {
|
||||||
|
assert_eq!(NodeState::Up.as_str(), "UP");
|
||||||
|
assert_eq!(NodeState::Down.as_str(), "DOWN");
|
||||||
|
assert_eq!(NodeState::Unhealthy.as_str(), "UNHEALTHY");
|
||||||
|
assert_eq!(NodeState::Banned.as_str(), "BANNED");
|
||||||
|
assert_eq!(NodeState::Disabled.as_str(), "DISABLED");
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_set_node_state() {
|
||||||
|
let mgr = NodeManager::new(NodeId(0), addr(4000));
|
||||||
|
mgr.add_node(NodeId(1), addr(4001));
|
||||||
|
mgr.set_node_state(NodeId(1), NodeState::Banned);
|
||||||
|
assert_eq!(mgr.get_node(NodeId(1)).unwrap().state, NodeState::Banned);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_generation_increment() {
|
||||||
|
let mgr = NodeManager::new(NodeId(0), addr(4000));
|
||||||
|
mgr.add_node(NodeId(1), addr(4001));
|
||||||
|
let gen0 = mgr.get_node(NodeId(1)).unwrap().generation;
|
||||||
|
mgr.set_node_state(NodeId(1), NodeState::Down);
|
||||||
|
let gen1 = mgr.get_node(NodeId(1)).unwrap().generation;
|
||||||
|
assert!(gen1 > gen0);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_active_nodes() {
|
||||||
|
let mgr = NodeManager::new(NodeId(0), addr(4000));
|
||||||
|
mgr.add_node(NodeId(1), addr(4001));
|
||||||
|
mgr.add_node(NodeId(2), addr(4002));
|
||||||
|
mgr.record_heartbeat(NodeId(1));
|
||||||
|
let active = mgr.active_nodes();
|
||||||
|
assert_eq!(active.len(), 2);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_is_alive() {
|
||||||
|
let node = NodeInfo::new(NodeId(0), addr(4000));
|
||||||
|
assert!(!node.is_alive(Duration::from_secs(5)));
|
||||||
|
let mut node = node;
|
||||||
|
node.last_heartbeat = Some(Instant::now());
|
||||||
|
assert!(node.is_alive(Duration::from_secs(5)));
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,514 @@
|
|||||||
|
use std::io::{self, Read, Write};
|
||||||
|
use std::net::TcpStream;
|
||||||
|
|
||||||
|
pub const CTDB_MAGIC: u32 = 0x43544442;
|
||||||
|
pub const CTDB_VERSION: u32 = 1;
|
||||||
|
pub const CTDB_HEADER_SIZE: usize = 24;
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||||
|
#[repr(u32)]
|
||||||
|
pub enum CtdbCommand {
|
||||||
|
Connect = 1,
|
||||||
|
Disconnect = 2,
|
||||||
|
Ping = 3,
|
||||||
|
Pong = 4,
|
||||||
|
GetDb = 10,
|
||||||
|
Fetch = 11,
|
||||||
|
Store = 12,
|
||||||
|
Delete = 13,
|
||||||
|
Keys = 14,
|
||||||
|
SetNodeMask = 20,
|
||||||
|
GetNodeMask = 21,
|
||||||
|
NodeStatus = 22,
|
||||||
|
TakeIp = 30,
|
||||||
|
ReleaseIp = 31,
|
||||||
|
Monitor = 40,
|
||||||
|
Recovery = 50,
|
||||||
|
RecoveryDone = 51,
|
||||||
|
Unknown = 0xFFFF,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl CtdbCommand {
|
||||||
|
pub fn from_u32(v: u32) -> Self {
|
||||||
|
match v {
|
||||||
|
1 => CtdbCommand::Connect,
|
||||||
|
2 => CtdbCommand::Disconnect,
|
||||||
|
3 => CtdbCommand::Ping,
|
||||||
|
4 => CtdbCommand::Pong,
|
||||||
|
10 => CtdbCommand::GetDb,
|
||||||
|
11 => CtdbCommand::Fetch,
|
||||||
|
12 => CtdbCommand::Store,
|
||||||
|
13 => CtdbCommand::Delete,
|
||||||
|
14 => CtdbCommand::Keys,
|
||||||
|
20 => CtdbCommand::SetNodeMask,
|
||||||
|
21 => CtdbCommand::GetNodeMask,
|
||||||
|
22 => CtdbCommand::NodeStatus,
|
||||||
|
30 => CtdbCommand::TakeIp,
|
||||||
|
31 => CtdbCommand::ReleaseIp,
|
||||||
|
40 => CtdbCommand::Monitor,
|
||||||
|
50 => CtdbCommand::Recovery,
|
||||||
|
51 => CtdbCommand::RecoveryDone,
|
||||||
|
_ => CtdbCommand::Unknown,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||||
|
pub enum CtdbStatus {
|
||||||
|
Success = 0,
|
||||||
|
Error = 1,
|
||||||
|
NotFound = 2,
|
||||||
|
Exists = 3,
|
||||||
|
Corrupt = 4,
|
||||||
|
Timeout = 5,
|
||||||
|
NotActive = 6,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl CtdbStatus {
|
||||||
|
pub fn from_u32(v: u32) -> Self {
|
||||||
|
match v {
|
||||||
|
0 => CtdbStatus::Success,
|
||||||
|
2 => CtdbStatus::NotFound,
|
||||||
|
3 => CtdbStatus::Exists,
|
||||||
|
4 => CtdbStatus::Corrupt,
|
||||||
|
5 => CtdbStatus::Timeout,
|
||||||
|
6 => CtdbStatus::NotActive,
|
||||||
|
_ => CtdbStatus::Error,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Clone)]
|
||||||
|
pub struct CtdbHeader {
|
||||||
|
pub magic: u32,
|
||||||
|
pub version: u32,
|
||||||
|
pub command: u32,
|
||||||
|
pub status: u32,
|
||||||
|
pub length: u64,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl CtdbHeader {
|
||||||
|
pub fn new(command: CtdbCommand, status: CtdbStatus, length: u64) -> Self {
|
||||||
|
Self {
|
||||||
|
magic: CTDB_MAGIC,
|
||||||
|
version: CTDB_VERSION,
|
||||||
|
command: command as u32,
|
||||||
|
status: status as u32,
|
||||||
|
length,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn to_bytes(&self) -> Vec<u8> {
|
||||||
|
let mut buf = Vec::with_capacity(CTDB_HEADER_SIZE);
|
||||||
|
buf.extend_from_slice(&self.magic.to_le_bytes());
|
||||||
|
buf.extend_from_slice(&self.version.to_le_bytes());
|
||||||
|
buf.extend_from_slice(&self.command.to_le_bytes());
|
||||||
|
buf.extend_from_slice(&self.status.to_le_bytes());
|
||||||
|
buf.extend_from_slice(&self.length.to_le_bytes());
|
||||||
|
buf
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn from_bytes(buf: &[u8]) -> Result<Self, CtdbProtoError> {
|
||||||
|
if buf.len() < CTDB_HEADER_SIZE {
|
||||||
|
return Err(CtdbProtoError::HeaderTooShort);
|
||||||
|
}
|
||||||
|
let magic = u32::from_le_bytes(buf[0..4].try_into().unwrap());
|
||||||
|
let version = u32::from_le_bytes(buf[4..8].try_into().unwrap());
|
||||||
|
let command = u32::from_le_bytes(buf[8..12].try_into().unwrap());
|
||||||
|
let status = u32::from_le_bytes(buf[12..16].try_into().unwrap());
|
||||||
|
let length = u64::from_le_bytes(buf[16..24].try_into().unwrap());
|
||||||
|
Ok(Self {
|
||||||
|
magic,
|
||||||
|
version,
|
||||||
|
command,
|
||||||
|
status,
|
||||||
|
length,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn is_valid(&self) -> bool {
|
||||||
|
self.magic == CTDB_MAGIC && self.version == CTDB_VERSION
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Clone)]
|
||||||
|
pub struct CtdbMessage {
|
||||||
|
pub header: CtdbHeader,
|
||||||
|
pub payload: Vec<u8>,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl CtdbMessage {
|
||||||
|
pub fn new(command: CtdbCommand, status: CtdbStatus, payload: Vec<u8>) -> Self {
|
||||||
|
let length = payload.len() as u64;
|
||||||
|
Self {
|
||||||
|
header: CtdbHeader::new(command, status, length),
|
||||||
|
payload,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn to_bytes(&self) -> Vec<u8> {
|
||||||
|
let mut buf = self.header.to_bytes();
|
||||||
|
buf.extend_from_slice(&self.payload);
|
||||||
|
buf
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn from_bytes(buf: &[u8]) -> Result<Self, CtdbProtoError> {
|
||||||
|
let header = CtdbHeader::from_bytes(buf)?;
|
||||||
|
if !header.is_valid() {
|
||||||
|
return Err(CtdbProtoError::InvalidMagic);
|
||||||
|
}
|
||||||
|
let payload = if buf.len() > CTDB_HEADER_SIZE {
|
||||||
|
buf[CTDB_HEADER_SIZE..].to_vec()
|
||||||
|
} else {
|
||||||
|
Vec::new()
|
||||||
|
};
|
||||||
|
Ok(Self { header, payload })
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn command(&self) -> CtdbCommand {
|
||||||
|
CtdbCommand::from_u32(self.header.command)
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn status(&self) -> CtdbStatus {
|
||||||
|
CtdbStatus::from_u32(self.header.status)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||||
|
pub enum CtdbProtoError {
|
||||||
|
HeaderTooShort,
|
||||||
|
InvalidMagic,
|
||||||
|
IoError,
|
||||||
|
InvalidPayload,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl std::fmt::Display for CtdbProtoError {
|
||||||
|
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||||
|
match self {
|
||||||
|
CtdbProtoError::HeaderTooShort => write!(f, "header too short"),
|
||||||
|
CtdbProtoError::InvalidMagic => write!(f, "invalid magic number"),
|
||||||
|
CtdbProtoError::IoError => write!(f, "I/O error"),
|
||||||
|
CtdbProtoError::InvalidPayload => write!(f, "invalid payload"),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl std::error::Error for CtdbProtoError {}
|
||||||
|
impl From<io::Error> for CtdbProtoError {
|
||||||
|
fn from(_: io::Error) -> Self {
|
||||||
|
CtdbProtoError::IoError
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub mod payload {
|
||||||
|
use super::CtdbProtoError;
|
||||||
|
|
||||||
|
pub fn encode_kv(key: &[u8], value: &[u8]) -> Vec<u8> {
|
||||||
|
let mut buf = Vec::new();
|
||||||
|
buf.extend_from_slice(&(key.len() as u32).to_le_bytes());
|
||||||
|
buf.extend_from_slice(key);
|
||||||
|
buf.extend_from_slice(&(value.len() as u32).to_le_bytes());
|
||||||
|
buf.extend_from_slice(value);
|
||||||
|
buf
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn decode_kv(payload: &[u8]) -> Result<(Vec<u8>, Vec<u8>), CtdbProtoError> {
|
||||||
|
if payload.len() < 4 {
|
||||||
|
return Err(CtdbProtoError::InvalidPayload);
|
||||||
|
}
|
||||||
|
let key_len = u32::from_le_bytes(payload[0..4].try_into().unwrap()) as usize;
|
||||||
|
if payload.len() < 4 + key_len + 4 {
|
||||||
|
return Err(CtdbProtoError::InvalidPayload);
|
||||||
|
}
|
||||||
|
let key = payload[4..4 + key_len].to_vec();
|
||||||
|
let val_len_offset = 4 + key_len;
|
||||||
|
let val_len = u32::from_le_bytes(
|
||||||
|
payload[val_len_offset..val_len_offset + 4].try_into().unwrap(),
|
||||||
|
) as usize;
|
||||||
|
let val_start = val_len_offset + 4;
|
||||||
|
if payload.len() < val_start + val_len {
|
||||||
|
return Err(CtdbProtoError::InvalidPayload);
|
||||||
|
}
|
||||||
|
let value = payload[val_start..val_start + val_len].to_vec();
|
||||||
|
Ok((key, value))
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn encode_key(key: &[u8]) -> Vec<u8> {
|
||||||
|
let mut buf = Vec::new();
|
||||||
|
buf.extend_from_slice(&(key.len() as u32).to_le_bytes());
|
||||||
|
buf.extend_from_slice(key);
|
||||||
|
buf
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn decode_key(payload: &[u8]) -> Result<Vec<u8>, CtdbProtoError> {
|
||||||
|
if payload.len() < 4 {
|
||||||
|
return Err(CtdbProtoError::InvalidPayload);
|
||||||
|
}
|
||||||
|
let key_len = u32::from_le_bytes(payload[0..4].try_into().unwrap()) as usize;
|
||||||
|
if payload.len() < 4 + key_len {
|
||||||
|
return Err(CtdbProtoError::InvalidPayload);
|
||||||
|
}
|
||||||
|
Ok(payload[4..4 + key_len].to_vec())
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn encode_node_id(id: u32) -> Vec<u8> {
|
||||||
|
id.to_le_bytes().to_vec()
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn decode_node_id(payload: &[u8]) -> Result<u32, CtdbProtoError> {
|
||||||
|
if payload.len() < 4 {
|
||||||
|
return Err(CtdbProtoError::InvalidPayload);
|
||||||
|
}
|
||||||
|
Ok(u32::from_le_bytes(payload[0..4].try_into().unwrap()))
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn encode_nodemask(active: &[u32]) -> Vec<u8> {
|
||||||
|
let mut buf = Vec::new();
|
||||||
|
buf.extend_from_slice(&(active.len() as u32).to_le_bytes());
|
||||||
|
for &id in active {
|
||||||
|
buf.extend_from_slice(&id.to_le_bytes());
|
||||||
|
}
|
||||||
|
buf
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn decode_nodemask(payload: &[u8]) -> Result<Vec<u32>, CtdbProtoError> {
|
||||||
|
if payload.len() < 4 {
|
||||||
|
return Err(CtdbProtoError::InvalidPayload);
|
||||||
|
}
|
||||||
|
let count = u32::from_le_bytes(payload[0..4].try_into().unwrap()) as usize;
|
||||||
|
let mut ids = Vec::with_capacity(count);
|
||||||
|
for i in 0..count {
|
||||||
|
let offset = 4 + i * 4;
|
||||||
|
if payload.len() < offset + 4 {
|
||||||
|
return Err(CtdbProtoError::InvalidPayload);
|
||||||
|
}
|
||||||
|
ids.push(u32::from_le_bytes(payload[offset..offset + 4].try_into().unwrap()));
|
||||||
|
}
|
||||||
|
Ok(ids)
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn encode_ip(ip: &str, interface: &str) -> Vec<u8> {
|
||||||
|
let ip_bytes = ip.as_bytes();
|
||||||
|
let if_bytes = interface.as_bytes();
|
||||||
|
let mut buf = Vec::new();
|
||||||
|
buf.extend_from_slice(&(ip_bytes.len() as u32).to_le_bytes());
|
||||||
|
buf.extend_from_slice(ip_bytes);
|
||||||
|
buf.extend_from_slice(&(if_bytes.len() as u32).to_le_bytes());
|
||||||
|
buf.extend_from_slice(if_bytes);
|
||||||
|
buf
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn decode_ip(payload: &[u8]) -> Result<(String, String), CtdbProtoError> {
|
||||||
|
if payload.len() < 4 {
|
||||||
|
return Err(CtdbProtoError::InvalidPayload);
|
||||||
|
}
|
||||||
|
let ip_len = u32::from_le_bytes(payload[0..4].try_into().unwrap()) as usize;
|
||||||
|
if payload.len() < 4 + ip_len + 4 {
|
||||||
|
return Err(CtdbProtoError::InvalidPayload);
|
||||||
|
}
|
||||||
|
let ip = String::from_utf8_lossy(&payload[4..4 + ip_len]).to_string();
|
||||||
|
let if_offset = 4 + ip_len;
|
||||||
|
let if_len = u32::from_le_bytes(
|
||||||
|
payload[if_offset..if_offset + 4].try_into().unwrap(),
|
||||||
|
) as usize;
|
||||||
|
let if_start = if_offset + 4;
|
||||||
|
if payload.len() < if_start + if_len {
|
||||||
|
return Err(CtdbProtoError::InvalidPayload);
|
||||||
|
}
|
||||||
|
let interface = String::from_utf8_lossy(&payload[if_start..if_start + if_len]).to_string();
|
||||||
|
Ok((ip, interface))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub struct CtdbConnection {
|
||||||
|
stream: TcpStream,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl CtdbConnection {
|
||||||
|
pub fn new(stream: TcpStream) -> Self {
|
||||||
|
Self { stream }
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn connect(addr: &str) -> Result<Self, CtdbProtoError> {
|
||||||
|
let stream = TcpStream::connect(addr)?;
|
||||||
|
Ok(Self { stream })
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn send_message(&mut self, msg: &CtdbMessage) -> Result<(), CtdbProtoError> {
|
||||||
|
let bytes = msg.to_bytes();
|
||||||
|
self.stream.write_all(&bytes)?;
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn recv_message(&mut self) -> Result<CtdbMessage, CtdbProtoError> {
|
||||||
|
let mut header_buf = [0u8; CTDB_HEADER_SIZE];
|
||||||
|
self.stream.read_exact(&mut header_buf)?;
|
||||||
|
let header = CtdbHeader::from_bytes(&header_buf)?;
|
||||||
|
if !header.is_valid() {
|
||||||
|
return Err(CtdbProtoError::InvalidMagic);
|
||||||
|
}
|
||||||
|
let payload_len = header.length as usize;
|
||||||
|
let mut payload = vec![0u8; payload_len];
|
||||||
|
if payload_len > 0 {
|
||||||
|
self.stream.read_exact(&mut payload)?;
|
||||||
|
}
|
||||||
|
Ok(CtdbMessage { header, payload })
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn ping(&mut self) -> Result<(), CtdbProtoError> {
|
||||||
|
let msg = CtdbMessage::new(CtdbCommand::Ping, CtdbStatus::Success, vec![]);
|
||||||
|
self.send_message(&msg)?;
|
||||||
|
let resp = self.recv_message()?;
|
||||||
|
if resp.command() == CtdbCommand::Pong && resp.status() == CtdbStatus::Success {
|
||||||
|
Ok(())
|
||||||
|
} else {
|
||||||
|
Err(CtdbProtoError::InvalidPayload)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn store(&mut self, key: &[u8], value: &[u8]) -> Result<bool, CtdbProtoError> {
|
||||||
|
let payload = payload::encode_kv(key, value);
|
||||||
|
let msg = CtdbMessage::new(CtdbCommand::Store, CtdbStatus::Success, payload);
|
||||||
|
self.send_message(&msg)?;
|
||||||
|
let resp = self.recv_message()?;
|
||||||
|
Ok(resp.status() == CtdbStatus::Success)
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn fetch(&mut self, key: &[u8]) -> Result<Vec<u8>, CtdbProtoError> {
|
||||||
|
let payload = payload::encode_key(key);
|
||||||
|
let msg = CtdbMessage::new(CtdbCommand::Fetch, CtdbStatus::Success, payload);
|
||||||
|
self.send_message(&msg)?;
|
||||||
|
let resp = self.recv_message()?;
|
||||||
|
if resp.status() == CtdbStatus::Success {
|
||||||
|
Ok(resp.payload)
|
||||||
|
} else {
|
||||||
|
Err(CtdbProtoError::InvalidPayload)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn delete(&mut self, key: &[u8]) -> Result<bool, CtdbProtoError> {
|
||||||
|
let payload = payload::encode_key(key);
|
||||||
|
let msg = CtdbMessage::new(CtdbCommand::Delete, CtdbStatus::Success, payload);
|
||||||
|
self.send_message(&msg)?;
|
||||||
|
let resp = self.recv_message()?;
|
||||||
|
Ok(resp.status() == CtdbStatus::Success)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use super::*;
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_header_roundtrip() {
|
||||||
|
let header = CtdbHeader::new(CtdbCommand::Ping, CtdbStatus::Success, 42);
|
||||||
|
let bytes = header.to_bytes();
|
||||||
|
let restored = CtdbHeader::from_bytes(&bytes).unwrap();
|
||||||
|
assert_eq!(restored.magic, CTDB_MAGIC);
|
||||||
|
assert_eq!(restored.version, CTDB_VERSION);
|
||||||
|
assert_eq!(restored.command, CtdbCommand::Ping as u32);
|
||||||
|
assert_eq!(restored.status, CtdbStatus::Success as u32);
|
||||||
|
assert_eq!(restored.length, 42);
|
||||||
|
assert!(restored.is_valid());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_message_roundtrip() {
|
||||||
|
let msg = CtdbMessage::new(
|
||||||
|
CtdbCommand::Store,
|
||||||
|
CtdbStatus::Success,
|
||||||
|
b"test_payload".to_vec(),
|
||||||
|
);
|
||||||
|
let bytes = msg.to_bytes();
|
||||||
|
let restored = CtdbMessage::from_bytes(&bytes).unwrap();
|
||||||
|
assert_eq!(restored.command(), CtdbCommand::Store);
|
||||||
|
assert_eq!(restored.payload, b"test_payload");
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_command_from_u32() {
|
||||||
|
assert_eq!(CtdbCommand::from_u32(1), CtdbCommand::Connect);
|
||||||
|
assert_eq!(CtdbCommand::from_u32(3), CtdbCommand::Ping);
|
||||||
|
assert_eq!(CtdbCommand::from_u32(0xFFFF), CtdbCommand::Unknown);
|
||||||
|
assert_eq!(CtdbCommand::from_u32(999), CtdbCommand::Unknown);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_status_from_u32() {
|
||||||
|
assert_eq!(CtdbStatus::from_u32(0), CtdbStatus::Success);
|
||||||
|
assert_eq!(CtdbStatus::from_u32(2), CtdbStatus::NotFound);
|
||||||
|
assert_eq!(CtdbStatus::from_u32(99), CtdbStatus::Error);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_payload_encode_decode_kv() {
|
||||||
|
let (key, val) = (b"mykey", b"myvalue");
|
||||||
|
let encoded = payload::encode_kv(key, val);
|
||||||
|
let (k, v) = payload::decode_kv(&encoded).unwrap();
|
||||||
|
assert_eq!(k, key);
|
||||||
|
assert_eq!(v, val);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_payload_encode_decode_key() {
|
||||||
|
let key = b"test_key";
|
||||||
|
let encoded = payload::encode_key(key);
|
||||||
|
let decoded = payload::decode_key(&encoded).unwrap();
|
||||||
|
assert_eq!(decoded, key);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_payload_node_id() {
|
||||||
|
let encoded = payload::encode_node_id(42);
|
||||||
|
let decoded = payload::decode_node_id(&encoded).unwrap();
|
||||||
|
assert_eq!(decoded, 42);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_payload_nodemask() {
|
||||||
|
let ids = vec![0u32, 1, 2, 3];
|
||||||
|
let encoded = payload::encode_nodemask(&ids);
|
||||||
|
let decoded = payload::decode_nodemask(&encoded).unwrap();
|
||||||
|
assert_eq!(decoded, ids);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_payload_ip() {
|
||||||
|
let encoded = payload::encode_ip("192.168.1.100", "eth0");
|
||||||
|
let (ip, iface) = payload::decode_ip(&encoded).unwrap();
|
||||||
|
assert_eq!(ip, "192.168.1.100");
|
||||||
|
assert_eq!(iface, "eth0");
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_invalid_magic() {
|
||||||
|
let mut bad_header = CtdbHeader::new(CtdbCommand::Ping, CtdbStatus::Success, 0);
|
||||||
|
bad_header.magic = 0xDEADBEEF;
|
||||||
|
assert!(!bad_header.is_valid());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_empty_message() {
|
||||||
|
let msg = CtdbMessage::new(CtdbCommand::Connect, CtdbStatus::Success, vec![]);
|
||||||
|
let bytes = msg.to_bytes();
|
||||||
|
let restored = CtdbMessage::from_bytes(&bytes).unwrap();
|
||||||
|
assert!(restored.payload.is_empty());
|
||||||
|
assert_eq!(restored.header.length, 0);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_header_too_short() {
|
||||||
|
let result = CtdbHeader::from_bytes(&[0u8; 10]);
|
||||||
|
assert!(result.is_err());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_large_payload() {
|
||||||
|
let large = vec![0xABu8; 65000];
|
||||||
|
let msg = CtdbMessage::new(CtdbCommand::Fetch, CtdbStatus::Success, large.clone());
|
||||||
|
let bytes = msg.to_bytes();
|
||||||
|
let restored = CtdbMessage::from_bytes(&bytes).unwrap();
|
||||||
|
assert_eq!(restored.payload.len(), 65000);
|
||||||
|
assert_eq!(restored.payload, large);
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,417 @@
|
|||||||
|
use std::sync::RwLock;
|
||||||
|
use std::time::{Duration, Instant};
|
||||||
|
|
||||||
|
use super::ip_manager::IpManager;
|
||||||
|
use super::node::{NodeManager, NodeId, NodeState};
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||||
|
pub enum RecoveryState {
|
||||||
|
Idle,
|
||||||
|
Initiated,
|
||||||
|
InProgress,
|
||||||
|
Verifying,
|
||||||
|
Completed,
|
||||||
|
Failed,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl RecoveryState {
|
||||||
|
pub fn as_str(&self) -> &'static str {
|
||||||
|
match self {
|
||||||
|
RecoveryState::Idle => "IDLE",
|
||||||
|
RecoveryState::Initiated => "INITIATED",
|
||||||
|
RecoveryState::InProgress => "IN_PROGRESS",
|
||||||
|
RecoveryState::Verifying => "VERIFYING",
|
||||||
|
RecoveryState::Completed => "COMPLETED",
|
||||||
|
RecoveryState::Failed => "FAILED",
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Clone)]
|
||||||
|
pub struct RecoveryEvent {
|
||||||
|
pub timestamp: Instant,
|
||||||
|
pub event_type: RecoveryEventType,
|
||||||
|
pub node_id: Option<NodeId>,
|
||||||
|
pub message: String,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||||
|
pub enum RecoveryEventType {
|
||||||
|
NodeDown,
|
||||||
|
NodeRejoin,
|
||||||
|
IpMigration,
|
||||||
|
RecoveryStart,
|
||||||
|
RecoveryComplete,
|
||||||
|
RecoveryFailed,
|
||||||
|
}
|
||||||
|
|
||||||
|
pub struct RecoveryManager {
|
||||||
|
state: RwLock<RecoveryState>,
|
||||||
|
events: RwLock<Vec<RecoveryEvent>>,
|
||||||
|
recovery_cooldown: Duration,
|
||||||
|
last_recovery: RwLock<Option<Instant>>,
|
||||||
|
max_events: usize,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl RecoveryManager {
|
||||||
|
pub fn new() -> Self {
|
||||||
|
Self {
|
||||||
|
state: RwLock::new(RecoveryState::Idle),
|
||||||
|
events: RwLock::new(Vec::new()),
|
||||||
|
recovery_cooldown: Duration::from_secs(30),
|
||||||
|
last_recovery: RwLock::new(None),
|
||||||
|
max_events: 100,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn with_cooldown(mut self, cooldown: Duration) -> Self {
|
||||||
|
self.recovery_cooldown = cooldown;
|
||||||
|
self
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn state(&self) -> RecoveryState {
|
||||||
|
*self.state.read().unwrap()
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn set_state(&self, state: RecoveryState) {
|
||||||
|
*self.state.write().unwrap() = state;
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn log_event(&self, event_type: RecoveryEventType, node_id: Option<NodeId>, msg: &str) {
|
||||||
|
let event = RecoveryEvent {
|
||||||
|
timestamp: Instant::now(),
|
||||||
|
event_type,
|
||||||
|
node_id,
|
||||||
|
message: msg.to_string(),
|
||||||
|
};
|
||||||
|
let mut events = self.events.write().unwrap();
|
||||||
|
events.push(event);
|
||||||
|
if events.len() > self.max_events {
|
||||||
|
events.remove(0);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn events(&self) -> Vec<RecoveryEvent> {
|
||||||
|
self.events.read().unwrap().clone()
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn can_recover(&self) -> bool {
|
||||||
|
match *self.last_recovery.read().unwrap() {
|
||||||
|
Some(t) => t.elapsed() > self.recovery_cooldown,
|
||||||
|
None => true,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn handle_node_failure(
|
||||||
|
&self,
|
||||||
|
failed_node: NodeId,
|
||||||
|
node_mgr: &NodeManager,
|
||||||
|
ip_mgr: &IpManager,
|
||||||
|
) -> Vec<(super::ip_manager::PublicIpId, NodeId)> {
|
||||||
|
self.log_event(RecoveryEventType::NodeDown, Some(failed_node), &format!(
|
||||||
|
"Node {:?} marked DOWN", failed_node
|
||||||
|
));
|
||||||
|
|
||||||
|
if !self.can_recover() {
|
||||||
|
self.log_event(
|
||||||
|
RecoveryEventType::RecoveryFailed,
|
||||||
|
Some(failed_node),
|
||||||
|
"Recovery skipped: cooldown active",
|
||||||
|
);
|
||||||
|
return Vec::new();
|
||||||
|
}
|
||||||
|
|
||||||
|
self.set_state(RecoveryState::Initiated);
|
||||||
|
self.log_event(RecoveryEventType::RecoveryStart, Some(failed_node), "Starting recovery");
|
||||||
|
|
||||||
|
*self.last_recovery.write().unwrap() = Some(Instant::now());
|
||||||
|
self.set_state(RecoveryState::InProgress);
|
||||||
|
|
||||||
|
let active_nodes: Vec<NodeId> = node_mgr
|
||||||
|
.active_nodes()
|
||||||
|
.iter()
|
||||||
|
.filter(|n| n.id != failed_node)
|
||||||
|
.map(|n| n.id)
|
||||||
|
.collect();
|
||||||
|
|
||||||
|
if active_nodes.is_empty() {
|
||||||
|
self.log_event(
|
||||||
|
RecoveryEventType::RecoveryFailed,
|
||||||
|
Some(failed_node),
|
||||||
|
"No active nodes available for IP migration",
|
||||||
|
);
|
||||||
|
self.set_state(RecoveryState::Failed);
|
||||||
|
return Vec::new();
|
||||||
|
}
|
||||||
|
|
||||||
|
let reassignments = ip_mgr.reassign_on_failure(failed_node, &active_nodes);
|
||||||
|
|
||||||
|
for (ip_id, new_owner) in &reassignments {
|
||||||
|
self.log_event(
|
||||||
|
RecoveryEventType::IpMigration,
|
||||||
|
Some(*new_owner),
|
||||||
|
&format!("IP {:?} migrated to node {:?}", ip_id, new_owner),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
self.set_state(RecoveryState::Completed);
|
||||||
|
self.log_event(
|
||||||
|
RecoveryEventType::RecoveryComplete,
|
||||||
|
Some(failed_node),
|
||||||
|
&format!("Recovery complete: {} IPs migrated", reassignments.len()),
|
||||||
|
);
|
||||||
|
|
||||||
|
reassignments
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn handle_node_rejoin(
|
||||||
|
&self,
|
||||||
|
rejoining_node: NodeId,
|
||||||
|
node_mgr: &NodeManager,
|
||||||
|
ip_mgr: &IpManager,
|
||||||
|
) -> Vec<(super::ip_manager::PublicIpId, NodeId)> {
|
||||||
|
self.log_event(RecoveryEventType::NodeRejoin, Some(rejoining_node), &format!(
|
||||||
|
"Node {:?} rejoining cluster", rejoining_node
|
||||||
|
));
|
||||||
|
|
||||||
|
let active_nodes: Vec<NodeId> = node_mgr.active_nodes().iter().map(|n| n.id).collect();
|
||||||
|
|
||||||
|
self.set_state(RecoveryState::InProgress);
|
||||||
|
let rebalance = ip_mgr.balance(&active_nodes);
|
||||||
|
|
||||||
|
for (ip_id, new_owner) in &rebalance {
|
||||||
|
self.log_event(
|
||||||
|
RecoveryEventType::IpMigration,
|
||||||
|
Some(*new_owner),
|
||||||
|
&format!("IP {:?} rebalanced to node {:?}", ip_id, new_owner),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
self.set_state(RecoveryState::Completed);
|
||||||
|
self.log_event(
|
||||||
|
RecoveryEventType::RecoveryComplete,
|
||||||
|
Some(rejoining_node),
|
||||||
|
&format!("Rebalance complete: {} IPs reassigned", rebalance.len()),
|
||||||
|
);
|
||||||
|
|
||||||
|
rebalance
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn check_and_recover(
|
||||||
|
&self,
|
||||||
|
node_mgr: &NodeManager,
|
||||||
|
ip_mgr: &IpManager,
|
||||||
|
) -> Vec<(super::ip_manager::PublicIpId, NodeId)> {
|
||||||
|
let transitions = node_mgr.check_health();
|
||||||
|
|
||||||
|
if transitions.is_empty() {
|
||||||
|
return Vec::new();
|
||||||
|
}
|
||||||
|
|
||||||
|
let mut all_reassignments = Vec::new();
|
||||||
|
|
||||||
|
for (node_id, new_state) in &transitions {
|
||||||
|
match new_state {
|
||||||
|
NodeState::Down => {
|
||||||
|
let reassignments =
|
||||||
|
self.handle_node_failure(*node_id, node_mgr, ip_mgr);
|
||||||
|
all_reassignments.extend(reassignments);
|
||||||
|
}
|
||||||
|
NodeState::Up => {
|
||||||
|
let reassignments =
|
||||||
|
self.handle_node_rejoin(*node_id, node_mgr, ip_mgr);
|
||||||
|
all_reassignments.extend(reassignments);
|
||||||
|
}
|
||||||
|
_ => {}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
all_reassignments
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn reset(&self) {
|
||||||
|
*self.state.write().unwrap() = RecoveryState::Idle;
|
||||||
|
*self.last_recovery.write().unwrap() = None;
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn events_by_node(&self, node_id: NodeId) -> Vec<RecoveryEvent> {
|
||||||
|
self.events
|
||||||
|
.read()
|
||||||
|
.unwrap()
|
||||||
|
.iter()
|
||||||
|
.filter(|e| e.node_id == Some(node_id))
|
||||||
|
.cloned()
|
||||||
|
.collect()
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn last_event(&self) -> Option<RecoveryEvent> {
|
||||||
|
self.events.read().unwrap().last().cloned()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Default for RecoveryManager {
|
||||||
|
fn default() -> Self {
|
||||||
|
Self::new()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use super::*;
|
||||||
|
use std::net::{Ipv4Addr, SocketAddr, SocketAddrV4};
|
||||||
|
use std::thread;
|
||||||
|
|
||||||
|
fn addr(port: u16) -> SocketAddr {
|
||||||
|
SocketAddr::V4(SocketAddrV4::new(Ipv4Addr::LOCALHOST, port))
|
||||||
|
}
|
||||||
|
|
||||||
|
fn ip(a: u8, b: u8, c: u8, d: u8) -> std::net::IpAddr {
|
||||||
|
std::net::IpAddr::V4(Ipv4Addr::new(a, b, c, d))
|
||||||
|
}
|
||||||
|
|
||||||
|
fn setup() -> (NodeManager, IpManager, RecoveryManager) {
|
||||||
|
let mgr = NodeManager::new(NodeId(0), addr(4000));
|
||||||
|
mgr.add_node(NodeId(1), addr(4001));
|
||||||
|
mgr.add_node(NodeId(2), addr(4002));
|
||||||
|
mgr.record_heartbeat(NodeId(1));
|
||||||
|
mgr.record_heartbeat(NodeId(2));
|
||||||
|
|
||||||
|
let ip_mgr = IpManager::new();
|
||||||
|
ip_mgr.add_ip(ip(192, 168, 1, 100), "eth0");
|
||||||
|
ip_mgr.add_ip(ip(192, 168, 1, 101), "eth0");
|
||||||
|
ip_mgr.add_ip(ip(192, 168, 1, 102), "eth1");
|
||||||
|
ip_mgr.assign_ip(super::super::ip_manager::PublicIpId(0), NodeId(0)).unwrap();
|
||||||
|
ip_mgr.assign_ip(super::super::ip_manager::PublicIpId(1), NodeId(1)).unwrap();
|
||||||
|
ip_mgr.assign_ip(super::super::ip_manager::PublicIpId(2), NodeId(2)).unwrap();
|
||||||
|
|
||||||
|
let recovery = RecoveryManager::new();
|
||||||
|
(mgr, ip_mgr, recovery)
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_initial_state() {
|
||||||
|
let recovery = RecoveryManager::new();
|
||||||
|
assert_eq!(recovery.state(), RecoveryState::Idle);
|
||||||
|
assert!(recovery.can_recover());
|
||||||
|
assert!(recovery.events().is_empty());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_node_failure_triggers_recovery() {
|
||||||
|
let (mgr, ip_mgr, recovery) = setup();
|
||||||
|
|
||||||
|
let reassignments = recovery.handle_node_failure(NodeId(1), &mgr, &ip_mgr);
|
||||||
|
|
||||||
|
assert!(!reassignments.is_empty());
|
||||||
|
assert_eq!(recovery.state(), RecoveryState::Completed);
|
||||||
|
|
||||||
|
let events = recovery.events();
|
||||||
|
assert!(events.iter().any(|e| e.event_type == RecoveryEventType::NodeDown));
|
||||||
|
assert!(events.iter().any(|e| e.event_type == RecoveryEventType::RecoveryStart));
|
||||||
|
assert!(events.iter().any(|e| e.event_type == RecoveryEventType::RecoveryComplete));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_no_active_nodes_fails() {
|
||||||
|
let mgr = NodeManager::new(NodeId(0), addr(4000));
|
||||||
|
let ip_mgr = IpManager::new();
|
||||||
|
ip_mgr.add_ip(ip(192, 168, 1, 100), "eth0");
|
||||||
|
let recovery = RecoveryManager::new();
|
||||||
|
|
||||||
|
let reassignments = recovery.handle_node_failure(NodeId(0), &mgr, &ip_mgr);
|
||||||
|
assert!(reassignments.is_empty());
|
||||||
|
assert_eq!(recovery.state(), RecoveryState::Failed);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_cooldown_prevents_rapid_recovery() {
|
||||||
|
let (mgr, ip_mgr, recovery) = setup();
|
||||||
|
recovery.handle_node_failure(NodeId(1), &mgr, &ip_mgr);
|
||||||
|
assert!(!recovery.can_recover());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_cooldown_expires() {
|
||||||
|
let recovery = RecoveryManager::new().with_cooldown(Duration::from_millis(10));
|
||||||
|
*recovery.last_recovery.write().unwrap() = Some(Instant::now());
|
||||||
|
assert!(!recovery.can_recover());
|
||||||
|
thread::sleep(Duration::from_millis(20));
|
||||||
|
assert!(recovery.can_recover());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_node_rejoin_balances() {
|
||||||
|
let (mgr, ip_mgr, recovery) = setup();
|
||||||
|
|
||||||
|
recovery.handle_node_failure(NodeId(1), &mgr, &ip_mgr);
|
||||||
|
thread::sleep(Duration::from_millis(10));
|
||||||
|
|
||||||
|
let recovery2 = RecoveryManager::new();
|
||||||
|
let reassignments = recovery2.handle_node_rejoin(NodeId(1), &mgr, &ip_mgr);
|
||||||
|
assert!(!reassignments.is_empty());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_log_event() {
|
||||||
|
let recovery = RecoveryManager::new();
|
||||||
|
recovery.log_event(RecoveryEventType::NodeDown, Some(NodeId(5)), "test message");
|
||||||
|
assert_eq!(recovery.events().len(), 1);
|
||||||
|
let event = &recovery.events()[0];
|
||||||
|
assert_eq!(event.node_id, Some(NodeId(5)));
|
||||||
|
assert_eq!(event.message, "test message");
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_event_max_limit() {
|
||||||
|
let recovery = RecoveryManager::new();
|
||||||
|
for i in 0..200 {
|
||||||
|
recovery.log_event(RecoveryEventType::NodeDown, Some(NodeId(i)), "msg");
|
||||||
|
}
|
||||||
|
assert_eq!(recovery.events().len(), 100);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_events_by_node() {
|
||||||
|
let recovery = RecoveryManager::new();
|
||||||
|
recovery.log_event(RecoveryEventType::NodeDown, Some(NodeId(1)), "a");
|
||||||
|
recovery.log_event(RecoveryEventType::NodeDown, Some(NodeId(2)), "b");
|
||||||
|
recovery.log_event(RecoveryEventType::IpMigration, Some(NodeId(1)), "c");
|
||||||
|
|
||||||
|
let node1_events = recovery.events_by_node(NodeId(1));
|
||||||
|
assert_eq!(node1_events.len(), 2);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_last_event() {
|
||||||
|
let recovery = RecoveryManager::new();
|
||||||
|
recovery.log_event(RecoveryEventType::NodeDown, Some(NodeId(0)), "first");
|
||||||
|
recovery.log_event(RecoveryEventType::RecoveryComplete, Some(NodeId(0)), "last");
|
||||||
|
let last = recovery.last_event().unwrap();
|
||||||
|
assert_eq!(last.message, "last");
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_reset() {
|
||||||
|
let (mgr, ip_mgr, recovery) = setup();
|
||||||
|
recovery.handle_node_failure(NodeId(1), &mgr, &ip_mgr);
|
||||||
|
assert_ne!(recovery.state(), RecoveryState::Idle);
|
||||||
|
recovery.reset();
|
||||||
|
assert_eq!(recovery.state(), RecoveryState::Idle);
|
||||||
|
assert!(recovery.can_recover());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_check_and_recover_no_transitions() {
|
||||||
|
let (mgr, ip_mgr, recovery) = setup();
|
||||||
|
let result = recovery.check_and_recover(&mgr, &ip_mgr);
|
||||||
|
assert!(result.is_empty());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_recovery_state_string() {
|
||||||
|
assert_eq!(RecoveryState::Idle.as_str(), "IDLE");
|
||||||
|
assert_eq!(RecoveryState::InProgress.as_str(), "IN_PROGRESS");
|
||||||
|
assert_eq!(RecoveryState::Failed.as_str(), "FAILED");
|
||||||
|
assert_eq!(RecoveryState::Completed.as_str(), "COMPLETED");
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,678 @@
|
|||||||
|
use std::collections::HashMap;
|
||||||
|
use std::io::{self, Read, Write, Seek, SeekFrom};
|
||||||
|
use std::path::Path;
|
||||||
|
use std::sync::{Mutex, RwLock};
|
||||||
|
|
||||||
|
const TDB_MAGIC: u32 = 0x1BADFACE;
|
||||||
|
const TDB_VERSION: u32 = 1;
|
||||||
|
const DEFAULT_HASH_SIZE: u32 = 1024;
|
||||||
|
const TDB_HEADER_SIZE: u64 = 128;
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||||
|
pub enum RecordFlag {
|
||||||
|
Active,
|
||||||
|
Free,
|
||||||
|
Deleted,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl RecordFlag {
|
||||||
|
fn as_u32(&self) -> u32 {
|
||||||
|
match self {
|
||||||
|
RecordFlag::Active => 0,
|
||||||
|
RecordFlag::Free => 1,
|
||||||
|
RecordFlag::Deleted => 2,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn from_u32(v: u32) -> Self {
|
||||||
|
match v {
|
||||||
|
1 => RecordFlag::Free,
|
||||||
|
2 => RecordFlag::Deleted,
|
||||||
|
_ => RecordFlag::Active,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Clone)]
|
||||||
|
pub struct TdbRecord {
|
||||||
|
pub key: Vec<u8>,
|
||||||
|
pub data: Vec<u8>,
|
||||||
|
pub flag: RecordFlag,
|
||||||
|
pub hash_next: u64,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl TdbRecord {
|
||||||
|
pub fn new(key: Vec<u8>, data: Vec<u8>) -> Self {
|
||||||
|
Self {
|
||||||
|
key,
|
||||||
|
data,
|
||||||
|
flag: RecordFlag::Active,
|
||||||
|
hash_next: 0,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn key_str(&self) -> &str {
|
||||||
|
std::str::from_utf8(&self.key).unwrap_or("")
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn data_str(&self) -> &str {
|
||||||
|
std::str::from_utf8(&self.data).unwrap_or("")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||||
|
pub enum TdbError {
|
||||||
|
IoError,
|
||||||
|
NotFound,
|
||||||
|
Corrupt,
|
||||||
|
Exists,
|
||||||
|
LockFailed,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl std::fmt::Display for TdbError {
|
||||||
|
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||||
|
match self {
|
||||||
|
TdbError::IoError => write!(f, "I/O error"),
|
||||||
|
TdbError::NotFound => write!(f, "record not found"),
|
||||||
|
TdbError::Corrupt => write!(f, "database corrupt"),
|
||||||
|
TdbError::Exists => write!(f, "record already exists"),
|
||||||
|
TdbError::LockFailed => write!(f, "lock failed"),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl std::error::Error for TdbError {}
|
||||||
|
|
||||||
|
impl From<io::Error> for TdbError {
|
||||||
|
fn from(_: io::Error) -> Self {
|
||||||
|
TdbError::IoError
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub type TdbResult<T> = Result<T, TdbError>;
|
||||||
|
|
||||||
|
fn hash_key(key: &[u8], hash_size: u32) -> u32 {
|
||||||
|
let mut h: u32 = 0;
|
||||||
|
for &b in key {
|
||||||
|
h = h.wrapping_mul(31).wrapping_add(b as u32);
|
||||||
|
}
|
||||||
|
h % hash_size
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Clone)]
|
||||||
|
struct HashEntry {
|
||||||
|
offset: u64,
|
||||||
|
}
|
||||||
|
|
||||||
|
struct TdbHeader {
|
||||||
|
magic: u32,
|
||||||
|
version: u32,
|
||||||
|
hash_size: u32,
|
||||||
|
record_count: u64,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl TdbHeader {
|
||||||
|
fn new(hash_size: u32) -> Self {
|
||||||
|
Self {
|
||||||
|
magic: TDB_MAGIC,
|
||||||
|
version: TDB_VERSION,
|
||||||
|
hash_size,
|
||||||
|
record_count: 0,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn to_bytes(&self) -> Vec<u8> {
|
||||||
|
let mut buf = Vec::with_capacity(TDB_HEADER_SIZE as usize);
|
||||||
|
buf.extend_from_slice(&self.magic.to_le_bytes());
|
||||||
|
buf.extend_from_slice(&self.version.to_le_bytes());
|
||||||
|
buf.extend_from_slice(&self.hash_size.to_le_bytes());
|
||||||
|
buf.extend_from_slice(&self.record_count.to_le_bytes());
|
||||||
|
while buf.len() < TDB_HEADER_SIZE as usize {
|
||||||
|
buf.push(0);
|
||||||
|
}
|
||||||
|
buf
|
||||||
|
}
|
||||||
|
|
||||||
|
fn from_bytes(buf: &[u8]) -> TdbResult<Self> {
|
||||||
|
if buf.len() < 20 {
|
||||||
|
return Err(TdbError::Corrupt);
|
||||||
|
}
|
||||||
|
let magic = u32::from_le_bytes(buf[0..4].try_into().unwrap());
|
||||||
|
if magic != TDB_MAGIC {
|
||||||
|
return Err(TdbError::Corrupt);
|
||||||
|
}
|
||||||
|
let version = u32::from_le_bytes(buf[4..8].try_into().unwrap());
|
||||||
|
let hash_size = u32::from_le_bytes(buf[8..12].try_into().unwrap());
|
||||||
|
let record_count = u64::from_le_bytes(buf[12..20].try_into().unwrap());
|
||||||
|
Ok(Self {
|
||||||
|
magic,
|
||||||
|
version,
|
||||||
|
hash_size,
|
||||||
|
record_count,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub struct TdbEngine {
|
||||||
|
header: RwLock<TdbHeader>,
|
||||||
|
records: RwLock<HashMap<Vec<u8>, TdbRecord>>,
|
||||||
|
hash_table: RwLock<Vec<Vec<u64>>>,
|
||||||
|
file_path: Option<std::path::PathBuf>,
|
||||||
|
dirty: Mutex<bool>,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl TdbEngine {
|
||||||
|
pub fn new() -> Self {
|
||||||
|
Self::with_hash_size(DEFAULT_HASH_SIZE)
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn with_hash_size(hash_size: u32) -> Self {
|
||||||
|
Self {
|
||||||
|
header: RwLock::new(TdbHeader::new(hash_size)),
|
||||||
|
records: RwLock::new(HashMap::new()),
|
||||||
|
hash_table: RwLock::new(vec![Vec::new(); 1]),
|
||||||
|
file_path: None,
|
||||||
|
dirty: Mutex::new(false),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn open<P: AsRef<Path>>(path: P) -> TdbResult<Self> {
|
||||||
|
let path = path.as_ref().to_path_buf();
|
||||||
|
if path.exists() {
|
||||||
|
Self::load_from_file(&path)
|
||||||
|
} else {
|
||||||
|
let engine = Self::new();
|
||||||
|
let mut engine = engine;
|
||||||
|
engine.file_path = Some(path.clone());
|
||||||
|
engine.save_to_file(&path)?;
|
||||||
|
Ok(engine)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn create<P: AsRef<Path>>(path: P, hash_size: u32) -> TdbResult<Self> {
|
||||||
|
let path = path.as_ref().to_path_buf();
|
||||||
|
let engine = Self::with_hash_size(hash_size);
|
||||||
|
let mut engine = engine;
|
||||||
|
engine.file_path = Some(path.clone());
|
||||||
|
engine.save_to_file(&path)?;
|
||||||
|
Ok(engine)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn load_from_file(path: &Path) -> TdbResult<Self> {
|
||||||
|
let mut file = std::fs::File::open(path)?;
|
||||||
|
let mut header_buf = vec![0u8; TDB_HEADER_SIZE as usize];
|
||||||
|
file.read_exact(&mut header_buf)?;
|
||||||
|
let header = TdbHeader::from_bytes(&header_buf)?;
|
||||||
|
|
||||||
|
let hash_size = header.hash_size;
|
||||||
|
let mut hash_table = vec![Vec::new(); hash_size as usize];
|
||||||
|
|
||||||
|
let mut records = HashMap::new();
|
||||||
|
|
||||||
|
let mut pos = TDB_HEADER_SIZE;
|
||||||
|
let file_meta = file.metadata()?;
|
||||||
|
let file_size = file_meta.len();
|
||||||
|
|
||||||
|
while pos < file_size {
|
||||||
|
file.seek(SeekFrom::Start(pos))?;
|
||||||
|
let mut flag_buf = [0u8; 4];
|
||||||
|
let mut key_len_buf = [0u8; 4];
|
||||||
|
let mut data_len_buf = [0u8; 4];
|
||||||
|
let mut hash_next_buf = [0u8; 8];
|
||||||
|
|
||||||
|
if file.read_exact(&mut flag_buf).is_err() {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
let _ = file.read_exact(&mut key_len_buf);
|
||||||
|
let _ = file.read_exact(&mut data_len_buf);
|
||||||
|
let _ = file.read_exact(&mut hash_next_buf);
|
||||||
|
|
||||||
|
let flag = RecordFlag::from_u32(u32::from_le_bytes(flag_buf));
|
||||||
|
let key_len = u32::from_le_bytes(key_len_buf) as usize;
|
||||||
|
let data_len = u32::from_le_bytes(data_len_buf) as usize;
|
||||||
|
let hash_next = u64::from_le_bytes(hash_next_buf);
|
||||||
|
|
||||||
|
let mut key = vec![0u8; key_len];
|
||||||
|
let mut data = vec![0u8; data_len];
|
||||||
|
let _ = file.read_exact(&mut key);
|
||||||
|
let _ = file.read_exact(&mut data);
|
||||||
|
|
||||||
|
let rec_offset = pos;
|
||||||
|
let bucket = hash_key(&key, hash_size) as usize;
|
||||||
|
if bucket < hash_table.len() {
|
||||||
|
hash_table[bucket].push(rec_offset);
|
||||||
|
}
|
||||||
|
|
||||||
|
if flag == RecordFlag::Active {
|
||||||
|
let record = TdbRecord {
|
||||||
|
key: key.clone(),
|
||||||
|
data,
|
||||||
|
flag,
|
||||||
|
hash_next,
|
||||||
|
};
|
||||||
|
records.insert(key, record);
|
||||||
|
}
|
||||||
|
|
||||||
|
let rec_size = 4 + 4 + 4 + 8 + key_len as u64 + data_len as u64;
|
||||||
|
pos += rec_size;
|
||||||
|
}
|
||||||
|
|
||||||
|
let record_count = records.len() as u64;
|
||||||
|
let mut header = header;
|
||||||
|
header.record_count = record_count;
|
||||||
|
|
||||||
|
Ok(Self {
|
||||||
|
header: RwLock::new(header),
|
||||||
|
records: RwLock::new(records),
|
||||||
|
hash_table: RwLock::new(hash_table),
|
||||||
|
file_path: Some(path.to_path_buf()),
|
||||||
|
dirty: Mutex::new(false),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
fn save_to_file(&self, path: &Path) -> TdbResult<()> {
|
||||||
|
let header = self.header.read().unwrap();
|
||||||
|
let header_bytes = header.to_bytes();
|
||||||
|
|
||||||
|
let mut file = std::fs::File::create(path)?;
|
||||||
|
file.write_all(&header_bytes)?;
|
||||||
|
|
||||||
|
let records = self.records.read().unwrap();
|
||||||
|
for (_, record) in records.iter() {
|
||||||
|
let flag = record.flag.as_u32().to_le_bytes();
|
||||||
|
let key_len = (record.key.len() as u32).to_le_bytes();
|
||||||
|
let data_len = (record.data.len() as u32).to_le_bytes();
|
||||||
|
let hash_next = record.hash_next.to_le_bytes();
|
||||||
|
|
||||||
|
file.write_all(&flag)?;
|
||||||
|
file.write_all(&key_len)?;
|
||||||
|
file.write_all(&data_len)?;
|
||||||
|
file.write_all(&hash_next)?;
|
||||||
|
file.write_all(&record.key)?;
|
||||||
|
file.write_all(&record.data)?;
|
||||||
|
}
|
||||||
|
file.flush()?;
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn store(&self, key: Vec<u8>, data: Vec<u8>) -> TdbResult<bool> {
|
||||||
|
let mut records = self.records.write().unwrap();
|
||||||
|
let existed = records.contains_key(&key);
|
||||||
|
let record = TdbRecord::new(key.clone(), data);
|
||||||
|
records.insert(key, record);
|
||||||
|
*self.dirty.lock().unwrap() = true;
|
||||||
|
let mut header = self.header.write().unwrap();
|
||||||
|
header.record_count = records.len() as u64;
|
||||||
|
drop(header);
|
||||||
|
drop(records);
|
||||||
|
self.try_flush()?;
|
||||||
|
Ok(!existed)
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn fetch(&self, key: &[u8]) -> TdbResult<Vec<u8>> {
|
||||||
|
let records = self.records.read().unwrap();
|
||||||
|
match records.get(key) {
|
||||||
|
Some(record) => Ok(record.data.clone()),
|
||||||
|
None => Err(TdbError::NotFound),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn delete(&self, key: &[u8]) -> TdbResult<()> {
|
||||||
|
let mut records = self.records.write().unwrap();
|
||||||
|
if records.remove(key).is_some() {
|
||||||
|
*self.dirty.lock().unwrap() = true;
|
||||||
|
let mut header = self.header.write().unwrap();
|
||||||
|
header.record_count = records.len() as u64;
|
||||||
|
drop(header);
|
||||||
|
drop(records);
|
||||||
|
self.try_flush()?;
|
||||||
|
Ok(())
|
||||||
|
} else {
|
||||||
|
Err(TdbError::NotFound)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn exists(&self, key: &[u8]) -> bool {
|
||||||
|
self.records.read().unwrap().contains_key(key)
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn keys(&self) -> Vec<Vec<u8>> {
|
||||||
|
self.records.read().unwrap().keys().cloned().collect()
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn record_count(&self) -> u64 {
|
||||||
|
self.header.read().unwrap().record_count
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn iter(&self) -> Vec<(Vec<u8>, Vec<u8>)> {
|
||||||
|
self.records
|
||||||
|
.read()
|
||||||
|
.unwrap()
|
||||||
|
.iter()
|
||||||
|
.map(|(k, v)| (k.clone(), v.data.clone()))
|
||||||
|
.collect()
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn try_flush(&self) -> TdbResult<()> {
|
||||||
|
let dirty = *self.dirty.lock().unwrap();
|
||||||
|
if dirty {
|
||||||
|
if let Some(ref path) = self.file_path {
|
||||||
|
self.save_to_file(path)?;
|
||||||
|
*self.dirty.lock().unwrap() = false;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn flush(&self) -> TdbResult<()> {
|
||||||
|
if let Some(ref path) = self.file_path {
|
||||||
|
self.save_to_file(path)?;
|
||||||
|
*self.dirty.lock().unwrap() = false;
|
||||||
|
}
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Default for TdbEngine {
|
||||||
|
fn default() -> Self {
|
||||||
|
Self::new()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Clone)]
|
||||||
|
pub struct TdbStats {
|
||||||
|
pub record_count: u64,
|
||||||
|
pub hash_size: u32,
|
||||||
|
pub active_records: u64,
|
||||||
|
pub total_data_size: u64,
|
||||||
|
pub avg_key_size: f64,
|
||||||
|
pub avg_data_size: f64,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl TdbEngine {
|
||||||
|
pub fn stats(&self) -> TdbStats {
|
||||||
|
let records = self.records.read().unwrap();
|
||||||
|
let header = self.header.read().unwrap();
|
||||||
|
let count = records.len();
|
||||||
|
let total_data: u64 = records.values().map(|r| r.data.len() as u64).sum();
|
||||||
|
let total_key: u64 = records.values().map(|r| r.key.len() as u64).sum();
|
||||||
|
|
||||||
|
TdbStats {
|
||||||
|
record_count: header.record_count,
|
||||||
|
hash_size: header.hash_size,
|
||||||
|
active_records: count as u64,
|
||||||
|
total_data_size: total_data,
|
||||||
|
avg_key_size: if count > 0 {
|
||||||
|
total_key as f64 / count as f64
|
||||||
|
} else {
|
||||||
|
0.0
|
||||||
|
},
|
||||||
|
avg_data_size: if count > 0 {
|
||||||
|
total_data as f64 / count as f64
|
||||||
|
} else {
|
||||||
|
0.0
|
||||||
|
},
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub struct TdbTransaction<'a> {
|
||||||
|
engine: &'a TdbEngine,
|
||||||
|
snapshot: HashMap<Vec<u8>, TdbRecord>,
|
||||||
|
committed: bool,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl<'a> TdbTransaction<'a> {
|
||||||
|
pub fn begin(engine: &'a TdbEngine) -> Self {
|
||||||
|
let snapshot = engine.records.read().unwrap().clone();
|
||||||
|
Self {
|
||||||
|
engine,
|
||||||
|
snapshot,
|
||||||
|
committed: false,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn store(&self, key: Vec<u8>, data: Vec<u8>) -> TdbResult<bool> {
|
||||||
|
self.engine.store(key, data)
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn fetch(&self, key: &[u8]) -> TdbResult<Vec<u8>> {
|
||||||
|
self.engine.fetch(key)
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn delete(&self, key: &[u8]) -> TdbResult<()> {
|
||||||
|
self.engine.delete(key)
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn commit(&mut self) -> TdbResult<()> {
|
||||||
|
self.committed = true;
|
||||||
|
self.engine.flush()
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn rollback(&mut self) -> TdbResult<()> {
|
||||||
|
if !self.committed {
|
||||||
|
let mut records = self.engine.records.write().unwrap();
|
||||||
|
*records = self.snapshot.clone();
|
||||||
|
let mut header = self.engine.header.write().unwrap();
|
||||||
|
header.record_count = records.len() as u64;
|
||||||
|
*self.engine.dirty.lock().unwrap() = true;
|
||||||
|
self.engine.flush()?;
|
||||||
|
}
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl<'a> Drop for TdbTransaction<'a> {
|
||||||
|
fn drop(&mut self) {
|
||||||
|
if !self.committed {
|
||||||
|
let _ = self.rollback();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use super::*;
|
||||||
|
use tempfile::TempDir;
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_tdb_create_and_store() {
|
||||||
|
let engine = TdbEngine::new();
|
||||||
|
let created = engine.store(b"key1".to_vec(), b"value1".to_vec()).unwrap();
|
||||||
|
assert!(created);
|
||||||
|
assert_eq!(engine.fetch(b"key1").unwrap(), b"value1");
|
||||||
|
assert_eq!(engine.record_count(), 1);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_tdb_overwrite() {
|
||||||
|
let engine = TdbEngine::new();
|
||||||
|
engine.store(b"key1".to_vec(), b"v1".to_vec()).unwrap();
|
||||||
|
let created = engine.store(b"key1".to_vec(), b"v2".to_vec()).unwrap();
|
||||||
|
assert!(!created);
|
||||||
|
assert_eq!(engine.fetch(b"key1").unwrap(), b"v2");
|
||||||
|
assert_eq!(engine.record_count(), 1);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_tdb_delete() {
|
||||||
|
let engine = TdbEngine::new();
|
||||||
|
engine.store(b"key1".to_vec(), b"v1".to_vec()).unwrap();
|
||||||
|
engine.delete(b"key1").unwrap();
|
||||||
|
assert!(!engine.exists(b"key1"));
|
||||||
|
assert_eq!(engine.record_count(), 0);
|
||||||
|
assert!(engine.fetch(b"key1").is_err());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_tdb_not_found() {
|
||||||
|
let engine = TdbEngine::new();
|
||||||
|
assert!(engine.fetch(b"missing").is_err());
|
||||||
|
assert!(engine.delete(b"missing").is_err());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_tdb_multiple_records() {
|
||||||
|
let engine = TdbEngine::new();
|
||||||
|
for i in 0..100 {
|
||||||
|
engine.store(
|
||||||
|
format!("key{}", i).into_bytes(),
|
||||||
|
format!("value{}", i).into_bytes(),
|
||||||
|
).unwrap();
|
||||||
|
}
|
||||||
|
assert_eq!(engine.record_count(), 100);
|
||||||
|
assert_eq!(engine.fetch(b"key50").unwrap(), b"value50");
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_tdb_persist_to_disk() {
|
||||||
|
let tmp = TempDir::new().unwrap();
|
||||||
|
let db_path = tmp.path().join("test.tdb");
|
||||||
|
|
||||||
|
{
|
||||||
|
let engine = TdbEngine::open(&db_path).unwrap();
|
||||||
|
engine.store(b"persistent".to_vec(), b"yes".to_vec()).unwrap();
|
||||||
|
engine.store(b"key2".to_vec(), b"val2".to_vec()).unwrap();
|
||||||
|
engine.flush().unwrap();
|
||||||
|
}
|
||||||
|
|
||||||
|
let engine2 = TdbEngine::open(&db_path).unwrap();
|
||||||
|
assert_eq!(engine2.fetch(b"persistent").unwrap(), b"yes");
|
||||||
|
assert_eq!(engine2.fetch(b"key2").unwrap(), b"val2");
|
||||||
|
assert_eq!(engine2.record_count(), 2);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_tdb_keys() {
|
||||||
|
let engine = TdbEngine::new();
|
||||||
|
engine.store(b"a".to_vec(), b"1".to_vec()).unwrap();
|
||||||
|
engine.store(b"b".to_vec(), b"2".to_vec()).unwrap();
|
||||||
|
engine.store(b"c".to_vec(), b"3".to_vec()).unwrap();
|
||||||
|
let mut keys = engine.keys();
|
||||||
|
keys.sort();
|
||||||
|
assert_eq!(keys, vec![b"a".to_vec(), b"b".to_vec(), b"c".to_vec()]);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_tdb_iter() {
|
||||||
|
let engine = TdbEngine::new();
|
||||||
|
engine.store(b"k1".to_vec(), b"d1".to_vec()).unwrap();
|
||||||
|
engine.store(b"k2".to_vec(), b"d2".to_vec()).unwrap();
|
||||||
|
let entries: Vec<_> = engine.iter();
|
||||||
|
assert_eq!(entries.len(), 2);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_tdb_stats() {
|
||||||
|
let engine = TdbEngine::new();
|
||||||
|
engine.store(b"key1".to_vec(), b"12345".to_vec()).unwrap();
|
||||||
|
engine.store(b"key2".to_vec(), b"67890".to_vec()).unwrap();
|
||||||
|
let stats = engine.stats();
|
||||||
|
assert_eq!(stats.active_records, 2);
|
||||||
|
assert_eq!(stats.total_data_size, 10);
|
||||||
|
assert_eq!(stats.avg_key_size, 4.0);
|
||||||
|
assert_eq!(stats.avg_data_size, 5.0);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_tdb_create_with_hash_size() {
|
||||||
|
let engine = TdbEngine::with_hash_size(256);
|
||||||
|
let header = engine.header.read().unwrap();
|
||||||
|
assert_eq!(header.hash_size, 256);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_tdb_transaction_commit() {
|
||||||
|
let engine = TdbEngine::new();
|
||||||
|
engine.store(b"init".to_vec(), b"data".to_vec()).unwrap();
|
||||||
|
{
|
||||||
|
let mut tx = TdbTransaction::begin(&engine);
|
||||||
|
tx.store(b"tx_key".to_vec(), b"tx_data".to_vec()).unwrap();
|
||||||
|
tx.commit().unwrap();
|
||||||
|
}
|
||||||
|
assert!(engine.exists(b"tx_key"));
|
||||||
|
assert_eq!(engine.fetch(b"tx_key").unwrap(), b"tx_data");
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_tdb_transaction_rollback() {
|
||||||
|
let engine = TdbEngine::new();
|
||||||
|
engine.store(b"keep".to_vec(), b"yes".to_vec()).unwrap();
|
||||||
|
{
|
||||||
|
let mut tx = TdbTransaction::begin(&engine);
|
||||||
|
tx.store(b"temp".to_vec(), b"no".to_vec()).unwrap();
|
||||||
|
tx.delete(b"keep").unwrap();
|
||||||
|
tx.rollback().unwrap();
|
||||||
|
}
|
||||||
|
assert!(!engine.exists(b"temp"));
|
||||||
|
assert!(engine.exists(b"keep"));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_tdb_transaction_auto_rollback_on_drop() {
|
||||||
|
let engine = TdbEngine::new();
|
||||||
|
engine.store(b"orig".to_vec(), b"val".to_vec()).unwrap();
|
||||||
|
{
|
||||||
|
let _tx = TdbTransaction::begin(&engine);
|
||||||
|
_tx.store(b"temp".to_vec(), b"x".to_vec()).unwrap();
|
||||||
|
}
|
||||||
|
assert!(!engine.exists(b"temp"));
|
||||||
|
assert!(engine.exists(b"orig"));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_hash_key_distribution() {
|
||||||
|
let hash_size = 1024u32;
|
||||||
|
let mut buckets = std::collections::HashSet::new();
|
||||||
|
for i in 0..1000 {
|
||||||
|
let key = format!("key_{}", i);
|
||||||
|
let bucket = hash_key(key.as_bytes(), hash_size);
|
||||||
|
buckets.insert(bucket);
|
||||||
|
}
|
||||||
|
assert!(buckets.len() > 200);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_tdb_corrupt_file_detection() {
|
||||||
|
let tmp = TempDir::new().unwrap();
|
||||||
|
let db_path = tmp.path().join("corrupt.tdb");
|
||||||
|
std::fs::write(&db_path, b"NOT_A_TDB_FILE_GARBAGE_DATA_HERE").unwrap();
|
||||||
|
assert!(TdbEngine::open(&db_path).is_err());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_record_flag_conversion() {
|
||||||
|
assert_eq!(RecordFlag::Active.as_u32(), 0);
|
||||||
|
assert_eq!(RecordFlag::Free.as_u32(), 1);
|
||||||
|
assert_eq!(RecordFlag::Deleted.as_u32(), 2);
|
||||||
|
assert_eq!(RecordFlag::from_u32(0), RecordFlag::Active);
|
||||||
|
assert_eq!(RecordFlag::from_u32(1), RecordFlag::Free);
|
||||||
|
assert_eq!(RecordFlag::from_u32(2), RecordFlag::Deleted);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_tdb_empty_keys_and_values() {
|
||||||
|
let engine = TdbEngine::new();
|
||||||
|
engine.store(vec![], vec![]).unwrap();
|
||||||
|
assert!(engine.exists(&[]));
|
||||||
|
assert_eq!(engine.fetch(&[]).unwrap(), b"");
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_tdb_large_value() {
|
||||||
|
let engine = TdbEngine::new();
|
||||||
|
let large_data = vec![0x42u8; 100_000];
|
||||||
|
engine.store(b"big".to_vec(), large_data.clone()).unwrap();
|
||||||
|
let fetched = engine.fetch(b"big").unwrap();
|
||||||
|
assert_eq!(fetched.len(), 100_000);
|
||||||
|
assert_eq!(fetched, large_data);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_tdb_binary_keys() {
|
||||||
|
let engine = TdbEngine::new();
|
||||||
|
let key = vec![0u8, 1u8, 255u8, 128u8, 64u8];
|
||||||
|
engine.store(key.clone(), b"binary_val".to_vec()).unwrap();
|
||||||
|
assert_eq!(engine.fetch(&key).unwrap(), b"binary_val");
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -5,10 +5,12 @@ pub mod audit;
|
|||||||
pub mod auth;
|
pub mod auth;
|
||||||
pub mod category_view;
|
pub mod category_view;
|
||||||
pub mod cli;
|
pub mod cli;
|
||||||
|
pub mod ctdb;
|
||||||
pub mod command;
|
pub mod command;
|
||||||
pub mod config;
|
pub mod config;
|
||||||
pub mod download;
|
pub mod download;
|
||||||
pub mod import_markdown;
|
pub mod import_markdown;
|
||||||
|
pub mod myfiles;
|
||||||
pub mod pg_client;
|
pub mod pg_client;
|
||||||
pub mod provider;
|
pub mod provider;
|
||||||
pub mod render;
|
pub mod render;
|
||||||
@@ -16,15 +18,21 @@ pub mod rsync;
|
|||||||
pub mod s3;
|
pub mod s3;
|
||||||
pub mod s3_auth;
|
pub mod s3_auth;
|
||||||
pub mod s3_config;
|
pub mod s3_config;
|
||||||
|
pub mod s3_policy;
|
||||||
pub mod s3_xml;
|
pub mod s3_xml;
|
||||||
pub mod scan;
|
pub mod scan;
|
||||||
|
pub mod smb_config;
|
||||||
pub mod server;
|
pub mod server;
|
||||||
pub mod ssh_server;
|
pub mod ssh_server;
|
||||||
pub mod sync;
|
pub mod sync;
|
||||||
pub mod vfs;
|
pub mod vfs;
|
||||||
pub mod webdav;
|
pub mod webdav;
|
||||||
|
pub mod webdav_locks;
|
||||||
pub mod webdav_version;
|
pub mod webdav_version;
|
||||||
|
|
||||||
|
#[cfg(feature = "async-vfs")]
|
||||||
|
pub mod async_webdav;
|
||||||
|
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
mod security_audit;
|
mod security_audit;
|
||||||
|
|
||||||
|
|||||||
@@ -0,0 +1,692 @@
|
|||||||
|
use axum::{
|
||||||
|
body::Body,
|
||||||
|
extract::{Path, Query, State},
|
||||||
|
http::{header, StatusCode, HeaderMap},
|
||||||
|
response::{Html, IntoResponse, Json, Response},
|
||||||
|
};
|
||||||
|
use rusqlite::{params, Connection};
|
||||||
|
use serde::{Deserialize, Serialize};
|
||||||
|
use std::path::PathBuf;
|
||||||
|
use std::sync::OnceLock;
|
||||||
|
|
||||||
|
use crate::server::AppState;
|
||||||
|
|
||||||
|
const SCHEMA: &str = "
|
||||||
|
CREATE TABLE IF NOT EXISTS virtual_folders (
|
||||||
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||||
|
folder TEXT NOT NULL UNIQUE,
|
||||||
|
description TEXT DEFAULT '',
|
||||||
|
created_at TEXT DEFAULT (datetime('now'))
|
||||||
|
);
|
||||||
|
CREATE TABLE IF NOT EXISTS file_tags (
|
||||||
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||||
|
filename TEXT NOT NULL,
|
||||||
|
tag TEXT NOT NULL,
|
||||||
|
UNIQUE(filename, tag)
|
||||||
|
);
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_file_tags_tag ON file_tags(tag);
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_file_tags_filename ON file_tags(filename);
|
||||||
|
";
|
||||||
|
|
||||||
|
static MYFILES_UPLOAD_PATH: OnceLock<String> = OnceLock::new();
|
||||||
|
|
||||||
|
pub fn init_upload_path(path: String) {
|
||||||
|
let _ = MYFILES_UPLOAD_PATH.set(path);
|
||||||
|
}
|
||||||
|
|
||||||
|
fn upload_base_path() -> &'static str {
|
||||||
|
MYFILES_UPLOAD_PATH.get().map(|s| s.as_str())
|
||||||
|
.unwrap_or("/Users/accusys/momentry/var/sftpgo/data")
|
||||||
|
}
|
||||||
|
|
||||||
|
fn user_db_path(state: &AppState, username: &str) -> PathBuf {
|
||||||
|
PathBuf::from(&state.upload_path)
|
||||||
|
.join(username)
|
||||||
|
.join("webdav_virtual.sqlite")
|
||||||
|
}
|
||||||
|
|
||||||
|
fn user_root(base_path: &str, username: &str) -> PathBuf {
|
||||||
|
PathBuf::from(base_path).join(username)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn ensure_schema(db_path: &PathBuf) -> anyhow::Result<Connection> {
|
||||||
|
let conn = Connection::open(db_path)
|
||||||
|
.map_err(|e| anyhow::anyhow!("Failed to open DB: {}", e))?;
|
||||||
|
conn.execute_batch(SCHEMA)
|
||||||
|
.map_err(|e| anyhow::anyhow!("Failed to create schema: {}", e))?;
|
||||||
|
Ok(conn)
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Serialize)]
|
||||||
|
pub struct FolderInfo {
|
||||||
|
pub name: String,
|
||||||
|
pub description: String,
|
||||||
|
pub file_count: usize,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Serialize)]
|
||||||
|
pub struct FileInfo {
|
||||||
|
pub name: String,
|
||||||
|
pub size: u64,
|
||||||
|
pub tags: Vec<String>,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Deserialize)]
|
||||||
|
pub struct FolderRequest {
|
||||||
|
pub name: String,
|
||||||
|
pub description: Option<String>,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Deserialize)]
|
||||||
|
pub struct TagRequest {
|
||||||
|
pub file: String,
|
||||||
|
pub tag: String,
|
||||||
|
}
|
||||||
|
|
||||||
|
pub async fn list_folders(
|
||||||
|
State(state): State<AppState>,
|
||||||
|
Path(username): Path<String>,
|
||||||
|
) -> Result<Json<Vec<FolderInfo>>, (StatusCode, String)> {
|
||||||
|
let db_path = user_db_path(&state, &username);
|
||||||
|
let conn = ensure_schema(&db_path).map_err(|e| (StatusCode::INTERNAL_SERVER_ERROR, e.to_string()))?;
|
||||||
|
|
||||||
|
let mut stmt = conn
|
||||||
|
.prepare("SELECT folder, description FROM virtual_folders ORDER BY folder")
|
||||||
|
.map_err(|e| (StatusCode::INTERNAL_SERVER_ERROR, e.to_string()))?;
|
||||||
|
let folders: Vec<(String, String)> = stmt
|
||||||
|
.query_map([], |row| Ok((row.get(0)?, row.get(1)?)))
|
||||||
|
.map_err(|e| (StatusCode::INTERNAL_SERVER_ERROR, e.to_string()))?
|
||||||
|
.filter_map(|r| r.ok())
|
||||||
|
.collect();
|
||||||
|
|
||||||
|
let mut result = Vec::new();
|
||||||
|
for (folder, desc) in folders {
|
||||||
|
let tag = folder.trim_start_matches('/').to_string();
|
||||||
|
let count: usize = conn
|
||||||
|
.query_row(
|
||||||
|
"SELECT COUNT(*) FROM file_tags WHERE tag = ?1",
|
||||||
|
params![tag],
|
||||||
|
|row| row.get(0),
|
||||||
|
)
|
||||||
|
.unwrap_or(0);
|
||||||
|
result.push(FolderInfo {
|
||||||
|
name: folder,
|
||||||
|
description: desc,
|
||||||
|
file_count: count,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
Ok(Json(result))
|
||||||
|
}
|
||||||
|
|
||||||
|
pub async fn create_folder(
|
||||||
|
State(state): State<AppState>,
|
||||||
|
Path(username): Path<String>,
|
||||||
|
Json(req): Json<FolderRequest>,
|
||||||
|
) -> Result<Json<serde_json::Value>, (StatusCode, String)> {
|
||||||
|
let db_path = user_db_path(&state, &username);
|
||||||
|
let conn = ensure_schema(&db_path).map_err(|e| (StatusCode::INTERNAL_SERVER_ERROR, e.to_string()))?;
|
||||||
|
|
||||||
|
let folder = if req.name.starts_with('/') {
|
||||||
|
req.name
|
||||||
|
} else {
|
||||||
|
format!("/{}", req.name)
|
||||||
|
};
|
||||||
|
let desc = req.description.unwrap_or_default();
|
||||||
|
|
||||||
|
conn.execute(
|
||||||
|
"INSERT OR IGNORE INTO virtual_folders (folder, description) VALUES (?1, ?2)",
|
||||||
|
params![folder, desc],
|
||||||
|
)
|
||||||
|
.map_err(|e| (StatusCode::INTERNAL_SERVER_ERROR, e.to_string()))?;
|
||||||
|
|
||||||
|
Ok(Json(serde_json::json!({
|
||||||
|
"status": "ok",
|
||||||
|
"folder": folder,
|
||||||
|
"description": desc
|
||||||
|
})))
|
||||||
|
}
|
||||||
|
|
||||||
|
pub async fn delete_folder(
|
||||||
|
State(state): State<AppState>,
|
||||||
|
Path((username, folder_name)): Path<(String, String)>,
|
||||||
|
) -> Result<Json<serde_json::Value>, (StatusCode, String)> {
|
||||||
|
let db_path = user_db_path(&state, &username);
|
||||||
|
let conn = ensure_schema(&db_path).map_err(|e| (StatusCode::INTERNAL_SERVER_ERROR, e.to_string()))?;
|
||||||
|
|
||||||
|
let folder = if folder_name.starts_with('/') {
|
||||||
|
folder_name
|
||||||
|
} else {
|
||||||
|
format!("/{}", folder_name)
|
||||||
|
};
|
||||||
|
let tag = folder.trim_start_matches('/').to_string();
|
||||||
|
|
||||||
|
conn.execute("DELETE FROM file_tags WHERE tag = ?1", params![tag])
|
||||||
|
.map_err(|e| (StatusCode::INTERNAL_SERVER_ERROR, e.to_string()))?;
|
||||||
|
conn.execute("DELETE FROM virtual_folders WHERE folder = ?1", params![folder])
|
||||||
|
.map_err(|e| (StatusCode::INTERNAL_SERVER_ERROR, e.to_string()))?;
|
||||||
|
|
||||||
|
Ok(Json(serde_json::json!({"status": "ok", "deleted": folder})))
|
||||||
|
}
|
||||||
|
|
||||||
|
pub async fn delete_file(
|
||||||
|
State(state): State<AppState>,
|
||||||
|
Path((username, filename)): Path<(String, String)>,
|
||||||
|
) -> Result<Json<serde_json::Value>, (StatusCode, String)> {
|
||||||
|
let root = user_root(&state.upload_path, &username);
|
||||||
|
let file_path = root.join(&filename);
|
||||||
|
let db_path = user_db_path(&state, &username);
|
||||||
|
|
||||||
|
if tokio::fs::remove_file(&file_path).await.is_err() {
|
||||||
|
return Err((StatusCode::NOT_FOUND, "File not found".to_string()));
|
||||||
|
}
|
||||||
|
|
||||||
|
// Remove tags associated with this file
|
||||||
|
if let Ok(conn) = ensure_schema(&db_path) {
|
||||||
|
let _ = conn.execute("DELETE FROM file_tags WHERE filename = ?1", params![filename]);
|
||||||
|
}
|
||||||
|
|
||||||
|
Ok(Json(serde_json::json!({"status": "ok", "deleted": filename})))
|
||||||
|
}
|
||||||
|
|
||||||
|
pub async fn list_files(
|
||||||
|
State(state): State<AppState>,
|
||||||
|
Path(username): Path<String>,
|
||||||
|
Query(q): Query<serde_json::Map<String, serde_json::Value>>,
|
||||||
|
) -> Result<Json<Vec<FileInfo>>, (StatusCode, String)> {
|
||||||
|
let root = user_root(&state.upload_path, &username);
|
||||||
|
let db_path = user_db_path(&state, &username);
|
||||||
|
let conn = ensure_schema(&db_path).map_err(|e| (StatusCode::INTERNAL_SERVER_ERROR, e.to_string()))?;
|
||||||
|
|
||||||
|
let folder_filter = q.get("folder").and_then(|v| v.as_str()).map(|s| s.to_string());
|
||||||
|
|
||||||
|
let filenames: Vec<String> = if let Some(folder) = &folder_filter {
|
||||||
|
let tag = folder.trim_start_matches('/');
|
||||||
|
let rows: Vec<String> = conn
|
||||||
|
.prepare("SELECT filename FROM file_tags WHERE tag = ?1 ORDER BY filename")
|
||||||
|
.map_err(|e| (StatusCode::INTERNAL_SERVER_ERROR, e.to_string()))?
|
||||||
|
.query_map(params![tag], |row| row.get::<_, String>(0))
|
||||||
|
.map_err(|e| (StatusCode::INTERNAL_SERVER_ERROR, e.to_string()))?
|
||||||
|
.filter_map(|r| r.ok())
|
||||||
|
.collect();
|
||||||
|
rows
|
||||||
|
} else {
|
||||||
|
let mut entries = Vec::new();
|
||||||
|
if let Ok(rd) = std::fs::read_dir(&root) {
|
||||||
|
for entry in rd.flatten() {
|
||||||
|
if entry.path().is_file() {
|
||||||
|
if let Some(name) = entry.file_name().to_str() {
|
||||||
|
entries.push(name.to_string());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
entries.sort();
|
||||||
|
entries
|
||||||
|
};
|
||||||
|
|
||||||
|
let mut result = Vec::new();
|
||||||
|
for fname in filenames {
|
||||||
|
let size = std::fs::metadata(root.join(&fname))
|
||||||
|
.map(|m| m.len())
|
||||||
|
.unwrap_or(0);
|
||||||
|
|
||||||
|
let mut tags_stmt = conn
|
||||||
|
.prepare("SELECT tag FROM file_tags WHERE filename = ?1 ORDER BY tag")
|
||||||
|
.map_err(|e| (StatusCode::INTERNAL_SERVER_ERROR, e.to_string()))?;
|
||||||
|
let tags: Vec<String> = tags_stmt
|
||||||
|
.query_map(params![fname], |row| row.get::<_, String>(0))
|
||||||
|
.map_err(|e| (StatusCode::INTERNAL_SERVER_ERROR, e.to_string()))?
|
||||||
|
.filter_map(|r| r.ok())
|
||||||
|
.collect();
|
||||||
|
|
||||||
|
result.push(FileInfo {
|
||||||
|
name: fname,
|
||||||
|
size,
|
||||||
|
tags,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
Ok(Json(result))
|
||||||
|
}
|
||||||
|
|
||||||
|
pub async fn add_tag(
|
||||||
|
State(state): State<AppState>,
|
||||||
|
Path(username): Path<String>,
|
||||||
|
Json(req): Json<TagRequest>,
|
||||||
|
) -> Result<Json<serde_json::Value>, (StatusCode, String)> {
|
||||||
|
let db_path = user_db_path(&state, &username);
|
||||||
|
let conn = ensure_schema(&db_path).map_err(|e| (StatusCode::INTERNAL_SERVER_ERROR, e.to_string()))?;
|
||||||
|
|
||||||
|
let folder = if req.tag.starts_with('/') {
|
||||||
|
req.tag.clone()
|
||||||
|
} else {
|
||||||
|
format!("/{}", req.tag)
|
||||||
|
};
|
||||||
|
let tag = folder.trim_start_matches('/').to_string();
|
||||||
|
|
||||||
|
conn.execute(
|
||||||
|
"INSERT OR IGNORE INTO virtual_folders (folder, description) VALUES (?1, '')",
|
||||||
|
params![folder],
|
||||||
|
)
|
||||||
|
.ok();
|
||||||
|
|
||||||
|
conn.execute(
|
||||||
|
"INSERT OR IGNORE INTO file_tags (filename, tag) VALUES (?1, ?2)",
|
||||||
|
params![req.file, tag],
|
||||||
|
)
|
||||||
|
.map_err(|e| (StatusCode::INTERNAL_SERVER_ERROR, e.to_string()))?;
|
||||||
|
|
||||||
|
Ok(Json(serde_json::json!({
|
||||||
|
"status": "ok",
|
||||||
|
"file": req.file,
|
||||||
|
"tag": tag
|
||||||
|
})))
|
||||||
|
}
|
||||||
|
|
||||||
|
pub async fn remove_tag(
|
||||||
|
State(state): State<AppState>,
|
||||||
|
Path(username): Path<String>,
|
||||||
|
Json(req): Json<TagRequest>,
|
||||||
|
) -> Result<Json<serde_json::Value>, (StatusCode, String)> {
|
||||||
|
let db_path = user_db_path(&state, &username);
|
||||||
|
let conn = ensure_schema(&db_path).map_err(|e| (StatusCode::INTERNAL_SERVER_ERROR, e.to_string()))?;
|
||||||
|
|
||||||
|
let tag = req.tag.trim_start_matches('/');
|
||||||
|
|
||||||
|
conn.execute(
|
||||||
|
"DELETE FROM file_tags WHERE filename = ?1 AND tag = ?2",
|
||||||
|
params![req.file, tag],
|
||||||
|
)
|
||||||
|
.map_err(|e| (StatusCode::INTERNAL_SERVER_ERROR, e.to_string()))?;
|
||||||
|
|
||||||
|
Ok(Json(serde_json::json!({
|
||||||
|
"status": "ok",
|
||||||
|
"file": req.file,
|
||||||
|
"removed_tag": tag
|
||||||
|
})))
|
||||||
|
}
|
||||||
|
|
||||||
|
pub async fn file_tags(
|
||||||
|
State(state): State<AppState>,
|
||||||
|
Path((username, filename)): Path<(String, String)>,
|
||||||
|
) -> Result<Json<Vec<String>>, (StatusCode, String)> {
|
||||||
|
let db_path = user_db_path(&state, &username);
|
||||||
|
let conn = ensure_schema(&db_path).map_err(|e| (StatusCode::INTERNAL_SERVER_ERROR, e.to_string()))?;
|
||||||
|
|
||||||
|
let mut stmt = conn
|
||||||
|
.prepare("SELECT tag FROM file_tags WHERE filename = ?1 ORDER BY tag")
|
||||||
|
.map_err(|e| (StatusCode::INTERNAL_SERVER_ERROR, e.to_string()))?;
|
||||||
|
let tags: Vec<String> = stmt
|
||||||
|
.query_map(params![filename], |row| row.get::<_, String>(0))
|
||||||
|
.map_err(|e| (StatusCode::INTERNAL_SERVER_ERROR, e.to_string()))?
|
||||||
|
.filter_map(|r| r.ok())
|
||||||
|
.collect();
|
||||||
|
|
||||||
|
Ok(Json(tags))
|
||||||
|
}
|
||||||
|
|
||||||
|
pub async fn preview_file(
|
||||||
|
Path((username, filename)): Path<(String, String)>,
|
||||||
|
) -> Response {
|
||||||
|
let root = user_root(upload_base_path(), &username);
|
||||||
|
let file_path = root.join(&filename);
|
||||||
|
|
||||||
|
if !file_path.exists() || !file_path.is_file() {
|
||||||
|
return (StatusCode::NOT_FOUND, "File not found").into_response();
|
||||||
|
}
|
||||||
|
|
||||||
|
let ext = file_path
|
||||||
|
.extension()
|
||||||
|
.and_then(|e| e.to_str())
|
||||||
|
.unwrap_or("")
|
||||||
|
.to_lowercase();
|
||||||
|
|
||||||
|
let mime = match ext.as_str() {
|
||||||
|
"png" => "image/png",
|
||||||
|
"jpg" | "jpeg" => "image/jpeg",
|
||||||
|
"gif" => "image/gif",
|
||||||
|
"webp" => "image/webp",
|
||||||
|
"svg" => "image/svg+xml",
|
||||||
|
"pdf" => "application/pdf",
|
||||||
|
"mp4" | "m4v" => "video/mp4",
|
||||||
|
"webm" => "video/webm",
|
||||||
|
"mov" => "video/quicktime",
|
||||||
|
"avi" => "video/x-msvideo",
|
||||||
|
"mkv" => "video/x-matroska",
|
||||||
|
"mp3" => "audio/mpeg",
|
||||||
|
"m4a" => "audio/mp4",
|
||||||
|
"wav" => "audio/wav",
|
||||||
|
"flac" => "audio/flac",
|
||||||
|
"ogg" => "audio/ogg",
|
||||||
|
"aac" => "audio/aac",
|
||||||
|
"txt" | "md" | "json" | "yaml" | "yml" | "toml" | "log" | "csv" | "xml" | "html" | "js" | "ts" | "rs" | "py" | "sh" => "text/plain; charset=utf-8",
|
||||||
|
_ => "application/octet-stream",
|
||||||
|
};
|
||||||
|
|
||||||
|
let is_text = mime.starts_with("text/");
|
||||||
|
if is_text {
|
||||||
|
match tokio::fs::read_to_string(&file_path).await {
|
||||||
|
Ok(content) => {
|
||||||
|
let headers = [(header::CONTENT_TYPE, "text/plain; charset=utf-8")];
|
||||||
|
(headers, content).into_response()
|
||||||
|
}
|
||||||
|
Err(_) => (StatusCode::INTERNAL_SERVER_ERROR, "Failed to read file").into_response(),
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
match tokio::fs::read(&file_path).await {
|
||||||
|
Ok(data) => {
|
||||||
|
let headers = [(header::CONTENT_TYPE, mime)];
|
||||||
|
(headers, Body::from(data)).into_response()
|
||||||
|
}
|
||||||
|
Err(_) => (StatusCode::INTERNAL_SERVER_ERROR, "Failed to read file").into_response(),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub async fn ui_page() -> Html<String> {
|
||||||
|
Html(MYFILES_HTML.to_string())
|
||||||
|
}
|
||||||
|
|
||||||
|
const MYFILES_HTML: &str = r#"<!DOCTYPE html>
|
||||||
|
<html lang="zh-TW">
|
||||||
|
<head>
|
||||||
|
<meta charset="utf-8">
|
||||||
|
<meta name="viewport" content="width=device-width, initial-scale=1">
|
||||||
|
<title>MyFiles — MarkBase</title>
|
||||||
|
<style>
|
||||||
|
* { margin: 0; padding: 0; box-sizing: border-box; }
|
||||||
|
body { font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", sans-serif; background: #f5f5f7; color: #1d1d1f; }
|
||||||
|
.header { background: #fff; border-bottom: 1px solid #d2d2d7; padding: 12px 20px; display: flex; align-items: center; justify-content: space-between; }
|
||||||
|
.header h1 { font-size: 20px; font-weight: 600; }
|
||||||
|
.header .user-info { font-size: 14px; color: #6e6e73; }
|
||||||
|
.container { display: flex; max-width: 1200px; margin: 0 auto; padding: 20px; gap: 20px; }
|
||||||
|
.sidebar { width: 220px; flex-shrink: 0; }
|
||||||
|
.sidebar h2 { font-size: 14px; color: #6e6e73; text-transform: uppercase; margin-bottom: 10px; }
|
||||||
|
.folder-list { list-style: none; }
|
||||||
|
.folder-list li { padding: 8px 12px; border-radius: 8px; cursor: pointer; display: flex; align-items: center; gap: 8px; font-size: 14px; }
|
||||||
|
.folder-list li:hover { background: #e8e8ed; }
|
||||||
|
.folder-list li.active { background: #0071e3; color: #fff; }
|
||||||
|
.folder-list .count { margin-left: auto; font-size: 12px; opacity: 0.6; }
|
||||||
|
.main { flex: 1; min-width: 0; }
|
||||||
|
.toolbar { display: flex; gap: 8px; margin-bottom: 16px; align-items: center; }
|
||||||
|
.toolbar input[type="text"] { flex: 1; padding: 8px 12px; border: 1px solid #d2d2d7; border-radius: 8px; font-size: 14px; }
|
||||||
|
.btn { padding: 8px 16px; border: none; border-radius: 8px; cursor: pointer; font-size: 14px; font-weight: 500; }
|
||||||
|
.btn-primary { background: #0071e3; color: #fff; }
|
||||||
|
.btn-primary:hover { background: #0058b0; }
|
||||||
|
.btn-secondary { background: #e8e8ed; color: #1d1d1f; }
|
||||||
|
.btn-secondary:hover { background: #d2d2d7; }
|
||||||
|
.btn-danger { background: #ff3b30; color: #fff; }
|
||||||
|
.btn-danger:hover { background: #cc2918; }
|
||||||
|
.file-grid { display: grid; grid-template-columns: repeat(auto-fill, minmax(180px, 1fr)); gap: 12px; }
|
||||||
|
.file-card { background: #fff; border-radius: 12px; padding: 12px; border: 1px solid #e8e8ed; }
|
||||||
|
.file-card .name { font-size: 14px; font-weight: 500; word-break: break-all; margin-bottom: 4px; }
|
||||||
|
.file-card .size { font-size: 12px; color: #6e6e73; margin-bottom: 8px; }
|
||||||
|
.file-card .tags { display: flex; flex-wrap: wrap; gap: 4px; }
|
||||||
|
.tag { font-size: 11px; padding: 2px 8px; border-radius: 10px; background: #e8e8ed; }
|
||||||
|
.tag.blue { background: #d1e8ff; color: #0058b0; }
|
||||||
|
.empty { text-align: center; padding: 60px; color: #6e6e73; }
|
||||||
|
.modal-overlay { display: none; position: fixed; inset: 0; background: rgba(0,0,0,0.4); z-index: 100; align-items: center; justify-content: center; }
|
||||||
|
.modal-overlay.show { display: flex; }
|
||||||
|
.modal { background: #fff; border-radius: 16px; padding: 24px; min-width: 320px; max-width: 400px; }
|
||||||
|
.modal h3 { font-size: 18px; margin-bottom: 16px; }
|
||||||
|
.modal label { font-size: 14px; color: #6e6e73; display: block; margin-bottom: 4px; }
|
||||||
|
.modal input { width: 100%; padding: 8px 12px; border: 1px solid #d2d2d7; border-radius: 8px; font-size: 14px; margin-bottom: 12px; }
|
||||||
|
.modal .actions { display: flex; gap: 8px; justify-content: flex-end; margin-top: 16px; }
|
||||||
|
.preview-modal { max-width: 90vw; max-height: 90vh; width: 800px; display: flex; flex-direction: column; padding: 0; overflow: hidden; }
|
||||||
|
.preview-header { display: flex; align-items: center; justify-content: space-between; padding: 16px 20px; border-bottom: 1px solid #d2d2d7; }
|
||||||
|
.preview-header h3 { font-size: 16px; margin: 0; }
|
||||||
|
.btn-sm { padding: 4px 12px; font-size: 13px; }
|
||||||
|
.preview-content { flex: 1; overflow: auto; padding: 20px; min-height: 200px; max-height: calc(90vh - 60px); }
|
||||||
|
.preview-loading { text-align: center; padding: 40px; color: #6e6e73; }
|
||||||
|
.preview-content img { max-width: 100%; height: auto; display: block; margin: 0 auto; }
|
||||||
|
.preview-content pre { background: #f5f5f7; padding: 16px; border-radius: 8px; overflow: auto; font-size: 13px; line-height: 1.5; max-height: 60vh; }
|
||||||
|
.preview-content iframe { width: 100%; height: 70vh; border: none; }
|
||||||
|
.preview-content .file-meta { font-size: 14px; color: #6e6e73; text-align: center; padding: 40px; }
|
||||||
|
.preview-content .file-meta a { color: #0071e3; text-decoration: none; }
|
||||||
|
.preview-content .file-meta a:hover { text-decoration: underline; }
|
||||||
|
</style>
|
||||||
|
</head>
|
||||||
|
<body>
|
||||||
|
<div class="header">
|
||||||
|
<h1>📁 MyFiles</h1>
|
||||||
|
<div class="user-info" id="user-info">Loading...</div>
|
||||||
|
</div>
|
||||||
|
<div class="container">
|
||||||
|
<div class="sidebar">
|
||||||
|
<h2>Folders</h2>
|
||||||
|
<ul class="folder-list" id="folder-list">
|
||||||
|
<li class="active" data-folder="">All Files</li>
|
||||||
|
</ul>
|
||||||
|
<div style="margin-top:16px">
|
||||||
|
<button class="btn btn-secondary" onclick="showNewFolderModal()" style="width:100%">+ New Folder</button>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div class="main">
|
||||||
|
<div class="toolbar">
|
||||||
|
<input type="text" id="search" placeholder="Search files..." oninput="loadFiles()">
|
||||||
|
<button class="btn btn-primary" onclick="showUploadModal()">Upload</button>
|
||||||
|
</div>
|
||||||
|
<div class="file-grid" id="file-grid"></div>
|
||||||
|
<div class="empty" id="empty-state" style="display:none">No files found</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div class="modal-overlay" id="folder-modal">
|
||||||
|
<div class="modal">
|
||||||
|
<h3>New Folder</h3>
|
||||||
|
<label>Folder Name</label>
|
||||||
|
<input type="text" id="folder-name" placeholder="e.g. photos">
|
||||||
|
<label>Description</label>
|
||||||
|
<input type="text" id="folder-desc" placeholder="Optional description">
|
||||||
|
<div class="actions">
|
||||||
|
<button class="btn btn-secondary" onclick="hideFolderModal()">Cancel</button>
|
||||||
|
<button class="btn btn-primary" onclick="createFolder()">Create</button>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div class="modal-overlay" id="tag-modal">
|
||||||
|
<div class="modal">
|
||||||
|
<h3>Tag File</h3>
|
||||||
|
<p style="margin-bottom:12px;font-size:14px" id="tag-filename"></p>
|
||||||
|
<label>Tag / Folder</label>
|
||||||
|
<input type="text" id="tag-name" placeholder="e.g. photos">
|
||||||
|
<div class="actions">
|
||||||
|
<button class="btn btn-secondary" onclick="hideTagModal()">Cancel</button>
|
||||||
|
<button class="btn btn-primary" onclick="addTag()">Tag</button>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div class="modal-overlay" id="preview-modal" onclick="hidePreview()">
|
||||||
|
<div class="modal preview-modal" onclick="event.stopPropagation()">
|
||||||
|
<div class="preview-header">
|
||||||
|
<h3 id="preview-filename"></h3>
|
||||||
|
<button class="btn btn-secondary btn-sm" onclick="hidePreview()">✕</button>
|
||||||
|
</div>
|
||||||
|
<div class="preview-content" id="preview-content">
|
||||||
|
<div class="preview-loading">Loading preview...</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<script>
|
||||||
|
const API = '/api/v2/myfiles';
|
||||||
|
let username = 'demo';
|
||||||
|
let currentFolder = '';
|
||||||
|
let allFiles = [];
|
||||||
|
|
||||||
|
async function init() {
|
||||||
|
try {
|
||||||
|
const res = await fetch('/api/v2/auth/verify');
|
||||||
|
const data = await res.json();
|
||||||
|
if (data.user) username = data.user;
|
||||||
|
} catch(e) {}
|
||||||
|
document.getElementById('user-info').textContent = 'User: ' + username;
|
||||||
|
loadFolders();
|
||||||
|
loadFiles();
|
||||||
|
}
|
||||||
|
|
||||||
|
async function loadFolders() {
|
||||||
|
try {
|
||||||
|
const res = await fetch(`${API}/${username}/folders`);
|
||||||
|
const folders = await res.json();
|
||||||
|
const list = document.getElementById('folder-list');
|
||||||
|
list.innerHTML = '<li class="active" data-folder="" onclick="selectFolder(\'\')">All Files</li>';
|
||||||
|
for (const f of folders) {
|
||||||
|
const li = document.createElement('li');
|
||||||
|
li.dataset.folder = f.name;
|
||||||
|
li.onclick = () => selectFolder(f.name);
|
||||||
|
li.innerHTML = `📁 ${f.name} <span class="count">${f.file_count}</span>`;
|
||||||
|
list.appendChild(li);
|
||||||
|
}
|
||||||
|
} catch(e) { console.error(e); }
|
||||||
|
}
|
||||||
|
|
||||||
|
async function loadFiles() {
|
||||||
|
const search = document.getElementById('search').value;
|
||||||
|
let url = `${API}/${username}/files`;
|
||||||
|
if (currentFolder) url += `?folder=${encodeURIComponent(currentFolder)}`;
|
||||||
|
try {
|
||||||
|
const res = await fetch(url);
|
||||||
|
allFiles = await res.json();
|
||||||
|
const filtered = search ? allFiles.filter(f => f.name.toLowerCase().includes(search.toLowerCase())) : allFiles;
|
||||||
|
renderFiles(filtered);
|
||||||
|
} catch(e) { console.error(e); }
|
||||||
|
}
|
||||||
|
|
||||||
|
function renderFiles(files) {
|
||||||
|
const grid = document.getElementById('file-grid');
|
||||||
|
const empty = document.getElementById('empty-state');
|
||||||
|
grid.innerHTML = '';
|
||||||
|
if (files.length === 0) { empty.style.display = 'block'; return; }
|
||||||
|
empty.style.display = 'none';
|
||||||
|
for (const f of files) {
|
||||||
|
const card = document.createElement('div');
|
||||||
|
card.className = 'file-card';
|
||||||
|
let tagHtml = '';
|
||||||
|
for (const t of f.tags) {
|
||||||
|
tagHtml += `<span class="tag blue" onclick="event.stopPropagation();removeTag('${f.name}','${t}')" style="cursor:pointer">${t} ×</span>`;
|
||||||
|
}
|
||||||
|
tagHtml += `<span class="tag" onclick="showTagModal('${f.name}')" style="cursor:pointer">+ tag</span>`;
|
||||||
|
card.innerHTML = `
|
||||||
|
<div class="name" style="cursor:pointer;color:#0071e3" onclick="previewFile('${f.name}')">${f.name}</div>
|
||||||
|
<div class="size">${formatSize(f.size)}</div>
|
||||||
|
<div class="tags">${tagHtml}</div>
|
||||||
|
`;
|
||||||
|
grid.appendChild(card);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function formatSize(bytes) {
|
||||||
|
if (bytes < 1024) return bytes + ' B';
|
||||||
|
if (bytes < 1048576) return (bytes/1024).toFixed(1) + ' KB';
|
||||||
|
if (bytes < 1073741824) return (bytes/1048576).toFixed(1) + ' MB';
|
||||||
|
return (bytes/1073741824).toFixed(1) + ' GB';
|
||||||
|
}
|
||||||
|
|
||||||
|
function selectFolder(folder) {
|
||||||
|
currentFolder = folder;
|
||||||
|
document.querySelectorAll('.folder-list li').forEach(li => li.classList.remove('active'));
|
||||||
|
const target = document.querySelector(`[data-folder="${folder}"]`);
|
||||||
|
if (target) target.classList.add('active');
|
||||||
|
loadFiles();
|
||||||
|
}
|
||||||
|
|
||||||
|
function showNewFolderModal() { document.getElementById('folder-modal').classList.add('show'); }
|
||||||
|
function hideFolderModal() { document.getElementById('folder-modal').classList.remove('show'); }
|
||||||
|
|
||||||
|
async function createFolder() {
|
||||||
|
const name = document.getElementById('folder-name').value.trim();
|
||||||
|
const desc = document.getElementById('folder-desc').value.trim();
|
||||||
|
if (!name) return;
|
||||||
|
await fetch(`${API}/${username}/folders`, {
|
||||||
|
method: 'POST', headers: {'Content-Type': 'application/json'},
|
||||||
|
body: JSON.stringify({name, description: desc})
|
||||||
|
});
|
||||||
|
document.getElementById('folder-name').value = '';
|
||||||
|
document.getElementById('folder-desc').value = '';
|
||||||
|
hideFolderModal();
|
||||||
|
loadFolders();
|
||||||
|
}
|
||||||
|
|
||||||
|
function showTagModal(filename) {
|
||||||
|
document.getElementById('tag-filename').textContent = filename;
|
||||||
|
document.getElementById('tag-name').value = '';
|
||||||
|
document.getElementById('tag-modal').classList.add('show');
|
||||||
|
}
|
||||||
|
function hideTagModal() { document.getElementById('tag-modal').classList.remove('show'); }
|
||||||
|
|
||||||
|
async function addTag() {
|
||||||
|
const filename = document.getElementById('tag-filename').textContent;
|
||||||
|
const tag = document.getElementById('tag-name').value.trim();
|
||||||
|
if (!tag) return;
|
||||||
|
await fetch(`${API}/${username}/tags`, {
|
||||||
|
method: 'POST', headers: {'Content-Type': 'application/json'},
|
||||||
|
body: JSON.stringify({file: filename, tag})
|
||||||
|
});
|
||||||
|
hideTagModal();
|
||||||
|
loadFolders();
|
||||||
|
loadFiles();
|
||||||
|
}
|
||||||
|
|
||||||
|
async function removeTag(file, tag) {
|
||||||
|
await fetch(`${API}/${username}/tags`, {
|
||||||
|
method: 'DELETE', headers: {'Content-Type': 'application/json'},
|
||||||
|
body: JSON.stringify({file, tag})
|
||||||
|
});
|
||||||
|
loadFolders();
|
||||||
|
loadFiles();
|
||||||
|
}
|
||||||
|
|
||||||
|
function showUploadModal() { alert('Upload via WebDAV at http://webdav.momentry.ddns.net (user: ' + username + ')'); }
|
||||||
|
|
||||||
|
async function previewFile(filename) {
|
||||||
|
document.getElementById('preview-filename').textContent = filename;
|
||||||
|
document.getElementById('preview-content').innerHTML = '<div class="preview-loading">Loading preview...</div>';
|
||||||
|
document.getElementById('preview-modal').classList.add('show');
|
||||||
|
|
||||||
|
const ext = filename.split('.').pop()?.toLowerCase() || '';
|
||||||
|
const imageExts = ['png','jpg','jpeg','gif','webp','svg'];
|
||||||
|
const videoExts = ['mp4','webm','mov','avi','mkv','m4v'];
|
||||||
|
const audioExts = ['mp3','m4a','wav','flac','ogg','aac'];
|
||||||
|
const textExts = ['txt','md','json','yaml','yml','toml','log','csv','xml','html','js','ts','rs','py','sh','css','ini','cfg','conf'];
|
||||||
|
|
||||||
|
if (imageExts.includes(ext)) {
|
||||||
|
document.getElementById('preview-content').innerHTML = `<img src="${API}/${username}/preview/${encodeURIComponent(filename)}" alt="${filename}" style="max-width:100%;height:auto">`;
|
||||||
|
} else if (videoExts.includes(ext)) {
|
||||||
|
document.getElementById('preview-content').innerHTML = `<video controls autoplay style="max-width:100%;max-height:70vh"><source src="${API}/${username}/preview/${encodeURIComponent(filename)}" type="video/${ext === 'm4v' ? 'mp4' : ext}"></video>`;
|
||||||
|
} else if (audioExts.includes(ext)) {
|
||||||
|
document.getElementById('preview-content').innerHTML = `<audio controls autoplay style="width:100%"><source src="${API}/${username}/preview/${encodeURIComponent(filename)}" type="audio/${ext === 'm4a' ? 'mp4' : ext}"></audio>`;
|
||||||
|
} else if (ext === 'pdf') {
|
||||||
|
document.getElementById('preview-content').innerHTML = `<iframe src="${API}/${username}/preview/${encodeURIComponent(filename)}"></iframe>`;
|
||||||
|
} else if (textExts.includes(ext)) {
|
||||||
|
try {
|
||||||
|
const res = await fetch(`${API}/${username}/preview/${encodeURIComponent(filename)}`);
|
||||||
|
if (!res.ok) throw new Error('Preview failed');
|
||||||
|
const text = await res.text();
|
||||||
|
document.getElementById('preview-content').innerHTML = `<pre>${escapeHtml(text)}</pre>`;
|
||||||
|
} catch(e) {
|
||||||
|
document.getElementById('preview-content').innerHTML = '<div class="file-meta">Preview not available</div>';
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
const size = allFiles.find(f => f.name === filename)?.size || 0;
|
||||||
|
document.getElementById('preview-content').innerHTML = `
|
||||||
|
<div class="file-meta">
|
||||||
|
<p>${filename}</p>
|
||||||
|
<p>${formatSize(size)}</p>
|
||||||
|
<p style="margin-top:12px"><a href="${API}/${username}/preview/${encodeURIComponent(filename)}" download="${filename}">⬇ Download</a></p>
|
||||||
|
</div>`;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function hidePreview() {
|
||||||
|
document.getElementById('preview-modal').classList.remove('show');
|
||||||
|
}
|
||||||
|
|
||||||
|
function escapeHtml(text) {
|
||||||
|
const d = document.createElement('div');
|
||||||
|
d.textContent = text;
|
||||||
|
return d.innerHTML;
|
||||||
|
}
|
||||||
|
|
||||||
|
init();
|
||||||
|
</script>
|
||||||
|
</body>
|
||||||
|
</html>"#;
|
||||||
@@ -489,7 +489,7 @@ function showTreeLoginModal(){
|
|||||||
'<div style="color:#60a5fa;font-size:16px;font-weight:600;margin-bottom:16px">File Tree Authentication</div>'+
|
'<div style="color:#60a5fa;font-size:16px;font-weight:600;margin-bottom:16px">File Tree Authentication</div>'+
|
||||||
'<div style="margin-bottom:12px">'+
|
'<div style="margin-bottom:12px">'+
|
||||||
'<label style="color:#94a3b8;font-size:13px;display:block;margin-bottom:4px">User ID</label>'+
|
'<label style="color:#94a3b8;font-size:13px;display:block;margin-bottom:4px">User ID</label>'+
|
||||||
'<input style="background:#0f172a;border:1px solid #60a5fa;border-radius:4px;color:#e2e8f0;padding:8px 12px;width:100%;font-size:13px" type=text id=tree-user placeholder="Enter user ID (e.g., demo)">'+
|
'<input style="background:#0f172a;border:1px solid #60a5fa;border-radius:4px;color:#e2e8f0;padding:8px 12px;width:100%;font-size:13px" type=text id=tree-user placeholder="Enter user ID" value="demo">'+
|
||||||
'</div>'+
|
'</div>'+
|
||||||
'<div style="margin-bottom:12px;position:relative">'+
|
'<div style="margin-bottom:12px;position:relative">'+
|
||||||
'<label style="color:#94a3b8;font-size:13px;display:block;margin-bottom:4px">Password</label>'+
|
'<label style="color:#94a3b8;font-size:13px;display:block;margin-bottom:4px">Password</label>'+
|
||||||
@@ -501,13 +501,13 @@ function showTreeLoginModal(){
|
|||||||
document.body.appendChild(m);
|
document.body.appendChild(m);
|
||||||
}
|
}
|
||||||
|
|
||||||
document.getElementById('tree-user').value='';
|
document.getElementById('tree-user').value='demo';
|
||||||
document.getElementById('tree-password').value='';
|
document.getElementById('tree-password').value='';
|
||||||
document.getElementById('tree-password').type='password';
|
document.getElementById('tree-password').type='password';
|
||||||
document.getElementById('tree-error').textContent='';
|
document.getElementById('tree-error').textContent='';
|
||||||
m.classList.add('active');
|
m.classList.add('active');
|
||||||
m.style.display='block';
|
m.style.display='block';
|
||||||
document.getElementById('tree-user').focus();
|
document.getElementById('tree-password').focus();
|
||||||
}
|
}
|
||||||
|
|
||||||
function handleTreeKeyPress(e){
|
function handleTreeKeyPress(e){
|
||||||
|
|||||||
@@ -0,0 +1,374 @@
|
|||||||
|
//! LDAP Authentication Provider
|
||||||
|
//!
|
||||||
|
//! Implements DataProvider trait for LDAP/Active Directory authentication.
|
||||||
|
//! Compatible with OpenLDAP and Microsoft Active Directory.
|
||||||
|
//! Uses tokio::spawn_blocking to wrap async LDAP operations.
|
||||||
|
|
||||||
|
use std::path::PathBuf;
|
||||||
|
use tracing::{info, warn, debug};
|
||||||
|
|
||||||
|
use super::{User, ProviderError, DataProvider};
|
||||||
|
|
||||||
|
/// LDAP Provider Configuration
|
||||||
|
#[derive(Debug, Clone)]
|
||||||
|
pub struct LdapConfig {
|
||||||
|
/// LDAP server URL (e.g., ldap://server:389 or ldaps://server:636)
|
||||||
|
pub ldap_url: String,
|
||||||
|
|
||||||
|
/// Base DN for user searches (e.g., dc=example,dc=com)
|
||||||
|
pub base_dn: String,
|
||||||
|
|
||||||
|
/// Bind DN for authenticated searches (e.g., cn=admin,dc=example,dc=com)
|
||||||
|
pub bind_dn: String,
|
||||||
|
|
||||||
|
/// Bind password for authenticated searches
|
||||||
|
pub bind_password: String,
|
||||||
|
|
||||||
|
/// User search base (e.g., ou=users,dc=example,dc=com)
|
||||||
|
pub user_search_base: String,
|
||||||
|
|
||||||
|
/// Group search base (e.g., ou=groups,dc=example,dc=com)
|
||||||
|
pub group_search_base: String,
|
||||||
|
|
||||||
|
/// User object class filter (default: (objectClass=person))
|
||||||
|
pub user_filter: String,
|
||||||
|
|
||||||
|
/// Group object class filter (default: (objectClass=group))
|
||||||
|
pub group_filter: String,
|
||||||
|
|
||||||
|
/// User ID attribute (default: uid for OpenLDAP, sAMAccountName for AD)
|
||||||
|
pub user_id_attr: String,
|
||||||
|
|
||||||
|
/// User groups attribute (default: memberOf for AD)
|
||||||
|
pub user_groups_attr: String,
|
||||||
|
|
||||||
|
/// Home directory attribute (default: homeDirectory)
|
||||||
|
pub home_dir_attr: String,
|
||||||
|
|
||||||
|
/// Default home directory path prefix
|
||||||
|
pub home_dir_prefix: String,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Default for LdapConfig {
|
||||||
|
fn default() -> Self {
|
||||||
|
Self {
|
||||||
|
ldap_url: "ldap://localhost:389".to_string(),
|
||||||
|
base_dn: "dc=example,dc=com".to_string(),
|
||||||
|
bind_dn: "cn=admin,dc=example,dc=com".to_string(),
|
||||||
|
bind_password: "".to_string(),
|
||||||
|
user_search_base: "ou=users,dc=example,dc=com".to_string(),
|
||||||
|
group_search_base: "ou=groups,dc=example,dc=com".to_string(),
|
||||||
|
user_filter: "(objectClass=person)".to_string(),
|
||||||
|
group_filter: "(objectClass=group)".to_string(),
|
||||||
|
user_id_attr: "uid".to_string(),
|
||||||
|
user_groups_attr: "memberOf".to_string(),
|
||||||
|
home_dir_attr: "homeDirectory".to_string(),
|
||||||
|
home_dir_prefix: "/home".to_string(),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl LdapConfig {
|
||||||
|
/// Create Active Directory configuration
|
||||||
|
pub fn for_ad(ldap_url: String, base_dn: String, bind_dn: String, bind_password: String) -> Self {
|
||||||
|
let user_search_base = base_dn.clone();
|
||||||
|
let group_search_base = base_dn.clone();
|
||||||
|
|
||||||
|
Self {
|
||||||
|
ldap_url,
|
||||||
|
base_dn,
|
||||||
|
bind_dn,
|
||||||
|
bind_password,
|
||||||
|
user_search_base,
|
||||||
|
group_search_base,
|
||||||
|
user_filter: "(objectClass=user)".to_string(),
|
||||||
|
group_filter: "(objectClass=group)".to_string(),
|
||||||
|
user_id_attr: "sAMAccountName".to_string(),
|
||||||
|
user_groups_attr: "memberOf".to_string(),
|
||||||
|
home_dir_attr: "homeDirectory".to_string(),
|
||||||
|
home_dir_prefix: "/home".to_string(),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// LDAP Provider (uses blocking wrapper for async LDAP operations)
|
||||||
|
pub struct LdapProvider {
|
||||||
|
config: LdapConfig,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl LdapProvider {
|
||||||
|
pub fn new(config: LdapConfig) -> Self {
|
||||||
|
Self { config }
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Async implementation of get_user (internal)
|
||||||
|
async fn get_user_async(&self, username: &str) -> Result<Option<User>, ProviderError> {
|
||||||
|
use ldap3::{Ldap, LdapConnAsync, Scope, SearchEntry};
|
||||||
|
|
||||||
|
// Connect to LDAP
|
||||||
|
let (conn, mut ldap) = LdapConnAsync::new(&self.config.ldap_url).await
|
||||||
|
.map_err(|e| ProviderError::Internal(format!("LDAP connection failed: {}", e)))?;
|
||||||
|
ldap3::drive!(conn);
|
||||||
|
|
||||||
|
// Bind with admin credentials
|
||||||
|
ldap.simple_bind(&self.config.bind_dn, &self.config.bind_password).await
|
||||||
|
.map_err(|e| ProviderError::AuthFailed(format!("LDAP bind failed: {}", e)))?;
|
||||||
|
|
||||||
|
// Search for user
|
||||||
|
let filter = format!("(&{}({}={}))",
|
||||||
|
self.config.user_filter,
|
||||||
|
self.config.user_id_attr,
|
||||||
|
username
|
||||||
|
);
|
||||||
|
|
||||||
|
let result = ldap
|
||||||
|
.search(
|
||||||
|
&self.config.user_search_base,
|
||||||
|
Scope::Subtree,
|
||||||
|
&filter,
|
||||||
|
&[
|
||||||
|
&self.config.user_id_attr,
|
||||||
|
&self.config.user_groups_attr,
|
||||||
|
&self.config.home_dir_attr,
|
||||||
|
"uidNumber",
|
||||||
|
"gidNumber",
|
||||||
|
],
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.map_err(|e| ProviderError::Internal(format!("LDAP search failed: {}", e)))?;
|
||||||
|
|
||||||
|
let (entries, _controls) = result.success()
|
||||||
|
.map_err(|e| ProviderError::Internal(format!("LDAP search result error: {}", e)))?;
|
||||||
|
|
||||||
|
ldap.unbind().await.ok();
|
||||||
|
|
||||||
|
if entries.is_empty() {
|
||||||
|
return Ok(None);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Parse first entry using SearchEntry::construct()
|
||||||
|
let search_entry = SearchEntry::construct(entries.into_iter().next().unwrap());
|
||||||
|
|
||||||
|
// Extract attributes
|
||||||
|
let home_dir = search_entry.attrs.get(&self.config.home_dir_attr)
|
||||||
|
.and_then(|v| v.first().cloned())
|
||||||
|
.map(PathBuf::from)
|
||||||
|
.unwrap_or_else(|| {
|
||||||
|
PathBuf::from(&self.config.home_dir_prefix).join(username)
|
||||||
|
});
|
||||||
|
|
||||||
|
let uid = search_entry.attrs.get("uidNumber")
|
||||||
|
.and_then(|v| v.first().and_then(|s| s.parse().ok()))
|
||||||
|
.unwrap_or(1000);
|
||||||
|
|
||||||
|
let gid = search_entry.attrs.get("gidNumber")
|
||||||
|
.and_then(|v| v.first().and_then(|s| s.parse().ok()))
|
||||||
|
.unwrap_or(1000);
|
||||||
|
|
||||||
|
info!(username, uid, gid, home_dir = %home_dir.display(), dn = %search_entry.dn, "LDAP user found");
|
||||||
|
|
||||||
|
Ok(Some(User {
|
||||||
|
username: username.to_string(),
|
||||||
|
password_hash: "".to_string(),
|
||||||
|
home_dir,
|
||||||
|
uid,
|
||||||
|
gid,
|
||||||
|
permissions: "read-write".to_string(),
|
||||||
|
status: 1,
|
||||||
|
}))
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Async implementation of check_password (internal)
|
||||||
|
async fn check_password_async(&self, username: &str, password: &str) -> Result<bool, ProviderError> {
|
||||||
|
use ldap3::{LdapConnAsync, Scope, SearchEntry};
|
||||||
|
|
||||||
|
// First get user DN
|
||||||
|
let (conn, mut ldap) = LdapConnAsync::new(&self.config.ldap_url).await
|
||||||
|
.map_err(|e| ProviderError::Internal(format!("LDAP connection failed: {}", e)))?;
|
||||||
|
ldap3::drive!(conn);
|
||||||
|
|
||||||
|
ldap.simple_bind(&self.config.bind_dn, &self.config.bind_password).await
|
||||||
|
.map_err(|e| ProviderError::AuthFailed(format!("LDAP bind failed: {}", e)))?;
|
||||||
|
|
||||||
|
let filter = format!("(&{}({}={}))",
|
||||||
|
self.config.user_filter,
|
||||||
|
self.config.user_id_attr,
|
||||||
|
username
|
||||||
|
);
|
||||||
|
|
||||||
|
let result = ldap
|
||||||
|
.search(
|
||||||
|
&self.config.user_search_base,
|
||||||
|
Scope::Subtree,
|
||||||
|
&filter,
|
||||||
|
&["dn"],
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.map_err(|e| ProviderError::Internal(format!("LDAP search failed: {}", e)))?;
|
||||||
|
|
||||||
|
let (entries, _controls) = result.success()
|
||||||
|
.map_err(|e| ProviderError::Internal(format!("LDAP search result error: {}", e)))?;
|
||||||
|
|
||||||
|
if entries.is_empty() {
|
||||||
|
ldap.unbind().await.ok();
|
||||||
|
return Ok(false);
|
||||||
|
}
|
||||||
|
|
||||||
|
let search_entry = SearchEntry::construct(entries.into_iter().next().unwrap());
|
||||||
|
let user_dn = search_entry.dn;
|
||||||
|
|
||||||
|
ldap.unbind().await.ok();
|
||||||
|
|
||||||
|
// Try to bind as the user
|
||||||
|
let (conn2, mut user_ldap) = LdapConnAsync::new(&self.config.ldap_url).await
|
||||||
|
.map_err(|e| ProviderError::Internal(format!("LDAP connection failed: {}", e)))?;
|
||||||
|
ldap3::drive!(conn2);
|
||||||
|
|
||||||
|
let result = user_ldap.simple_bind(&user_dn, password).await;
|
||||||
|
user_ldap.unbind().await.ok();
|
||||||
|
|
||||||
|
match result {
|
||||||
|
Ok(_) => {
|
||||||
|
debug!(username, "LDAP password verification successful");
|
||||||
|
Ok(true)
|
||||||
|
}
|
||||||
|
Err(e) => {
|
||||||
|
debug!(username, error = %e, "LDAP password verification failed");
|
||||||
|
Ok(false)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Async implementation of get_user_groups (internal)
|
||||||
|
async fn get_user_groups_async(&self, username: &str) -> Result<Vec<String>, ProviderError> {
|
||||||
|
use ldap3::{LdapConnAsync, Scope, SearchEntry};
|
||||||
|
|
||||||
|
let (conn, mut ldap) = LdapConnAsync::new(&self.config.ldap_url).await
|
||||||
|
.map_err(|e| ProviderError::Internal(format!("LDAP connection failed: {}", e)))?;
|
||||||
|
ldap3::drive!(conn);
|
||||||
|
|
||||||
|
ldap.simple_bind(&self.config.bind_dn, &self.config.bind_password).await
|
||||||
|
.map_err(|e| ProviderError::AuthFailed(format!("LDAP bind failed: {}", e)))?;
|
||||||
|
|
||||||
|
let filter = format!("(&{}({}={}))",
|
||||||
|
self.config.user_filter,
|
||||||
|
self.config.user_id_attr,
|
||||||
|
username
|
||||||
|
);
|
||||||
|
|
||||||
|
let result = ldap
|
||||||
|
.search(
|
||||||
|
&self.config.user_search_base,
|
||||||
|
Scope::Subtree,
|
||||||
|
&filter,
|
||||||
|
&[&self.config.user_groups_attr],
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.map_err(|e| ProviderError::Internal(format!("LDAP search failed: {}", e)))?;
|
||||||
|
|
||||||
|
let (entries, _controls) = result.success()
|
||||||
|
.map_err(|e| ProviderError::Internal(format!("LDAP search result error: {}", e)))?;
|
||||||
|
|
||||||
|
ldap.unbind().await.ok();
|
||||||
|
|
||||||
|
if entries.is_empty() {
|
||||||
|
return Ok(Vec::new());
|
||||||
|
}
|
||||||
|
|
||||||
|
let search_entry = SearchEntry::construct(entries.into_iter().next().unwrap());
|
||||||
|
let attrs = search_entry.attrs;
|
||||||
|
|
||||||
|
let groups: Vec<String> = attrs
|
||||||
|
.get(&self.config.user_groups_attr)
|
||||||
|
.map(|v| v.clone())
|
||||||
|
.unwrap_or_default();
|
||||||
|
|
||||||
|
// Extract group names from DN (e.g., CN=group1,OU=groups,DC=example,DC=com -> group1)
|
||||||
|
let group_names: Vec<String> = groups.iter()
|
||||||
|
.filter_map(|dn| {
|
||||||
|
dn.split(',')
|
||||||
|
.next()
|
||||||
|
.and_then(|s| s.strip_prefix("CN="))
|
||||||
|
.map(|s| s.to_string())
|
||||||
|
})
|
||||||
|
.collect();
|
||||||
|
|
||||||
|
debug!(username, groups = ?group_names, "LDAP groups found");
|
||||||
|
Ok(group_names)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl DataProvider for LdapProvider {
|
||||||
|
fn get_user(&self, username: &str) -> Result<Option<User>, ProviderError> {
|
||||||
|
// Use tokio runtime to run async operation
|
||||||
|
let config = self.config.clone();
|
||||||
|
let username = username.to_string();
|
||||||
|
|
||||||
|
tokio::task::block_in_place(|| {
|
||||||
|
tokio::runtime::Handle::current().block_on(async {
|
||||||
|
let provider = LdapProvider::new(config);
|
||||||
|
provider.get_user_async(&username).await
|
||||||
|
})
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
fn check_password(&self, username: &str, password: &str) -> Result<bool, ProviderError> {
|
||||||
|
let config = self.config.clone();
|
||||||
|
let username = username.to_string();
|
||||||
|
let password = password.to_string();
|
||||||
|
|
||||||
|
tokio::task::block_in_place(|| {
|
||||||
|
tokio::runtime::Handle::current().block_on(async {
|
||||||
|
let provider = LdapProvider::new(config);
|
||||||
|
provider.check_password_async(&username, &password).await
|
||||||
|
})
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
fn get_home_dir(&self, username: &str) -> Result<Option<String>, ProviderError> {
|
||||||
|
let user = self.get_user(username)?;
|
||||||
|
Ok(user.map(|u| u.home_dir.to_string_lossy().to_string()))
|
||||||
|
}
|
||||||
|
|
||||||
|
fn get_user_groups(&self, username: &str) -> Result<Vec<String>, ProviderError> {
|
||||||
|
let config = self.config.clone();
|
||||||
|
let username = username.to_string();
|
||||||
|
|
||||||
|
tokio::task::block_in_place(|| {
|
||||||
|
tokio::runtime::Handle::current().block_on(async {
|
||||||
|
let provider = LdapProvider::new(config);
|
||||||
|
provider.get_user_groups_async(&username).await
|
||||||
|
})
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
fn get_public_keys(&self, username: &str) -> Result<Vec<String>, ProviderError> {
|
||||||
|
// LDAP typically doesn't store SSH public keys
|
||||||
|
debug!(username, "LDAP provider doesn't support public keys");
|
||||||
|
Ok(Vec::new())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use super::*;
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_ldap_config_default() {
|
||||||
|
let config = LdapConfig::default();
|
||||||
|
assert_eq!(config.user_id_attr, "uid");
|
||||||
|
assert_eq!(config.user_groups_attr, "memberOf");
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_ldap_config_for_ad() {
|
||||||
|
let config = LdapConfig::for_ad(
|
||||||
|
"ldap://ad.example.com:389".to_string(),
|
||||||
|
"dc=example,dc=com".to_string(),
|
||||||
|
"cn=admin,dc=example,dc=com".to_string(),
|
||||||
|
"password".to_string(),
|
||||||
|
);
|
||||||
|
assert_eq!(config.user_id_attr, "sAMAccountName");
|
||||||
|
assert_eq!(config.user_filter, "(objectClass=user)");
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -1,8 +1,13 @@
|
|||||||
pub mod pg;
|
pub mod pg;
|
||||||
pub mod sqlite;
|
pub mod sqlite;
|
||||||
|
#[cfg(feature = "ldap")]
|
||||||
|
#[cfg(feature = "ldap")]
|
||||||
|
pub mod ldap;
|
||||||
|
|
||||||
pub use pg::PgProvider;
|
pub use pg::PgProvider;
|
||||||
pub use sqlite::SqliteProvider;
|
pub use sqlite::SqliteProvider;
|
||||||
|
#[cfg(feature = "ldap")]
|
||||||
|
pub use ldap::{LdapProvider, LdapConfig};
|
||||||
|
|
||||||
use std::path::PathBuf;
|
use std::path::PathBuf;
|
||||||
|
|
||||||
@@ -68,4 +73,19 @@ pub trait DataProvider: Send + Sync {
|
|||||||
let _ = username;
|
let _ = username;
|
||||||
Ok(Vec::new())
|
Ok(Vec::new())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// 列出所有用户
|
||||||
|
fn list_users(&self) -> Result<Vec<User>, ProviderError>;
|
||||||
|
|
||||||
|
/// 创建用户
|
||||||
|
fn create_user(&self, user: &User, password: &str) -> Result<(), ProviderError>;
|
||||||
|
|
||||||
|
/// 更新用户
|
||||||
|
fn update_user(&self, user: &User, new_password: Option<&str>) -> Result<(), ProviderError>;
|
||||||
|
|
||||||
|
/// 删除用户
|
||||||
|
fn delete_user(&self, username: &str) -> Result<(), ProviderError>;
|
||||||
|
|
||||||
|
/// 重置密码
|
||||||
|
fn reset_password(&self, username: &str, new_password: &str) -> Result<(), ProviderError>;
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -115,6 +115,102 @@ impl DataProvider for PgProvider {
|
|||||||
None => Ok(Vec::new()),
|
None => Ok(Vec::new()),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn list_users(&self) -> Result<Vec<User>, ProviderError> {
|
||||||
|
let mut conn = self.open_conn()?;
|
||||||
|
|
||||||
|
let rows = conn
|
||||||
|
.query(
|
||||||
|
"SELECT username, password, home_dir, permissions, uid, gid, status
|
||||||
|
FROM users ORDER BY username",
|
||||||
|
&[],
|
||||||
|
)
|
||||||
|
.map_err(|e| ProviderError::Internal(format!("Query error: {}", e)))?;
|
||||||
|
|
||||||
|
let users = rows
|
||||||
|
.iter()
|
||||||
|
.map(|row| User {
|
||||||
|
username: row.get(0),
|
||||||
|
password_hash: row.get::<_, Option<String>>(1).unwrap_or_default(),
|
||||||
|
home_dir: PathBuf::from(row.get::<_, String>(2)),
|
||||||
|
permissions: row
|
||||||
|
.get::<_, Option<String>>(3)
|
||||||
|
.unwrap_or_else(|| "*".to_string()),
|
||||||
|
uid: row.get::<_, i64>(4) as u32,
|
||||||
|
gid: row.get::<_, i64>(5) as u32,
|
||||||
|
status: row.get(6),
|
||||||
|
})
|
||||||
|
.collect();
|
||||||
|
|
||||||
|
Ok(users)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn create_user(&self, user: &User, password: &str) -> Result<(), ProviderError> {
|
||||||
|
let mut conn = self.open_conn()?;
|
||||||
|
|
||||||
|
let hash = bcrypt::hash(password, bcrypt::DEFAULT_COST)
|
||||||
|
.map_err(|e| ProviderError::Internal(format!("bcrypt hash error: {}", e)))?;
|
||||||
|
|
||||||
|
conn.execute(
|
||||||
|
"INSERT INTO users (username, password, home_dir, permissions, uid, gid, status)
|
||||||
|
VALUES ($1, $2, $3, $4, $5, $6, $7)",
|
||||||
|
&[&user.username, &hash, &user.home_dir.to_string_lossy(), &user.permissions, &(user.uid as i64), &(user.gid as i64), &user.status],
|
||||||
|
)
|
||||||
|
.map_err(|e| ProviderError::Internal(format!("Insert error: {}", e)))?;
|
||||||
|
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
fn update_user(&self, user: &User, new_password: Option<&str>) -> Result<(), ProviderError> {
|
||||||
|
let mut conn = self.open_conn()?;
|
||||||
|
|
||||||
|
if let Some(pwd) = new_password {
|
||||||
|
let hash = bcrypt::hash(pwd, bcrypt::DEFAULT_COST)
|
||||||
|
.map_err(|e| ProviderError::Internal(format!("bcrypt hash error: {}", e)))?;
|
||||||
|
|
||||||
|
conn.execute(
|
||||||
|
"UPDATE users
|
||||||
|
SET password = $2, home_dir = $3, permissions = $4, uid = $5, gid = $6, status = $7
|
||||||
|
WHERE username = $1",
|
||||||
|
&[&user.username, &hash, &user.home_dir.to_string_lossy(), &user.permissions, &(user.uid as i64), &(user.gid as i64), &user.status],
|
||||||
|
)
|
||||||
|
.map_err(|e| ProviderError::Internal(format!("Update error: {}", e)))?;
|
||||||
|
} else {
|
||||||
|
conn.execute(
|
||||||
|
"UPDATE users
|
||||||
|
SET home_dir = $2, permissions = $3, uid = $4, gid = $5, status = $6
|
||||||
|
WHERE username = $1",
|
||||||
|
&[&user.username, &user.home_dir.to_string_lossy(), &user.permissions, &(user.uid as i64), &(user.gid as i64), &user.status],
|
||||||
|
)
|
||||||
|
.map_err(|e| ProviderError::Internal(format!("Update error: {}", e)))?;
|
||||||
|
}
|
||||||
|
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
fn delete_user(&self, username: &str) -> Result<(), ProviderError> {
|
||||||
|
let mut conn = self.open_conn()?;
|
||||||
|
|
||||||
|
conn.execute("DELETE FROM users WHERE username = $1", &[&username])
|
||||||
|
.map_err(|e| ProviderError::Internal(format!("Delete error: {}", e)))?;
|
||||||
|
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
fn reset_password(&self, username: &str, new_password: &str) -> Result<(), ProviderError> {
|
||||||
|
let mut conn = self.open_conn()?;
|
||||||
|
|
||||||
|
let hash = bcrypt::hash(new_password, bcrypt::DEFAULT_COST)
|
||||||
|
.map_err(|e| ProviderError::Internal(format!("bcrypt hash error: {}", e)))?;
|
||||||
|
|
||||||
|
conn.execute(
|
||||||
|
"UPDATE users SET password = $2 WHERE username = $1",
|
||||||
|
&[&username, &hash],
|
||||||
|
)
|
||||||
|
.map_err(|e| ProviderError::Internal(format!("Update error: {}", e)))?;
|
||||||
|
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
|
|||||||
@@ -89,6 +89,123 @@ impl DataProvider for SqliteProvider {
|
|||||||
.collect();
|
.collect();
|
||||||
Ok(groups)
|
Ok(groups)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn list_users(&self) -> Result<Vec<User>, ProviderError> {
|
||||||
|
let conn = self.open_conn()?;
|
||||||
|
|
||||||
|
let users = conn
|
||||||
|
.prepare(
|
||||||
|
"SELECT username, password_hash, home_dir, permissions, uid, gid, status
|
||||||
|
FROM sftpgo_users ORDER BY username",
|
||||||
|
)
|
||||||
|
.map_err(|e| ProviderError::Internal(format!("Query prepare error: {}", e)))?
|
||||||
|
.query_map([], |row| {
|
||||||
|
Ok(User {
|
||||||
|
username: row.get(0)?,
|
||||||
|
password_hash: row.get(1)?,
|
||||||
|
home_dir: PathBuf::from(row.get::<_, String>(2)?),
|
||||||
|
permissions: row.get(3)?,
|
||||||
|
uid: row.get::<_, i64>(4)? as u32,
|
||||||
|
gid: row.get::<_, i64>(5)? as u32,
|
||||||
|
status: row.get(6)?,
|
||||||
|
})
|
||||||
|
})
|
||||||
|
.map_err(|e| ProviderError::Internal(format!("Query map error: {}", e)))?
|
||||||
|
.filter_map(|r| r.ok())
|
||||||
|
.collect();
|
||||||
|
|
||||||
|
Ok(users)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn create_user(&self, user: &User, password: &str) -> Result<(), ProviderError> {
|
||||||
|
let conn = self.open_conn()?;
|
||||||
|
|
||||||
|
let hash = bcrypt::hash(password, bcrypt::DEFAULT_COST)
|
||||||
|
.map_err(|e| ProviderError::Internal(format!("bcrypt hash error: {}", e)))?;
|
||||||
|
|
||||||
|
conn.execute(
|
||||||
|
"INSERT INTO sftpgo_users (username, password_hash, home_dir, permissions, uid, gid, status)
|
||||||
|
VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7)",
|
||||||
|
params![
|
||||||
|
user.username,
|
||||||
|
hash,
|
||||||
|
user.home_dir.to_string_lossy(),
|
||||||
|
user.permissions,
|
||||||
|
user.uid as i64,
|
||||||
|
user.gid as i64,
|
||||||
|
user.status,
|
||||||
|
],
|
||||||
|
)
|
||||||
|
.map_err(|e| ProviderError::Internal(format!("Insert error: {}", e)))?;
|
||||||
|
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
fn update_user(&self, user: &User, new_password: Option<&str>) -> Result<(), ProviderError> {
|
||||||
|
let conn = self.open_conn()?;
|
||||||
|
|
||||||
|
if let Some(pwd) = new_password {
|
||||||
|
let hash = bcrypt::hash(pwd, bcrypt::DEFAULT_COST)
|
||||||
|
.map_err(|e| ProviderError::Internal(format!("bcrypt hash error: {}", e)))?;
|
||||||
|
|
||||||
|
conn.execute(
|
||||||
|
"UPDATE sftpgo_users
|
||||||
|
SET password_hash = ?2, home_dir = ?3, permissions = ?4, uid = ?5, gid = ?6, status = ?7
|
||||||
|
WHERE username = ?1",
|
||||||
|
params![
|
||||||
|
user.username,
|
||||||
|
hash,
|
||||||
|
user.home_dir.to_string_lossy(),
|
||||||
|
user.permissions,
|
||||||
|
user.uid as i64,
|
||||||
|
user.gid as i64,
|
||||||
|
user.status,
|
||||||
|
],
|
||||||
|
)
|
||||||
|
.map_err(|e| ProviderError::Internal(format!("Update error: {}", e)))?;
|
||||||
|
} else {
|
||||||
|
conn.execute(
|
||||||
|
"UPDATE sftpgo_users
|
||||||
|
SET home_dir = ?2, permissions = ?3, uid = ?4, gid = ?5, status = ?6
|
||||||
|
WHERE username = ?1",
|
||||||
|
params![
|
||||||
|
user.username,
|
||||||
|
user.home_dir.to_string_lossy(),
|
||||||
|
user.permissions,
|
||||||
|
user.uid as i64,
|
||||||
|
user.gid as i64,
|
||||||
|
user.status,
|
||||||
|
],
|
||||||
|
)
|
||||||
|
.map_err(|e| ProviderError::Internal(format!("Update error: {}", e)))?;
|
||||||
|
}
|
||||||
|
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
fn delete_user(&self, username: &str) -> Result<(), ProviderError> {
|
||||||
|
let conn = self.open_conn()?;
|
||||||
|
|
||||||
|
conn.execute("DELETE FROM sftpgo_users WHERE username = ?1", params![username])
|
||||||
|
.map_err(|e| ProviderError::Internal(format!("Delete error: {}", e)))?;
|
||||||
|
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
fn reset_password(&self, username: &str, new_password: &str) -> Result<(), ProviderError> {
|
||||||
|
let conn = self.open_conn()?;
|
||||||
|
|
||||||
|
let hash = bcrypt::hash(new_password, bcrypt::DEFAULT_COST)
|
||||||
|
.map_err(|e| ProviderError::Internal(format!("bcrypt hash error: {}", e)))?;
|
||||||
|
|
||||||
|
conn.execute(
|
||||||
|
"UPDATE sftpgo_users SET password_hash = ?2 WHERE username = ?1",
|
||||||
|
params![username, hash],
|
||||||
|
)
|
||||||
|
.map_err(|e| ProviderError::Internal(format!("Update error: {}", e)))?;
|
||||||
|
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
|
|||||||
@@ -12,6 +12,8 @@ use futures_util::StreamExt;
|
|||||||
use serde::{Deserialize, Serialize};
|
use serde::{Deserialize, Serialize};
|
||||||
use serde_json::Value;
|
use serde_json::Value;
|
||||||
use sha2::{Digest, Sha256};
|
use sha2::{Digest, Sha256};
|
||||||
|
use std::collections::HashMap;
|
||||||
|
use std::io::Write;
|
||||||
use tokio::io::{AsyncReadExt, AsyncWriteExt};
|
use tokio::io::{AsyncReadExt, AsyncWriteExt};
|
||||||
use tokio_util::io::ReaderStream;
|
use tokio_util::io::ReaderStream;
|
||||||
|
|
||||||
@@ -90,6 +92,12 @@ pub async fn get_object(
|
|||||||
) -> impl IntoResponse {
|
) -> impl IntoResponse {
|
||||||
println!("S3 GET Object: bucket={}, key={}", bucket, key);
|
println!("S3 GET Object: bucket={}, key={}", bucket, key);
|
||||||
|
|
||||||
|
// Policy check - user needs GetObject permission
|
||||||
|
let user_id = extract_user_from_auth(&headers).unwrap_or_else(|| "anonymous".to_string());
|
||||||
|
if !check_bucket_policy(&bucket, "s3:GetObject", &format!("arn:aws:s3:::{}", bucket), &user_id) {
|
||||||
|
return (StatusCode::FORBIDDEN, "Policy denied").into_response();
|
||||||
|
}
|
||||||
|
|
||||||
let conn = match FileTree::open_user_db(&bucket) {
|
let conn = match FileTree::open_user_db(&bucket) {
|
||||||
Ok(c) => c,
|
Ok(c) => c,
|
||||||
Err(e) => {
|
Err(e) => {
|
||||||
@@ -167,10 +175,17 @@ pub async fn get_object(
|
|||||||
pub async fn put_object(
|
pub async fn put_object(
|
||||||
Path((bucket, key)): Path<(String, String)>,
|
Path((bucket, key)): Path<(String, String)>,
|
||||||
State(_state): State<crate::server::AppState>,
|
State(_state): State<crate::server::AppState>,
|
||||||
|
headers: HeaderMap,
|
||||||
body: Body,
|
body: Body,
|
||||||
) -> impl IntoResponse {
|
) -> impl IntoResponse {
|
||||||
println!("S3 PUT Object: bucket={}, key={}", bucket, key);
|
println!("S3 PUT Object: bucket={}, key={}", bucket, key);
|
||||||
|
|
||||||
|
// Policy check - user needs PutObject permission
|
||||||
|
let user_id = extract_user_from_auth(&headers).unwrap_or_else(|| "anonymous".to_string());
|
||||||
|
if !check_bucket_policy(&bucket, "s3:PutObject", &format!("arn:aws:s3:::{}", bucket), &user_id) {
|
||||||
|
return (StatusCode::FORBIDDEN, "Policy denied").into_response();
|
||||||
|
}
|
||||||
|
|
||||||
let base_dir = "/Users/accusys/momentry/var/sftpgo/data";
|
let base_dir = "/Users/accusys/momentry/var/sftpgo/data";
|
||||||
let file_path = format!("{}/{}/{}", base_dir, bucket, key);
|
let file_path = format!("{}/{}/{}", base_dir, bucket, key);
|
||||||
|
|
||||||
@@ -364,9 +379,16 @@ pub async fn generate_s3_key(State(state): State<crate::server::AppState>) -> im
|
|||||||
pub async fn delete_object(
|
pub async fn delete_object(
|
||||||
Path((bucket, key)): Path<(String, String)>,
|
Path((bucket, key)): Path<(String, String)>,
|
||||||
State(_state): State<crate::server::AppState>,
|
State(_state): State<crate::server::AppState>,
|
||||||
|
headers: HeaderMap,
|
||||||
) -> impl IntoResponse {
|
) -> impl IntoResponse {
|
||||||
println!("S3 DELETE Object: bucket={}, key={}", bucket, key);
|
println!("S3 DELETE Object: bucket={}, key={}", bucket, key);
|
||||||
|
|
||||||
|
// Policy check - user needs DeleteObject permission
|
||||||
|
let user_id = extract_user_from_auth(&headers).unwrap_or_else(|| "anonymous".to_string());
|
||||||
|
if !check_bucket_policy(&bucket, "s3:DeleteObject", &format!("arn:aws:s3:::{}", bucket), &user_id) {
|
||||||
|
return (StatusCode::FORBIDDEN, "Policy denied").into_response();
|
||||||
|
}
|
||||||
|
|
||||||
let result = tokio::task::spawn_blocking(move || -> anyhow::Result<()> {
|
let result = tokio::task::spawn_blocking(move || -> anyhow::Result<()> {
|
||||||
let conn = FileTree::open_user_db(&bucket)?;
|
let conn = FileTree::open_user_db(&bucket)?;
|
||||||
let mut tree = FileTree::load(&conn, &bucket, "untitled folder")?;
|
let mut tree = FileTree::load(&conn, &bucket, "untitled folder")?;
|
||||||
@@ -556,3 +578,528 @@ fn parse_range_header(range: &str, file_size: i64) -> Option<(u64, u64)> {
|
|||||||
|
|
||||||
Some((start, end))
|
Some((start, end))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// ===== Multipart Upload Support =====
|
||||||
|
|
||||||
|
use std::sync::Arc;
|
||||||
|
use tokio::sync::RwLock;
|
||||||
|
use uuid::Uuid;
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||||
|
pub struct MultipartUpload {
|
||||||
|
pub upload_id: String,
|
||||||
|
pub bucket: String,
|
||||||
|
pub key: String,
|
||||||
|
pub parts: Vec<UploadedPart>,
|
||||||
|
pub created_at: chrono::DateTime<chrono::Utc>,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||||
|
pub struct UploadedPart {
|
||||||
|
pub part_number: u32,
|
||||||
|
pub etag: String,
|
||||||
|
pub size: u64,
|
||||||
|
}
|
||||||
|
|
||||||
|
static MULTIPART_UPLOADS: once_cell::sync::Lazy<Arc<RwLock<HashMap<String, MultipartUpload>>>> =
|
||||||
|
once_cell::sync::Lazy::new(|| Arc::new(RwLock::new(HashMap::new())));
|
||||||
|
|
||||||
|
pub async fn initiate_multipart_upload(
|
||||||
|
Path((bucket, key)): Path<(String, String)>,
|
||||||
|
State(_state): State<crate::server::AppState>,
|
||||||
|
headers: HeaderMap,
|
||||||
|
) -> impl IntoResponse {
|
||||||
|
// Authentication check
|
||||||
|
if !crate::s3_auth::verify_signature(headers.clone(), "POST", &format!("/s3/multipart/{}/{}?uploads", bucket, key)) {
|
||||||
|
return (StatusCode::FORBIDDEN, "Access denied").into_response();
|
||||||
|
}
|
||||||
|
|
||||||
|
// Policy check - user needs PutObject permission
|
||||||
|
let user_id = extract_user_from_auth(&headers).unwrap_or_else(|| "anonymous".to_string());
|
||||||
|
if !check_bucket_policy(&bucket, "s3:PutObject", &format!("arn:aws:s3:::{}/*", bucket), &user_id) {
|
||||||
|
return (StatusCode::FORBIDDEN, "Policy denied").into_response();
|
||||||
|
}
|
||||||
|
|
||||||
|
let upload_id = Uuid::new_v4().to_string();
|
||||||
|
|
||||||
|
let upload = MultipartUpload {
|
||||||
|
upload_id: upload_id.clone(),
|
||||||
|
bucket: bucket.clone(),
|
||||||
|
key: key.clone(),
|
||||||
|
parts: Vec::new(),
|
||||||
|
created_at: chrono::Utc::now(),
|
||||||
|
};
|
||||||
|
|
||||||
|
{
|
||||||
|
let mut uploads = MULTIPART_UPLOADS.write().await;
|
||||||
|
uploads.insert(upload_id.clone(), upload);
|
||||||
|
}
|
||||||
|
|
||||||
|
let (headers, xml_body) = crate::s3_xml::initiate_multipart_upload_xml(&bucket, &key, &upload_id);
|
||||||
|
(StatusCode::OK, headers, xml_body).into_response()
|
||||||
|
}
|
||||||
|
|
||||||
|
pub async fn upload_part(
|
||||||
|
Path((bucket, key)): Path<(String, String)>,
|
||||||
|
State(_state): State<crate::server::AppState>,
|
||||||
|
query: axum::extract::Query<UploadPartQuery>,
|
||||||
|
headers: HeaderMap,
|
||||||
|
body: Body,
|
||||||
|
) -> impl IntoResponse {
|
||||||
|
// Authentication check
|
||||||
|
if !crate::s3_auth::verify_signature(headers.clone(), "PUT", &format!("/s3/multipart/{}/{}?uploadId={}&partNumber={}", bucket, key, query.upload_id, query.part_number)) {
|
||||||
|
return (StatusCode::FORBIDDEN, "Access denied").into_response();
|
||||||
|
}
|
||||||
|
|
||||||
|
// Policy check
|
||||||
|
let user_id = extract_user_from_auth(&headers).unwrap_or_else(|| "anonymous".to_string());
|
||||||
|
if !check_bucket_policy(&bucket, "s3:PutObject", &format!("arn:aws:s3:::{}/*", bucket), &user_id) {
|
||||||
|
return (StatusCode::FORBIDDEN, "Policy denied").into_response();
|
||||||
|
}
|
||||||
|
|
||||||
|
let upload_id = query.upload_id.clone();
|
||||||
|
let part_number = query.part_number;
|
||||||
|
|
||||||
|
let uploads = MULTIPART_UPLOADS.read().await;
|
||||||
|
let upload = uploads.get(&upload_id);
|
||||||
|
|
||||||
|
if upload.is_none() {
|
||||||
|
return (StatusCode::NOT_FOUND, "Upload not found").into_response();
|
||||||
|
}
|
||||||
|
|
||||||
|
let upload = upload.unwrap();
|
||||||
|
if upload.bucket != bucket || upload.key != key {
|
||||||
|
return (StatusCode::BAD_REQUEST, "Bucket/key mismatch").into_response();
|
||||||
|
}
|
||||||
|
|
||||||
|
// Collect body data
|
||||||
|
let mut total_size: u64 = 0;
|
||||||
|
let mut hasher = Sha256::new();
|
||||||
|
let mut stream = body.into_data_stream();
|
||||||
|
|
||||||
|
// Create temp file for part data
|
||||||
|
let temp_dir = std::env::temp_dir();
|
||||||
|
let part_file_path = temp_dir.join(format!("s3_multipart_{}_{}_{}.tmp", upload_id, part_number, Uuid::new_v4()));
|
||||||
|
let part_file = match tokio::fs::File::create(&part_file_path).await {
|
||||||
|
Ok(f) => f,
|
||||||
|
Err(e) => return (StatusCode::INTERNAL_SERVER_ERROR, format!("Failed to create temp file: {}", e)).into_response(),
|
||||||
|
};
|
||||||
|
let mut writer = tokio::io::BufWriter::new(part_file);
|
||||||
|
|
||||||
|
while let Some(chunk_result) = stream.next().await {
|
||||||
|
let chunk = match chunk_result {
|
||||||
|
Ok(c) => c,
|
||||||
|
Err(e) => return (StatusCode::BAD_REQUEST, format!("Failed to read chunk: {}", e)).into_response(),
|
||||||
|
};
|
||||||
|
|
||||||
|
total_size += chunk.len() as u64;
|
||||||
|
hasher.update(&chunk);
|
||||||
|
|
||||||
|
if let Err(e) = writer.write_all(&chunk).await {
|
||||||
|
return (StatusCode::INTERNAL_SERVER_ERROR, format!("Failed to write chunk: {}", e)).into_response();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if let Err(e) = writer.flush().await {
|
||||||
|
return (StatusCode::INTERNAL_SERVER_ERROR, format!("Failed to flush: {}", e)).into_response();
|
||||||
|
}
|
||||||
|
|
||||||
|
let etag = format!("{:x}", hasher.finalize());
|
||||||
|
|
||||||
|
// Update multipart upload with new part
|
||||||
|
{
|
||||||
|
let mut uploads = MULTIPART_UPLOADS.write().await;
|
||||||
|
if let Some(upload) = uploads.get_mut(&upload_id) {
|
||||||
|
upload.parts.push(UploadedPart {
|
||||||
|
part_number,
|
||||||
|
etag: etag.clone(),
|
||||||
|
size: total_size,
|
||||||
|
});
|
||||||
|
upload.parts.sort_by_key(|p| p.part_number);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
let mut headers = HeaderMap::new();
|
||||||
|
headers.insert("ETag", format!("\"{}\"", etag).parse().unwrap());
|
||||||
|
(StatusCode::OK, headers).into_response()
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, serde::Deserialize)]
|
||||||
|
pub struct UploadPartQuery {
|
||||||
|
pub upload_id: String,
|
||||||
|
pub part_number: u32,
|
||||||
|
}
|
||||||
|
|
||||||
|
pub async fn complete_multipart_upload(
|
||||||
|
Path((bucket, key)): Path<(String, String)>,
|
||||||
|
State(_state): State<crate::server::AppState>,
|
||||||
|
query: axum::extract::Query<CompleteMultipartQuery>,
|
||||||
|
headers: HeaderMap,
|
||||||
|
body: Body,
|
||||||
|
) -> impl IntoResponse {
|
||||||
|
// Authentication check
|
||||||
|
if !crate::s3_auth::verify_signature(headers.clone(), "POST", &format!("/s3/multipart/{}/{}?uploadId={}", bucket, key, query.upload_id)) {
|
||||||
|
return (StatusCode::FORBIDDEN, "Access denied").into_response();
|
||||||
|
}
|
||||||
|
|
||||||
|
// Policy check
|
||||||
|
let user_id = extract_user_from_auth(&headers).unwrap_or_else(|| "anonymous".to_string());
|
||||||
|
if !check_bucket_policy(&bucket, "s3:PutObject", &format!("arn:aws:s3:::{}/*", bucket), &user_id) {
|
||||||
|
return (StatusCode::FORBIDDEN, "Policy denied").into_response();
|
||||||
|
}
|
||||||
|
|
||||||
|
let upload_id = query.upload_id.clone();
|
||||||
|
|
||||||
|
let uploads = MULTIPART_UPLOADS.read().await;
|
||||||
|
let upload = uploads.get(&upload_id);
|
||||||
|
|
||||||
|
if upload.is_none() {
|
||||||
|
return (StatusCode::NOT_FOUND, "Upload not found").into_response();
|
||||||
|
}
|
||||||
|
|
||||||
|
let upload = upload.unwrap();
|
||||||
|
if upload.bucket != bucket || upload.key != key {
|
||||||
|
return (StatusCode::BAD_REQUEST, "Bucket/key mismatch").into_response();
|
||||||
|
}
|
||||||
|
|
||||||
|
// Parse CompleteMultipartUpload XML from body
|
||||||
|
let body_bytes = axum::body::to_bytes(body, 10000).await.ok();
|
||||||
|
let part_list = body_bytes.as_ref().and_then(|b| parse_complete_multipart_xml(b));
|
||||||
|
|
||||||
|
if part_list.is_none() {
|
||||||
|
return (StatusCode::BAD_REQUEST, "Invalid CompleteMultipartUpload XML").into_response();
|
||||||
|
}
|
||||||
|
|
||||||
|
// Combine parts into final file
|
||||||
|
let base_dir = "/Users/accusys/momentry/var/sftpgo/data";
|
||||||
|
let file_path = format!("{}/{}/{}", base_dir, bucket, key);
|
||||||
|
|
||||||
|
if let Err(e) = tokio::fs::create_dir_all(&format!("{}/{}", base_dir, bucket)).await {
|
||||||
|
return (StatusCode::INTERNAL_SERVER_ERROR, format!("Failed to create dir: {}", e)).into_response();
|
||||||
|
}
|
||||||
|
|
||||||
|
let final_file = match tokio::fs::File::create(&file_path).await {
|
||||||
|
Ok(f) => f,
|
||||||
|
Err(e) => return (StatusCode::INTERNAL_SERVER_ERROR, format!("Failed to create file: {}", e)).into_response(),
|
||||||
|
};
|
||||||
|
let mut final_writer = tokio::io::BufWriter::new(final_file);
|
||||||
|
|
||||||
|
let temp_dir = std::env::temp_dir();
|
||||||
|
let mut final_hasher = Sha256::new();
|
||||||
|
let mut final_size: u64 = 0;
|
||||||
|
|
||||||
|
for part in &upload.parts {
|
||||||
|
let _part_file_path = temp_dir.join(format!("s3_multipart_{}_{}_*.tmp", upload_id, part.part_number));
|
||||||
|
|
||||||
|
// Find the actual part file (with UUID suffix)
|
||||||
|
let part_files: Option<Vec<_>> = std::fs::read_dir(&temp_dir).ok().map(|dir| dir.filter_map(|e| e.ok())
|
||||||
|
.filter(|e| e.file_name().to_str().unwrap_or("").starts_with(&format!("s3_multipart_{}_{}_", upload_id, part.part_number)))
|
||||||
|
.collect::<Vec<_>>());
|
||||||
|
|
||||||
|
if let Some(files) = part_files {
|
||||||
|
if let Some(part_file_entry) = files.first() {
|
||||||
|
let part_file = part_file_entry.path();
|
||||||
|
if let Ok(data) = tokio::fs::read(&part_file).await {
|
||||||
|
final_hasher.update(&data);
|
||||||
|
final_size += data.len() as u64;
|
||||||
|
if let Err(e) = final_writer.write_all(&data).await {
|
||||||
|
return (StatusCode::INTERNAL_SERVER_ERROR, format!("Failed to write part: {}", e)).into_response();
|
||||||
|
}
|
||||||
|
// Clean up temp file
|
||||||
|
let _ = tokio::fs::remove_file(&part_file).await;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if let Err(e) = final_writer.flush().await {
|
||||||
|
return (StatusCode::INTERNAL_SERVER_ERROR, format!("Failed to flush final: {}", e)).into_response();
|
||||||
|
}
|
||||||
|
|
||||||
|
let final_etag = format!("{:x}", final_hasher.finalize());
|
||||||
|
|
||||||
|
// Remove upload from tracking
|
||||||
|
{
|
||||||
|
let mut uploads = MULTIPART_UPLOADS.write().await;
|
||||||
|
uploads.remove(&upload_id);
|
||||||
|
}
|
||||||
|
|
||||||
|
let (headers, xml_body) = crate::s3_xml::complete_multipart_upload_xml(&bucket, &key, &final_etag);
|
||||||
|
(StatusCode::OK, headers, xml_body).into_response()
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, serde::Deserialize)]
|
||||||
|
pub struct CompleteMultipartQuery {
|
||||||
|
pub upload_id: String,
|
||||||
|
}
|
||||||
|
|
||||||
|
pub async fn abort_multipart_upload(
|
||||||
|
Path((bucket, key)): Path<(String, String)>,
|
||||||
|
State(_state): State<crate::server::AppState>,
|
||||||
|
query: axum::extract::Query<AbortMultipartQuery>,
|
||||||
|
headers: HeaderMap,
|
||||||
|
) -> impl IntoResponse {
|
||||||
|
// Authentication check
|
||||||
|
if !crate::s3_auth::verify_signature(headers.clone(), "DELETE", &format!("/s3/multipart/{}/{}?uploadId={}", bucket, key, query.upload_id)) {
|
||||||
|
return (StatusCode::FORBIDDEN, "Access denied").into_response();
|
||||||
|
}
|
||||||
|
|
||||||
|
let upload_id = query.upload_id.clone();
|
||||||
|
|
||||||
|
let uploads = MULTIPART_UPLOADS.read().await;
|
||||||
|
let upload = uploads.get(&upload_id);
|
||||||
|
|
||||||
|
if upload.is_none() {
|
||||||
|
return (StatusCode::NOT_FOUND, "Upload not found").into_response();
|
||||||
|
}
|
||||||
|
|
||||||
|
let upload = upload.unwrap();
|
||||||
|
if upload.bucket != bucket || upload.key != key {
|
||||||
|
return (StatusCode::BAD_REQUEST, "Bucket/key mismatch").into_response();
|
||||||
|
}
|
||||||
|
|
||||||
|
// Clean up temp files
|
||||||
|
let temp_dir = std::env::temp_dir();
|
||||||
|
if let Ok(dir) = std::fs::read_dir(&temp_dir) {
|
||||||
|
for entry in dir.filter_map(|e| e.ok()) {
|
||||||
|
if entry.file_name().to_str().unwrap_or("").starts_with(&format!("s3_multipart_{}_", upload_id)) {
|
||||||
|
let _ = tokio::fs::remove_file(entry.path()).await;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Remove upload from tracking
|
||||||
|
{
|
||||||
|
let mut uploads = MULTIPART_UPLOADS.write().await;
|
||||||
|
uploads.remove(&upload_id);
|
||||||
|
}
|
||||||
|
|
||||||
|
(StatusCode::NO_CONTENT, HeaderMap::new()).into_response()
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, serde::Deserialize)]
|
||||||
|
pub struct AbortMultipartQuery {
|
||||||
|
pub upload_id: String,
|
||||||
|
}
|
||||||
|
|
||||||
|
fn parse_complete_multipart_xml(xml: &[u8]) -> Option<Vec<(u32, String)>> {
|
||||||
|
let xml_str = std::str::from_utf8(xml).ok()?;
|
||||||
|
let mut parts = Vec::new();
|
||||||
|
|
||||||
|
for part_elem in xml_str.split("<Part>") {
|
||||||
|
if part_elem.contains("</Part>") {
|
||||||
|
let part_number = part_elem.split("<PartNumber>")
|
||||||
|
.nth(1)
|
||||||
|
.and_then(|s| s.split("</PartNumber>").next())
|
||||||
|
.and_then(|s| s.parse().ok());
|
||||||
|
|
||||||
|
let etag = part_elem.split("<ETag>")
|
||||||
|
.nth(1)
|
||||||
|
.and_then(|s| s.split("</ETag>").next())
|
||||||
|
.map(|s| s.replace("\"", ""));
|
||||||
|
|
||||||
|
if let (Some(num), Some(tag)) = (part_number, etag) {
|
||||||
|
parts.push((num, tag));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
Some(parts)
|
||||||
|
}
|
||||||
|
|
||||||
|
// ===== Bucket Policy Support =====
|
||||||
|
|
||||||
|
use crate::s3_policy::BucketPolicy;
|
||||||
|
|
||||||
|
static BUCKET_POLICIES: once_cell::sync::Lazy<Arc<RwLock<HashMap<String, BucketPolicy>>>> =
|
||||||
|
once_cell::sync::Lazy::new(|| Arc::new(RwLock::new(HashMap::new())));
|
||||||
|
|
||||||
|
pub async fn get_bucket_policy(
|
||||||
|
Path(bucket): Path<String>,
|
||||||
|
State(_state): State<crate::server::AppState>,
|
||||||
|
) -> impl IntoResponse {
|
||||||
|
let policies = BUCKET_POLICIES.read().await;
|
||||||
|
let policy = policies.get(&bucket);
|
||||||
|
|
||||||
|
if policy.is_none() {
|
||||||
|
return (StatusCode::NOT_FOUND, "Bucket policy not found").into_response();
|
||||||
|
}
|
||||||
|
|
||||||
|
let policy = policy.unwrap();
|
||||||
|
let json = serde_json::to_string_pretty(policy)
|
||||||
|
.unwrap_or_else(|_| "{}".to_string());
|
||||||
|
|
||||||
|
let mut headers = HeaderMap::new();
|
||||||
|
headers.insert("Content-Type", "application/json".parse().unwrap());
|
||||||
|
|
||||||
|
(StatusCode::OK, headers, json).into_response()
|
||||||
|
}
|
||||||
|
|
||||||
|
pub async fn put_bucket_policy(
|
||||||
|
Path(bucket): Path<String>,
|
||||||
|
State(_state): State<crate::server::AppState>,
|
||||||
|
body: Body,
|
||||||
|
) -> impl IntoResponse {
|
||||||
|
let body_bytes = axum::body::to_bytes(body, 100000).await.ok();
|
||||||
|
|
||||||
|
if body_bytes.is_none() {
|
||||||
|
return (StatusCode::BAD_REQUEST, "Empty body").into_response();
|
||||||
|
}
|
||||||
|
|
||||||
|
let policy: BucketPolicy = match serde_json::from_slice(&body_bytes.unwrap()) {
|
||||||
|
Ok(p) => p,
|
||||||
|
Err(e) => return (StatusCode::BAD_REQUEST, format!("Invalid policy JSON: {}", e)).into_response(),
|
||||||
|
};
|
||||||
|
|
||||||
|
// Persist to file first (before moving policy)
|
||||||
|
let policy_path = format!("data/s3_policies/{}/policy.json", bucket);
|
||||||
|
if let Err(e) = std::fs::create_dir_all(format!("data/s3_policies/{}", bucket)) {
|
||||||
|
return (StatusCode::INTERNAL_SERVER_ERROR, format!("Failed to create policy dir: {}", e)).into_response();
|
||||||
|
}
|
||||||
|
|
||||||
|
let policy_json = serde_json::to_string_pretty(&policy).unwrap_or_default();
|
||||||
|
if let Err(e) = std::fs::write(&policy_path, &policy_json) {
|
||||||
|
return (StatusCode::INTERNAL_SERVER_ERROR, format!("Failed to write policy: {}", e)).into_response();
|
||||||
|
}
|
||||||
|
|
||||||
|
// Now move policy to in-memory storage
|
||||||
|
{
|
||||||
|
let mut policies = BUCKET_POLICIES.write().await;
|
||||||
|
policies.insert(bucket.clone(), policy);
|
||||||
|
}
|
||||||
|
|
||||||
|
(StatusCode::NO_CONTENT, HeaderMap::new()).into_response()
|
||||||
|
}
|
||||||
|
|
||||||
|
pub async fn delete_bucket_policy(
|
||||||
|
Path(bucket): Path<String>,
|
||||||
|
State(_state): State<crate::server::AppState>,
|
||||||
|
) -> impl IntoResponse {
|
||||||
|
{
|
||||||
|
let mut policies = BUCKET_POLICIES.write().await;
|
||||||
|
policies.remove(&bucket);
|
||||||
|
}
|
||||||
|
|
||||||
|
let policy_path = format!("data/s3_policies/{}/policy.json", bucket);
|
||||||
|
let _ = std::fs::remove_file(&policy_path);
|
||||||
|
|
||||||
|
(StatusCode::NO_CONTENT, HeaderMap::new()).into_response()
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn check_bucket_policy(bucket: &str, action: &str, resource: &str, user_id: &str) -> bool {
|
||||||
|
let policies = BUCKET_POLICIES.blocking_read();
|
||||||
|
|
||||||
|
if let Some(policy) = policies.get(bucket) {
|
||||||
|
return policy.is_allowed(action, resource, user_id);
|
||||||
|
}
|
||||||
|
|
||||||
|
true
|
||||||
|
}
|
||||||
|
|
||||||
|
fn extract_user_from_auth(headers: &HeaderMap) -> Option<String> {
|
||||||
|
let auth_header = headers
|
||||||
|
.get("Authorization")
|
||||||
|
.and_then(|v| v.to_str().ok())
|
||||||
|
.unwrap_or("");
|
||||||
|
|
||||||
|
if auth_header.starts_with("AWS4-HMAC-SHA256") {
|
||||||
|
// Extract from Credential=access_key/date/region/service
|
||||||
|
let credential_part = auth_header.split(',')
|
||||||
|
.find(|p| p.trim().starts_with("Credential="))?;
|
||||||
|
let credential_str = credential_part.trim().strip_prefix("Credential=")?;
|
||||||
|
let access_key = credential_str.split('/').next()?;
|
||||||
|
|
||||||
|
// Look up user_id from s3_keys.json
|
||||||
|
let s3_keys_path = "data/s3_keys.json";
|
||||||
|
let s3_keys_json = std::fs::read_to_string(s3_keys_path).ok()?;
|
||||||
|
|
||||||
|
#[derive(serde::Deserialize)]
|
||||||
|
struct S3Key {
|
||||||
|
access_key: String,
|
||||||
|
user_id: String,
|
||||||
|
}
|
||||||
|
|
||||||
|
let s3_keys: Vec<S3Key> = serde_json::from_str(&s3_keys_json).ok()?;
|
||||||
|
s3_keys.iter()
|
||||||
|
.find(|k| k.access_key == access_key)
|
||||||
|
.map(|k| k.user_id.clone())
|
||||||
|
} else {
|
||||||
|
None
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Unified multipart handler using query param for action
|
||||||
|
#[derive(serde::Deserialize)]
|
||||||
|
pub struct MultipartActionQuery {
|
||||||
|
action: Option<String>, // init, part, complete, abort
|
||||||
|
upload_id: Option<String>,
|
||||||
|
part_number: Option<u32>,
|
||||||
|
}
|
||||||
|
|
||||||
|
pub async fn multipart_handler(
|
||||||
|
method: axum::http::Method,
|
||||||
|
Path((bucket, key)): Path<(String, String)>,
|
||||||
|
State(state): State<crate::server::AppState>,
|
||||||
|
query: axum::extract::Query<MultipartActionQuery>,
|
||||||
|
headers: HeaderMap,
|
||||||
|
body: Body,
|
||||||
|
) -> axum::response::Response {
|
||||||
|
let action = query.action.as_deref().unwrap_or("");
|
||||||
|
|
||||||
|
match action {
|
||||||
|
"init" => {
|
||||||
|
initiate_multipart_upload(
|
||||||
|
Path((bucket, key)),
|
||||||
|
State(state),
|
||||||
|
headers,
|
||||||
|
).await.into_response()
|
||||||
|
}
|
||||||
|
"part" => {
|
||||||
|
let upload_query = axum::extract::Query(UploadPartQuery {
|
||||||
|
upload_id: query.upload_id.clone().unwrap_or_default(),
|
||||||
|
part_number: query.part_number.unwrap_or(1),
|
||||||
|
});
|
||||||
|
upload_part(
|
||||||
|
Path((bucket, key)),
|
||||||
|
State(state),
|
||||||
|
upload_query,
|
||||||
|
headers,
|
||||||
|
body,
|
||||||
|
).await.into_response()
|
||||||
|
}
|
||||||
|
"complete" => {
|
||||||
|
let complete_query = axum::extract::Query(CompleteMultipartQuery {
|
||||||
|
upload_id: query.upload_id.clone().unwrap_or_default(),
|
||||||
|
});
|
||||||
|
complete_multipart_upload(
|
||||||
|
Path((bucket, key)),
|
||||||
|
State(state),
|
||||||
|
complete_query,
|
||||||
|
headers,
|
||||||
|
body,
|
||||||
|
).await.into_response()
|
||||||
|
}
|
||||||
|
"abort" => {
|
||||||
|
let abort_query = axum::extract::Query(AbortMultipartQuery {
|
||||||
|
upload_id: query.upload_id.clone().unwrap_or_default(),
|
||||||
|
});
|
||||||
|
abort_multipart_upload(
|
||||||
|
Path((bucket, key)),
|
||||||
|
State(state),
|
||||||
|
abort_query,
|
||||||
|
headers,
|
||||||
|
).await.into_response()
|
||||||
|
}
|
||||||
|
_ => {
|
||||||
|
if method == axum::http::Method::POST {
|
||||||
|
initiate_multipart_upload(
|
||||||
|
Path((bucket, key)),
|
||||||
|
State(state),
|
||||||
|
headers,
|
||||||
|
).await.into_response()
|
||||||
|
} else {
|
||||||
|
(StatusCode::BAD_REQUEST, "Missing action parameter").into_response()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
+161
-69
@@ -6,19 +6,14 @@ use std::fs;
|
|||||||
type HmacSha256 = Hmac<Sha256>;
|
type HmacSha256 = Hmac<Sha256>;
|
||||||
|
|
||||||
pub fn verify_signature(headers: HeaderMap, method: &str, path: &str) -> bool {
|
pub fn verify_signature(headers: HeaderMap, method: &str, path: &str) -> bool {
|
||||||
// Load S3 config and check require_auth flag
|
|
||||||
let config = crate::s3_config::S3Config::load_default().unwrap_or_default();
|
let config = crate::s3_config::S3Config::load_default().unwrap_or_default();
|
||||||
|
|
||||||
// Merge environment variables (allows override via MB_S3_REQUIRE_AUTH)
|
|
||||||
let mut config = config;
|
let mut config = config;
|
||||||
config.merge_env();
|
config.merge_env();
|
||||||
|
|
||||||
if !config.s3.require_auth {
|
if !config.s3.require_auth {
|
||||||
// Development mode: allow access without authentication
|
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
|
|
||||||
// 生产模式:必须提供Authorization header
|
|
||||||
let auth_header = headers
|
let auth_header = headers
|
||||||
.get("Authorization")
|
.get("Authorization")
|
||||||
.and_then(|v| v.to_str().ok())
|
.and_then(|v| v.to_str().ok())
|
||||||
@@ -28,41 +23,55 @@ pub fn verify_signature(headers: HeaderMap, method: &str, path: &str) -> bool {
|
|||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
|
|
||||||
// 2. Parse Credential
|
|
||||||
let credential = extract_credential(auth_header);
|
let credential = extract_credential(auth_header);
|
||||||
if credential.is_none() {
|
if credential.is_none() {
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
|
|
||||||
let credential = credential.unwrap();
|
let credential = credential.unwrap();
|
||||||
|
|
||||||
// 3. Get secret_key from S3AccessKey database
|
|
||||||
let secret_key = get_secret_key(&credential.access_key);
|
let secret_key = get_secret_key(&credential.access_key);
|
||||||
if secret_key.is_none() {
|
if secret_key.is_none() {
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
|
|
||||||
let secret_key = secret_key.unwrap();
|
let secret_key = secret_key.unwrap();
|
||||||
|
|
||||||
// 4. Calculate Signature
|
let x_amz_date = headers
|
||||||
let calculated_signature = calculate_signature(
|
.get("X-Amz-Date")
|
||||||
headers.clone(),
|
.and_then(|v| v.to_str().ok())
|
||||||
|
.unwrap_or(&credential.date);
|
||||||
|
|
||||||
|
let signed_headers = extract_signed_headers(auth_header);
|
||||||
|
if signed_headers.is_none() {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
let signed_headers = signed_headers.unwrap();
|
||||||
|
|
||||||
|
let payload_hash = get_payload_hash(&headers);
|
||||||
|
|
||||||
|
let canonical_request = create_canonical_request(
|
||||||
|
&headers,
|
||||||
method,
|
method,
|
||||||
path,
|
path,
|
||||||
&credential.access_key,
|
&signed_headers,
|
||||||
&secret_key,
|
&payload_hash,
|
||||||
&credential.region,
|
|
||||||
&credential.service,
|
|
||||||
&credential.date,
|
|
||||||
);
|
);
|
||||||
|
|
||||||
// 5. Extract Signature from header
|
let string_to_sign = create_string_to_sign(
|
||||||
|
x_amz_date,
|
||||||
|
&credential.region,
|
||||||
|
&credential.service,
|
||||||
|
&canonical_request,
|
||||||
|
);
|
||||||
|
|
||||||
|
let signing_key = calculate_signing_key(&secret_key, &credential.date, &credential.region, &credential.service);
|
||||||
|
|
||||||
|
let calculated_signature = hmac_sha256_hex(&signing_key, &string_to_sign);
|
||||||
|
|
||||||
let provided_signature = extract_signature(auth_header);
|
let provided_signature = extract_signature(auth_header);
|
||||||
if provided_signature.is_none() {
|
if provided_signature.is_none() {
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
|
|
||||||
// 6. Compare signatures
|
|
||||||
calculated_signature == provided_signature.unwrap()
|
calculated_signature == provided_signature.unwrap()
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -74,14 +83,11 @@ struct Credential {
|
|||||||
}
|
}
|
||||||
|
|
||||||
fn extract_credential(auth_header: &str) -> Option<Credential> {
|
fn extract_credential(auth_header: &str) -> Option<Credential> {
|
||||||
let parts: Vec<&str> = auth_header.split_whitespace().collect();
|
let credential_part = auth_header
|
||||||
if parts.len() < 2 {
|
.split(',')
|
||||||
return None;
|
.find(|p| p.trim().starts_with("Credential="))?;
|
||||||
}
|
|
||||||
|
|
||||||
let credential_part = parts.iter().find(|p| p.starts_with("Credential="))?;
|
let credential_str = credential_part.trim().strip_prefix("Credential=")?;
|
||||||
|
|
||||||
let credential_str = credential_part.strip_prefix("Credential=")?;
|
|
||||||
let credential_parts: Vec<&str> = credential_str.split('/').collect();
|
let credential_parts: Vec<&str> = credential_str.split('/').collect();
|
||||||
|
|
||||||
if credential_parts.len() < 5 {
|
if credential_parts.len() < 5 {
|
||||||
@@ -96,16 +102,24 @@ fn extract_credential(auth_header: &str) -> Option<Credential> {
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn extract_signed_headers(auth_header: &str) -> Option<Vec<String>> {
|
||||||
|
let signed_headers_part = auth_header
|
||||||
|
.split(',')
|
||||||
|
.find(|p| p.trim().starts_with("SignedHeaders="))?;
|
||||||
|
|
||||||
|
let signed_headers_str = signed_headers_part.trim().strip_prefix("SignedHeaders=")?;
|
||||||
|
Some(signed_headers_str.split(';').map(|s| s.to_lowercase()).collect())
|
||||||
|
}
|
||||||
|
|
||||||
fn extract_signature(auth_header: &str) -> Option<String> {
|
fn extract_signature(auth_header: &str) -> Option<String> {
|
||||||
let parts: Vec<&str> = auth_header.split_whitespace().collect();
|
let signature_part = auth_header
|
||||||
|
.split(',')
|
||||||
|
.find(|p| p.trim().starts_with("Signature="))?;
|
||||||
|
|
||||||
let signature_part = parts.iter().find(|p| p.starts_with("Signature="))?;
|
Some(signature_part.trim().strip_prefix("Signature=")?.to_string())
|
||||||
|
|
||||||
Some(signature_part.strip_prefix("Signature=")?.to_string())
|
|
||||||
}
|
}
|
||||||
|
|
||||||
fn get_secret_key(access_key: &str) -> Option<String> {
|
fn get_secret_key(access_key: &str) -> Option<String> {
|
||||||
// Load S3AccessKey database from data/s3_keys.json
|
|
||||||
let s3_keys_path = "data/s3_keys.json";
|
let s3_keys_path = "data/s3_keys.json";
|
||||||
let s3_keys_json = fs::read_to_string(s3_keys_path).ok()?;
|
let s3_keys_json = fs::read_to_string(s3_keys_path).ok()?;
|
||||||
|
|
||||||
@@ -116,62 +130,97 @@ fn get_secret_key(access_key: &str) -> Option<String> {
|
|||||||
}
|
}
|
||||||
|
|
||||||
let s3_keys: Vec<S3Key> = serde_json::from_str(&s3_keys_json).ok()?;
|
let s3_keys: Vec<S3Key> = serde_json::from_str(&s3_keys_json).ok()?;
|
||||||
|
|
||||||
s3_keys
|
s3_keys
|
||||||
.iter()
|
.iter()
|
||||||
.find(|k| k.access_key == access_key)
|
.find(|k| k.access_key == access_key)
|
||||||
.map(|k| k.secret_key.clone())
|
.map(|k| k.secret_key.clone())
|
||||||
}
|
}
|
||||||
|
|
||||||
fn calculate_signature(
|
fn get_payload_hash(headers: &HeaderMap) -> String {
|
||||||
headers: HeaderMap,
|
headers
|
||||||
method: &str,
|
.get("X-Amz-Content-Sha256")
|
||||||
path: &str,
|
.and_then(|v| v.to_str().ok())
|
||||||
_access_key: &str,
|
.map(|s| s.to_string())
|
||||||
secret_key: &str,
|
.unwrap_or_else(|| sha256_hex(""))
|
||||||
region: &str,
|
|
||||||
service: &str,
|
|
||||||
date: &str,
|
|
||||||
) -> String {
|
|
||||||
// 1. Create Canonical Request
|
|
||||||
let canonical_request = create_canonical_request(headers, method, path);
|
|
||||||
|
|
||||||
// 2. Create String to Sign
|
|
||||||
let string_to_sign = create_string_to_sign(date, region, service, &canonical_request);
|
|
||||||
|
|
||||||
// 3. Calculate Signing Key
|
|
||||||
let signing_key = calculate_signing_key(secret_key, date, region, service);
|
|
||||||
|
|
||||||
// 4. Calculate Signature
|
|
||||||
|
|
||||||
|
|
||||||
hmac_sha256_hex(&signing_key, &string_to_sign)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
fn create_canonical_request(headers: HeaderMap, method: &str, path: &str) -> String {
|
fn create_canonical_request(
|
||||||
// Simplified implementation for POC
|
headers: &HeaderMap,
|
||||||
let host = headers
|
method: &str,
|
||||||
.get("Host")
|
path: &str,
|
||||||
.and_then(|v| v.to_str().ok())
|
signed_headers: &[String],
|
||||||
.unwrap_or("localhost:11438");
|
payload_hash: &str,
|
||||||
|
) -> String {
|
||||||
|
let canonical_uri = uri_encode(path, false);
|
||||||
|
|
||||||
|
let canonical_query_string = build_canonical_query_string(headers);
|
||||||
|
|
||||||
|
let canonical_headers = build_canonical_headers(headers, signed_headers);
|
||||||
|
|
||||||
|
let signed_headers_str = signed_headers.join(";");
|
||||||
|
|
||||||
format!(
|
format!(
|
||||||
"{}\n{}\n\nhost:{}\n\nhost\nUNSIGNED-PAYLOAD",
|
"{}\n{}\n{}\n{}\n{}\n{}",
|
||||||
method, path, host
|
method,
|
||||||
|
canonical_uri,
|
||||||
|
canonical_query_string,
|
||||||
|
canonical_headers,
|
||||||
|
signed_headers_str,
|
||||||
|
payload_hash
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn uri_encode(input: &str, encode_slash: bool) -> String {
|
||||||
|
input
|
||||||
|
.chars()
|
||||||
|
.map(|c| {
|
||||||
|
if c.is_ascii_alphanumeric() || c == '-' || c == '_' || c == '.' || c == '~' {
|
||||||
|
c.to_string()
|
||||||
|
} else if c == '/' && !encode_slash {
|
||||||
|
c.to_string()
|
||||||
|
} else {
|
||||||
|
format!("%{:02X}", c as u8)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
.collect()
|
||||||
|
}
|
||||||
|
|
||||||
|
fn build_canonical_query_string(_headers: &HeaderMap) -> String {
|
||||||
|
// For S3, query string is typically empty for basic operations
|
||||||
|
// This can be extended for presigned URLs
|
||||||
|
String::new()
|
||||||
|
}
|
||||||
|
|
||||||
|
fn build_canonical_headers(headers: &HeaderMap, signed_headers: &[String]) -> String {
|
||||||
|
signed_headers
|
||||||
|
.iter()
|
||||||
|
.map(|h| {
|
||||||
|
let value = headers
|
||||||
|
.get(h)
|
||||||
|
.and_then(|v| v.to_str().ok())
|
||||||
|
.unwrap_or("");
|
||||||
|
format!("{}:{}\n", h, value.trim())
|
||||||
|
})
|
||||||
|
.collect()
|
||||||
|
}
|
||||||
|
|
||||||
fn create_string_to_sign(
|
fn create_string_to_sign(
|
||||||
date: &str,
|
amz_date: &str,
|
||||||
region: &str,
|
region: &str,
|
||||||
service: &str,
|
service: &str,
|
||||||
canonical_request: &str,
|
canonical_request: &str,
|
||||||
) -> String {
|
) -> String {
|
||||||
let canonical_request_hash = sha256_hex(canonical_request);
|
let canonical_request_hash = sha256_hex(canonical_request);
|
||||||
|
|
||||||
|
let date_stamp = &amz_date[..8];
|
||||||
|
|
||||||
format!(
|
format!(
|
||||||
"AWS4-HMAC-SHA256\n{}T000000Z\n{}/{}/{}/aws4_request\n{}",
|
"AWS4-HMAC-SHA256\n{}\n{}/{}/{}/aws4_request\n{}",
|
||||||
date, date, region, service, canonical_request_hash
|
amz_date,
|
||||||
|
date_stamp,
|
||||||
|
region,
|
||||||
|
service,
|
||||||
|
canonical_request_hash
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -203,7 +252,50 @@ fn sha256_hex(data: &str) -> String {
|
|||||||
}
|
}
|
||||||
|
|
||||||
fn hex_encode(data: &[u8]) -> String {
|
fn hex_encode(data: &[u8]) -> String {
|
||||||
data.iter()
|
data.iter().map(|b| format!("{:02x}", b)).collect()
|
||||||
.map(|b| format!("{:02x}", b))
|
}
|
||||||
.collect::<String>()
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use super::*;
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_uri_encode() {
|
||||||
|
assert_eq!(uri_encode("/bucket/key", false), "/bucket/key");
|
||||||
|
assert_eq!(uri_encode("/bucket/key", true), "%2Fbucket%2Fkey");
|
||||||
|
assert_eq!(uri_encode("test file.txt", false), "test%20file.txt");
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_sha256_hex() {
|
||||||
|
let empty_hash = sha256_hex("");
|
||||||
|
assert_eq!(
|
||||||
|
empty_hash,
|
||||||
|
"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_calculate_signing_key() {
|
||||||
|
let key = calculate_signing_key("secret", "20260621", "us-east-1", "s3");
|
||||||
|
assert_eq!(key.len(), 32);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_create_canonical_request() {
|
||||||
|
let mut headers = HeaderMap::new();
|
||||||
|
headers.insert("Host", "localhost:11438".parse().unwrap());
|
||||||
|
let signed_headers = vec!["host".to_string()];
|
||||||
|
|
||||||
|
let canonical = create_canonical_request(
|
||||||
|
&headers,
|
||||||
|
"GET",
|
||||||
|
"/bucket/key",
|
||||||
|
&signed_headers,
|
||||||
|
"UNSIGNED-PAYLOAD",
|
||||||
|
);
|
||||||
|
|
||||||
|
assert!(canonical.contains("GET"));
|
||||||
|
assert!(canonical.contains("host:localhost:11438"));
|
||||||
|
}
|
||||||
}
|
}
|
||||||
@@ -0,0 +1,250 @@
|
|||||||
|
use serde::{Deserialize, Serialize};
|
||||||
|
use std::collections::HashMap;
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||||
|
pub struct BucketPolicy {
|
||||||
|
#[serde(rename = "Version")]
|
||||||
|
pub version: String,
|
||||||
|
#[serde(rename = "Statement")]
|
||||||
|
pub statement: Vec<PolicyStatement>,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Default for BucketPolicy {
|
||||||
|
fn default() -> Self {
|
||||||
|
Self {
|
||||||
|
version: "2012-10-17".to_string(),
|
||||||
|
statement: Vec::new(),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||||
|
pub struct PolicyStatement {
|
||||||
|
#[serde(rename = "Sid")]
|
||||||
|
pub sid: Option<String>,
|
||||||
|
#[serde(rename = "Effect")]
|
||||||
|
pub effect: PolicyEffect,
|
||||||
|
#[serde(rename = "Principal")]
|
||||||
|
pub principal: Principal,
|
||||||
|
#[serde(rename = "Action")]
|
||||||
|
pub action: Vec<String>,
|
||||||
|
#[serde(rename = "Resource")]
|
||||||
|
pub resource: Vec<String>,
|
||||||
|
#[serde(rename = "Condition")]
|
||||||
|
pub condition: Option<HashMap<String, HashMap<String, String>>>,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
|
||||||
|
#[serde(rename_all = "lowercase")]
|
||||||
|
pub enum PolicyEffect {
|
||||||
|
Allow,
|
||||||
|
Deny,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||||
|
#[serde(untagged)]
|
||||||
|
pub enum Principal {
|
||||||
|
Wildcard(String),
|
||||||
|
Specific(HashMap<String, Vec<String>>),
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Principal {
|
||||||
|
pub fn is_public(&self) -> bool {
|
||||||
|
match self {
|
||||||
|
Principal::Wildcard(s) => s == "*",
|
||||||
|
Principal::Specific(_) => false,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn matches_user(&self, user_id: &str) -> bool {
|
||||||
|
match self {
|
||||||
|
Principal::Wildcard(s) => s == "*",
|
||||||
|
Principal::Specific(map) => {
|
||||||
|
if let Some(aws_users) = map.get("AWS") {
|
||||||
|
aws_users.iter().any(|u| u == user_id || u == "*")
|
||||||
|
} else {
|
||||||
|
false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl BucketPolicy {
|
||||||
|
pub fn new() -> Self {
|
||||||
|
Self::default()
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn is_allowed(&self, action: &str, resource: &str, user_id: &str) -> bool {
|
||||||
|
let mut allowed = false;
|
||||||
|
|
||||||
|
for stmt in &self.statement {
|
||||||
|
if stmt.matches_action(action) && stmt.matches_resource(resource)
|
||||||
|
&& stmt.principal.matches_user(user_id) {
|
||||||
|
match stmt.effect {
|
||||||
|
PolicyEffect::Allow => {
|
||||||
|
if stmt.matches_condition(user_id) {
|
||||||
|
allowed = true;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
PolicyEffect::Deny => {
|
||||||
|
if stmt.matches_condition(user_id) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
allowed
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl PolicyStatement {
|
||||||
|
pub fn matches_action(&self, action: &str) -> bool {
|
||||||
|
self.action.iter().any(|a| {
|
||||||
|
a == action || a == "s3:*" || a == "*" ||
|
||||||
|
(a.ends_with('*') && action.starts_with(&a[..a.len()-1]))
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn matches_resource(&self, resource: &str) -> bool {
|
||||||
|
self.resource.iter().any(|r| {
|
||||||
|
r == resource || r == "*" ||
|
||||||
|
(r.ends_with('*') && resource.starts_with(&r[..r.len()-1]))
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn matches_condition(&self, _user_id: &str) -> bool {
|
||||||
|
if let Some(cond) = &self.condition {
|
||||||
|
for (operator, values) in cond {
|
||||||
|
for (key, value) in values {
|
||||||
|
if operator == "StringEquals" && key == "aws:userid"
|
||||||
|
&& value != _user_id {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn default_public_policy(bucket: &str) -> BucketPolicy {
|
||||||
|
BucketPolicy {
|
||||||
|
version: "2012-10-17".to_string(),
|
||||||
|
statement: vec![
|
||||||
|
PolicyStatement {
|
||||||
|
sid: Some("PublicRead".to_string()),
|
||||||
|
effect: PolicyEffect::Allow,
|
||||||
|
principal: Principal::Wildcard("*".to_string()),
|
||||||
|
action: vec!["s3:GetObject".to_string()],
|
||||||
|
resource: vec![format!("arn:aws:s3:::{}/*", bucket)],
|
||||||
|
condition: None,
|
||||||
|
},
|
||||||
|
],
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn default_private_policy(bucket: &str, user_id: &str) -> BucketPolicy {
|
||||||
|
BucketPolicy {
|
||||||
|
version: "2012-10-17".to_string(),
|
||||||
|
statement: vec![
|
||||||
|
PolicyStatement {
|
||||||
|
sid: Some("OwnerFullAccess".to_string()),
|
||||||
|
effect: PolicyEffect::Allow,
|
||||||
|
principal: Principal::Specific({
|
||||||
|
let mut map = HashMap::new();
|
||||||
|
map.insert("AWS".to_string(), vec![format!("arn:aws:iam:::user/{}", user_id)]);
|
||||||
|
map
|
||||||
|
}),
|
||||||
|
action: vec!["s3:*".to_string()],
|
||||||
|
resource: vec![
|
||||||
|
format!("arn:aws:s3:::{}/*", bucket),
|
||||||
|
format!("arn:aws:s3:::{}/*", bucket),
|
||||||
|
],
|
||||||
|
condition: None,
|
||||||
|
},
|
||||||
|
],
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use super::*;
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_policy_parse() {
|
||||||
|
let policy_json = r#"{
|
||||||
|
"Version": "2012-10-17",
|
||||||
|
"Statement": [
|
||||||
|
{
|
||||||
|
"Effect": "allow",
|
||||||
|
"Principal": "*",
|
||||||
|
"Action": ["s3:GetObject"],
|
||||||
|
"Resource": ["arn:aws:s3:::mybucket/*"]
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}"#;
|
||||||
|
|
||||||
|
let policy: BucketPolicy = serde_json::from_str(policy_json).unwrap();
|
||||||
|
assert_eq!(policy.version, "2012-10-17");
|
||||||
|
assert_eq!(policy.statement.len(), 1);
|
||||||
|
assert_eq!(policy.statement[0].effect, PolicyEffect::Allow);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_policy_evaluation_allow() {
|
||||||
|
let policy = default_public_policy("testbucket");
|
||||||
|
assert!(policy.is_allowed("s3:GetObject", "arn:aws:s3:::testbucket/file.txt", "anonymous"));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_policy_evaluation_deny() {
|
||||||
|
let policy = default_public_policy("testbucket");
|
||||||
|
assert!(!policy.is_allowed("s3:PutObject", "arn:aws:s3:::testbucket/file.txt", "anonymous"));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_action_wildcard() {
|
||||||
|
let stmt = PolicyStatement {
|
||||||
|
sid: None,
|
||||||
|
effect: PolicyEffect::Allow,
|
||||||
|
principal: Principal::Wildcard("*".to_string()),
|
||||||
|
action: vec!["s3:*".to_string()],
|
||||||
|
resource: vec!["*".to_string()],
|
||||||
|
condition: None,
|
||||||
|
};
|
||||||
|
|
||||||
|
assert!(stmt.matches_action("s3:GetObject"));
|
||||||
|
assert!(stmt.matches_action("s3:PutObject"));
|
||||||
|
assert!(stmt.matches_action("s3:DeleteObject"));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_resource_pattern() {
|
||||||
|
let stmt = PolicyStatement {
|
||||||
|
sid: None,
|
||||||
|
effect: PolicyEffect::Allow,
|
||||||
|
principal: Principal::Wildcard("*".to_string()),
|
||||||
|
action: vec!["s3:GetObject".to_string()],
|
||||||
|
resource: vec!["arn:aws:s3:::mybucket/home/*".to_string()],
|
||||||
|
condition: None,
|
||||||
|
};
|
||||||
|
|
||||||
|
assert!(stmt.matches_resource("arn:aws:s3:::mybucket/home/user/file.txt"));
|
||||||
|
assert!(!stmt.matches_resource("arn:aws:s3:::mybucket/public/file.txt"));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_principal_user_match() {
|
||||||
|
let principal = Principal::Specific({
|
||||||
|
let mut map = HashMap::new();
|
||||||
|
map.insert("AWS".to_string(), vec!["warren".to_string()]);
|
||||||
|
map
|
||||||
|
});
|
||||||
|
|
||||||
|
assert!(principal.matches_user("warren"));
|
||||||
|
assert!(!principal.matches_user("demo"));
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -76,3 +76,38 @@ pub fn list_objects_xml(bucket_name: &str, objects: &[Value]) -> (HeaderMap, Str
|
|||||||
|
|
||||||
(headers, xml)
|
(headers, xml)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
pub fn initiate_multipart_upload_xml(bucket: &str, key: &str, upload_id: &str) -> (HeaderMap, String) {
|
||||||
|
let mut headers = HeaderMap::new();
|
||||||
|
headers.insert("Content-Type", "application/xml".parse().unwrap());
|
||||||
|
|
||||||
|
let xml = format!(
|
||||||
|
"<?xml version=\"1.0\" encoding=\"UTF-8\"?>
|
||||||
|
<InitiateMultipartUploadResult xmlns=\"http://s3.amazonaws.com/doc/2006-03-01/\">
|
||||||
|
<Bucket>{}</Bucket>
|
||||||
|
<Key>{}</Key>
|
||||||
|
<UploadId>{}</UploadId>
|
||||||
|
</InitiateMultipartUploadResult>",
|
||||||
|
bucket, key, upload_id
|
||||||
|
);
|
||||||
|
|
||||||
|
(headers, xml)
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn complete_multipart_upload_xml(bucket: &str, key: &str, etag: &str) -> (HeaderMap, String) {
|
||||||
|
let mut headers = HeaderMap::new();
|
||||||
|
headers.insert("Content-Type", "application/xml".parse().unwrap());
|
||||||
|
|
||||||
|
let xml = format!(
|
||||||
|
"<?xml version=\"1.0\" encoding=\"UTF-8\"?>
|
||||||
|
<CompleteMultipartUploadResult xmlns=\"http://s3.amazonaws.com/doc/2006-03-01/\">
|
||||||
|
<Location>http://localhost:11438/s3/{}/{}</Location>
|
||||||
|
<Bucket>{}</Bucket>
|
||||||
|
<Key>{}</Key>
|
||||||
|
<ETag>{}</ETag>
|
||||||
|
</CompleteMultipartUploadResult>",
|
||||||
|
bucket, key, bucket, key, etag
|
||||||
|
);
|
||||||
|
|
||||||
|
(headers, xml)
|
||||||
|
}
|
||||||
|
|||||||
+568
-409
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,339 @@
|
|||||||
|
//! SMB Server Configuration Templates
|
||||||
|
//! Provides preset configurations for common deployment scenarios
|
||||||
|
|
||||||
|
use std::path::PathBuf;
|
||||||
|
|
||||||
|
/// SMB Server Configuration
|
||||||
|
#[derive(Clone, Debug)]
|
||||||
|
pub struct SmbConfig {
|
||||||
|
pub port: u16,
|
||||||
|
pub root: PathBuf,
|
||||||
|
pub share_name: String,
|
||||||
|
pub read_only: bool,
|
||||||
|
pub users: Vec<(String, String)>,
|
||||||
|
pub backend: SmbBackend,
|
||||||
|
pub ldap: Option<LdapConfig>,
|
||||||
|
pub cache: CacheConfig,
|
||||||
|
pub encryption: EncryptionConfig,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Clone, Debug)]
|
||||||
|
pub enum SmbBackend {
|
||||||
|
LocalFs,
|
||||||
|
S3 {
|
||||||
|
endpoint: String,
|
||||||
|
bucket: String,
|
||||||
|
access_key: String,
|
||||||
|
secret_key: String,
|
||||||
|
region: String,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Clone, Debug)]
|
||||||
|
pub struct LdapConfig {
|
||||||
|
pub url: String,
|
||||||
|
pub base_dn: String,
|
||||||
|
pub bind_dn: String,
|
||||||
|
pub bind_password: String,
|
||||||
|
pub user_search_base: String,
|
||||||
|
pub group_search_base: String,
|
||||||
|
pub user_id_attr: String,
|
||||||
|
pub user_filter: String,
|
||||||
|
pub group_filter: String,
|
||||||
|
pub home_dir_attr: String,
|
||||||
|
pub home_dir_prefix: String,
|
||||||
|
pub user_groups_attr: String,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Clone, Debug)]
|
||||||
|
pub struct CacheConfig {
|
||||||
|
pub read_cache_size_mb: usize,
|
||||||
|
pub write_cache_size_mb: usize,
|
||||||
|
pub cache_ttl_secs: u64,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Clone, Debug)]
|
||||||
|
pub enum EncryptionConfig {
|
||||||
|
Disabled,
|
||||||
|
Aes128Ctr,
|
||||||
|
Aes256Gcm,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Default for SmbConfig {
|
||||||
|
fn default() -> Self {
|
||||||
|
Self {
|
||||||
|
port: 4445,
|
||||||
|
root: PathBuf::from("/data/smb"),
|
||||||
|
share_name: "share".to_string(),
|
||||||
|
read_only: false,
|
||||||
|
users: vec![("demo".to_string(), "demo123".to_string())],
|
||||||
|
backend: SmbBackend::LocalFs,
|
||||||
|
ldap: None,
|
||||||
|
cache: CacheConfig::default(),
|
||||||
|
encryption: EncryptionConfig::Aes128Ctr,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Default for LdapConfig {
|
||||||
|
fn default() -> Self {
|
||||||
|
Self {
|
||||||
|
url: "ldap://localhost:389".to_string(),
|
||||||
|
base_dn: "dc=example,dc=com".to_string(),
|
||||||
|
bind_dn: "cn=admin,dc=example,dc=com".to_string(),
|
||||||
|
bind_password: "".to_string(),
|
||||||
|
user_search_base: "ou=users,dc=example,dc=com".to_string(),
|
||||||
|
group_search_base: "ou=groups,dc=example,dc=com".to_string(),
|
||||||
|
user_id_attr: "uid".to_string(),
|
||||||
|
user_filter: "(objectClass=person)".to_string(),
|
||||||
|
group_filter: "(objectClass=group)".to_string(),
|
||||||
|
home_dir_attr: "homeDirectory".to_string(),
|
||||||
|
home_dir_prefix: "/home".to_string(),
|
||||||
|
user_groups_attr: "memberOf".to_string(),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Default for CacheConfig {
|
||||||
|
fn default() -> Self {
|
||||||
|
Self {
|
||||||
|
read_cache_size_mb: 64,
|
||||||
|
write_cache_size_mb: 32,
|
||||||
|
cache_ttl_secs: 300,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl SmbConfig {
|
||||||
|
/// Generate TOML configuration template
|
||||||
|
pub fn generate_template() -> String {
|
||||||
|
let config = Self::default();
|
||||||
|
|
||||||
|
format!(
|
||||||
|
"# === SMB Server Configuration ===
|
||||||
|
# MarkBase SMB2/3 File Server
|
||||||
|
|
||||||
|
[smb]
|
||||||
|
# === Network Settings ===
|
||||||
|
port = {} # SMB server port (default: 4445)
|
||||||
|
share_name = \"{}\" # Share name visible to clients
|
||||||
|
read_only = {} # Read-only mode (default: false)
|
||||||
|
|
||||||
|
# === Storage Backend ===
|
||||||
|
# Options: localfs, s3
|
||||||
|
backend = \"localfs\" # Default: local filesystem
|
||||||
|
root = \"{}\" # Local filesystem root path
|
||||||
|
|
||||||
|
# === User Authentication ===
|
||||||
|
# Format: [[smb.users]]
|
||||||
|
# name = \"username\"
|
||||||
|
# password = \"password\" (bcrypt hashed in production)
|
||||||
|
|
||||||
|
[[smb.users]]
|
||||||
|
name = \"{}\"
|
||||||
|
password = \"{}\" # ⚠️ Use bcrypt hash in production
|
||||||
|
|
||||||
|
# === LDAP Integration (Optional) ===
|
||||||
|
# Uncomment to enable LDAP authentication
|
||||||
|
# [smb.ldap]
|
||||||
|
# url = \"ldap://localhost:389\" # LDAP server URL
|
||||||
|
# base_dn = \"dc=example,dc=com\" # Base DN
|
||||||
|
# bind_dn = \"cn=admin,dc=example,dc=com\" # Bind DN for search
|
||||||
|
# bind_password = \"admin_password\" # Bind password
|
||||||
|
# user_search_base = \"ou=users,dc=example,dc=com\"
|
||||||
|
# group_search_base = \"ou=groups,dc=example,dc=com\"
|
||||||
|
# user_id_attr = \"uid\" # User ID attribute
|
||||||
|
# user_filter = \"(objectClass=person)\" # User object filter
|
||||||
|
# group_filter = \"(objectClass=group)\" # Group object filter
|
||||||
|
# home_dir_attr = \"homeDirectory\" # Home directory attribute
|
||||||
|
# home_dir_prefix = \"/home\" # Home directory prefix
|
||||||
|
# user_groups_attr = \"memberOf\" # Group membership attribute
|
||||||
|
|
||||||
|
# === Performance Settings ===
|
||||||
|
[smb.cache]
|
||||||
|
read_cache_size_mb = {} # Read cache size (default: 64MB)
|
||||||
|
write_cache_size_mb = {} # Write cache size (default: 32MB)
|
||||||
|
cache_ttl_secs = {} # Cache TTL (default: 300s)
|
||||||
|
|
||||||
|
# === Security Settings ===
|
||||||
|
[smb.encryption]
|
||||||
|
# Options: disabled, aes128-ctr, aes256-gcm
|
||||||
|
mode = \"aes128-ctr\" # SMB3 encryption mode (default: AES-128-CTR)
|
||||||
|
|
||||||
|
# === S3 Backend (Optional) ===
|
||||||
|
# Uncomment to use S3 backend instead of local filesystem
|
||||||
|
# [smb.s3]
|
||||||
|
# endpoint = \"https://s3.amazonaws.com\"
|
||||||
|
# bucket = \"my-bucket\"
|
||||||
|
# access_key = \"AKIAIOSFODNN7EXAMPLE\"
|
||||||
|
# secret_key = \"wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY\"
|
||||||
|
# region = \"us-east-1\"
|
||||||
|
",
|
||||||
|
config.port,
|
||||||
|
config.share_name,
|
||||||
|
config.read_only,
|
||||||
|
config.root.display(),
|
||||||
|
config.users[0].0,
|
||||||
|
config.users[0].1,
|
||||||
|
config.cache.read_cache_size_mb,
|
||||||
|
config.cache.write_cache_size_mb,
|
||||||
|
config.cache.cache_ttl_secs,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Preset: Local File Server (simple deployment)
|
||||||
|
pub fn preset_local_file_server() -> Self {
|
||||||
|
Self {
|
||||||
|
port: 4445,
|
||||||
|
root: PathBuf::from("/data/smb"),
|
||||||
|
share_name: "files".to_string(),
|
||||||
|
read_only: false,
|
||||||
|
users: vec![
|
||||||
|
("alice".to_string(), "alice123".to_string()),
|
||||||
|
("bob".to_string(), "bob123".to_string()),
|
||||||
|
],
|
||||||
|
backend: SmbBackend::LocalFs,
|
||||||
|
ldap: None,
|
||||||
|
cache: CacheConfig {
|
||||||
|
read_cache_size_mb: 64,
|
||||||
|
write_cache_size_mb: 32,
|
||||||
|
cache_ttl_secs: 300,
|
||||||
|
},
|
||||||
|
encryption: EncryptionConfig::Aes128Ctr,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Preset: S3 Backend (cloud storage)
|
||||||
|
pub fn preset_s3_backend() -> Self {
|
||||||
|
Self {
|
||||||
|
port: 4445,
|
||||||
|
root: PathBuf::from("demo/"),
|
||||||
|
share_name: "s3share".to_string(),
|
||||||
|
read_only: false,
|
||||||
|
users: vec![("demo".to_string(), "demo123".to_string())],
|
||||||
|
backend: SmbBackend::S3 {
|
||||||
|
endpoint: "https://s3.amazonaws.com".to_string(),
|
||||||
|
bucket: "my-bucket".to_string(),
|
||||||
|
access_key: "AKIAIOSFODNN7EXAMPLE".to_string(),
|
||||||
|
secret_key: "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY".to_string(),
|
||||||
|
region: "us-east-1".to_string(),
|
||||||
|
},
|
||||||
|
ldap: None,
|
||||||
|
cache: CacheConfig {
|
||||||
|
read_cache_size_mb: 128,
|
||||||
|
write_cache_size_mb: 64,
|
||||||
|
cache_ttl_secs: 600,
|
||||||
|
},
|
||||||
|
encryption: EncryptionConfig::Aes256Gcm,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Preset: LDAP Enterprise (Active Directory integration)
|
||||||
|
pub fn preset_ldap_enterprise() -> Self {
|
||||||
|
Self {
|
||||||
|
port: 4445,
|
||||||
|
root: PathBuf::from("/data/smb"),
|
||||||
|
share_name: "enterprise".to_string(),
|
||||||
|
read_only: false,
|
||||||
|
users: vec![], // LDAP handles authentication
|
||||||
|
backend: SmbBackend::LocalFs,
|
||||||
|
ldap: Some(LdapConfig {
|
||||||
|
url: "ldap://ad.example.com:389".to_string(),
|
||||||
|
base_dn: "dc=example,dc=com".to_string(),
|
||||||
|
bind_dn: "cn=admin,cn=users,dc=example,dc=com".to_string(),
|
||||||
|
bind_password: "admin_password".to_string(),
|
||||||
|
user_search_base: "cn=users,dc=example,dc=com".to_string(),
|
||||||
|
group_search_base: "cn=groups,dc=example,dc=com".to_string(),
|
||||||
|
user_id_attr: "sAMAccountName".to_string(),
|
||||||
|
user_filter: "(objectClass=user)".to_string(),
|
||||||
|
group_filter: "(objectClass=group)".to_string(),
|
||||||
|
home_dir_attr: "homeDirectory".to_string(),
|
||||||
|
home_dir_prefix: "/home".to_string(),
|
||||||
|
user_groups_attr: "memberOf".to_string(),
|
||||||
|
}),
|
||||||
|
cache: CacheConfig {
|
||||||
|
read_cache_size_mb: 128,
|
||||||
|
write_cache_size_mb: 64,
|
||||||
|
cache_ttl_secs: 300,
|
||||||
|
},
|
||||||
|
encryption: EncryptionConfig::Aes256Gcm,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Preset: Read-Only Archive (public documents)
|
||||||
|
pub fn preset_read_only_archive() -> Self {
|
||||||
|
Self {
|
||||||
|
port: 4445,
|
||||||
|
root: PathBuf::from("/data/archive"),
|
||||||
|
share_name: "archive".to_string(),
|
||||||
|
read_only: true,
|
||||||
|
users: vec![("public".to_string(), "".to_string())],
|
||||||
|
backend: SmbBackend::LocalFs,
|
||||||
|
ldap: None,
|
||||||
|
cache: CacheConfig {
|
||||||
|
read_cache_size_mb: 256,
|
||||||
|
write_cache_size_mb: 0,
|
||||||
|
cache_ttl_secs: 3600,
|
||||||
|
},
|
||||||
|
encryption: EncryptionConfig::Disabled,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use super::*;
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_default_config() {
|
||||||
|
let config = SmbConfig::default();
|
||||||
|
assert_eq!(config.port, 4445);
|
||||||
|
assert_eq!(config.share_name, "share");
|
||||||
|
assert!(!config.read_only);
|
||||||
|
assert_eq!(config.users.len(), 1);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_generate_template() {
|
||||||
|
let template = SmbConfig::generate_template();
|
||||||
|
assert!(template.contains("port = 4445"));
|
||||||
|
assert!(template.contains("share_name = \"share\""));
|
||||||
|
assert!(template.contains("backend = \"localfs\""));
|
||||||
|
assert!(template.contains("[smb.users]"));
|
||||||
|
assert!(template.contains("[smb.cache]"));
|
||||||
|
assert!(template.contains("[smb.encryption]"));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_preset_local_file_server() {
|
||||||
|
let config = SmbConfig::preset_local_file_server();
|
||||||
|
assert_eq!(config.port, 4445);
|
||||||
|
assert_eq!(config.share_name, "files");
|
||||||
|
assert_eq!(config.users.len(), 2);
|
||||||
|
assert!(config.ldap.is_none());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_preset_s3_backend() {
|
||||||
|
let config = SmbConfig::preset_s3_backend();
|
||||||
|
assert_eq!(config.share_name, "s3share");
|
||||||
|
assert!(matches!(config.backend, SmbBackend::S3 { .. }));
|
||||||
|
assert!(matches!(config.encryption, EncryptionConfig::Aes256Gcm));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_preset_ldap_enterprise() {
|
||||||
|
let config = SmbConfig::preset_ldap_enterprise();
|
||||||
|
assert_eq!(config.share_name, "enterprise");
|
||||||
|
assert!(config.ldap.is_some());
|
||||||
|
assert_eq!(config.users.len(), 0);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_preset_read_only_archive() {
|
||||||
|
let config = SmbConfig::preset_read_only_archive();
|
||||||
|
assert!(config.read_only);
|
||||||
|
assert_eq!(config.share_name, "archive");
|
||||||
|
assert!(matches!(config.encryption, EncryptionConfig::Disabled));
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -320,7 +320,7 @@ impl ChannelManager {
|
|||||||
|
|
||||||
// 解析forwarded-tcpip参数
|
// 解析forwarded-tcpip参数
|
||||||
let mut port_forward_manager = PortForwardManager::new();
|
let mut port_forward_manager = PortForwardManager::new();
|
||||||
let forwarded_tcpip =
|
let _forwarded_tcpip =
|
||||||
port_forward_manager.handle_forwarded_tcpip_channel(&packet.payload)?;
|
port_forward_manager.handle_forwarded_tcpip_channel(&packet.payload)?;
|
||||||
|
|
||||||
let server_channel = self.next_channel_id;
|
let server_channel = self.next_channel_id;
|
||||||
@@ -398,7 +398,7 @@ direct_tcpip: None,
|
|||||||
|
|
||||||
// 创建 X11ForwardContext(从 DISPLAY 环境变量)
|
// 创建 X11ForwardContext(从 DISPLAY 环境变量)
|
||||||
let display = std::env::var("DISPLAY").unwrap_or_else(|_| ":0".to_string());
|
let display = std::env::var("DISPLAY").unwrap_or_else(|_| ":0".to_string());
|
||||||
let x11_ctx = super::x11_forward::X11ForwardContext::new(&display)?;
|
let _x11_ctx = super::x11_forward::X11ForwardContext::new(&display)?;
|
||||||
|
|
||||||
let server_channel = self.next_channel_id;
|
let server_channel = self.next_channel_id;
|
||||||
self.next_channel_id += 1;
|
self.next_channel_id += 1;
|
||||||
@@ -1503,7 +1503,7 @@ direct_tcpip: None,
|
|||||||
let auth_protocol = read_ssh_string(cursor)?;
|
let auth_protocol = read_ssh_string(cursor)?;
|
||||||
|
|
||||||
// auth_cookie: SSH string (hex-encoded cookie)
|
// auth_cookie: SSH string (hex-encoded cookie)
|
||||||
let auth_cookie_hex = read_ssh_string(cursor)?;
|
let _auth_cookie_hex = read_ssh_string(cursor)?;
|
||||||
|
|
||||||
// screen_number: u32
|
// screen_number: u32
|
||||||
let screen_number = cursor.read_u32::<BigEndian>()?;
|
let screen_number = cursor.read_u32::<BigEndian>()?;
|
||||||
|
|||||||
@@ -20,6 +20,7 @@ use hmac::{Hmac, Mac};
|
|||||||
use log::info;
|
use log::info;
|
||||||
use sha2::Sha256;
|
use sha2::Sha256;
|
||||||
use std::io::Write;
|
use std::io::Write;
|
||||||
|
use tokio::io::{AsyncReadExt, AsyncWriteExt};
|
||||||
|
|
||||||
type Aes128Ctr = Ctr128BE<Aes128>; // AES-128-CTR(16字节密钥)
|
type Aes128Ctr = Ctr128BE<Aes128>; // AES-128-CTR(16字节密钥)
|
||||||
type HmacSha256 = Hmac<Sha256>;
|
type HmacSha256 = Hmac<Sha256>;
|
||||||
@@ -1167,6 +1168,187 @@ impl EncryptedPacket {
|
|||||||
pub fn take_payload(&mut self) -> Vec<u8> {
|
pub fn take_payload(&mut self) -> Vec<u8> {
|
||||||
std::mem::take(&mut self.payload)
|
std::mem::take(&mut self.payload)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// ── Async I/O (tokio) ─────────────────────────────────────────────
|
||||||
|
|
||||||
|
/// Async write encrypted packet (tokio::io::AsyncWriteExt)
|
||||||
|
pub async fn write_async<W: AsyncWriteExt + Unpin>(&self, stream: &mut W) -> Result<()> {
|
||||||
|
if self.payload.len() > 4 && self.payload[0..4] == self.packet_length.to_be_bytes() {
|
||||||
|
stream.write_all(&self.payload).await?;
|
||||||
|
} else {
|
||||||
|
stream.write_all(&self.payload).await?;
|
||||||
|
stream.write_all(&self.mac).await?;
|
||||||
|
}
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Async read encrypted packet (tokio::io::AsyncReadExt)
|
||||||
|
pub async fn read_async<R: AsyncReadExt + Unpin>(
|
||||||
|
stream: &mut R,
|
||||||
|
encryption_ctx: &mut EncryptionContext,
|
||||||
|
is_client_to_server: bool,
|
||||||
|
) -> Result<Self> {
|
||||||
|
if encryption_ctx.cipher_mode == CipherMode::AesGcm {
|
||||||
|
let mut packet_length_bytes = [0u8; 4];
|
||||||
|
stream.read_exact(&mut packet_length_bytes).await?;
|
||||||
|
let packet_length = u32::from_be_bytes(packet_length_bytes);
|
||||||
|
if packet_length > 35000 {
|
||||||
|
return Err(anyhow!("Invalid packet_length: {}", packet_length));
|
||||||
|
}
|
||||||
|
let ciphertext_length = packet_length as usize + 16;
|
||||||
|
let mut ciphertext = vec![0u8; ciphertext_length];
|
||||||
|
stream.read_exact(&mut ciphertext).await?;
|
||||||
|
|
||||||
|
let sequence_number = if is_client_to_server {
|
||||||
|
encryption_ctx.sequence_number_ctos
|
||||||
|
} else {
|
||||||
|
encryption_ctx.sequence_number_stoc
|
||||||
|
};
|
||||||
|
let iv_bytes = if is_client_to_server {
|
||||||
|
&encryption_ctx.iv_ctos
|
||||||
|
} else {
|
||||||
|
&encryption_ctx.iv_stoc
|
||||||
|
};
|
||||||
|
let mut nonce_bytes = [0u8; 12];
|
||||||
|
nonce_bytes.copy_from_slice(&iv_bytes[..12]);
|
||||||
|
let mut carry = sequence_number;
|
||||||
|
for i in (8..12).rev() {
|
||||||
|
let sum = nonce_bytes[i] as u16 + (carry & 0xFF) as u16;
|
||||||
|
nonce_bytes[i] = (sum & 0xFF) as u8;
|
||||||
|
carry = (carry >> 8) + ((sum >> 8) as u32);
|
||||||
|
}
|
||||||
|
if carry > 0 {
|
||||||
|
for i in (4..8).rev() {
|
||||||
|
let sum = nonce_bytes[i] as u16 + (carry & 0xFF) as u16;
|
||||||
|
nonce_bytes[i] = (sum & 0xFF) as u8;
|
||||||
|
carry = (carry >> 8) + ((sum >> 8) as u32);
|
||||||
|
if carry == 0 { break; }
|
||||||
|
}
|
||||||
|
}
|
||||||
|
let key_bytes = if is_client_to_server {
|
||||||
|
&encryption_ctx.encryption_key_ctos
|
||||||
|
} else {
|
||||||
|
&encryption_ctx.encryption_key_stoc
|
||||||
|
};
|
||||||
|
let cipher = Aes256GcmAead::new_from_slice(&key_bytes[..32])
|
||||||
|
.map_err(|e| anyhow!("AES-GCM key init failed: {}", e))?;
|
||||||
|
let nonce = Nonce::from_slice(&nonce_bytes);
|
||||||
|
let plaintext_payload_buffer = cipher.decrypt(nonce, Payload {
|
||||||
|
msg: ciphertext.as_slice(),
|
||||||
|
aad: &packet_length_bytes,
|
||||||
|
}).map_err(|e| anyhow!("AES-GCM decrypt failed: {}", e))?;
|
||||||
|
|
||||||
|
let padding_length = plaintext_payload_buffer[0];
|
||||||
|
let payload_len = packet_length as usize - padding_length as usize - 1;
|
||||||
|
let compressed_payload = plaintext_payload_buffer[1..1 + payload_len].to_vec();
|
||||||
|
let payload = if is_client_to_server {
|
||||||
|
if encryption_ctx.compression_ctos.is_enabled() {
|
||||||
|
encryption_ctx.compression_ctos.decompress(&compressed_payload)?
|
||||||
|
} else { compressed_payload }
|
||||||
|
} else { compressed_payload };
|
||||||
|
let mac = ciphertext[ciphertext.len() - 16..].to_vec();
|
||||||
|
if is_client_to_server {
|
||||||
|
encryption_ctx.sequence_number_ctos += 1;
|
||||||
|
} else {
|
||||||
|
encryption_ctx.sequence_number_stoc += 1;
|
||||||
|
}
|
||||||
|
return Ok(Self { packet_length, padding_length, payload, padding: Vec::new(), mac });
|
||||||
|
} else if encryption_ctx.cipher_mode == CipherMode::ChaChaPoly {
|
||||||
|
let mut packet_length_bytes = [0u8; 4];
|
||||||
|
stream.read_exact(&mut packet_length_bytes).await?;
|
||||||
|
let packet_length = u32::from_be_bytes(packet_length_bytes);
|
||||||
|
if packet_length > 35000 {
|
||||||
|
return Err(anyhow!("Invalid packet_length: {}", packet_length));
|
||||||
|
}
|
||||||
|
let ciphertext_length = packet_length as usize + 16;
|
||||||
|
let mut ciphertext = vec![0u8; ciphertext_length];
|
||||||
|
stream.read_exact(&mut ciphertext).await?;
|
||||||
|
|
||||||
|
let sequence_number = if is_client_to_server {
|
||||||
|
encryption_ctx.sequence_number_ctos
|
||||||
|
} else {
|
||||||
|
encryption_ctx.sequence_number_stoc
|
||||||
|
};
|
||||||
|
let iv_bytes = if is_client_to_server {
|
||||||
|
&encryption_ctx.iv_ctos
|
||||||
|
} else {
|
||||||
|
&encryption_ctx.iv_stoc
|
||||||
|
};
|
||||||
|
let nonce_bytes: [u8; 12] = {
|
||||||
|
let mut n = [0u8; 12];
|
||||||
|
n[0..4].copy_from_slice(&sequence_number.to_be_bytes());
|
||||||
|
n[4..12].copy_from_slice(&iv_bytes[..8]);
|
||||||
|
n
|
||||||
|
};
|
||||||
|
let key_bytes = if is_client_to_server {
|
||||||
|
&encryption_ctx.encryption_key_ctos
|
||||||
|
} else {
|
||||||
|
&encryption_ctx.encryption_key_stoc
|
||||||
|
};
|
||||||
|
let cipher_cha = ChaCha20Poly1305::new(ChaKey::from_slice(&key_bytes[..32]));
|
||||||
|
let nonce = ChaNonce::from_slice(&nonce_bytes);
|
||||||
|
let plaintext_payload_buffer = cipher_cha.decrypt(nonce, ChaPayload {
|
||||||
|
msg: ciphertext.as_slice(),
|
||||||
|
aad: &packet_length_bytes,
|
||||||
|
}).map_err(|e| anyhow!("ChaCha20Poly1305 decrypt failed: {}", e))?;
|
||||||
|
|
||||||
|
let padding_length = plaintext_payload_buffer[0];
|
||||||
|
let payload_len = packet_length as usize - padding_length as usize - 1;
|
||||||
|
let payload = plaintext_payload_buffer[1..1 + payload_len].to_vec();
|
||||||
|
let mac = ciphertext[ciphertext.len() - 16..].to_vec();
|
||||||
|
if is_client_to_server {
|
||||||
|
encryption_ctx.sequence_number_ctos += 1;
|
||||||
|
} else {
|
||||||
|
encryption_ctx.sequence_number_stoc += 1;
|
||||||
|
}
|
||||||
|
return Ok(Self { packet_length, padding_length, payload, padding: Vec::new(), mac });
|
||||||
|
} else {
|
||||||
|
let mut first_block_encrypted = [0u8; 16];
|
||||||
|
stream.read_exact(&mut first_block_encrypted).await?;
|
||||||
|
let cipher = if is_client_to_server {
|
||||||
|
encryption_ctx.cipher_ctos.as_mut()
|
||||||
|
.ok_or_else(|| anyhow!("cipher_ctos not initialized"))?
|
||||||
|
} else {
|
||||||
|
encryption_ctx.cipher_stoc.as_mut()
|
||||||
|
.ok_or_else(|| anyhow!("cipher_stoc not initialized"))?
|
||||||
|
};
|
||||||
|
let mut first_block_decrypted = first_block_encrypted;
|
||||||
|
cipher.apply_keystream(&mut first_block_decrypted);
|
||||||
|
let packet_length = u32::from_be_bytes([first_block_decrypted[0], first_block_decrypted[1], first_block_decrypted[2], first_block_decrypted[3]]);
|
||||||
|
let padding_length = first_block_decrypted[4];
|
||||||
|
if packet_length > 35000 {
|
||||||
|
return Err(anyhow!("Invalid packet_length: {}", packet_length));
|
||||||
|
}
|
||||||
|
let total_encrypted_size = packet_length as usize + 4;
|
||||||
|
let remaining_size = total_encrypted_size.saturating_sub(16);
|
||||||
|
let mut remaining_encrypted = vec![0u8; remaining_size];
|
||||||
|
if remaining_size > 0 {
|
||||||
|
stream.read_exact(&mut remaining_encrypted).await?;
|
||||||
|
}
|
||||||
|
cipher.apply_keystream(&mut remaining_encrypted);
|
||||||
|
let payload_len = packet_length as usize - padding_length as usize - 1;
|
||||||
|
let part1_len = std::cmp::min(payload_len, 11);
|
||||||
|
let part1 = &first_block_decrypted[5..5 + part1_len];
|
||||||
|
let part2 = &remaining_encrypted[..payload_len.saturating_sub(part1_len)];
|
||||||
|
let mut payload = Vec::with_capacity(payload_len);
|
||||||
|
payload.extend_from_slice(part1);
|
||||||
|
payload.extend_from_slice(part2);
|
||||||
|
let payload = if is_client_to_server {
|
||||||
|
if encryption_ctx.compression_ctos.is_enabled() {
|
||||||
|
encryption_ctx.compression_ctos.decompress(&payload)?
|
||||||
|
} else { payload }
|
||||||
|
} else { payload };
|
||||||
|
let padding = remaining_encrypted[payload_len.saturating_sub(part1_len)..].to_vec();
|
||||||
|
let mut mac = vec![0u8; 32];
|
||||||
|
stream.read_exact(&mut mac).await?;
|
||||||
|
if is_client_to_server {
|
||||||
|
encryption_ctx.sequence_number_ctos += 1;
|
||||||
|
} else {
|
||||||
|
encryption_ctx.sequence_number_stoc += 1;
|
||||||
|
}
|
||||||
|
return Ok(Self { packet_length, padding_length, payload, padding, mac });
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
|
|||||||
@@ -4,7 +4,7 @@
|
|||||||
//! Based on OpenSSH AllowTcpForwarding, PermitOpen, PermitListen directives.
|
//! Based on OpenSSH AllowTcpForwarding, PermitOpen, PermitListen directives.
|
||||||
|
|
||||||
use std::collections::HashMap;
|
use std::collections::HashMap;
|
||||||
use std::net::{IpAddr, Ipv4Addr, Ipv6Addr};
|
use std::net::{IpAddr, Ipv4Addr};
|
||||||
use std::sync::{Arc, RwLock};
|
use std::sync::{Arc, RwLock};
|
||||||
|
|
||||||
/// Forward rule type
|
/// Forward rule type
|
||||||
@@ -175,7 +175,7 @@ impl ForwardAcl {
|
|||||||
.write()
|
.write()
|
||||||
.unwrap()
|
.unwrap()
|
||||||
.entry(rule.direction)
|
.entry(rule.direction)
|
||||||
.or_insert_with(Vec::new)
|
.or_default()
|
||||||
.push(rule);
|
.push(rule);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -3,7 +3,6 @@ use ed25519_dalek::{Signer, SigningKey};
|
|||||||
use log::{info, warn};
|
use log::{info, warn};
|
||||||
use rand::rngs::OsRng;
|
use rand::rngs::OsRng;
|
||||||
use std::fs;
|
use std::fs;
|
||||||
use std::io::{Read, Write};
|
|
||||||
use std::path::{Path, PathBuf};
|
use std::path::{Path, PathBuf};
|
||||||
use std::time::{Duration, SystemTime};
|
use std::time::{Duration, SystemTime};
|
||||||
|
|
||||||
@@ -153,7 +152,7 @@ impl HostKeyManager {
|
|||||||
self.ensure_keys_dir()?;
|
self.ensure_keys_dir()?;
|
||||||
|
|
||||||
let signing_key = SigningKey::generate(&mut OsRng);
|
let signing_key = SigningKey::generate(&mut OsRng);
|
||||||
let verifying_key = signing_key.verifying_key();
|
let _verifying_key = signing_key.verifying_key();
|
||||||
|
|
||||||
self.save_ed25519_private_key(&signing_key, &key_path)?;
|
self.save_ed25519_private_key(&signing_key, &key_path)?;
|
||||||
self.save_ed25519_public_key(&signing_key, &pub_path)?;
|
self.save_ed25519_public_key(&signing_key, &pub_path)?;
|
||||||
@@ -227,7 +226,7 @@ impl HostKeyManager {
|
|||||||
fn save_ed25519_private_key(&self, key: &SigningKey, path: &Path) -> Result<()> {
|
fn save_ed25519_private_key(&self, key: &SigningKey, path: &Path) -> Result<()> {
|
||||||
let key_bytes = key.to_bytes();
|
let key_bytes = key.to_bytes();
|
||||||
|
|
||||||
fs::write(path, &key_bytes)?;
|
fs::write(path, key_bytes)?;
|
||||||
|
|
||||||
#[cfg(unix)]
|
#[cfg(unix)]
|
||||||
{
|
{
|
||||||
|
|||||||
@@ -2,7 +2,7 @@
|
|||||||
// 参考OpenSSH kex.c: kex_input_kex_init(), kex_send_kex_reply()
|
// 参考OpenSSH kex.c: kex_input_kex_init(), kex_send_kex_reply()
|
||||||
|
|
||||||
use crate::ssh_server::crypto::{Curve25519Kex, SessionKeys};
|
use crate::ssh_server::crypto::{Curve25519Kex, SessionKeys};
|
||||||
use crate::ssh_server::host_key::{HostKey, HostKeyManager, HostKeyType};
|
use crate::ssh_server::host_key::{HostKey, HostKeyManager};
|
||||||
use crate::ssh_server::kex::KexResult;
|
use crate::ssh_server::kex::KexResult;
|
||||||
use crate::ssh_server::packet::{PacketType, SshPacket};
|
use crate::ssh_server::packet::{PacketType, SshPacket};
|
||||||
use anyhow::{anyhow, Result};
|
use anyhow::{anyhow, Result};
|
||||||
|
|||||||
@@ -1,9 +1,8 @@
|
|||||||
use anyhow::{anyhow, Result};
|
use anyhow::{anyhow, Result};
|
||||||
use log::{info, warn};
|
use log::{info, warn};
|
||||||
use std::collections::HashMap;
|
|
||||||
use std::fs;
|
use std::fs;
|
||||||
use std::io::{BufRead, BufReader};
|
use std::io::{BufRead, BufReader};
|
||||||
use std::net::{IpAddr, Ipv4Addr, Ipv6Addr};
|
use std::net::{IpAddr, Ipv4Addr};
|
||||||
use std::path::{Path, PathBuf};
|
use std::path::{Path, PathBuf};
|
||||||
|
|
||||||
#[derive(Debug, Clone, PartialEq)]
|
#[derive(Debug, Clone, PartialEq)]
|
||||||
@@ -48,18 +47,17 @@ impl KnownHostEntry {
|
|||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
if let Some(ip_addr) = ip {
|
if let Some(ip_addr) = ip {
|
||||||
if part == &ip_addr.to_string() {
|
if part == ip_addr.to_string() {
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
if host.starts_with('|') {
|
if host.starts_with('|')
|
||||||
if self.matches_pattern_host(host, hostname) {
|
&& self.matches_pattern_host(host, hostname) {
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
false
|
false
|
||||||
@@ -154,6 +152,12 @@ pub struct KnownHostsParser {
|
|||||||
entries: Vec<KnownHostEntry>,
|
entries: Vec<KnownHostEntry>,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
impl Default for KnownHostsParser {
|
||||||
|
fn default() -> Self {
|
||||||
|
Self::new()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
impl KnownHostsParser {
|
impl KnownHostsParser {
|
||||||
pub fn new() -> Self {
|
pub fn new() -> Self {
|
||||||
Self {
|
Self {
|
||||||
@@ -214,7 +218,7 @@ impl KnownHostsParser {
|
|||||||
(parts[0], parts[1], parts[2], &parts[3..])
|
(parts[0], parts[1], parts[2], &parts[3..])
|
||||||
};
|
};
|
||||||
|
|
||||||
let comment = if rest_parts.len() > 0 {
|
let comment = if !rest_parts.is_empty() {
|
||||||
Some(rest_parts.join(" "))
|
Some(rest_parts.join(" "))
|
||||||
} else {
|
} else {
|
||||||
None
|
None
|
||||||
@@ -325,14 +329,14 @@ impl KnownHostsParser {
|
|||||||
let salt: [u8; 20] = rand::rngs::OsRng.gen();
|
let salt: [u8; 20] = rand::rngs::OsRng.gen();
|
||||||
|
|
||||||
let mut hasher = Sha256::new();
|
let mut hasher = Sha256::new();
|
||||||
hasher.update(&salt);
|
hasher.update(salt);
|
||||||
hasher.update(hostname.as_bytes());
|
hasher.update(hostname.as_bytes());
|
||||||
let hash = hasher.finalize();
|
let hash = hasher.finalize();
|
||||||
|
|
||||||
Ok(format!(
|
Ok(format!(
|
||||||
"|1|{}|{}|{}",
|
"|1|{}|{}|{}",
|
||||||
STANDARD.encode(&salt),
|
STANDARD.encode(salt),
|
||||||
STANDARD.encode(&hash),
|
STANDARD.encode(hash),
|
||||||
hostname
|
hostname
|
||||||
))
|
))
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,6 +1,5 @@
|
|||||||
use std::collections::HashMap;
|
use std::collections::HashMap;
|
||||||
use std::net::SocketAddr;
|
use std::net::SocketAddr;
|
||||||
use std::sync::Arc;
|
|
||||||
use std::time::{Duration, Instant};
|
use std::time::{Duration, Instant};
|
||||||
use tokio::sync::RwLock;
|
use tokio::sync::RwLock;
|
||||||
|
|
||||||
@@ -195,7 +194,7 @@ impl MultiplexManager {
|
|||||||
pub async fn add_channel_to_session(&self, session_id: u64) -> Result<(), MultiplexError> {
|
pub async fn add_channel_to_session(&self, session_id: u64) -> Result<(), MultiplexError> {
|
||||||
let mut connections = self.connections.write().await;
|
let mut connections = self.connections.write().await;
|
||||||
|
|
||||||
for connection in connections.values_mut() {
|
if let Some(connection) = connections.values_mut().next() {
|
||||||
let session = connection
|
let session = connection
|
||||||
.sessions
|
.sessions
|
||||||
.get_mut(&session_id)
|
.get_mut(&session_id)
|
||||||
@@ -216,7 +215,7 @@ impl MultiplexManager {
|
|||||||
pub async fn remove_channel_from_session(&self, session_id: u64) -> Result<(), MultiplexError> {
|
pub async fn remove_channel_from_session(&self, session_id: u64) -> Result<(), MultiplexError> {
|
||||||
let mut connections = self.connections.write().await;
|
let mut connections = self.connections.write().await;
|
||||||
|
|
||||||
for connection in connections.values_mut() {
|
if let Some(connection) = connections.values_mut().next() {
|
||||||
let session = connection
|
let session = connection
|
||||||
.sessions
|
.sessions
|
||||||
.get_mut(&session_id)
|
.get_mut(&session_id)
|
||||||
|
|||||||
@@ -4,6 +4,7 @@
|
|||||||
use anyhow::{anyhow, Result};
|
use anyhow::{anyhow, Result};
|
||||||
use byteorder::{BigEndian, ReadBytesExt, WriteBytesExt};
|
use byteorder::{BigEndian, ReadBytesExt, WriteBytesExt};
|
||||||
use std::io::{Read, Write};
|
use std::io::{Read, Write};
|
||||||
|
use tokio::io::{AsyncReadExt, AsyncWriteExt};
|
||||||
|
|
||||||
/// SSH Packet类型(参考OpenSSH SSH_MSG_*定义)
|
/// SSH Packet类型(参考OpenSSH SSH_MSG_*定义)
|
||||||
#[derive(Debug, Clone, Copy, PartialEq)]
|
#[derive(Debug, Clone, Copy, PartialEq)]
|
||||||
@@ -160,6 +161,39 @@ impl SshPacket {
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Async write (tokio)
|
||||||
|
pub async fn write_async<W: AsyncWriteExt + Unpin>(&self, stream: &mut W) -> Result<()> {
|
||||||
|
stream.write_all(&self.packet_length.to_be_bytes()).await?;
|
||||||
|
stream.write_all(&[self.padding_length]).await?;
|
||||||
|
stream.write_all(&self.payload).await?;
|
||||||
|
stream.write_all(&self.padding).await?;
|
||||||
|
stream.flush().await?;
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Async read (tokio)
|
||||||
|
pub async fn read_async<R: AsyncReadExt + Unpin>(stream: &mut R) -> Result<Self> {
|
||||||
|
let mut len_buf = [0u8; 4];
|
||||||
|
stream.read_exact(&mut len_buf).await?;
|
||||||
|
let packet_length = u32::from_be_bytes(len_buf);
|
||||||
|
if packet_length > 256 * 1024 {
|
||||||
|
return Err(anyhow!("Packet too large: {}", packet_length));
|
||||||
|
}
|
||||||
|
let mut pad_buf = [0u8; 1];
|
||||||
|
stream.read_exact(&mut pad_buf).await?;
|
||||||
|
let padding_length = pad_buf[0];
|
||||||
|
let payload_len = packet_length.saturating_sub(padding_length as u32 + 1);
|
||||||
|
let mut payload = vec![0u8; payload_len as usize];
|
||||||
|
if !payload.is_empty() {
|
||||||
|
stream.read_exact(&mut payload).await?;
|
||||||
|
}
|
||||||
|
let mut padding = vec![0u8; padding_length as usize];
|
||||||
|
if !padding.is_empty() {
|
||||||
|
stream.read_exact(&mut padding).await?;
|
||||||
|
}
|
||||||
|
Ok(Self { packet_length, padding_length, payload, padding })
|
||||||
|
}
|
||||||
|
|
||||||
/// 获取payload中的packet type
|
/// 获取payload中的packet type
|
||||||
pub fn get_type(&self) -> Result<PacketType> {
|
pub fn get_type(&self) -> Result<PacketType> {
|
||||||
if self.payload.is_empty() {
|
if self.payload.is_empty() {
|
||||||
|
|||||||
@@ -1,6 +1,5 @@
|
|||||||
use std::collections::HashMap;
|
use std::collections::HashMap;
|
||||||
use std::net::IpAddr;
|
use std::net::IpAddr;
|
||||||
use std::sync::Arc;
|
|
||||||
use std::time::{Duration, Instant};
|
use std::time::{Duration, Instant};
|
||||||
use tokio::sync::RwLock;
|
use tokio::sync::RwLock;
|
||||||
|
|
||||||
|
|||||||
@@ -17,10 +17,10 @@ use crate::ssh_server::version::VersionExchange;
|
|||||||
use anyhow::{anyhow, Result};
|
use anyhow::{anyhow, Result};
|
||||||
use log::{error, info, warn};
|
use log::{error, info, warn};
|
||||||
use std::io::{Read, Write};
|
use std::io::{Read, Write};
|
||||||
use std::net::{TcpListener, TcpStream};
|
use std::net::TcpStream;
|
||||||
use std::path::PathBuf;
|
use std::path::PathBuf;
|
||||||
use std::sync::{Arc, Mutex};
|
use std::sync::{Arc, Mutex};
|
||||||
use std::thread;
|
use tokio::net::TcpListener;
|
||||||
|
|
||||||
pub struct SshServerConfig {
|
pub struct SshServerConfig {
|
||||||
pub port: u16,
|
pub port: u16,
|
||||||
@@ -71,11 +71,11 @@ impl SshServer {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn run(&self) -> Result<()> {
|
pub async fn run(&self) -> Result<()> {
|
||||||
let bind_addr = format!("{}:{}", self.config.bind_address, self.config.port);
|
let bind_addr = format!("{}:{}", self.config.bind_address, self.config.port);
|
||||||
let listener = TcpListener::bind(&bind_addr)?;
|
let listener = TcpListener::bind(&bind_addr).await?;
|
||||||
|
|
||||||
info!("MarkBaseSSH server listening on {}", bind_addr);
|
info!("MarkBaseSSH server listening on {} (async tokio)", bind_addr);
|
||||||
info!("Implementation: Complete SSH/SFTP + Port Forwarding (Phase 1-13)");
|
info!("Implementation: Complete SSH/SFTP + Port Forwarding (Phase 1-13)");
|
||||||
info!(
|
info!(
|
||||||
"Security config: GatewayPorts={}, PermitOpen={:?}, MaxSessions={}",
|
"Security config: GatewayPorts={}, PermitOpen={:?}, MaxSessions={}",
|
||||||
@@ -88,23 +88,30 @@ impl SshServer {
|
|||||||
let pg_conn = self.config.pg_conn.clone();
|
let pg_conn = self.config.pg_conn.clone();
|
||||||
let upload_hook_config = self.config.upload_hook_config.clone();
|
let upload_hook_config = self.config.upload_hook_config.clone();
|
||||||
|
|
||||||
for stream in listener.incoming() {
|
loop {
|
||||||
match stream {
|
match listener.accept().await {
|
||||||
Ok(stream) => {
|
Ok((stream, addr)) => {
|
||||||
let client_addr = stream.peer_addr()?;
|
info!("New SSH connection from {}", addr);
|
||||||
info!("New SSH connection from {}", client_addr);
|
|
||||||
|
|
||||||
let security_config_clone = security_config.clone();
|
let security_config_clone = security_config.clone();
|
||||||
let pg_conn_clone = pg_conn.clone();
|
let pg_conn_clone = pg_conn.clone();
|
||||||
let upload_hook_config_clone = upload_hook_config.clone();
|
let upload_hook_config_clone = upload_hook_config.clone();
|
||||||
|
|
||||||
thread::spawn(move || {
|
// ⭐⭐⭐⭐⭐ Convert tokio TcpStream to std TcpStream for blocking handler
|
||||||
if let Err(e) = handle_connection_complete(
|
// Set blocking explicitly since into_std() may preserve non-blocking mode
|
||||||
stream,
|
let std_stream = stream.into_std()?;
|
||||||
security_config_clone,
|
std_stream.set_nonblocking(false)?;
|
||||||
pg_conn_clone,
|
|
||||||
upload_hook_config_clone,
|
tokio::spawn(async move {
|
||||||
)
|
// Run the existing sync connection handler in a blocking thread
|
||||||
|
if let Err(e) = tokio::task::spawn_blocking(move || {
|
||||||
|
handle_connection_complete(
|
||||||
|
std_stream,
|
||||||
|
security_config_clone,
|
||||||
|
pg_conn_clone,
|
||||||
|
upload_hook_config_clone,
|
||||||
|
)
|
||||||
|
}).await.unwrap_or(Err(anyhow!("Task join error")))
|
||||||
{
|
{
|
||||||
error!("SSH connection error: {}", e);
|
error!("SSH connection error: {}", e);
|
||||||
}
|
}
|
||||||
@@ -115,8 +122,6 @@ impl SshServer {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
Ok(())
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -787,7 +792,7 @@ fn extract_username_from_auth_request(
|
|||||||
}
|
}
|
||||||
|
|
||||||
/// SSH服务器CLI入口
|
/// SSH服务器CLI入口
|
||||||
pub fn run_ssh_server(port: Option<u16>, pg_conn: Option<&str>) -> Result<()> {
|
pub async fn run_ssh_server(port: Option<u16>, pg_conn: Option<&str>) -> Result<()> {
|
||||||
let config = SshServerConfig {
|
let config = SshServerConfig {
|
||||||
port: port.unwrap_or(2024),
|
port: port.unwrap_or(2024),
|
||||||
bind_address: "0.0.0.0".to_string(), // ⭐⭐⭐⭐⭐ Phase 8.3: Allow Docker container access
|
bind_address: "0.0.0.0".to_string(), // ⭐⭐⭐⭐⭐ Phase 8.3: Allow Docker container access
|
||||||
@@ -797,5 +802,5 @@ pub fn run_ssh_server(port: Option<u16>, pg_conn: Option<&str>) -> Result<()> {
|
|||||||
};
|
};
|
||||||
|
|
||||||
let server = SshServer::new(config);
|
let server = SshServer::new(config);
|
||||||
server.run()
|
server.run().await
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,9 +1,6 @@
|
|||||||
use serde::Serialize;
|
use serde::Serialize;
|
||||||
use std::net::IpAddr;
|
use std::net::IpAddr;
|
||||||
use std::time::SystemTime;
|
use tracing::Subscriber;
|
||||||
use tracing::{Event, Level, Subscriber};
|
|
||||||
use tracing_subscriber::fmt::FormatEvent;
|
|
||||||
use tracing_subscriber::fmt::format::{Format, Json};
|
|
||||||
use tracing_subscriber::layer::Layer;
|
use tracing_subscriber::layer::Layer;
|
||||||
|
|
||||||
pub struct SshAuditLog;
|
pub struct SshAuditLog;
|
||||||
|
|||||||
@@ -7,7 +7,7 @@ use anyhow::{Result, anyhow};
|
|||||||
use log::info;
|
use log::info;
|
||||||
use std::path::PathBuf;
|
use std::path::PathBuf;
|
||||||
use std::net::TcpStream;
|
use std::net::TcpStream;
|
||||||
use std::io::{Read, Write};
|
use std::io::Read;
|
||||||
|
|
||||||
/// X11 authentication cookie type (RFC 4254 §7.2).
|
/// X11 authentication cookie type (RFC 4254 §7.2).
|
||||||
#[derive(Debug, Clone)]
|
#[derive(Debug, Clone)]
|
||||||
|
|||||||
+119
-242
@@ -2,268 +2,145 @@
|
|||||||
<html lang="en">
|
<html lang="en">
|
||||||
<head>
|
<head>
|
||||||
<meta charset="UTF-8">
|
<meta charset="UTF-8">
|
||||||
|
<meta name="viewport" content="width=device-width, initial-scale=1">
|
||||||
<title>File Upload</title>
|
<title>File Upload</title>
|
||||||
<style>
|
<style>
|
||||||
body {
|
* { margin: 0; padding: 0; box-sizing: border-box; }
|
||||||
font-family: Arial, sans-serif;
|
body { font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", sans-serif; background: #f5f5f7; color: #1d1d1f; padding: 20px; }
|
||||||
max-width: 800px;
|
.container { max-width: 800px; margin: 0 auto; }
|
||||||
margin: 50px auto;
|
h1 { font-size: 28px; margin-bottom: 8px; }
|
||||||
padding: 20px;
|
.desc { color: #6e6e73; margin-bottom: 24px; }
|
||||||
background: #f5f5f5;
|
.card { background: #fff; border-radius: 12px; padding: 24px; box-shadow: 0 1px 4px rgba(0,0,0,0.08); margin-bottom: 16px; }
|
||||||
}
|
.form-group { margin-bottom: 16px; }
|
||||||
.upload-container {
|
label { display: block; font-weight: 600; margin-bottom: 6px; font-size: 14px; }
|
||||||
background: white;
|
input[type="text"] { width: 100%; padding: 10px 12px; border: 1px solid #d2d2d7; border-radius: 8px; font-size: 14px; }
|
||||||
padding: 30px;
|
input[type="text"]:focus { outline: none; border-color: #0071e3; }
|
||||||
border-radius: 8px;
|
.radio-group { display: flex; gap: 16px; margin-top: 6px; }
|
||||||
box-shadow: 0 2px 10px rgba(0,0,0,0.1);
|
.radio-group label { font-weight: 400; font-size: 14px; display: flex; align-items: center; gap: 6px; cursor: pointer; }
|
||||||
}
|
.file-input-wrap { margin-top: 8px; }
|
||||||
h1 {
|
.file-input-wrap input[type="file"] { width: 100%; padding: 8px; border: 1px solid #d2d2d7; border-radius: 8px; font-size: 14px; }
|
||||||
color: #333;
|
.hint { font-size: 12px; color: #6e6e73; margin-top: 4px; }
|
||||||
text-align: center;
|
.btn { padding: 10px 24px; border: none; border-radius: 8px; cursor: pointer; font-size: 14px; font-weight: 500; }
|
||||||
}
|
.btn-primary { background: #0071e3; color: #fff; }
|
||||||
.upload-form {
|
.btn-primary:hover { background: #0058b0; }
|
||||||
margin-top: 20px;
|
.btn-primary:disabled { opacity: 0.5; cursor: not-allowed; }
|
||||||
}
|
.progress-wrap { margin-top: 16px; display: none; }
|
||||||
.form-group {
|
.progress-bar { width: 100%; height: 8px; background: #e8e8ed; border-radius: 4px; overflow: hidden; }
|
||||||
margin-bottom: 15px;
|
.progress-fill { height: 100%; background: #0071e3; width: 0%; transition: width 0.3s; border-radius: 4px; }
|
||||||
}
|
.progress-text { font-size: 13px; color: #6e6e73; margin-top: 8px; }
|
||||||
label {
|
.result { margin-top: 16px; padding: 12px 16px; border-radius: 8px; font-size: 14px; display: none; }
|
||||||
display: block;
|
.result.success { background: #d1fae5; color: #065f46; }
|
||||||
margin-bottom: 5px;
|
.result.error { background: #fee2e2; color: #991b1b; }
|
||||||
font-weight: bold;
|
|
||||||
}
|
|
||||||
input[type="text"], input[type="file"] {
|
|
||||||
width: 100%;
|
|
||||||
padding: 10px;
|
|
||||||
border: 1px solid #ddd;
|
|
||||||
border-radius: 4px;
|
|
||||||
}
|
|
||||||
button {
|
|
||||||
background: #007bff;
|
|
||||||
color: white;
|
|
||||||
padding: 12px 24px;
|
|
||||||
border: none;
|
|
||||||
border-radius: 4px;
|
|
||||||
cursor: pointer;
|
|
||||||
font-size: 16px;
|
|
||||||
}
|
|
||||||
button:hover {
|
|
||||||
background: #0056b3;
|
|
||||||
}
|
|
||||||
.progress {
|
|
||||||
margin-top: 20px;
|
|
||||||
display: none;
|
|
||||||
}
|
|
||||||
.progress-bar {
|
|
||||||
width: 100%;
|
|
||||||
height: 20px;
|
|
||||||
background: #e0e0e0;
|
|
||||||
border-radius: 4px;
|
|
||||||
overflow: hidden;
|
|
||||||
}
|
|
||||||
.progress-fill {
|
|
||||||
height: 100%;
|
|
||||||
background: #007bff;
|
|
||||||
width: 0%;
|
|
||||||
transition: width 0.3s;
|
|
||||||
}
|
|
||||||
.result {
|
|
||||||
margin-top: 20px;
|
|
||||||
padding: 15px;
|
|
||||||
border-radius: 4px;
|
|
||||||
display: none;
|
|
||||||
}
|
|
||||||
.success {
|
|
||||||
background: #d4edda;
|
|
||||||
color: #155724;
|
|
||||||
border: 1px solid #c3e6cb;
|
|
||||||
}
|
|
||||||
.error {
|
|
||||||
background: #f8d7da;
|
|
||||||
color: #721c24;
|
|
||||||
border: 1px solid #f5c6cb;
|
|
||||||
}
|
|
||||||
</style>
|
</style>
|
||||||
</head>
|
</head>
|
||||||
<body>
|
<body>
|
||||||
<div class="upload-container">
|
<div class="container">
|
||||||
<h1>📁 File Upload Service</h1>
|
<h1>Upload</h1>
|
||||||
|
<p class="desc">Upload files to user storage directory</p>
|
||||||
|
|
||||||
<div class="upload-form">
|
<div class="card">
|
||||||
<div class="form-group">
|
<div class="form-group">
|
||||||
<label for="user_id">User ID:</label>
|
<label for="user_id">User ID</label>
|
||||||
<input type="text" id="user_id" value="accusys" placeholder="Enter User ID">
|
<input type="text" id="user_id" value="demo">
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
<div class="form-group">
|
<div class="form-group">
|
||||||
<label>Upload Mode:</label>
|
<label>Mode</label>
|
||||||
<div style="margin-top: 10px;">
|
<div class="radio-group">
|
||||||
<label style="margin-right: 20px;">
|
<label><input type="radio" name="mode" value="file" checked onchange="toggleMode()"> Single File</label>
|
||||||
<input type="radio" name="upload_mode" value="folder" checked onchange="toggleUploadMode()">
|
<label><input type="radio" name="mode" value="folder" onchange="toggleMode()"> Folder (all files)</label>
|
||||||
📁 Folder Upload (webkitdirectory)
|
</div>
|
||||||
</label>
|
</div>
|
||||||
<label>
|
|
||||||
<input type="radio" name="upload_mode" value="file" onchange="toggleUploadMode()">
|
|
||||||
📄 Single File Upload (supports ZIP)
|
|
||||||
</label>
|
|
||||||
</div>
|
|
||||||
</div>
|
|
||||||
|
|
||||||
<div class="form-group" id="folder-upload-group">
|
<div class="form-group">
|
||||||
<label for="folder">Select Folder:</label>
|
<div id="single-group">
|
||||||
<input type="file" id="folder" multiple webkitdirectory>
|
<div class="file-input-wrap">
|
||||||
<p style="color: #666; font-size: 12px; margin-top: 5px;">
|
<input type="file" id="single_file">
|
||||||
Upload entire folder with subdirectories
|
|
||||||
</p>
|
|
||||||
</div>
|
|
||||||
|
|
||||||
<div class="form-group" id="file-upload-group" style="display: none;">
|
|
||||||
<label for="file">Select File:</label>
|
|
||||||
<input type="file" id="single_file" accept=".zip,.rar,.7z,.tar,.gz,.bz2,.pdf,.doc,.docx,.xls,.xlsx,.ppt,.pptx,.txt,.md,.py,.rs,.js,.ts,.html,.css,.json,.xml,.yaml,.yml,.jpg,.jpeg,.png,.gif,.bmp,.svg,.mp4,.mov,.avi,.mkv,.mp3,.wav,.flac">
|
|
||||||
<p style="color: #666; font-size: 12px; margin-top: 5px;">
|
|
||||||
Supports: ZIP, RAR, 7Z, TAR, PDF, Office, Text, Code, Images, Videos, Audio files
|
|
||||||
</p>
|
|
||||||
</div>
|
|
||||||
|
|
||||||
<button onclick="uploadFiles()">Start Upload</button>
|
|
||||||
</div>
|
</div>
|
||||||
|
</div>
|
||||||
|
<div id="folder-group" style="display:none">
|
||||||
|
<div class="file-input-wrap">
|
||||||
|
<input type="file" id="folder" multiple webkitdirectory>
|
||||||
|
</div>
|
||||||
|
<p class="hint">Uploads all files in the selected folder</p>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<button class="btn btn-primary" id="upload-btn" onclick="uploadFiles()">Upload</button>
|
||||||
|
|
||||||
|
<div class="progress-wrap" id="progress">
|
||||||
|
<div class="progress-bar"><div class="progress-fill" id="progress-fill"></div></div>
|
||||||
|
<div class="progress-text" id="progress-text"></div>
|
||||||
|
</div>
|
||||||
|
<div class="result" id="result"></div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
<script>
|
<script>
|
||||||
function toggleUploadMode() {
|
function toggleMode() {
|
||||||
const mode = document.querySelector('input[name="upload_mode"]:checked').value;
|
const mode = document.querySelector('input[name="mode"]:checked').value;
|
||||||
const folderGroup = document.getElementById('folder-upload-group');
|
document.getElementById('single-group').style.display = mode === 'file' ? 'block' : 'none';
|
||||||
const fileGroup = document.getElementById('file-upload-group');
|
document.getElementById('folder-group').style.display = mode === 'folder' ? 'block' : 'none';
|
||||||
|
}
|
||||||
|
|
||||||
if (mode === 'folder') {
|
async function uploadFiles() {
|
||||||
folderGroup.style.display = 'block';
|
const uid = document.getElementById('user_id').value.trim();
|
||||||
fileGroup.style.display = 'none';
|
if (!uid) return showError('Enter a user ID');
|
||||||
} else {
|
|
||||||
folderGroup.style.display = 'none';
|
|
||||||
fileGroup.style.display = 'block';
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
async function uploadFiles() {
|
const mode = document.querySelector('input[name="mode"]:checked').value;
|
||||||
const userId = document.getElementById('user_id').value.trim();
|
const files = mode === 'folder'
|
||||||
if (!userId) {
|
? document.getElementById('folder').files
|
||||||
alert('Please enter User ID');
|
: document.getElementById('single_file').files;
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
const uploadMode = document.querySelector('input[name="upload_mode"]:checked').value;
|
if (!files || files.length === 0) return showError('Select a file or folder');
|
||||||
let files;
|
|
||||||
|
|
||||||
if (uploadMode === 'folder') {
|
const btn = document.getElementById('upload-btn');
|
||||||
files = document.getElementById('folder').files;
|
btn.disabled = true;
|
||||||
} else {
|
|
||||||
files = document.getElementById('single_file').files;
|
|
||||||
}
|
|
||||||
|
|
||||||
if (!files || files.length === 0) {
|
const progress = document.getElementById('progress');
|
||||||
alert('Please select files or folder');
|
const fill = document.getElementById('progress-fill');
|
||||||
return;
|
const ptext = document.getElementById('progress-text');
|
||||||
}
|
const result = document.getElementById('result');
|
||||||
const fileInput = document.getElementById('file');
|
progress.style.display = 'block';
|
||||||
const files = fileInput.files;
|
result.style.display = 'none';
|
||||||
|
|
||||||
if (!user_id || files.length === 0) {
|
let uploaded = 0;
|
||||||
showError('Please enter User ID and select at least one file');
|
const total = files.length;
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
const progressDiv = document.getElementById('progress');
|
for (let i = 0; i < total; i++) {
|
||||||
const progressFill = document.getElementById('progress-fill');
|
const f = files[i];
|
||||||
const progressText = document.getElementById('progress-text');
|
const fd = new FormData();
|
||||||
const resultDiv = document.getElementById('result');
|
fd.append('file', f);
|
||||||
|
ptext.textContent = `Uploading ${f.name} (${i+1}/${total})`;
|
||||||
|
|
||||||
progressDiv.style.display = 'block';
|
try {
|
||||||
resultDiv.style.display = 'none';
|
const res = await fetch(`/api/v2/upload-unlimited/${uid}`, { method: 'POST', body: fd });
|
||||||
|
if (!res.ok) { showError(`${f.name}: HTTP ${res.status}`); btn.disabled = false; return; }
|
||||||
|
const data = await res.json();
|
||||||
|
if (!data.ok) { showError(`${f.name}: ${data.error || 'unknown'}`); btn.disabled = false; return; }
|
||||||
|
uploaded++;
|
||||||
|
const pct = Math.round(uploaded / total * 100);
|
||||||
|
fill.style.width = pct + '%';
|
||||||
|
ptext.textContent = `${pct}% (${uploaded}/${total})`;
|
||||||
|
} catch(e) {
|
||||||
|
showError(`${f.name}: ${e.message}`);
|
||||||
|
btn.disabled = false;
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
let uploaded = 0;
|
showSuccess(`Uploaded ${uploaded} file${uploaded > 1 ? 's' : ''}`);
|
||||||
const total = files.length;
|
btn.disabled = false;
|
||||||
|
}
|
||||||
|
|
||||||
for (let i = 0; i < files.length; i++) {
|
function showSuccess(m) { showResult(m, 'success'); }
|
||||||
const file = files[i];
|
function showError(m) { showResult(m, 'error'); }
|
||||||
const formData = new FormData();
|
function showResult(m, t) {
|
||||||
formData.append('file', file);
|
const r = document.getElementById('result');
|
||||||
|
r.className = 'result ' + t;
|
||||||
// Create AbortController with timeout (30 minutes for large files)
|
r.textContent = m;
|
||||||
const controller = new AbortController();
|
r.style.display = 'block';
|
||||||
const timeoutId = setTimeout(() => {
|
}
|
||||||
controller.abort();
|
</script>
|
||||||
showError(`File ${file.name} upload timeout (30 minutes limit)`);
|
|
||||||
}, 30 * 60 * 1000); // 30 minutes
|
|
||||||
|
|
||||||
try {
|
|
||||||
progressText.textContent = `Uploading: ${file.name} (${uploaded + 1}/${total})`;
|
|
||||||
|
|
||||||
const response = await fetch(`/api/v2/upload-unlimited/${user_id}`, {
|
|
||||||
method: 'POST',
|
|
||||||
body: formData,
|
|
||||||
signal: controller.signal
|
|
||||||
});
|
|
||||||
|
|
||||||
clearTimeout(timeoutId); // Clear timeout if upload succeeds
|
|
||||||
|
|
||||||
// Check HTTP status
|
|
||||||
if (!response.ok) {
|
|
||||||
showError(`File ${file.name} upload failed: HTTP ${response.status} ${response.statusText}`);
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
// Check response body
|
|
||||||
const text = await response.text();
|
|
||||||
if (!text || text.trim() === '') {
|
|
||||||
showError(`File ${file.name} upload failed: Server returned empty response`);
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
// Parse JSON
|
|
||||||
let result;
|
|
||||||
try {
|
|
||||||
result = JSON.parse(text);
|
|
||||||
} catch (parseError) {
|
|
||||||
showError(`File ${file.name} upload failed: JSON parse error - ${parseError.message}`);
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
if (result.ok) {
|
|
||||||
uploaded++;
|
|
||||||
const percent = Math.round((uploaded / total) * 100);
|
|
||||||
progressFill.style.width = percent + '%';
|
|
||||||
progressText.textContent = `Upload progress: ${percent}% (${uploaded}/${total})`;
|
|
||||||
} else {
|
|
||||||
showError(`File ${file.name} upload failed: ${result.error || 'Unknown error'}`);
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
} catch (err) {
|
|
||||||
clearTimeout(timeoutId);
|
|
||||||
|
|
||||||
if (err.name === 'AbortError') {
|
|
||||||
showError(`File ${file.name} upload timeout (30 minutes limit)`);
|
|
||||||
} else {
|
|
||||||
showError(`File ${file.name} upload error: ${err.message}`);
|
|
||||||
}
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
showSuccess(`Successfully uploaded ${uploaded} files!`);
|
|
||||||
}
|
|
||||||
|
|
||||||
function showSuccess(message) {
|
|
||||||
const resultDiv = document.getElementById('result');
|
|
||||||
resultDiv.className = 'result success';
|
|
||||||
resultDiv.textContent = message;
|
|
||||||
resultDiv.style.display = 'block';
|
|
||||||
}
|
|
||||||
|
|
||||||
function showError(message) {
|
|
||||||
const resultDiv = document.getElementById('result');
|
|
||||||
resultDiv.className = 'result error';
|
|
||||||
resultDiv.textContent = message;
|
|
||||||
resultDiv.style.display = 'block';
|
|
||||||
}
|
|
||||||
</script>
|
|
||||||
</body>
|
</body>
|
||||||
</html>
|
</html>
|
||||||
@@ -0,0 +1,255 @@
|
|||||||
|
use std::path::{Path, PathBuf};
|
||||||
|
use std::time::SystemTime;
|
||||||
|
use std::pin::Pin;
|
||||||
|
use std::future::Future;
|
||||||
|
use std::io::{self, SeekFrom};
|
||||||
|
|
||||||
|
use tokio::fs;
|
||||||
|
use tokio::io::{AsyncReadExt, AsyncWriteExt, AsyncSeekExt};
|
||||||
|
|
||||||
|
use super::{VfsError, VfsStat, VfsDirEntry, open_flags::OpenFlags};
|
||||||
|
|
||||||
|
/// Async VFS 文件實現(使用 tokio::fs)
|
||||||
|
pub struct AsyncLocalFile {
|
||||||
|
file: fs::File,
|
||||||
|
path: PathBuf,
|
||||||
|
is_write: bool,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl AsyncLocalFile {
|
||||||
|
pub fn new(file: fs::File, path: PathBuf, is_write: bool) -> Self {
|
||||||
|
Self { file, path, is_write }
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl super::AsyncVfsFile for AsyncLocalFile {
|
||||||
|
fn read<'a>(&'a mut self, buf: &'a mut [u8]) -> Pin<Box<dyn Future<Output = Result<usize, VfsError>> + Send + 'a>> {
|
||||||
|
Box::pin(async move {
|
||||||
|
self.file.read(buf).await
|
||||||
|
.map_err(|e| VfsError::Io(e.to_string()))
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
fn write<'a>(&'a mut self, buf: &'a [u8]) -> Pin<Box<dyn Future<Output = Result<usize, VfsError>> + Send + 'a>> {
|
||||||
|
Box::pin(async move {
|
||||||
|
self.file.write(buf).await
|
||||||
|
.map_err(|e| VfsError::Io(e.to_string()))
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
fn seek<'a>(&'a mut self, pos: SeekFrom) -> Pin<Box<dyn Future<Output = Result<u64, VfsError>> + Send + 'a>> {
|
||||||
|
Box::pin(async move {
|
||||||
|
self.file.seek(pos).await
|
||||||
|
.map_err(|e| VfsError::Io(e.to_string()))
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
fn flush<'a>(&'a mut self) -> Pin<Box<dyn Future<Output = Result<(), VfsError>> + Send + 'a>> {
|
||||||
|
Box::pin(async move {
|
||||||
|
self.file.flush().await
|
||||||
|
.map_err(|e| VfsError::Io(e.to_string()))
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Async VFS 后端實現(使用 tokio::fs)
|
||||||
|
pub struct AsyncLocalFs {
|
||||||
|
root: PathBuf,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Default for AsyncLocalFs {
|
||||||
|
fn default() -> Self {
|
||||||
|
Self::new()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl AsyncLocalFs {
|
||||||
|
pub fn new() -> Self {
|
||||||
|
Self { root: PathBuf::new() }
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn with_root(root: PathBuf) -> Self {
|
||||||
|
Self { root }
|
||||||
|
}
|
||||||
|
|
||||||
|
fn map_io_error(path: &Path, e: io::Error) -> VfsError {
|
||||||
|
match e.kind() {
|
||||||
|
io::ErrorKind::NotFound => VfsError::NotFound(path.to_string_lossy().to_string()),
|
||||||
|
io::ErrorKind::PermissionDenied => VfsError::PermissionDenied(path.to_string_lossy().to_string()),
|
||||||
|
io::ErrorKind::AlreadyExists => VfsError::AlreadyExists(path.to_string_lossy().to_string()),
|
||||||
|
_ => VfsError::Io(e.to_string()),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn stat_from_metadata(meta: &std::fs::Metadata) -> VfsStat {
|
||||||
|
VfsStat {
|
||||||
|
size: meta.len(),
|
||||||
|
mode: 0o644,
|
||||||
|
uid: 0,
|
||||||
|
gid: 0,
|
||||||
|
atime: meta.accessed().unwrap_or(SystemTime::UNIX_EPOCH),
|
||||||
|
mtime: meta.modified().unwrap_or(SystemTime::UNIX_EPOCH),
|
||||||
|
is_dir: meta.is_dir(),
|
||||||
|
is_symlink: meta.file_type().is_symlink(),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Clone for AsyncLocalFs {
|
||||||
|
fn clone(&self) -> Self {
|
||||||
|
Self { root: self.root.clone() }
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl super::AsyncVfsBackend for AsyncLocalFs {
|
||||||
|
fn clone_boxed(&self) -> Box<dyn super::AsyncVfsBackend> {
|
||||||
|
Box::new(self.clone())
|
||||||
|
}
|
||||||
|
|
||||||
|
fn read_dir<'a>(&'a self, path: &'a Path) -> Pin<Box<dyn Future<Output = Result<Vec<VfsDirEntry>, VfsError>> + Send + 'a>> {
|
||||||
|
Box::pin(async move {
|
||||||
|
let mut entries = Vec::new();
|
||||||
|
let mut dir = fs::read_dir(path).await
|
||||||
|
.map_err(|e| Self::map_io_error(path, e))?;
|
||||||
|
|
||||||
|
while let Some(entry) = dir.next_entry().await.map_err(|e| Self::map_io_error(path, e))? {
|
||||||
|
let name = entry.file_name().to_string_lossy().to_string();
|
||||||
|
let long_name = name.clone();
|
||||||
|
let meta = entry.metadata().await.map_err(|e| Self::map_io_error(path, e))?;
|
||||||
|
let stat = Self::stat_from_metadata(&meta);
|
||||||
|
entries.push(VfsDirEntry { name, long_name, stat });
|
||||||
|
}
|
||||||
|
|
||||||
|
Ok(entries)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
fn open_file<'a>(&'a self, path: &'a Path, flags: &'a OpenFlags) -> Pin<Box<dyn Future<Output = Result<Box<dyn super::AsyncVfsFile>, VfsError>> + Send + 'a>> {
|
||||||
|
Box::pin(async move {
|
||||||
|
let mut options = fs::OpenOptions::new();
|
||||||
|
|
||||||
|
if flags.read {
|
||||||
|
options.read(true);
|
||||||
|
}
|
||||||
|
if flags.write {
|
||||||
|
options.write(true);
|
||||||
|
}
|
||||||
|
if flags.create {
|
||||||
|
options.create(true);
|
||||||
|
}
|
||||||
|
if flags.truncate {
|
||||||
|
options.truncate(true);
|
||||||
|
}
|
||||||
|
if flags.append {
|
||||||
|
options.append(true);
|
||||||
|
}
|
||||||
|
|
||||||
|
let file = options.open(path).await
|
||||||
|
.map_err(|e| Self::map_io_error(path, e))?;
|
||||||
|
|
||||||
|
Ok(Box::new(AsyncLocalFile::new(file, path.to_path_buf(), flags.write)) as Box<dyn super::AsyncVfsFile>)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
fn stat<'a>(&'a self, path: &'a Path) -> Pin<Box<dyn Future<Output = Result<VfsStat, VfsError>> + Send + 'a>> {
|
||||||
|
Box::pin(async move {
|
||||||
|
let meta = fs::metadata(path).await
|
||||||
|
.map_err(|e| Self::map_io_error(path, e))?;
|
||||||
|
Ok(Self::stat_from_metadata(&meta))
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
fn create_dir<'a>(&'a self, path: &'a Path, _mode: u32) -> Pin<Box<dyn Future<Output = Result<(), VfsError>> + Send + 'a>> {
|
||||||
|
Box::pin(async move {
|
||||||
|
fs::create_dir(path).await
|
||||||
|
.map_err(|e| Self::map_io_error(path, e))?;
|
||||||
|
Ok(())
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
fn remove_dir<'a>(&'a self, path: &'a Path) -> Pin<Box<dyn Future<Output = Result<(), VfsError>> + Send + 'a>> {
|
||||||
|
Box::pin(async move {
|
||||||
|
fs::remove_dir(path).await
|
||||||
|
.map_err(|e| Self::map_io_error(path, e))?;
|
||||||
|
Ok(())
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
fn remove_file<'a>(&'a self, path: &'a Path) -> Pin<Box<dyn Future<Output = Result<(), VfsError>> + Send + 'a>> {
|
||||||
|
Box::pin(async move {
|
||||||
|
fs::remove_file(path).await
|
||||||
|
.map_err(|e| Self::map_io_error(path, e))?;
|
||||||
|
Ok(())
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
fn rename<'a>(&'a self, from: &'a Path, to: &'a Path) -> Pin<Box<dyn Future<Output = Result<(), VfsError>> + Send + 'a>> {
|
||||||
|
Box::pin(async move {
|
||||||
|
fs::rename(from, to).await
|
||||||
|
.map_err(|e| Self::map_io_error(from, e))?;
|
||||||
|
Ok(())
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
fn exists<'a>(&'a self, path: &'a Path) -> Pin<Box<dyn Future<Output = bool> + Send + 'a>> {
|
||||||
|
Box::pin(async move {
|
||||||
|
fs::metadata(path).await.is_ok()
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use super::*;
|
||||||
|
use tempfile::TempDir;
|
||||||
|
use crate::vfs::AsyncVfsBackend;
|
||||||
|
use crate::vfs::AsyncVfsFile;
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn test_async_read_dir() {
|
||||||
|
let tmp = TempDir::new().unwrap();
|
||||||
|
fs::create_dir(tmp.path().join("subdir")).await.unwrap();
|
||||||
|
fs::write(tmp.path().join("test.txt"), "content").await.unwrap();
|
||||||
|
|
||||||
|
let vfs = AsyncLocalFs::with_root(tmp.path().to_path_buf());
|
||||||
|
let entries = AsyncVfsBackend::read_dir(&vfs, tmp.path()).await.unwrap();
|
||||||
|
|
||||||
|
assert_eq!(entries.len(), 2);
|
||||||
|
assert!(entries.iter().any(|e| e.name == "subdir" && e.stat.is_dir));
|
||||||
|
assert!(entries.iter().any(|e| e.name == "test.txt" && !e.stat.is_dir));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn test_async_open_read() {
|
||||||
|
let tmp = TempDir::new().unwrap();
|
||||||
|
fs::write(tmp.path().join("test.txt"), "hello world").await.unwrap();
|
||||||
|
|
||||||
|
let vfs = AsyncLocalFs::with_root(tmp.path().to_path_buf());
|
||||||
|
let flags = OpenFlags::new().read();
|
||||||
|
let mut file = AsyncVfsBackend::open_file(&vfs, &tmp.path().join("test.txt"), &flags).await.unwrap();
|
||||||
|
|
||||||
|
let mut buf = [0u8; 11];
|
||||||
|
let n = AsyncVfsFile::read(&mut *file, &mut buf).await.unwrap();
|
||||||
|
assert_eq!(n, 11);
|
||||||
|
assert_eq!(&buf, b"hello world");
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn test_async_create_dir() {
|
||||||
|
let tmp = TempDir::new().unwrap();
|
||||||
|
let vfs = AsyncLocalFs::with_root(tmp.path().to_path_buf());
|
||||||
|
|
||||||
|
AsyncVfsBackend::create_dir(&vfs, &tmp.path().join("newdir"), 0o755).await.unwrap();
|
||||||
|
assert!(AsyncVfsBackend::exists(&vfs, &tmp.path().join("newdir")).await);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn test_async_remove_file() {
|
||||||
|
let tmp = TempDir::new().unwrap();
|
||||||
|
fs::write(tmp.path().join("test.txt"), "content").await.unwrap();
|
||||||
|
|
||||||
|
let vfs = AsyncLocalFs::with_root(tmp.path().to_path_buf());
|
||||||
|
AsyncVfsBackend::remove_file(&vfs, &tmp.path().join("test.txt")).await.unwrap();
|
||||||
|
assert!(!AsyncVfsBackend::exists(&vfs, &tmp.path().join("test.txt")).await);
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,454 @@
|
|||||||
|
use std::path::Path;
|
||||||
|
use std::pin::Pin;
|
||||||
|
use std::future::Future;
|
||||||
|
use std::io::{SeekFrom};
|
||||||
|
use std::sync::Arc;
|
||||||
|
use std::time::Duration;
|
||||||
|
|
||||||
|
use tokio::sync::Mutex;
|
||||||
|
use reqwest::Client;
|
||||||
|
use rusty_s3::{Bucket, Credentials, S3Action, actions, UrlStyle};
|
||||||
|
use url::Url;
|
||||||
|
|
||||||
|
use super::{VfsError, VfsStat, VfsDirEntry, open_flags::OpenFlags};
|
||||||
|
|
||||||
|
pub struct AsyncS3Vfs {
|
||||||
|
bucket: Bucket,
|
||||||
|
credentials: Credentials,
|
||||||
|
client: Client,
|
||||||
|
}
|
||||||
|
|
||||||
|
struct AsyncS3FileState {
|
||||||
|
key: String,
|
||||||
|
mode: FileMode,
|
||||||
|
position: u64,
|
||||||
|
size: u64,
|
||||||
|
data: Vec<u8>,
|
||||||
|
write_buffer: Vec<u8>,
|
||||||
|
mtime: std::time::SystemTime,
|
||||||
|
}
|
||||||
|
|
||||||
|
enum FileMode {
|
||||||
|
Read,
|
||||||
|
Write,
|
||||||
|
}
|
||||||
|
|
||||||
|
pub struct AsyncS3File {
|
||||||
|
inner: Arc<Mutex<AsyncS3FileState>>,
|
||||||
|
vfs: AsyncS3Vfs,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl AsyncS3Vfs {
|
||||||
|
pub fn new(
|
||||||
|
endpoint: &str,
|
||||||
|
region: &str,
|
||||||
|
bucket_name: &str,
|
||||||
|
access_key: &str,
|
||||||
|
secret_key: &str,
|
||||||
|
) -> Result<Self, VfsError> {
|
||||||
|
let endpoint_url = Url::parse(endpoint.trim_end_matches('/'))
|
||||||
|
.map_err(|e| VfsError::Io(format!("Invalid S3 endpoint URL: {}", e)))?;
|
||||||
|
|
||||||
|
let bucket = Bucket::new(
|
||||||
|
endpoint_url,
|
||||||
|
UrlStyle::Path,
|
||||||
|
bucket_name.to_string(),
|
||||||
|
region.to_string(),
|
||||||
|
).map_err(|e| VfsError::Io(format!("Failed to create S3 bucket config: {}", e)))?;
|
||||||
|
|
||||||
|
let credentials = Credentials::new(access_key, secret_key);
|
||||||
|
let client = Client::new();
|
||||||
|
|
||||||
|
Ok(Self { bucket, credentials, client })
|
||||||
|
}
|
||||||
|
|
||||||
|
fn path_to_key(path: &Path) -> String {
|
||||||
|
let s = path.to_string_lossy();
|
||||||
|
s.strip_prefix('/').unwrap_or(&s).to_string()
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn head_object(&self, key: &str) -> Result<(u64, std::time::SystemTime, String), VfsError> {
|
||||||
|
let action = actions::HeadObject::new(&self.bucket, Some(&self.credentials), key);
|
||||||
|
let url = action.sign(Duration::from_secs(3600));
|
||||||
|
|
||||||
|
let resp = self.client
|
||||||
|
.head(url.as_str())
|
||||||
|
.send()
|
||||||
|
.await
|
||||||
|
.map_err(|e| VfsError::Io(format!("S3 HEAD failed: {}", e)))?;
|
||||||
|
|
||||||
|
let status = resp.status();
|
||||||
|
if status == 404 {
|
||||||
|
return Err(VfsError::NotFound(key.to_string()));
|
||||||
|
}
|
||||||
|
if !status.is_success() {
|
||||||
|
return Err(VfsError::Io(format!("HeadObject returned {}", status)));
|
||||||
|
}
|
||||||
|
|
||||||
|
let content_len: u64 = resp
|
||||||
|
.headers()
|
||||||
|
.get("Content-Length")
|
||||||
|
.and_then(|v| v.to_str().ok())
|
||||||
|
.and_then(|v| v.parse().ok())
|
||||||
|
.unwrap_or(0);
|
||||||
|
|
||||||
|
let last_modified = parse_last_modified(
|
||||||
|
resp.headers()
|
||||||
|
.get("Last-Modified")
|
||||||
|
.and_then(|v| v.to_str().ok())
|
||||||
|
);
|
||||||
|
|
||||||
|
let etag = resp
|
||||||
|
.headers()
|
||||||
|
.get("ETag")
|
||||||
|
.and_then(|v| v.to_str().ok())
|
||||||
|
.map(|s| s.replace('"', ""))
|
||||||
|
.unwrap_or_default();
|
||||||
|
|
||||||
|
Ok((content_len, last_modified, etag))
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn get_object(&self, key: &str) -> Result<Vec<u8>, VfsError> {
|
||||||
|
let action = actions::GetObject::new(&self.bucket, Some(&self.credentials), key);
|
||||||
|
let url = action.sign(Duration::from_secs(3600));
|
||||||
|
|
||||||
|
let resp = self.client
|
||||||
|
.get(url.as_str())
|
||||||
|
.send()
|
||||||
|
.await
|
||||||
|
.map_err(|e| VfsError::Io(format!("S3 GET failed: {}", e)))?;
|
||||||
|
|
||||||
|
let status = resp.status();
|
||||||
|
if status == 404 {
|
||||||
|
return Err(VfsError::NotFound(key.to_string()));
|
||||||
|
}
|
||||||
|
if !status.is_success() {
|
||||||
|
return Err(VfsError::Io(format!("GetObject returned {}", status)));
|
||||||
|
}
|
||||||
|
|
||||||
|
let bytes = resp.bytes().await
|
||||||
|
.map_err(|e| VfsError::Io(format!("Failed to read response body: {}", e)))?;
|
||||||
|
|
||||||
|
Ok(bytes.to_vec())
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn put_object(&self, key: &str, data: &[u8]) -> Result<String, VfsError> {
|
||||||
|
let action = actions::PutObject::new(&self.bucket, Some(&self.credentials), key);
|
||||||
|
let url = action.sign(Duration::from_secs(3600));
|
||||||
|
|
||||||
|
let resp = self.client
|
||||||
|
.put(url.as_str())
|
||||||
|
.body(data.to_vec())
|
||||||
|
.send()
|
||||||
|
.await
|
||||||
|
.map_err(|e| VfsError::Io(format!("S3 PUT failed: {}", e)))?;
|
||||||
|
|
||||||
|
if !resp.status().is_success() {
|
||||||
|
return Err(VfsError::Io(format!("PutObject returned {}", resp.status())));
|
||||||
|
}
|
||||||
|
|
||||||
|
let etag = resp
|
||||||
|
.headers()
|
||||||
|
.get("ETag")
|
||||||
|
.and_then(|v| v.to_str().ok())
|
||||||
|
.map(|s| s.replace('"', ""))
|
||||||
|
.unwrap_or_default();
|
||||||
|
|
||||||
|
Ok(etag)
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn delete_object(&self, key: &str) -> Result<(), VfsError> {
|
||||||
|
let action = actions::DeleteObject::new(&self.bucket, Some(&self.credentials), key);
|
||||||
|
let url = action.sign(Duration::from_secs(3600));
|
||||||
|
|
||||||
|
let resp = self.client
|
||||||
|
.delete(url.as_str())
|
||||||
|
.send()
|
||||||
|
.await
|
||||||
|
.map_err(|e| VfsError::Io(format!("S3 DELETE failed: {}", e)))?;
|
||||||
|
|
||||||
|
if !resp.status().is_success() {
|
||||||
|
return Err(VfsError::Io(format!("DeleteObject returned {}", resp.status())));
|
||||||
|
}
|
||||||
|
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn list_objects(&self, prefix: &str) -> Result<Vec<VfsDirEntry>, VfsError> {
|
||||||
|
let mut action = actions::ListObjectsV2::new(&self.bucket, Some(&self.credentials));
|
||||||
|
if !prefix.is_empty() {
|
||||||
|
action.with_prefix(prefix);
|
||||||
|
}
|
||||||
|
action.with_delimiter("/");
|
||||||
|
|
||||||
|
let url = action.sign(Duration::from_secs(3600));
|
||||||
|
|
||||||
|
let resp = self.client
|
||||||
|
.get(url.as_str())
|
||||||
|
.send()
|
||||||
|
.await
|
||||||
|
.map_err(|e| VfsError::Io(format!("S3 LIST failed: {}", e)))?;
|
||||||
|
|
||||||
|
if !resp.status().is_success() {
|
||||||
|
return Err(VfsError::Io(format!("ListObjectsV2 returned {}", resp.status())));
|
||||||
|
}
|
||||||
|
|
||||||
|
let body = resp.text().await
|
||||||
|
.map_err(|e| VfsError::Io(format!("Failed to read LIST response: {}", e)))?;
|
||||||
|
|
||||||
|
// Use rusty-s3's built-in parser
|
||||||
|
let list_response = actions::ListObjectsV2::parse_response(&body)
|
||||||
|
.map_err(|e| VfsError::Io(format!("Failed to parse LIST response: {}", e)))?;
|
||||||
|
|
||||||
|
// Convert to VfsDirEntry
|
||||||
|
let mut entries = Vec::new();
|
||||||
|
for obj in list_response.contents {
|
||||||
|
let name = obj.key.strip_prefix(prefix).unwrap_or(&obj.key).to_string();
|
||||||
|
entries.push(VfsDirEntry {
|
||||||
|
name,
|
||||||
|
long_name: obj.key.clone(),
|
||||||
|
stat: VfsStat {
|
||||||
|
size: obj.size,
|
||||||
|
mode: 0o644,
|
||||||
|
uid: 0,
|
||||||
|
gid: 0,
|
||||||
|
atime: std::time::SystemTime::UNIX_EPOCH,
|
||||||
|
mtime: std::time::SystemTime::UNIX_EPOCH,
|
||||||
|
is_dir: false,
|
||||||
|
is_symlink: false,
|
||||||
|
},
|
||||||
|
});
|
||||||
|
}
|
||||||
|
for prefix_elem in list_response.common_prefixes {
|
||||||
|
let name = prefix_elem.prefix.strip_prefix(prefix).unwrap_or(&prefix_elem.prefix).trim_end_matches('/').to_string();
|
||||||
|
entries.push(VfsDirEntry {
|
||||||
|
name,
|
||||||
|
long_name: prefix_elem.prefix.clone(),
|
||||||
|
stat: VfsStat {
|
||||||
|
size: 0,
|
||||||
|
mode: 0o755,
|
||||||
|
uid: 0,
|
||||||
|
gid: 0,
|
||||||
|
atime: std::time::SystemTime::UNIX_EPOCH,
|
||||||
|
mtime: std::time::SystemTime::UNIX_EPOCH,
|
||||||
|
is_dir: true,
|
||||||
|
is_symlink: false,
|
||||||
|
},
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
Ok(entries)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Clone for AsyncS3Vfs {
|
||||||
|
fn clone(&self) -> Self {
|
||||||
|
Self {
|
||||||
|
bucket: self.bucket.clone(),
|
||||||
|
credentials: self.credentials.clone(),
|
||||||
|
client: self.client.clone(),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl AsyncS3File {
|
||||||
|
pub async fn new_read(vfs: AsyncS3Vfs, key: String) -> Result<Self, VfsError> {
|
||||||
|
let (size, mtime, _) = vfs.head_object(&key).await?;
|
||||||
|
|
||||||
|
Ok(Self {
|
||||||
|
inner: Arc::new(Mutex::new(AsyncS3FileState {
|
||||||
|
key,
|
||||||
|
mode: FileMode::Read,
|
||||||
|
position: 0,
|
||||||
|
size,
|
||||||
|
data: Vec::new(),
|
||||||
|
write_buffer: Vec::new(),
|
||||||
|
mtime,
|
||||||
|
})),
|
||||||
|
vfs,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn new_write(vfs: AsyncS3Vfs, key: String) -> Self {
|
||||||
|
Self {
|
||||||
|
inner: Arc::new(Mutex::new(AsyncS3FileState {
|
||||||
|
key,
|
||||||
|
mode: FileMode::Write,
|
||||||
|
position: 0,
|
||||||
|
size: 0,
|
||||||
|
data: Vec::new(),
|
||||||
|
write_buffer: Vec::new(),
|
||||||
|
mtime: std::time::SystemTime::now(),
|
||||||
|
})),
|
||||||
|
vfs,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl super::AsyncVfsFile for AsyncS3File {
|
||||||
|
fn read<'a>(&'a mut self, buf: &'a mut [u8]) -> Pin<Box<dyn Future<Output = Result<usize, VfsError>> + Send + 'a>> {
|
||||||
|
let inner = self.inner.clone();
|
||||||
|
let vfs = self.vfs.clone();
|
||||||
|
Box::pin(async move {
|
||||||
|
let mut state = inner.lock().await;
|
||||||
|
|
||||||
|
if state.position >= state.size {
|
||||||
|
return Ok(0);
|
||||||
|
}
|
||||||
|
|
||||||
|
if state.data.is_empty() {
|
||||||
|
let key = state.key.clone();
|
||||||
|
state.data = vfs.get_object(&key).await?;
|
||||||
|
}
|
||||||
|
|
||||||
|
let remaining = state.size - state.position;
|
||||||
|
let to_read = buf.len().min(remaining as usize);
|
||||||
|
let start = state.position as usize;
|
||||||
|
let end = start + to_read;
|
||||||
|
|
||||||
|
buf[..to_read].copy_from_slice(&state.data[start..end]);
|
||||||
|
state.position += to_read as u64;
|
||||||
|
|
||||||
|
Ok(to_read)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
fn write<'a>(&'a mut self, buf: &'a [u8]) -> Pin<Box<dyn Future<Output = Result<usize, VfsError>> + Send + 'a>> {
|
||||||
|
let inner = self.inner.clone();
|
||||||
|
Box::pin(async move {
|
||||||
|
let mut state = inner.lock().await;
|
||||||
|
state.write_buffer.extend_from_slice(buf);
|
||||||
|
Ok(buf.len())
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
fn seek<'a>(&'a mut self, pos: SeekFrom) -> Pin<Box<dyn Future<Output = Result<u64, VfsError>> + Send + 'a>> {
|
||||||
|
let inner = self.inner.clone();
|
||||||
|
Box::pin(async move {
|
||||||
|
let mut state = inner.lock().await;
|
||||||
|
let new_pos = match pos {
|
||||||
|
SeekFrom::Start(offset) => offset,
|
||||||
|
SeekFrom::Current(offset) => {
|
||||||
|
((state.position as i64) + offset).max(0) as u64
|
||||||
|
}
|
||||||
|
SeekFrom::End(offset) => {
|
||||||
|
((state.size as i64) + offset).max(0) as u64
|
||||||
|
}
|
||||||
|
};
|
||||||
|
state.position = new_pos.min(state.size);
|
||||||
|
Ok(state.position)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
fn flush<'a>(&'a mut self) -> Pin<Box<dyn Future<Output = Result<(), VfsError>> + Send + 'a>> {
|
||||||
|
let inner = self.inner.clone();
|
||||||
|
let vfs = self.vfs.clone();
|
||||||
|
Box::pin(async move {
|
||||||
|
let mut state = inner.lock().await;
|
||||||
|
if !state.write_buffer.is_empty() {
|
||||||
|
let key = state.key.clone();
|
||||||
|
let data = state.write_buffer.clone();
|
||||||
|
vfs.put_object(&key, &data).await?;
|
||||||
|
state.write_buffer.clear();
|
||||||
|
}
|
||||||
|
Ok(())
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl super::AsyncVfsBackend for AsyncS3Vfs {
|
||||||
|
fn clone_boxed(&self) -> Box<dyn super::AsyncVfsBackend> {
|
||||||
|
Box::new(self.clone())
|
||||||
|
}
|
||||||
|
|
||||||
|
fn read_dir<'a>(&'a self, path: &'a Path) -> Pin<Box<dyn Future<Output = Result<Vec<VfsDirEntry>, VfsError>> + Send + 'a>> {
|
||||||
|
let prefix = Self::path_to_key(path);
|
||||||
|
Box::pin(async move {
|
||||||
|
self.list_objects(&prefix).await
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
fn open_file<'a>(&'a self, path: &'a Path, flags: &'a OpenFlags) -> Pin<Box<dyn Future<Output = Result<Box<dyn super::AsyncVfsFile>, VfsError>> + Send + 'a>> {
|
||||||
|
let key = Self::path_to_key(path);
|
||||||
|
let vfs = self.clone();
|
||||||
|
let is_write = flags.write;
|
||||||
|
Box::pin(async move {
|
||||||
|
if is_write {
|
||||||
|
Ok(Box::new(AsyncS3File::new_write(vfs, key)) as Box<dyn super::AsyncVfsFile>)
|
||||||
|
} else {
|
||||||
|
let file = AsyncS3File::new_read(vfs, key).await?;
|
||||||
|
Ok(Box::new(file) as Box<dyn super::AsyncVfsFile>)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
fn stat<'a>(&'a self, path: &'a Path) -> Pin<Box<dyn Future<Output = Result<VfsStat, VfsError>> + Send + 'a>> {
|
||||||
|
let key = Self::path_to_key(path);
|
||||||
|
Box::pin(async move {
|
||||||
|
let (size, mtime, _) = self.head_object(&key).await?;
|
||||||
|
Ok(VfsStat {
|
||||||
|
size,
|
||||||
|
mode: 0o644,
|
||||||
|
uid: 0,
|
||||||
|
gid: 0,
|
||||||
|
atime: mtime,
|
||||||
|
mtime,
|
||||||
|
is_dir: false,
|
||||||
|
is_symlink: false,
|
||||||
|
})
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
fn create_dir<'a>(&'a self, path: &'a Path, _mode: u32) -> Pin<Box<dyn Future<Output = Result<(), VfsError>> + Send + 'a>> {
|
||||||
|
let key = Self::path_to_key(path);
|
||||||
|
if !key.ends_with('/') {
|
||||||
|
let _key = format!("{}/", key);
|
||||||
|
}
|
||||||
|
Box::pin(async move {
|
||||||
|
self.put_object(&key, &[]).await?;
|
||||||
|
Ok(())
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
fn remove_dir<'a>(&'a self, path: &'a Path) -> Pin<Box<dyn Future<Output = Result<(), VfsError>> + Send + 'a>> {
|
||||||
|
let key = Self::path_to_key(path);
|
||||||
|
let key = if key.ends_with('/') { key } else { format!("{}/", key) };
|
||||||
|
Box::pin(async move {
|
||||||
|
self.delete_object(&key).await?;
|
||||||
|
Ok(())
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
fn remove_file<'a>(&'a self, path: &'a Path) -> Pin<Box<dyn Future<Output = Result<(), VfsError>> + Send + 'a>> {
|
||||||
|
let key = Self::path_to_key(path);
|
||||||
|
Box::pin(async move {
|
||||||
|
self.delete_object(&key).await?;
|
||||||
|
Ok(())
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
fn rename<'a>(&'a self, from: &'a Path, to: &'a Path) -> Pin<Box<dyn Future<Output = Result<(), VfsError>> + Send + 'a>> {
|
||||||
|
let from_key = Self::path_to_key(from);
|
||||||
|
let to_key = Self::path_to_key(to);
|
||||||
|
Box::pin(async move {
|
||||||
|
let data = self.get_object(&from_key).await?;
|
||||||
|
self.put_object(&to_key, &data).await?;
|
||||||
|
self.delete_object(&from_key).await?;
|
||||||
|
Ok(())
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
fn exists<'a>(&'a self, path: &'a Path) -> Pin<Box<dyn Future<Output = bool> + Send + 'a>> {
|
||||||
|
let key = Self::path_to_key(path);
|
||||||
|
Box::pin(async move {
|
||||||
|
self.head_object(&key).await.is_ok()
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn parse_last_modified(header: Option<&str>) -> std::time::SystemTime {
|
||||||
|
header
|
||||||
|
.and_then(|s| chrono::DateTime::parse_from_rfc2822(s).ok())
|
||||||
|
.map(|dt| std::time::SystemTime::UNIX_EPOCH + std::time::Duration::from_secs(dt.timestamp() as u64))
|
||||||
|
.unwrap_or(std::time::SystemTime::now())
|
||||||
|
}
|
||||||
@@ -0,0 +1,260 @@
|
|||||||
|
use std::path::{Path, PathBuf};
|
||||||
|
use std::pin::Pin;
|
||||||
|
use std::future::Future;
|
||||||
|
use std::io::{SeekFrom, Read, Write};
|
||||||
|
use std::sync::Arc;
|
||||||
|
|
||||||
|
use tokio::task::spawn_blocking;
|
||||||
|
use tokio::sync::Mutex;
|
||||||
|
|
||||||
|
use super::{VfsError, VfsStat, VfsDirEntry, open_flags::OpenFlags, smb_fs::SmbVfs, VfsBackend};
|
||||||
|
|
||||||
|
/// Async SMB VFS 文件實現(spawn_blocking 包装)
|
||||||
|
pub struct AsyncSmbFile {
|
||||||
|
inner: Arc<Mutex<SmbFileState>>,
|
||||||
|
}
|
||||||
|
|
||||||
|
enum SmbFileState {
|
||||||
|
Read { vfs: SmbVfs, path: PathBuf, position: u64, size: u64, data: Vec<u8> },
|
||||||
|
Write { vfs: SmbVfs, path: PathBuf, buffer: Vec<u8> },
|
||||||
|
}
|
||||||
|
|
||||||
|
impl AsyncSmbFile {
|
||||||
|
pub async fn new_read(vfs: SmbVfs, path: PathBuf) -> Result<Self, VfsError> {
|
||||||
|
let stat = spawn_blocking({
|
||||||
|
let vfs = vfs.clone();
|
||||||
|
let path = path.clone();
|
||||||
|
move || VfsBackend::stat(&vfs, &path)
|
||||||
|
}).await.map_err(|_| VfsError::Io("spawn_blocking failed".to_string()))??;
|
||||||
|
|
||||||
|
Ok(Self {
|
||||||
|
inner: Arc::new(Mutex::new(SmbFileState::Read {
|
||||||
|
vfs, path, position: 0, size: stat.size, data: Vec::new(),
|
||||||
|
})),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn new_write(vfs: SmbVfs, path: PathBuf) -> Self {
|
||||||
|
Self {
|
||||||
|
inner: Arc::new(Mutex::new(SmbFileState::Write {
|
||||||
|
vfs, path, buffer: Vec::new(),
|
||||||
|
})),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl super::AsyncVfsFile for AsyncSmbFile {
|
||||||
|
fn read<'a>(&'a mut self, buf: &'a mut [u8]) -> Pin<Box<dyn Future<Output = Result<usize, VfsError>> + Send + 'a>> {
|
||||||
|
let inner = self.inner.clone();
|
||||||
|
let buf_len = buf.len();
|
||||||
|
Box::pin(async move {
|
||||||
|
let mut state = inner.lock().await;
|
||||||
|
match &mut *state {
|
||||||
|
SmbFileState::Read { vfs, path, position, size, data } => {
|
||||||
|
if *position >= *size {
|
||||||
|
return Ok(0);
|
||||||
|
}
|
||||||
|
|
||||||
|
if data.is_empty() {
|
||||||
|
let vfs_clone = vfs.clone();
|
||||||
|
let path_clone = path.clone();
|
||||||
|
let loaded_data = spawn_blocking(move || {
|
||||||
|
let flags = OpenFlags::new().read();
|
||||||
|
let mut file = VfsBackend::open_file(&vfs_clone, &path_clone, &flags)?;
|
||||||
|
let mut buf = Vec::new();
|
||||||
|
let mut chunk = [0u8; 8192];
|
||||||
|
loop {
|
||||||
|
match file.read(&mut chunk) {
|
||||||
|
Ok(0) => break,
|
||||||
|
Ok(n) => buf.extend_from_slice(&chunk[..n]),
|
||||||
|
Err(e) => return Err(e),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Ok(buf)
|
||||||
|
}).await.map_err(|_| VfsError::Io("spawn_blocking failed".to_string()))??;
|
||||||
|
*data = loaded_data;
|
||||||
|
}
|
||||||
|
|
||||||
|
let remaining = *size - *position;
|
||||||
|
let to_read = buf_len.min(remaining as usize);
|
||||||
|
let start = *position as usize;
|
||||||
|
let end = start + to_read;
|
||||||
|
|
||||||
|
buf[..to_read].copy_from_slice(&data[start..end]);
|
||||||
|
*position += to_read as u64;
|
||||||
|
Ok(to_read)
|
||||||
|
}
|
||||||
|
_ => Err(VfsError::Io("File not open for read".to_string())),
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
fn write<'a>(&'a mut self, buf: &'a [u8]) -> Pin<Box<dyn Future<Output = Result<usize, VfsError>> + Send + 'a>> {
|
||||||
|
let inner = self.inner.clone();
|
||||||
|
let buf_copy = buf.to_vec();
|
||||||
|
Box::pin(async move {
|
||||||
|
let mut state = inner.lock().await;
|
||||||
|
match &mut *state {
|
||||||
|
SmbFileState::Write { buffer, .. } => {
|
||||||
|
buffer.extend_from_slice(&buf_copy);
|
||||||
|
Ok(buf_copy.len())
|
||||||
|
}
|
||||||
|
_ => Err(VfsError::Io("File not open for write".to_string())),
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
fn seek<'a>(&'a mut self, pos: SeekFrom) -> Pin<Box<dyn Future<Output = Result<u64, VfsError>> + Send + 'a>> {
|
||||||
|
let inner = self.inner.clone();
|
||||||
|
Box::pin(async move {
|
||||||
|
let mut state = inner.lock().await;
|
||||||
|
match &mut *state {
|
||||||
|
SmbFileState::Read { position, size, .. } => {
|
||||||
|
let new_pos = match pos {
|
||||||
|
SeekFrom::Start(offset) => offset,
|
||||||
|
SeekFrom::Current(offset) => ((*position as i64) + offset).max(0) as u64,
|
||||||
|
SeekFrom::End(offset) => ((*size as i64) + offset).max(0) as u64,
|
||||||
|
};
|
||||||
|
*position = new_pos.min(*size);
|
||||||
|
Ok(*position)
|
||||||
|
}
|
||||||
|
_ => Err(VfsError::Io("File not open for read".to_string())),
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
fn flush<'a>(&'a mut self) -> Pin<Box<dyn Future<Output = Result<(), VfsError>> + Send + 'a>> {
|
||||||
|
let inner = self.inner.clone();
|
||||||
|
Box::pin(async move {
|
||||||
|
let mut state = inner.lock().await;
|
||||||
|
match &mut *state {
|
||||||
|
SmbFileState::Write { vfs, path, buffer } => {
|
||||||
|
let vfs_clone = vfs.clone();
|
||||||
|
let path_clone = path.clone();
|
||||||
|
let data = buffer.clone();
|
||||||
|
spawn_blocking(move || {
|
||||||
|
let flags = OpenFlags::new().write().create().truncate().mode(0o644);
|
||||||
|
let mut file = VfsBackend::open_file(&vfs_clone, &path_clone, &flags)?;
|
||||||
|
file.write_all(&data)?;
|
||||||
|
file.flush()?;
|
||||||
|
Ok(())
|
||||||
|
}).await.map_err(|_| VfsError::Io("spawn_blocking failed".to_string()))?
|
||||||
|
}
|
||||||
|
_ => Ok(()),
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Async SMB VFS 后端實現(spawn_blocking 包装 SmbVfs)
|
||||||
|
pub struct AsyncSmbVfs {
|
||||||
|
inner: SmbVfs,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl AsyncSmbVfs {
|
||||||
|
pub fn new(addr: &str, share: &str, username: &str, password: &str) -> Result<Self, VfsError> {
|
||||||
|
let inner = SmbVfs::new(addr, share, username, password)?;
|
||||||
|
Ok(Self { inner })
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn new_with_options(
|
||||||
|
addr: &str,
|
||||||
|
share: &str,
|
||||||
|
username: &str,
|
||||||
|
password: &str,
|
||||||
|
auto_reconnect: bool,
|
||||||
|
) -> Result<Self, VfsError> {
|
||||||
|
let inner = SmbVfs::new_with_options(addr, share, username, password, auto_reconnect)?;
|
||||||
|
Ok(Self { inner })
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Clone for AsyncSmbVfs {
|
||||||
|
fn clone(&self) -> Self {
|
||||||
|
Self { inner: self.inner.clone() }
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl super::AsyncVfsBackend for AsyncSmbVfs {
|
||||||
|
fn clone_boxed(&self) -> Box<dyn super::AsyncVfsBackend> {
|
||||||
|
Box::new(self.clone())
|
||||||
|
}
|
||||||
|
|
||||||
|
fn read_dir<'a>(&'a self, path: &'a Path) -> Pin<Box<dyn Future<Output = Result<Vec<VfsDirEntry>, VfsError>> + Send + 'a>> {
|
||||||
|
let inner = self.inner.clone();
|
||||||
|
let path_buf = path.to_path_buf();
|
||||||
|
Box::pin(async move {
|
||||||
|
spawn_blocking(move || VfsBackend::read_dir(&inner, &path_buf))
|
||||||
|
.await.map_err(|_| VfsError::Io("spawn_blocking failed".to_string()))?
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
fn open_file<'a>(&'a self, path: &'a Path, flags: &'a OpenFlags) -> Pin<Box<dyn Future<Output = Result<Box<dyn super::AsyncVfsFile>, VfsError>> + Send + 'a>> {
|
||||||
|
let inner = self.inner.clone();
|
||||||
|
let path_buf = path.to_path_buf();
|
||||||
|
let is_write = flags.write;
|
||||||
|
Box::pin(async move {
|
||||||
|
if is_write {
|
||||||
|
Ok(Box::new(AsyncSmbFile::new_write(inner, path_buf)) as Box<dyn super::AsyncVfsFile>)
|
||||||
|
} else {
|
||||||
|
let file = AsyncSmbFile::new_read(inner, path_buf).await?;
|
||||||
|
Ok(Box::new(file) as Box<dyn super::AsyncVfsFile>)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
fn stat<'a>(&'a self, path: &'a Path) -> Pin<Box<dyn Future<Output = Result<VfsStat, VfsError>> + Send + 'a>> {
|
||||||
|
let inner = self.inner.clone();
|
||||||
|
let path_buf = path.to_path_buf();
|
||||||
|
Box::pin(async move {
|
||||||
|
spawn_blocking(move || VfsBackend::stat(&inner, &path_buf))
|
||||||
|
.await.map_err(|_| VfsError::Io("spawn_blocking failed".to_string()))?
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
fn create_dir<'a>(&'a self, path: &'a Path, _mode: u32) -> Pin<Box<dyn Future<Output = Result<(), VfsError>> + Send + 'a>> {
|
||||||
|
let inner = self.inner.clone();
|
||||||
|
let path_buf = path.to_path_buf();
|
||||||
|
Box::pin(async move {
|
||||||
|
spawn_blocking(move || VfsBackend::create_dir(&inner, &path_buf, 0o755))
|
||||||
|
.await.map_err(|_| VfsError::Io("spawn_blocking failed".to_string()))?
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
fn remove_dir<'a>(&'a self, path: &'a Path) -> Pin<Box<dyn Future<Output = Result<(), VfsError>> + Send + 'a>> {
|
||||||
|
let inner = self.inner.clone();
|
||||||
|
let path_buf = path.to_path_buf();
|
||||||
|
Box::pin(async move {
|
||||||
|
spawn_blocking(move || VfsBackend::remove_dir(&inner, &path_buf))
|
||||||
|
.await.map_err(|_| VfsError::Io("spawn_blocking failed".to_string()))?
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
fn remove_file<'a>(&'a self, path: &'a Path) -> Pin<Box<dyn Future<Output = Result<(), VfsError>> + Send + 'a>> {
|
||||||
|
let inner = self.inner.clone();
|
||||||
|
let path_buf = path.to_path_buf();
|
||||||
|
Box::pin(async move {
|
||||||
|
spawn_blocking(move || VfsBackend::remove_file(&inner, &path_buf))
|
||||||
|
.await.map_err(|_| VfsError::Io("spawn_blocking failed".to_string()))?
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
fn rename<'a>(&'a self, from: &'a Path, to: &'a Path) -> Pin<Box<dyn Future<Output = Result<(), VfsError>> + Send + 'a>> {
|
||||||
|
let inner = self.inner.clone();
|
||||||
|
let from_buf = from.to_path_buf();
|
||||||
|
let to_buf = to.to_path_buf();
|
||||||
|
Box::pin(async move {
|
||||||
|
spawn_blocking(move || VfsBackend::rename(&inner, &from_buf, &to_buf))
|
||||||
|
.await.map_err(|_| VfsError::Io("spawn_blocking failed".to_string()))?
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
fn exists<'a>(&'a self, path: &'a Path) -> Pin<Box<dyn Future<Output = bool> + Send + 'a>> {
|
||||||
|
let inner = self.inner.clone();
|
||||||
|
let path_buf = path.to_path_buf();
|
||||||
|
Box::pin(async move {
|
||||||
|
spawn_blocking(move || VfsBackend::exists(&inner, &path_buf))
|
||||||
|
.await.unwrap_or(false)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,267 @@
|
|||||||
|
//! Backup Manifest - Snapshot metadata serialization
|
||||||
|
//!
|
||||||
|
//! Compatible with ZFS send/receive and Proxmox Backup Server format
|
||||||
|
|
||||||
|
use std::path::PathBuf;
|
||||||
|
|
||||||
|
use serde::{Serialize, Deserialize};
|
||||||
|
use sha2::Digest;
|
||||||
|
|
||||||
|
use super::{VfsCompression};
|
||||||
|
use super::checksum::VfsChecksumFile;
|
||||||
|
use super::dedup::DedupManifest;
|
||||||
|
|
||||||
|
pub const MANIFEST_VERSION: u32 = 1;
|
||||||
|
pub const MANIFEST_FILE: &str = ".manifest.json";
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
|
||||||
|
pub enum SendFormat {
|
||||||
|
#[serde(rename = "zfs_compatible")]
|
||||||
|
ZfsCompatible,
|
||||||
|
#[serde(rename = "custom_json")]
|
||||||
|
CustomJson,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||||
|
pub struct BackupFileEntry {
|
||||||
|
pub path: String,
|
||||||
|
pub size: u64,
|
||||||
|
pub checksums: Option<VfsChecksumFile>,
|
||||||
|
pub dedup_hash: Option<String>,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||||
|
pub struct EncryptionInfo {
|
||||||
|
pub algorithm: String,
|
||||||
|
pub enabled: bool,
|
||||||
|
pub key_hash: Option<String>,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||||
|
pub struct CompressionInfo {
|
||||||
|
pub algorithm: String,
|
||||||
|
pub level: u32,
|
||||||
|
pub original_size: u64,
|
||||||
|
pub compressed_size: u64,
|
||||||
|
pub ratio: f64,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||||
|
pub struct BackupManifest {
|
||||||
|
pub version: u32,
|
||||||
|
pub format: SendFormat,
|
||||||
|
pub snapshot_name: String,
|
||||||
|
pub created_at: u64,
|
||||||
|
pub root_path: String,
|
||||||
|
pub files: Vec<BackupFileEntry>,
|
||||||
|
pub dedup_manifest: Option<DedupManifest>,
|
||||||
|
pub encryption: Option<EncryptionInfo>,
|
||||||
|
pub compression: Option<CompressionInfo>,
|
||||||
|
pub total_size: u64,
|
||||||
|
pub stored_size: u64,
|
||||||
|
pub overall_ratio: f64,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl BackupManifest {
|
||||||
|
pub fn new(snapshot_name: String, root_path: PathBuf) -> Self {
|
||||||
|
Self {
|
||||||
|
version: MANIFEST_VERSION,
|
||||||
|
format: SendFormat::CustomJson,
|
||||||
|
snapshot_name,
|
||||||
|
created_at: current_time_secs(),
|
||||||
|
root_path: root_path.to_string_lossy().to_string(),
|
||||||
|
files: Vec::new(),
|
||||||
|
dedup_manifest: None,
|
||||||
|
encryption: None,
|
||||||
|
compression: None,
|
||||||
|
total_size: 0,
|
||||||
|
stored_size: 0,
|
||||||
|
overall_ratio: 1.0,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn add_file(&mut self, path: String, size: u64, checksums: Option<VfsChecksumFile>) {
|
||||||
|
self.files.push(BackupFileEntry {
|
||||||
|
path,
|
||||||
|
size,
|
||||||
|
checksums,
|
||||||
|
dedup_hash: None,
|
||||||
|
});
|
||||||
|
self.total_size += size;
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn set_dedup(&mut self, manifest: DedupManifest) {
|
||||||
|
self.dedup_manifest = Some(manifest.clone());
|
||||||
|
if manifest.original_size > 0 {
|
||||||
|
let stored = (manifest.block_hashes.len() as u64) * 4096; // Approximate
|
||||||
|
self.stored_size = stored;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn set_compression(&mut self, algorithm: VfsCompression, original: u64, compressed: u64) {
|
||||||
|
let ratio = if original > 0 { compressed as f64 / original as f64 } else { 1.0 };
|
||||||
|
self.compression = Some(CompressionInfo {
|
||||||
|
algorithm: algorithm_name(&algorithm),
|
||||||
|
level: 3,
|
||||||
|
original_size: original,
|
||||||
|
compressed_size: compressed,
|
||||||
|
ratio,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn set_encryption(&mut self, enabled: bool, key_hash: Option<String>) {
|
||||||
|
self.encryption = Some(EncryptionInfo {
|
||||||
|
algorithm: "AES-256-GCM".to_string(),
|
||||||
|
enabled,
|
||||||
|
key_hash,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn calculate_ratio(&mut self) {
|
||||||
|
if self.total_size > 0 && self.stored_size > 0 {
|
||||||
|
self.overall_ratio = self.stored_size as f64 / self.total_size as f64;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn to_bytes(&self) -> Result<Vec<u8>, String> {
|
||||||
|
serde_json::to_vec(self).map_err(|e| e.to_string())
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn from_bytes(data: &[u8]) -> Result<Self, String> {
|
||||||
|
serde_json::from_slice(data).map_err(|e| e.to_string())
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn save(&self, snapshot_dir: &PathBuf) -> Result<(), String> {
|
||||||
|
let manifest_path = snapshot_dir.join(MANIFEST_FILE);
|
||||||
|
let data = self.to_bytes()?;
|
||||||
|
std::fs::write(&manifest_path, data).map_err(|e| e.to_string())
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn load(snapshot_dir: &PathBuf) -> Result<Self, String> {
|
||||||
|
let manifest_path = snapshot_dir.join(MANIFEST_FILE);
|
||||||
|
let data = std::fs::read(&manifest_path).map_err(|e| e.to_string())?;
|
||||||
|
Self::from_bytes(&data)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn algorithm_name(compression: &VfsCompression) -> String {
|
||||||
|
match compression {
|
||||||
|
VfsCompression::None => "none".to_string(),
|
||||||
|
VfsCompression::Lz4 => "lz4".to_string(),
|
||||||
|
VfsCompression::Zstd => "zstd".to_string(),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn current_time_secs() -> u64 {
|
||||||
|
use std::time::{SystemTime, UNIX_EPOCH};
|
||||||
|
SystemTime::now()
|
||||||
|
.duration_since(UNIX_EPOCH)
|
||||||
|
.map(|d| d.as_secs())
|
||||||
|
.unwrap_or(0)
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Clone)]
|
||||||
|
pub struct BackupStream {
|
||||||
|
pub format: SendFormat,
|
||||||
|
pub manifest: BackupManifest,
|
||||||
|
pub data: Vec<u8>,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl BackupStream {
|
||||||
|
pub fn new(format: SendFormat, manifest: BackupManifest, data: Vec<u8>) -> Self {
|
||||||
|
Self { format, manifest, data }
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn to_bytes(&self) -> Result<Vec<u8>, String> {
|
||||||
|
match self.format {
|
||||||
|
SendFormat::CustomJson => {
|
||||||
|
let manifest_bytes = self.manifest.to_bytes()?;
|
||||||
|
let mut result = Vec::new();
|
||||||
|
result.extend_from_slice(&manifest_bytes.len().to_be_bytes());
|
||||||
|
result.extend_from_slice(&manifest_bytes);
|
||||||
|
result.extend_from_slice(&self.data);
|
||||||
|
Ok(result)
|
||||||
|
}
|
||||||
|
SendFormat::ZfsCompatible => {
|
||||||
|
Err("ZFS compatible format not yet implemented".to_string())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn from_bytes(data: &[u8]) -> Result<Self, String> {
|
||||||
|
if data.len() < 8 {
|
||||||
|
return Err("Stream too short".to_string());
|
||||||
|
}
|
||||||
|
|
||||||
|
let manifest_len = u64::from_be_bytes(data[0..8].try_into().map_err(|_| "Invalid length")?) as usize;
|
||||||
|
if data.len() < 8 + manifest_len {
|
||||||
|
return Err("Stream truncated".to_string());
|
||||||
|
}
|
||||||
|
|
||||||
|
let manifest_bytes = &data[8..8 + manifest_len];
|
||||||
|
let manifest = BackupManifest::from_bytes(manifest_bytes)?;
|
||||||
|
let payload = data[8 + manifest_len..].to_vec();
|
||||||
|
|
||||||
|
Ok(Self::new(manifest.format.clone(), manifest, payload))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use super::*;
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_manifest_creation() {
|
||||||
|
let manifest = BackupManifest::new("snap_2026-06-24".to_string(), PathBuf::from("/data"));
|
||||||
|
assert_eq!(manifest.version, MANIFEST_VERSION);
|
||||||
|
assert_eq!(manifest.format, SendFormat::CustomJson);
|
||||||
|
assert_eq!(manifest.snapshot_name, "snap_2026-06-24");
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_manifest_serialization() {
|
||||||
|
let mut manifest = BackupManifest::new("test_snap".to_string(), PathBuf::from("/data"));
|
||||||
|
manifest.add_file("file1.txt".to_string(), 1024, None);
|
||||||
|
manifest.add_file("file2.txt".to_string(), 2048, None);
|
||||||
|
manifest.calculate_ratio();
|
||||||
|
|
||||||
|
let bytes = manifest.to_bytes().unwrap();
|
||||||
|
let decoded = BackupManifest::from_bytes(&bytes).unwrap();
|
||||||
|
|
||||||
|
assert_eq!(decoded.files.len(), 2);
|
||||||
|
assert_eq!(decoded.total_size, 3072);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_backup_stream_roundtrip() {
|
||||||
|
let manifest = BackupManifest::new("test".to_string(), PathBuf::from("/"));
|
||||||
|
let stream = BackupStream::new(SendFormat::CustomJson, manifest, b"test data".to_vec());
|
||||||
|
|
||||||
|
let bytes = stream.to_bytes().unwrap();
|
||||||
|
let decoded = BackupStream::from_bytes(&bytes).unwrap();
|
||||||
|
|
||||||
|
assert_eq!(decoded.data, b"test data");
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_compression_info() {
|
||||||
|
let mut manifest = BackupManifest::new("test".to_string(), PathBuf::from("/"));
|
||||||
|
manifest.set_compression(VfsCompression::Zstd, 1000, 420);
|
||||||
|
|
||||||
|
assert!(manifest.compression.is_some());
|
||||||
|
let comp = manifest.compression.unwrap();
|
||||||
|
assert_eq!(comp.algorithm, "zstd");
|
||||||
|
assert_eq!(comp.ratio, 0.42);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_encryption_info() {
|
||||||
|
let mut manifest = BackupManifest::new("test".to_string(), PathBuf::from("/"));
|
||||||
|
manifest.set_encryption(true, Some("key_hash_abc".to_string()));
|
||||||
|
|
||||||
|
assert!(manifest.encryption.is_some());
|
||||||
|
let enc = manifest.encryption.unwrap();
|
||||||
|
assert!(enc.enabled);
|
||||||
|
assert_eq!(enc.algorithm, "AES-256-GCM");
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,630 @@
|
|||||||
|
//! Backup Scheduler - Automated snapshot creation
|
||||||
|
//!
|
||||||
|
//! Similar to Proxmox Backup Server scheduling
|
||||||
|
|
||||||
|
use std::sync::Arc;
|
||||||
|
use std::path::PathBuf;
|
||||||
|
use std::time::{SystemTime, UNIX_EPOCH};
|
||||||
|
use chrono::TimeZone;
|
||||||
|
|
||||||
|
use super::{VfsBackend, VfsError, VfsCompression};
|
||||||
|
|
||||||
|
pub struct BackupScheduleConfig {
|
||||||
|
pub enabled: bool,
|
||||||
|
pub interval_hours: u64,
|
||||||
|
pub max_snapshots: usize,
|
||||||
|
pub auto_cleanup: bool,
|
||||||
|
pub compress: VfsCompression,
|
||||||
|
pub encrypt: bool,
|
||||||
|
pub include_checksums: bool,
|
||||||
|
pub incremental: bool,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Default for BackupScheduleConfig {
|
||||||
|
fn default() -> Self {
|
||||||
|
Self {
|
||||||
|
enabled: true,
|
||||||
|
interval_hours: 24,
|
||||||
|
max_snapshots: 7,
|
||||||
|
auto_cleanup: true,
|
||||||
|
compress: VfsCompression::Zstd,
|
||||||
|
encrypt: false,
|
||||||
|
include_checksums: true,
|
||||||
|
incremental: true,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub struct BackupScheduler {
|
||||||
|
backend: Arc<dyn VfsBackend>,
|
||||||
|
root: PathBuf,
|
||||||
|
config: BackupScheduleConfig,
|
||||||
|
last_backup: Option<u64>,
|
||||||
|
next_backup: Option<u64>,
|
||||||
|
backup_count: usize,
|
||||||
|
snapshots: Vec<String>,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl BackupScheduler {
|
||||||
|
pub fn new(
|
||||||
|
backend: Arc<dyn VfsBackend>,
|
||||||
|
root: PathBuf,
|
||||||
|
config: BackupScheduleConfig,
|
||||||
|
) -> Self {
|
||||||
|
Self {
|
||||||
|
backend,
|
||||||
|
root,
|
||||||
|
config,
|
||||||
|
last_backup: None,
|
||||||
|
next_backup: None,
|
||||||
|
backup_count: 0,
|
||||||
|
snapshots: Vec::new(),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn with_defaults(backend: Arc<dyn VfsBackend>, root: PathBuf) -> Self {
|
||||||
|
Self::new(backend, root, BackupScheduleConfig::default())
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn start(&mut self) {
|
||||||
|
self.config.enabled = true;
|
||||||
|
self.schedule_next();
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn stop(&mut self) {
|
||||||
|
self.config.enabled = false;
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn is_enabled(&self) -> bool {
|
||||||
|
self.config.enabled
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn get_config(&self) -> &BackupScheduleConfig {
|
||||||
|
&self.config
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn set_config(&mut self, config: BackupScheduleConfig) {
|
||||||
|
self.config = config;
|
||||||
|
if self.config.enabled {
|
||||||
|
self.schedule_next();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn schedule_next(&mut self) {
|
||||||
|
let now = current_time_secs();
|
||||||
|
let interval_secs = self.config.interval_hours * 3600;
|
||||||
|
|
||||||
|
if let Some(last) = self.last_backup {
|
||||||
|
self.next_backup = Some(last + interval_secs);
|
||||||
|
} else {
|
||||||
|
self.next_backup = Some(now + interval_secs);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn should_run(&self) -> bool {
|
||||||
|
if !self.config.enabled {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
let now = current_time_secs();
|
||||||
|
|
||||||
|
match self.next_backup {
|
||||||
|
None => true,
|
||||||
|
Some(next) => now >= next,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn run_backup(&mut self) -> Result<String, VfsError> {
|
||||||
|
if !self.config.enabled {
|
||||||
|
return Err(VfsError::Io("Backup scheduler is disabled".to_string()));
|
||||||
|
}
|
||||||
|
|
||||||
|
let name = generate_snapshot_name();
|
||||||
|
|
||||||
|
let snapshot_dir = self.root.join(".snapshots").join(&name);
|
||||||
|
self.backend.create_dir(&snapshot_dir, 0o755)?;
|
||||||
|
|
||||||
|
if self.config.incremental && !self.snapshots.is_empty() {
|
||||||
|
let base_snapshot = self.snapshots.last().unwrap();
|
||||||
|
self.copy_incremental_to_snapshot(base_snapshot, &snapshot_dir)?;
|
||||||
|
} else {
|
||||||
|
self.copy_root_to_snapshot(&snapshot_dir)?;
|
||||||
|
}
|
||||||
|
|
||||||
|
if self.config.include_checksums {
|
||||||
|
self.generate_checksums(&snapshot_dir)?;
|
||||||
|
}
|
||||||
|
|
||||||
|
if self.config.auto_cleanup {
|
||||||
|
self.cleanup_old_snapshots()?;
|
||||||
|
}
|
||||||
|
|
||||||
|
self.last_backup = Some(current_time_secs());
|
||||||
|
self.backup_count += 1;
|
||||||
|
self.snapshots.push(name.clone());
|
||||||
|
self.schedule_next();
|
||||||
|
|
||||||
|
Ok(name)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn copy_incremental_to_snapshot(&self, base: &str, snapshot_dir: &PathBuf) -> Result<(), VfsError> {
|
||||||
|
let base_dir = self.root.join(".snapshots").join(base);
|
||||||
|
|
||||||
|
if !self.backend.exists(&base_dir) {
|
||||||
|
return self.copy_root_to_snapshot(snapshot_dir);
|
||||||
|
}
|
||||||
|
|
||||||
|
let entries = self.backend.read_dir(&self.root)?;
|
||||||
|
|
||||||
|
for entry in entries {
|
||||||
|
if entry.name == ".snapshots" || entry.name == ".checksums" {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
let src_path = self.root.join(&entry.name);
|
||||||
|
let dst_path = snapshot_dir.join(&entry.name);
|
||||||
|
let base_path = base_dir.join(&entry.name);
|
||||||
|
|
||||||
|
if entry.stat.is_dir {
|
||||||
|
self.copy_directory_incremental(&src_path, &dst_path, &base_path)?;
|
||||||
|
} else {
|
||||||
|
let needs_copy = !self.backend.exists(&base_path) ||
|
||||||
|
self.file_changed(&src_path, &base_path)?;
|
||||||
|
|
||||||
|
if needs_copy {
|
||||||
|
self.copy_file(&src_path, &dst_path)?;
|
||||||
|
} else {
|
||||||
|
self.create_hard_link(&base_path, &dst_path)?;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
fn file_changed(&self, src: &PathBuf, base: &PathBuf) -> Result<bool, VfsError> {
|
||||||
|
let src_stat = self.backend.stat(src)?;
|
||||||
|
let base_stat = self.backend.stat(base)?;
|
||||||
|
|
||||||
|
Ok(src_stat.size != base_stat.size ||
|
||||||
|
src_stat.mtime != base_stat.mtime)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn create_hard_link(&self, src: &PathBuf, dst: &PathBuf) -> Result<(), VfsError> {
|
||||||
|
self.backend.hard_link(src, dst)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn copy_directory_incremental(&self, src: &PathBuf, dst: &PathBuf, base: &PathBuf) -> Result<(), VfsError> {
|
||||||
|
self.backend.create_dir(dst, 0o755)?;
|
||||||
|
|
||||||
|
let entries = self.backend.read_dir(src)?;
|
||||||
|
|
||||||
|
for entry in entries {
|
||||||
|
let child_src = src.join(&entry.name);
|
||||||
|
let child_dst = dst.join(&entry.name);
|
||||||
|
let child_base = base.join(&entry.name);
|
||||||
|
|
||||||
|
if entry.stat.is_dir {
|
||||||
|
self.copy_directory_incremental(&child_src, &child_dst, &child_base)?;
|
||||||
|
} else {
|
||||||
|
let needs_copy = !self.backend.exists(&child_base) ||
|
||||||
|
self.file_changed(&child_src, &child_base)?;
|
||||||
|
|
||||||
|
if needs_copy {
|
||||||
|
self.copy_file(&child_src, &child_dst)?;
|
||||||
|
} else {
|
||||||
|
self.create_hard_link(&child_base, &child_dst)?;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
fn copy_root_to_snapshot(&self, snapshot_dir: &PathBuf) -> Result<(), VfsError> {
|
||||||
|
let entries = self.backend.read_dir(&self.root)?;
|
||||||
|
|
||||||
|
for entry in entries {
|
||||||
|
if entry.name == ".snapshots" || entry.name == ".checksums" {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
let src_path = self.root.join(&entry.name);
|
||||||
|
let dst_path = snapshot_dir.join(&entry.name);
|
||||||
|
|
||||||
|
if entry.stat.is_dir {
|
||||||
|
self.copy_directory(&src_path, &dst_path)?;
|
||||||
|
} else {
|
||||||
|
self.copy_file(&src_path, &dst_path)?;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
fn copy_directory(&self, src: &PathBuf, dst: &PathBuf) -> Result<(), VfsError> {
|
||||||
|
self.backend.create_dir(dst, 0o755)?;
|
||||||
|
|
||||||
|
let entries = self.backend.read_dir(src)?;
|
||||||
|
for entry in entries {
|
||||||
|
let src_path = src.join(&entry.name);
|
||||||
|
let dst_path = dst.join(&entry.name);
|
||||||
|
|
||||||
|
if entry.stat.is_dir {
|
||||||
|
self.copy_directory(&src_path, &dst_path)?;
|
||||||
|
} else {
|
||||||
|
self.copy_file(&src_path, &dst_path)?;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
fn copy_file(&self, src: &PathBuf, dst: &PathBuf) -> Result<(), VfsError> {
|
||||||
|
use super::compression::Compressor;
|
||||||
|
use super::VfsCompressionConfig;
|
||||||
|
|
||||||
|
let mut src_file = self.backend.open_file(src, &super::open_flags::OpenFlags::new().read())?;
|
||||||
|
let data = src_file.read_all()?;
|
||||||
|
|
||||||
|
let final_data = if self.config.compress != super::VfsCompression::None {
|
||||||
|
let compressor = Compressor::new(VfsCompressionConfig {
|
||||||
|
algorithm: self.config.compress,
|
||||||
|
min_size: 1024,
|
||||||
|
level: 3,
|
||||||
|
});
|
||||||
|
compressor.compress(&data)?
|
||||||
|
} else {
|
||||||
|
data
|
||||||
|
};
|
||||||
|
|
||||||
|
let mut dst_file = self.backend.open_file(
|
||||||
|
dst,
|
||||||
|
&super::open_flags::OpenFlags::new().write().create().truncate(),
|
||||||
|
)?;
|
||||||
|
dst_file.write_all(&final_data)?;
|
||||||
|
dst_file.flush()?;
|
||||||
|
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
fn generate_checksums(&self, snapshot_dir: &PathBuf) -> Result<(), VfsError> {
|
||||||
|
use super::checksum::create_checksums_for_file;
|
||||||
|
|
||||||
|
let entries = self.backend.read_dir(snapshot_dir)?;
|
||||||
|
for entry in entries {
|
||||||
|
if entry.name == ".manifest.json" || entry.name == ".meta" || entry.name == ".checksums" {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
let file_path = snapshot_dir.join(&entry.name);
|
||||||
|
|
||||||
|
if entry.stat.is_dir {
|
||||||
|
self.generate_checksums_recursive(&file_path, snapshot_dir)?;
|
||||||
|
} else {
|
||||||
|
create_checksums_for_file(self.backend.as_ref(), &file_path, snapshot_dir)?;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
fn generate_checksums_recursive(
|
||||||
|
&self,
|
||||||
|
dir: &PathBuf,
|
||||||
|
snapshot_dir: &PathBuf,
|
||||||
|
) -> Result<(), VfsError> {
|
||||||
|
use super::checksum::create_checksums_for_file;
|
||||||
|
|
||||||
|
let entries = self.backend.read_dir(dir)?;
|
||||||
|
for entry in entries {
|
||||||
|
let file_path = dir.join(&entry.name);
|
||||||
|
|
||||||
|
if entry.stat.is_dir {
|
||||||
|
self.generate_checksums_recursive(&file_path, snapshot_dir)?;
|
||||||
|
} else {
|
||||||
|
create_checksums_for_file(self.backend.as_ref(), &file_path, snapshot_dir)?;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
fn cleanup_old_snapshots(&mut self) -> Result<(), VfsError> {
|
||||||
|
let snapshots_dir = self.root.join(".snapshots");
|
||||||
|
|
||||||
|
if !self.backend.exists(&snapshots_dir) {
|
||||||
|
return Ok(());
|
||||||
|
}
|
||||||
|
|
||||||
|
let entries = self.backend.read_dir(&snapshots_dir)?;
|
||||||
|
let mut snapshot_names: Vec<String> = entries
|
||||||
|
.iter()
|
||||||
|
.filter(|e| e.stat.is_dir && e.name != ".checksums")
|
||||||
|
.map(|e| e.name.clone())
|
||||||
|
.collect();
|
||||||
|
|
||||||
|
snapshot_names.sort();
|
||||||
|
|
||||||
|
while snapshot_names.len() > self.config.max_snapshots {
|
||||||
|
let oldest = snapshot_names.remove(0);
|
||||||
|
let oldest_dir = snapshots_dir.join(&oldest);
|
||||||
|
|
||||||
|
self.remove_directory_recursive(&oldest_dir)?;
|
||||||
|
self.snapshots.retain(|s| s != &oldest);
|
||||||
|
}
|
||||||
|
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
fn remove_directory_recursive(&self, dir: &PathBuf) -> Result<(), VfsError> {
|
||||||
|
if !self.backend.exists(dir) {
|
||||||
|
return Ok(());
|
||||||
|
}
|
||||||
|
|
||||||
|
let entries = self.backend.read_dir(dir)?;
|
||||||
|
for entry in entries {
|
||||||
|
let path = dir.join(&entry.name);
|
||||||
|
|
||||||
|
if entry.stat.is_dir {
|
||||||
|
self.remove_directory_recursive(&path)?;
|
||||||
|
} else {
|
||||||
|
self.backend.remove_file(&path)?;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
self.backend.remove_dir(dir)?;
|
||||||
|
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn list_backups(&self) -> Result<Vec<BackupInfo>, VfsError> {
|
||||||
|
let snapshots_dir = self.root.join(".snapshots");
|
||||||
|
|
||||||
|
if !self.backend.exists(&snapshots_dir) {
|
||||||
|
return Ok(Vec::new());
|
||||||
|
}
|
||||||
|
|
||||||
|
let entries = self.backend.read_dir(&snapshots_dir)?;
|
||||||
|
let mut backups = Vec::new();
|
||||||
|
|
||||||
|
for entry in entries {
|
||||||
|
if !entry.stat.is_dir || entry.name == ".checksums" {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
let snapshot_dir = snapshots_dir.join(&entry.name);
|
||||||
|
let info = self.get_backup_info(&entry.name, &snapshot_dir)?;
|
||||||
|
backups.push(info);
|
||||||
|
}
|
||||||
|
|
||||||
|
backups.sort_by(|a, b| b.created_at.cmp(&a.created_at));
|
||||||
|
|
||||||
|
Ok(backups)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn get_backup_info(&self, name: &str, snapshot_dir: &PathBuf) -> Result<BackupInfo, VfsError> {
|
||||||
|
let manifest_path = snapshot_dir.join(".manifest.json");
|
||||||
|
|
||||||
|
let created_at = if self.backend.exists(&manifest_path) {
|
||||||
|
let mut file = self.backend.open_file(&manifest_path, &super::open_flags::OpenFlags::new().read())?;
|
||||||
|
let data = file.read_all()?;
|
||||||
|
|
||||||
|
if let Ok(manifest) = super::backup_manifest::BackupManifest::from_bytes(&data) {
|
||||||
|
manifest.created_at
|
||||||
|
} else {
|
||||||
|
current_time_secs()
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
current_time_secs()
|
||||||
|
};
|
||||||
|
|
||||||
|
let size = self.calculate_snapshot_size(snapshot_dir)?;
|
||||||
|
|
||||||
|
Ok(BackupInfo {
|
||||||
|
name: name.to_string(),
|
||||||
|
created_at,
|
||||||
|
size,
|
||||||
|
checksum_verified: false,
|
||||||
|
compressed: self.config.compress != VfsCompression::None,
|
||||||
|
encrypted: self.config.encrypt,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
fn calculate_snapshot_size(&self, dir: &PathBuf) -> Result<u64, VfsError> {
|
||||||
|
let mut total_size = 0u64;
|
||||||
|
|
||||||
|
let entries = self.backend.read_dir(dir)?;
|
||||||
|
for entry in entries {
|
||||||
|
let path = dir.join(&entry.name);
|
||||||
|
|
||||||
|
if entry.stat.is_dir {
|
||||||
|
total_size += self.calculate_snapshot_size(&path)?;
|
||||||
|
} else {
|
||||||
|
total_size += entry.stat.size;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
Ok(total_size)
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn get_stats(&self) -> BackupStats {
|
||||||
|
BackupStats {
|
||||||
|
enabled: self.config.enabled,
|
||||||
|
backup_count: self.backup_count,
|
||||||
|
last_backup: self.last_backup,
|
||||||
|
next_backup: self.next_backup,
|
||||||
|
interval_hours: self.config.interval_hours,
|
||||||
|
max_snapshots: self.config.max_snapshots,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn generate_snapshot_name() -> String {
|
||||||
|
let now = SystemTime::now()
|
||||||
|
.duration_since(UNIX_EPOCH)
|
||||||
|
.map(|d| d.as_secs())
|
||||||
|
.unwrap_or(0);
|
||||||
|
|
||||||
|
let datetime = chrono::Utc.timestamp_opt(now as i64, 0)
|
||||||
|
.single()
|
||||||
|
.map(|dt| dt.format("%Y-%m-%d_%H%M%S").to_string())
|
||||||
|
.unwrap_or_else(|| format!("{}", now));
|
||||||
|
|
||||||
|
format!("snap_{}", datetime)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn current_time_secs() -> u64 {
|
||||||
|
SystemTime::now()
|
||||||
|
.duration_since(UNIX_EPOCH)
|
||||||
|
.map(|d| d.as_secs())
|
||||||
|
.unwrap_or(0)
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Clone)]
|
||||||
|
pub struct BackupInfo {
|
||||||
|
pub name: String,
|
||||||
|
pub created_at: u64,
|
||||||
|
pub size: u64,
|
||||||
|
pub checksum_verified: bool,
|
||||||
|
pub compressed: bool,
|
||||||
|
pub encrypted: bool,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl BackupInfo {
|
||||||
|
pub fn format_created(&self) -> String {
|
||||||
|
chrono::Utc.timestamp_opt(self.created_at as i64, 0)
|
||||||
|
.single()
|
||||||
|
.map(|dt| dt.format("%Y-%m-%d %H:%M:%S UTC").to_string())
|
||||||
|
.unwrap_or_else(|| format!("{} seconds since epoch", self.created_at))
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn format_size(&self) -> String {
|
||||||
|
if self.size < 1024 {
|
||||||
|
format!("{} B", self.size)
|
||||||
|
} else if self.size < 1024 * 1024 {
|
||||||
|
format!("{:.2} KB", self.size as f64 / 1024.0)
|
||||||
|
} else if self.size < 1024 * 1024 * 1024 {
|
||||||
|
format!("{:.2} MB", self.size as f64 / (1024.0 * 1024.0))
|
||||||
|
} else {
|
||||||
|
format!("{:.2} GB", self.size as f64 / (1024.0 * 1024.0 * 1024.0))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Clone)]
|
||||||
|
pub struct BackupStats {
|
||||||
|
pub enabled: bool,
|
||||||
|
pub backup_count: usize,
|
||||||
|
pub last_backup: Option<u64>,
|
||||||
|
pub next_backup: Option<u64>,
|
||||||
|
pub interval_hours: u64,
|
||||||
|
pub max_snapshots: usize,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl BackupStats {
|
||||||
|
pub fn next_backup_in_secs(&self) -> Option<u64> {
|
||||||
|
if !self.enabled {
|
||||||
|
return None;
|
||||||
|
}
|
||||||
|
|
||||||
|
let now = current_time_secs();
|
||||||
|
let next = self.next_backup?;
|
||||||
|
|
||||||
|
if next > now {
|
||||||
|
Some(next - now)
|
||||||
|
} else {
|
||||||
|
Some(0)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn format_last_backup(&self) -> String {
|
||||||
|
match self.last_backup {
|
||||||
|
None => "Never".to_string(),
|
||||||
|
Some(t) => chrono::Utc.timestamp_opt(t as i64, 0)
|
||||||
|
.single()
|
||||||
|
.map(|dt| dt.format("%Y-%m-%d %H:%M:%S UTC").to_string())
|
||||||
|
.unwrap_or_else(|| format!("{} seconds since epoch", t)),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn format_next_backup(&self) -> String {
|
||||||
|
match self.next_backup {
|
||||||
|
None => "Not scheduled".to_string(),
|
||||||
|
Some(t) => chrono::Utc.timestamp_opt(t as i64, 0)
|
||||||
|
.single()
|
||||||
|
.map(|dt| dt.format("%Y-%m-%d %H:%M:%S UTC").to_string())
|
||||||
|
.unwrap_or_else(|| format!("{} seconds since epoch", t)),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use super::*;
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_default_config() {
|
||||||
|
let config = BackupScheduleConfig::default();
|
||||||
|
assert!(config.enabled);
|
||||||
|
assert_eq!(config.interval_hours, 24);
|
||||||
|
assert_eq!(config.max_snapshots, 7);
|
||||||
|
assert!(config.auto_cleanup);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_scheduler_creation() {
|
||||||
|
let backend: Arc<dyn VfsBackend> = Arc::new(super::super::local_fs::LocalFs::new());
|
||||||
|
let scheduler = BackupScheduler::with_defaults(backend, PathBuf::from("/tmp"));
|
||||||
|
|
||||||
|
assert!(scheduler.is_enabled());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_schedule_next() {
|
||||||
|
let backend: Arc<dyn VfsBackend> = Arc::new(super::super::local_fs::LocalFs::new());
|
||||||
|
let mut scheduler = BackupScheduler::with_defaults(backend, PathBuf::from("/tmp"));
|
||||||
|
|
||||||
|
scheduler.schedule_next();
|
||||||
|
assert!(scheduler.next_backup.is_some());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_backup_info_format() {
|
||||||
|
let info = BackupInfo {
|
||||||
|
name: "snap_test".to_string(),
|
||||||
|
created_at: 1719234567,
|
||||||
|
size: 1536,
|
||||||
|
checksum_verified: true,
|
||||||
|
compressed: true,
|
||||||
|
encrypted: false,
|
||||||
|
};
|
||||||
|
|
||||||
|
assert!(info.format_created().contains("2024"));
|
||||||
|
assert!(info.format_size().contains("KB"));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_backup_stats() {
|
||||||
|
let now = current_time_secs();
|
||||||
|
let stats = BackupStats {
|
||||||
|
enabled: true,
|
||||||
|
backup_count: 5,
|
||||||
|
last_backup: Some(now - 3600),
|
||||||
|
next_backup: Some(now + 3600),
|
||||||
|
interval_hours: 24,
|
||||||
|
max_snapshots: 7,
|
||||||
|
};
|
||||||
|
|
||||||
|
assert!(stats.enabled);
|
||||||
|
assert_eq!(stats.backup_count, 5);
|
||||||
|
assert!(stats.next_backup_in_secs().unwrap_or(0) > 0);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_snapshot_name_generation() {
|
||||||
|
let name = generate_snapshot_name();
|
||||||
|
assert!(name.starts_with("snap_"));
|
||||||
|
assert!(name.len() > "snap_".len());
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,334 @@
|
|||||||
|
use std::collections::HashMap;
|
||||||
|
use std::sync::{Arc, Mutex, RwLock};
|
||||||
|
use std::time::{Duration, Instant};
|
||||||
|
|
||||||
|
use bytes::Bytes;
|
||||||
|
|
||||||
|
const DEFAULT_READ_CACHE_SIZE: usize = 64 * 1024 * 1024; // 64MB
|
||||||
|
const DEFAULT_READ_CACHE_BLOCK_SIZE: usize = 64 * 1024; // 64KB blocks
|
||||||
|
const DEFAULT_WRITE_CACHE_SIZE: usize = 32 * 1024 * 1024; // 32MB
|
||||||
|
const DEFAULT_CACHE_TTL_SECS: u64 = 300; // 5 minutes
|
||||||
|
|
||||||
|
#[derive(Clone)]
|
||||||
|
pub struct CacheConfig {
|
||||||
|
pub read_cache_size: usize,
|
||||||
|
pub read_cache_block_size: usize,
|
||||||
|
pub write_cache_size: usize,
|
||||||
|
pub cache_ttl_secs: u64,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Default for CacheConfig {
|
||||||
|
fn default() -> Self {
|
||||||
|
Self {
|
||||||
|
read_cache_size: DEFAULT_READ_CACHE_SIZE,
|
||||||
|
read_cache_block_size: DEFAULT_READ_CACHE_BLOCK_SIZE,
|
||||||
|
write_cache_size: DEFAULT_WRITE_CACHE_SIZE,
|
||||||
|
cache_ttl_secs: DEFAULT_CACHE_TTL_SECS,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub struct ReadCache {
|
||||||
|
config: CacheConfig,
|
||||||
|
blocks: RwLock<HashMap<u64, CachedBlock>>,
|
||||||
|
total_size: RwLock<usize>,
|
||||||
|
last_access: RwLock<Instant>,
|
||||||
|
}
|
||||||
|
|
||||||
|
struct CachedBlock {
|
||||||
|
data: Bytes,
|
||||||
|
timestamp: Instant,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl ReadCache {
|
||||||
|
pub fn new(config: CacheConfig) -> Self {
|
||||||
|
Self {
|
||||||
|
config,
|
||||||
|
blocks: RwLock::new(HashMap::new()),
|
||||||
|
total_size: RwLock::new(0),
|
||||||
|
last_access: RwLock::new(Instant::now()),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn get(&self, block_offset: u64) -> Option<Bytes> {
|
||||||
|
let blocks = self.blocks.read().unwrap();
|
||||||
|
if let Some(block) = blocks.get(&block_offset) {
|
||||||
|
if block.timestamp.elapsed() < Duration::from_secs(self.config.cache_ttl_secs) {
|
||||||
|
*self.last_access.write().unwrap() = Instant::now();
|
||||||
|
return Some(block.data.clone());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
None
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn put(&self, block_offset: u64, data: Bytes) {
|
||||||
|
let mut blocks = self.blocks.write().unwrap();
|
||||||
|
let mut total_size = self.total_size.write().unwrap();
|
||||||
|
|
||||||
|
// Evict old blocks if cache is full
|
||||||
|
while *total_size + data.len() > self.config.read_cache_size && !blocks.is_empty() {
|
||||||
|
let oldest_key = blocks
|
||||||
|
.iter()
|
||||||
|
.min_by_key(|(_, b)| b.timestamp)
|
||||||
|
.map(|(k, _)| *k);
|
||||||
|
if let Some(key) = oldest_key {
|
||||||
|
if let Some(removed) = blocks.remove(&key) {
|
||||||
|
*total_size -= removed.data.len();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Insert new block
|
||||||
|
if *total_size + data.len() <= self.config.read_cache_size {
|
||||||
|
let data_clone = data.clone();
|
||||||
|
blocks.insert(
|
||||||
|
block_offset,
|
||||||
|
CachedBlock {
|
||||||
|
data: data_clone,
|
||||||
|
timestamp: Instant::now(),
|
||||||
|
},
|
||||||
|
);
|
||||||
|
*total_size += data.len();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn invalidate(&self, block_offset: u64) {
|
||||||
|
let mut blocks = self.blocks.write().unwrap();
|
||||||
|
let mut total_size = self.total_size.write().unwrap();
|
||||||
|
if let Some(removed) = blocks.remove(&block_offset) {
|
||||||
|
*total_size -= removed.data.len();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn invalidate_all(&self) {
|
||||||
|
let mut blocks = self.blocks.write().unwrap();
|
||||||
|
blocks.clear();
|
||||||
|
*self.total_size.write().unwrap() = 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn stats(&self) -> CacheStats {
|
||||||
|
let blocks = self.blocks.read().unwrap();
|
||||||
|
CacheStats {
|
||||||
|
block_count: blocks.len(),
|
||||||
|
total_size: *self.total_size.read().unwrap(),
|
||||||
|
max_size: self.config.read_cache_size,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn block_offset(offset: u64, block_size: usize) -> u64 {
|
||||||
|
offset / block_size as u64
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn block_range(offset: u64, len: u32, block_size: usize) -> Vec<u64> {
|
||||||
|
let start_block = Self::block_offset(offset, block_size);
|
||||||
|
let end_block = Self::block_offset(offset + len as u64 - 1, block_size);
|
||||||
|
(start_block..=end_block).collect()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub struct WriteCache {
|
||||||
|
config: CacheConfig,
|
||||||
|
pending_writes: Mutex<HashMap<u64, Vec<u8>>>,
|
||||||
|
total_size: RwLock<usize>,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl WriteCache {
|
||||||
|
pub fn new(config: CacheConfig) -> Self {
|
||||||
|
Self {
|
||||||
|
config,
|
||||||
|
pending_writes: Mutex::new(HashMap::new()),
|
||||||
|
total_size: RwLock::new(0),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn put(&self, offset: u64, data: &[u8]) -> bool {
|
||||||
|
let mut pending = self.pending_writes.lock().unwrap();
|
||||||
|
let mut total_size = self.total_size.write().unwrap();
|
||||||
|
|
||||||
|
// Check if we have space
|
||||||
|
if *total_size + data.len() > self.config.write_cache_size {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Merge with existing write at same offset
|
||||||
|
if let Some(existing) = pending.get_mut(&offset) {
|
||||||
|
// Overwrite overlapping range
|
||||||
|
for (i, byte) in data.iter().enumerate() {
|
||||||
|
if i < existing.len() {
|
||||||
|
existing[i] = *byte;
|
||||||
|
} else {
|
||||||
|
existing.push(*byte);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
pending.insert(offset, data.to_vec());
|
||||||
|
*total_size += data.len();
|
||||||
|
}
|
||||||
|
|
||||||
|
true
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn get_pending(&self) -> Vec<(u64, Vec<u8>)> {
|
||||||
|
let pending = self.pending_writes.lock().unwrap();
|
||||||
|
pending.iter().map(|(k, v)| (*k, v.clone())).collect()
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn clear(&self) {
|
||||||
|
let mut pending = self.pending_writes.lock().unwrap();
|
||||||
|
pending.clear();
|
||||||
|
*self.total_size.write().unwrap() = 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn stats(&self) -> CacheStats {
|
||||||
|
CacheStats {
|
||||||
|
block_count: self.pending_writes.lock().unwrap().len(),
|
||||||
|
total_size: *self.total_size.read().unwrap(),
|
||||||
|
max_size: self.config.write_cache_size,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub struct CacheStats {
|
||||||
|
pub block_count: usize,
|
||||||
|
pub total_size: usize,
|
||||||
|
pub max_size: usize,
|
||||||
|
}
|
||||||
|
|
||||||
|
pub struct FileManager {
|
||||||
|
read_cache: Arc<ReadCache>,
|
||||||
|
write_cache: Arc<WriteCache>,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl FileManager {
|
||||||
|
pub fn new(config: CacheConfig) -> Self {
|
||||||
|
Self {
|
||||||
|
read_cache: Arc::new(ReadCache::new(config.clone())),
|
||||||
|
write_cache: Arc::new(WriteCache::new(config)),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn read_cache(&self) -> Arc<ReadCache> {
|
||||||
|
self.read_cache.clone()
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn write_cache(&self) -> Arc<WriteCache> {
|
||||||
|
self.write_cache.clone()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use super::*;
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_read_cache_basic() {
|
||||||
|
let config = CacheConfig::default();
|
||||||
|
let cache = ReadCache::new(config);
|
||||||
|
|
||||||
|
let block_offset = 0;
|
||||||
|
let data1 = Bytes::from(vec![1, 2, 3, 4, 5]);
|
||||||
|
|
||||||
|
cache.put(block_offset, data1.clone());
|
||||||
|
|
||||||
|
let cached = cache.get(block_offset);
|
||||||
|
assert!(cached.is_some());
|
||||||
|
assert_eq!(cached.unwrap(), data1);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_read_cache_ttl_expiry() {
|
||||||
|
let config = CacheConfig {
|
||||||
|
cache_ttl_secs: 1,
|
||||||
|
..Default::default()
|
||||||
|
};
|
||||||
|
let cache = ReadCache::new(config);
|
||||||
|
|
||||||
|
let block_offset = 0;
|
||||||
|
let data1 = Bytes::from(vec![1, 2, 3, 4, 5]);
|
||||||
|
|
||||||
|
cache.put(block_offset, data1.clone());
|
||||||
|
|
||||||
|
// Should be available immediately
|
||||||
|
assert!(cache.get(block_offset).is_some());
|
||||||
|
|
||||||
|
// Wait for TTL
|
||||||
|
std::thread::sleep(Duration::from_secs(2));
|
||||||
|
|
||||||
|
// Should be expired
|
||||||
|
assert!(cache.get(block_offset).is_none());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_read_cache_eviction() {
|
||||||
|
let config = CacheConfig {
|
||||||
|
read_cache_size: 100,
|
||||||
|
..Default::default()
|
||||||
|
};
|
||||||
|
let cache = ReadCache::new(config);
|
||||||
|
|
||||||
|
// Fill cache
|
||||||
|
cache.put(0, Bytes::from(vec![0u8; 50]));
|
||||||
|
cache.put(1, Bytes::from(vec![1u8; 50]));
|
||||||
|
|
||||||
|
// Should evict oldest block
|
||||||
|
cache.put(2, Bytes::from(vec![2u8; 50]));
|
||||||
|
|
||||||
|
// Block 0 should be evicted
|
||||||
|
assert!(cache.get(0).is_none());
|
||||||
|
assert!(cache.get(1).is_some());
|
||||||
|
assert!(cache.get(2).is_some());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_write_cache_basic() {
|
||||||
|
let config = CacheConfig::default();
|
||||||
|
let cache = WriteCache::new(config);
|
||||||
|
|
||||||
|
let data1 = vec![1, 2, 3, 4, 5];
|
||||||
|
cache.put(0, &data1);
|
||||||
|
|
||||||
|
let pending = cache.get_pending();
|
||||||
|
assert_eq!(pending.len(), 1);
|
||||||
|
assert_eq!(pending[0], (0, data1));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_write_cache_merge() {
|
||||||
|
let config = CacheConfig::default();
|
||||||
|
let cache = WriteCache::new(config);
|
||||||
|
|
||||||
|
cache.put(0, &[1, 2, 3]);
|
||||||
|
cache.put(0, &[4, 5, 6, 7, 8]);
|
||||||
|
|
||||||
|
let pending = cache.get_pending();
|
||||||
|
assert_eq!(pending.len(), 1);
|
||||||
|
assert_eq!(pending[0].1, vec![4, 5, 6, 7, 8]);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_write_cache_full() {
|
||||||
|
let config = CacheConfig {
|
||||||
|
write_cache_size: 10,
|
||||||
|
..Default::default()
|
||||||
|
};
|
||||||
|
let cache = WriteCache::new(config);
|
||||||
|
|
||||||
|
assert!(cache.put(0, &[1, 2, 3, 4, 5]));
|
||||||
|
assert!(cache.put(5, &[6, 7, 8, 9, 10]));
|
||||||
|
assert!(!cache.put(10, &[11, 12])); // Cache full
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_block_range() {
|
||||||
|
let block_size = 1024;
|
||||||
|
|
||||||
|
let range = ReadCache::block_range(0, 512, block_size);
|
||||||
|
assert_eq!(range, vec![0]);
|
||||||
|
|
||||||
|
let range = ReadCache::block_range(0, 2048, block_size);
|
||||||
|
assert_eq!(range, vec![0, 1]);
|
||||||
|
|
||||||
|
let range = ReadCache::block_range(1024, 512, block_size);
|
||||||
|
assert_eq!(range, vec![1]);
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,448 @@
|
|||||||
|
//! Block-level Checksum for Data Integrity
|
||||||
|
//!
|
||||||
|
//! Reference: ZFS/Btrfs checksum verification
|
||||||
|
//! - ZFS: Fletcher4/SHA256 per-block checksum
|
||||||
|
//! - Btrfs: CRC32C per-block checksum
|
||||||
|
//!
|
||||||
|
//! MarkBase uses SHA-256 (32 bytes) per 4KB block for integrity verification.
|
||||||
|
|
||||||
|
use std::path::PathBuf;
|
||||||
|
use std::io::{Read, Write};
|
||||||
|
|
||||||
|
use sha2::{Sha256, Digest};
|
||||||
|
use serde::{Serialize, Deserialize};
|
||||||
|
|
||||||
|
use super::{VfsBackend, VfsFile, VfsError};
|
||||||
|
|
||||||
|
pub const BLOCK_SIZE: usize = 4096;
|
||||||
|
pub const HASH_SIZE: usize = 32; // SHA-256
|
||||||
|
pub const CHECKSUM_DIR: &str = ".checksums";
|
||||||
|
pub const CHECKSUM_EXT: &str = ".checksums";
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||||
|
pub struct VfsBlockChecksum {
|
||||||
|
pub offset: u64, // Block offset (multiple of BLOCK_SIZE)
|
||||||
|
pub hash: Vec<u8>, // SHA-256 hash (32 bytes)
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||||
|
pub struct VfsChecksumFile {
|
||||||
|
pub block_size: usize,
|
||||||
|
pub algorithm: String, // "sha256"
|
||||||
|
pub blocks: Vec<VfsBlockChecksum>,
|
||||||
|
pub file_size: u64, // Original file size
|
||||||
|
}
|
||||||
|
|
||||||
|
impl VfsChecksumFile {
|
||||||
|
pub fn new(file_size: u64) -> Self {
|
||||||
|
Self {
|
||||||
|
block_size: BLOCK_SIZE,
|
||||||
|
algorithm: "sha256".to_string(),
|
||||||
|
blocks: Vec::new(),
|
||||||
|
file_size,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn from_bytes(data: &[u8]) -> Result<Self, VfsError> {
|
||||||
|
serde_json::from_slice(data)
|
||||||
|
.map_err(|e| VfsError::Io(format!("checksum parse failed: {}", e)))
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn to_bytes(&self) -> Result<Vec<u8>, VfsError> {
|
||||||
|
serde_json::to_vec(self)
|
||||||
|
.map_err(|e| VfsError::Io(format!("checksum serialize failed: {}", e)))
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn get_checksum(&self, offset: u64) -> Option<&[u8]> {
|
||||||
|
self.blocks.iter()
|
||||||
|
.find(|b| b.offset == offset)
|
||||||
|
.map(|b| b.hash.as_slice())
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn set_checksum(&mut self, offset: u64, hash: Vec<u8>) {
|
||||||
|
if let Some(block) = self.blocks.iter_mut().find(|b| b.offset == offset) {
|
||||||
|
block.hash = hash;
|
||||||
|
} else {
|
||||||
|
self.blocks.push(VfsBlockChecksum { offset, hash });
|
||||||
|
self.blocks.sort_by_key(|b| b.offset);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn block_count(&self) -> usize {
|
||||||
|
(self.file_size as usize / BLOCK_SIZE) +
|
||||||
|
if !(self.file_size as usize).is_multiple_of(BLOCK_SIZE) { 1 } else { 0 }
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn compute_block_hash(data: &[u8]) -> Vec<u8> {
|
||||||
|
let mut hasher = Sha256::new();
|
||||||
|
hasher.update(data);
|
||||||
|
hasher.finalize().to_vec()
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn verify_block_hash(data: &[u8], expected: &[u8]) -> bool {
|
||||||
|
let actual = compute_block_hash(data);
|
||||||
|
actual == expected
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||||
|
pub enum ChecksumMode {
|
||||||
|
Lazy, // Only verify on scrub (default)
|
||||||
|
OnRead, // Verify every read
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Clone)]
|
||||||
|
pub struct ChecksumConfig {
|
||||||
|
pub mode: ChecksumMode,
|
||||||
|
pub cache_verified: bool,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Default for ChecksumConfig {
|
||||||
|
fn default() -> Self {
|
||||||
|
Self {
|
||||||
|
mode: ChecksumMode::Lazy,
|
||||||
|
cache_verified: true,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug)]
|
||||||
|
pub struct ScrubResult {
|
||||||
|
pub path: PathBuf,
|
||||||
|
pub total_blocks: usize,
|
||||||
|
pub verified_blocks: usize,
|
||||||
|
pub corrupted_blocks: Vec<u64>,
|
||||||
|
pub repaired_blocks: Vec<u64>,
|
||||||
|
pub repair_failed: bool,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl ScrubResult {
|
||||||
|
pub fn is_clean(&self) -> bool {
|
||||||
|
self.corrupted_blocks.is_empty()
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn repair_success_rate(&self) -> f64 {
|
||||||
|
if self.corrupted_blocks.is_empty() {
|
||||||
|
1.0
|
||||||
|
} else {
|
||||||
|
self.repaired_blocks.len() as f64 / self.corrupted_blocks.len() as f64
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn checksum_path_for_file(file_path: &PathBuf, root: &PathBuf) -> PathBuf {
|
||||||
|
let relative = file_path.strip_prefix(root)
|
||||||
|
.unwrap_or(file_path);
|
||||||
|
root.join(CHECKSUM_DIR)
|
||||||
|
.join(relative)
|
||||||
|
.with_extension(CHECKSUM_EXT)
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn ensure_checksum_dir(root: &PathBuf, backend: &dyn VfsBackend) -> Result<(), VfsError> {
|
||||||
|
let checksum_dir = root.join(CHECKSUM_DIR);
|
||||||
|
if !backend.exists(&checksum_dir) {
|
||||||
|
backend.create_dir(&checksum_dir, 0o755)?;
|
||||||
|
}
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Scrub a single file to verify integrity
|
||||||
|
///
|
||||||
|
/// This reads the file and verifies each block checksum.
|
||||||
|
/// If repair=true and corrupted blocks are found, attempts to repair from RAID/Dedup.
|
||||||
|
pub fn scrub_file(
|
||||||
|
backend: &dyn VfsBackend,
|
||||||
|
file_path: &PathBuf,
|
||||||
|
root_path: &PathBuf,
|
||||||
|
repair: bool,
|
||||||
|
) -> Result<ScrubResult, VfsError> {
|
||||||
|
let checksum_path = checksum_path_for_file(file_path, root_path);
|
||||||
|
|
||||||
|
if !backend.exists(&checksum_path) {
|
||||||
|
return Ok(ScrubResult {
|
||||||
|
path: file_path.clone(),
|
||||||
|
total_blocks: 0,
|
||||||
|
verified_blocks: 0,
|
||||||
|
corrupted_blocks: vec![],
|
||||||
|
repaired_blocks: vec![],
|
||||||
|
repair_failed: false,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
let checksum_file_data = {
|
||||||
|
let mut checksum_file = backend.open_file(&checksum_path, &super::open_flags::OpenFlags::new().read())?;
|
||||||
|
checksum_file.read_all()?
|
||||||
|
};
|
||||||
|
let checksum_data = VfsChecksumFile::from_bytes(&checksum_file_data)?;
|
||||||
|
|
||||||
|
let mut file_handle = backend.open_file(file_path, &super::open_flags::OpenFlags::new().read())?;
|
||||||
|
let stat = file_handle.stat()?;
|
||||||
|
let file_size = stat.size;
|
||||||
|
|
||||||
|
let block_count = checksum_data.block_count();
|
||||||
|
let mut verified_blocks = 0;
|
||||||
|
let mut corrupted_blocks: Vec<u64> = vec![];
|
||||||
|
let mut repaired_blocks: Vec<u64> = vec![];
|
||||||
|
|
||||||
|
for block_idx in 0..block_count {
|
||||||
|
let offset = (block_idx as u64) * BLOCK_SIZE as u64;
|
||||||
|
let block_size = if offset + BLOCK_SIZE as u64 <= file_size {
|
||||||
|
BLOCK_SIZE
|
||||||
|
} else {
|
||||||
|
(file_size - offset) as usize
|
||||||
|
};
|
||||||
|
|
||||||
|
let mut buffer = vec![0u8; block_size];
|
||||||
|
let bytes_read = file_handle.read_at(&mut buffer, offset)?;
|
||||||
|
|
||||||
|
if bytes_read != block_size {
|
||||||
|
corrupted_blocks.push(offset);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
let expected_hash = checksum_data.get_checksum(offset);
|
||||||
|
if expected_hash.is_none() {
|
||||||
|
verified_blocks += 1;
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
let is_valid = verify_block_hash(&buffer, expected_hash.unwrap());
|
||||||
|
if is_valid {
|
||||||
|
verified_blocks += 1;
|
||||||
|
} else {
|
||||||
|
corrupted_blocks.push(offset);
|
||||||
|
|
||||||
|
if repair {
|
||||||
|
if repair_block(backend, file_path, offset, &buffer).is_ok() {
|
||||||
|
repaired_blocks.push(offset);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
let corrupted_count = corrupted_blocks.len();
|
||||||
|
let repaired_count = repaired_blocks.len();
|
||||||
|
|
||||||
|
Ok(ScrubResult {
|
||||||
|
path: file_path.clone(),
|
||||||
|
total_blocks: block_count,
|
||||||
|
verified_blocks,
|
||||||
|
corrupted_blocks,
|
||||||
|
repaired_blocks,
|
||||||
|
repair_failed: repair && repaired_count < corrupted_count,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Scrub all files in a directory
|
||||||
|
///
|
||||||
|
/// Recursively walks the directory and scrubs all files with checksums.
|
||||||
|
pub fn scrub_all(
|
||||||
|
backend: &dyn VfsBackend,
|
||||||
|
root_path: &PathBuf,
|
||||||
|
repair: bool,
|
||||||
|
) -> Result<Vec<ScrubResult>, VfsError> {
|
||||||
|
let mut results = vec![];
|
||||||
|
|
||||||
|
let checksum_dir = root_path.join(CHECKSUM_DIR);
|
||||||
|
if !backend.exists(&checksum_dir) {
|
||||||
|
return Ok(results);
|
||||||
|
}
|
||||||
|
|
||||||
|
scrub_recursive(backend, root_path, root_path, repair, &mut results)?;
|
||||||
|
|
||||||
|
Ok(results)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn scrub_recursive(
|
||||||
|
backend: &dyn VfsBackend,
|
||||||
|
current_path: &PathBuf,
|
||||||
|
root_path: &PathBuf,
|
||||||
|
repair: bool,
|
||||||
|
results: &mut Vec<ScrubResult>,
|
||||||
|
) -> Result<(), VfsError> {
|
||||||
|
let entries = backend.read_dir(current_path)?;
|
||||||
|
|
||||||
|
for entry in entries {
|
||||||
|
let entry_path = current_path.join(&entry.name);
|
||||||
|
|
||||||
|
if entry.stat.is_dir {
|
||||||
|
if entry.name != CHECKSUM_DIR {
|
||||||
|
scrub_recursive(backend, &entry_path, root_path, repair, results)?;
|
||||||
|
}
|
||||||
|
} else if !entry.name.ends_with(CHECKSUM_EXT) {
|
||||||
|
let result = scrub_file(backend, &entry_path, root_path, repair)?;
|
||||||
|
results.push(result);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Attempt to repair a corrupted block
|
||||||
|
///
|
||||||
|
/// Tries RAID repair first (if backend is RAID), then Dedup repair.
|
||||||
|
pub fn repair_block(
|
||||||
|
_backend: &dyn VfsBackend,
|
||||||
|
_file_path: &PathBuf,
|
||||||
|
_offset: u64,
|
||||||
|
_expected_checksum: &[u8],
|
||||||
|
) -> Result<Vec<u8>, VfsError> {
|
||||||
|
// Try Dedup repair first (check if block exists in dedup store)
|
||||||
|
// This requires the backend to have dedup integration
|
||||||
|
|
||||||
|
// For now, return error - RAID/Dedup repair requires specific backend types
|
||||||
|
Err(VfsError::Io("block repair requires RAID or Dedup backend (Phase 4/6)".to_string()))
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Repair block from DedupStore
|
||||||
|
///
|
||||||
|
/// This is called when checksum detects corruption and dedup store is available.
|
||||||
|
pub fn repair_block_from_dedup(
|
||||||
|
dedup_store: &super::dedup::DedupStore,
|
||||||
|
checksum_hash: &[u8],
|
||||||
|
) -> Result<Vec<u8>, VfsError> {
|
||||||
|
dedup_store.repair_from_checksum(checksum_hash)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Create checksums for a file
|
||||||
|
///
|
||||||
|
/// This reads the file and computes checksums for all blocks.
|
||||||
|
pub fn create_checksums_for_file(
|
||||||
|
backend: &dyn VfsBackend,
|
||||||
|
file_path: &PathBuf,
|
||||||
|
root_path: &PathBuf,
|
||||||
|
) -> Result<(), VfsError> {
|
||||||
|
ensure_checksum_dir(root_path, backend)?;
|
||||||
|
|
||||||
|
let mut file_handle = backend.open_file(file_path, &super::open_flags::OpenFlags::new().read())?;
|
||||||
|
let stat = file_handle.stat()?;
|
||||||
|
let file_size = stat.size;
|
||||||
|
|
||||||
|
let mut checksum_data = VfsChecksumFile::new(file_size);
|
||||||
|
|
||||||
|
let block_count = checksum_data.block_count();
|
||||||
|
|
||||||
|
for block_idx in 0..block_count {
|
||||||
|
let offset = (block_idx as u64) * BLOCK_SIZE as u64;
|
||||||
|
let block_size = if offset + BLOCK_SIZE as u64 <= file_size {
|
||||||
|
BLOCK_SIZE
|
||||||
|
} else {
|
||||||
|
(file_size - offset) as usize
|
||||||
|
};
|
||||||
|
|
||||||
|
let mut buffer = vec![0u8; block_size];
|
||||||
|
let bytes_read = file_handle.read_at(&mut buffer, offset)?;
|
||||||
|
|
||||||
|
if bytes_read > 0 {
|
||||||
|
let hash = compute_block_hash(&buffer[..bytes_read]);
|
||||||
|
checksum_data.set_checksum(offset, hash);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
let checksum_path = checksum_path_for_file(file_path, root_path);
|
||||||
|
let checksum_bytes = checksum_data.to_bytes()?;
|
||||||
|
|
||||||
|
let mut checksum_file = backend.open_file(
|
||||||
|
&checksum_path,
|
||||||
|
&super::open_flags::OpenFlags::new().write().create().truncate(),
|
||||||
|
)?;
|
||||||
|
checksum_file.write_all(&checksum_bytes)?;
|
||||||
|
checksum_file.flush()?;
|
||||||
|
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use super::*;
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_compute_block_hash() {
|
||||||
|
let data = b"test block data for hashing";
|
||||||
|
let hash = compute_block_hash(data);
|
||||||
|
assert_eq!(hash.len(), HASH_SIZE);
|
||||||
|
|
||||||
|
let hash2 = compute_block_hash(data);
|
||||||
|
assert_eq!(hash, hash2);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_verify_block_hash() {
|
||||||
|
let data = b"test block data";
|
||||||
|
let hash = compute_block_hash(data);
|
||||||
|
assert!(verify_block_hash(data, &hash));
|
||||||
|
|
||||||
|
let wrong_data = b"wrong block data";
|
||||||
|
assert!(!verify_block_hash(wrong_data, &hash));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_checksum_file_roundtrip() {
|
||||||
|
let mut checksum_file = VfsChecksumFile::new(8192);
|
||||||
|
checksum_file.set_checksum(0, compute_block_hash(b"block0"));
|
||||||
|
checksum_file.set_checksum(4096, compute_block_hash(b"block1"));
|
||||||
|
|
||||||
|
let bytes = checksum_file.to_bytes().unwrap();
|
||||||
|
let decoded = VfsChecksumFile::from_bytes(&bytes).unwrap();
|
||||||
|
|
||||||
|
assert_eq!(decoded.block_size, BLOCK_SIZE);
|
||||||
|
assert_eq!(decoded.blocks.len(), 2);
|
||||||
|
assert_eq!(decoded.file_size, 8192);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_checksum_file_get_set() {
|
||||||
|
let mut checksum_file = VfsChecksumFile::new(4096);
|
||||||
|
|
||||||
|
let hash = compute_block_hash(b"test");
|
||||||
|
checksum_file.set_checksum(0, hash.clone());
|
||||||
|
|
||||||
|
let retrieved = checksum_file.get_checksum(0);
|
||||||
|
assert!(retrieved.is_some());
|
||||||
|
assert_eq!(retrieved.unwrap(), hash.as_slice());
|
||||||
|
|
||||||
|
checksum_file.set_checksum(0, compute_block_hash(b"new"));
|
||||||
|
let updated = checksum_file.get_checksum(0).unwrap();
|
||||||
|
assert_ne!(updated, hash.as_slice());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_block_count_calculation() {
|
||||||
|
let checksum_file = VfsChecksumFile::new(4096);
|
||||||
|
assert_eq!(checksum_file.block_count(), 1);
|
||||||
|
|
||||||
|
let checksum_file = VfsChecksumFile::new(8192);
|
||||||
|
assert_eq!(checksum_file.block_count(), 2);
|
||||||
|
|
||||||
|
let checksum_file = VfsChecksumFile::new(4097);
|
||||||
|
assert_eq!(checksum_file.block_count(), 2);
|
||||||
|
|
||||||
|
let checksum_file = VfsChecksumFile::new(0);
|
||||||
|
assert_eq!(checksum_file.block_count(), 0);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_scrub_result_metrics() {
|
||||||
|
let result = ScrubResult {
|
||||||
|
path: PathBuf::from("/test"),
|
||||||
|
total_blocks: 10,
|
||||||
|
verified_blocks: 10,
|
||||||
|
corrupted_blocks: vec![],
|
||||||
|
repaired_blocks: vec![],
|
||||||
|
repair_failed: false,
|
||||||
|
};
|
||||||
|
assert!(result.is_clean());
|
||||||
|
assert_eq!(result.repair_success_rate(), 1.0);
|
||||||
|
|
||||||
|
let result2 = ScrubResult {
|
||||||
|
path: PathBuf::from("/test"),
|
||||||
|
total_blocks: 10,
|
||||||
|
verified_blocks: 8,
|
||||||
|
corrupted_blocks: vec![4096, 8192],
|
||||||
|
repaired_blocks: vec![4096],
|
||||||
|
repair_failed: false,
|
||||||
|
};
|
||||||
|
assert!(!result2.is_clean());
|
||||||
|
assert_eq!(result2.repair_success_rate(), 0.5);
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,259 @@
|
|||||||
|
//! ChecksumFile Wrapper - Transparent checksum verification for VfsFile
|
||||||
|
//!
|
||||||
|
//! This wraps any VfsFile to provide:
|
||||||
|
//! - Automatic checksum calculation on write
|
||||||
|
//! - Optional verification on read (OnRead mode)
|
||||||
|
//! - Cache of verified blocks (Lazy mode)
|
||||||
|
//! - Scrub support for integrity checking
|
||||||
|
|
||||||
|
use std::collections::{HashMap, HashSet};
|
||||||
|
use std::path::PathBuf;
|
||||||
|
use std::io::{Seek, SeekFrom};
|
||||||
|
|
||||||
|
use super::{VfsBackend, VfsFile, VfsStat, VfsError};
|
||||||
|
use super::checksum::{
|
||||||
|
VfsChecksumFile, ChecksumConfig, ChecksumMode,
|
||||||
|
BLOCK_SIZE, compute_block_hash, verify_block_hash,
|
||||||
|
checksum_path_for_file, ensure_checksum_dir,
|
||||||
|
};
|
||||||
|
use sha2::Digest;
|
||||||
|
|
||||||
|
pub struct ChecksumFile {
|
||||||
|
inner: Box<dyn VfsFile>,
|
||||||
|
file_path: PathBuf,
|
||||||
|
root_path: PathBuf,
|
||||||
|
backend: Box<dyn VfsBackend>,
|
||||||
|
config: ChecksumConfig,
|
||||||
|
checksum_data: Option<VfsChecksumFile>,
|
||||||
|
verified_cache: HashMap<u64, Vec<u8>>,
|
||||||
|
modified_blocks: HashSet<u64>,
|
||||||
|
current_offset: u64,
|
||||||
|
file_size: u64,
|
||||||
|
loaded: bool,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl ChecksumFile {
|
||||||
|
pub fn new(
|
||||||
|
inner: Box<dyn VfsFile>,
|
||||||
|
file_path: PathBuf,
|
||||||
|
root_path: PathBuf,
|
||||||
|
backend: Box<dyn VfsBackend>,
|
||||||
|
config: ChecksumConfig,
|
||||||
|
) -> Self {
|
||||||
|
Self {
|
||||||
|
inner,
|
||||||
|
file_path,
|
||||||
|
root_path,
|
||||||
|
backend,
|
||||||
|
config,
|
||||||
|
checksum_data: None,
|
||||||
|
verified_cache: HashMap::new(),
|
||||||
|
modified_blocks: HashSet::new(),
|
||||||
|
current_offset: 0,
|
||||||
|
file_size: 0,
|
||||||
|
loaded: false,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn load_checksum_file(&mut self) -> Result<(), VfsError> {
|
||||||
|
if self.loaded {
|
||||||
|
return Ok(());
|
||||||
|
}
|
||||||
|
|
||||||
|
let checksum_path = checksum_path_for_file(&self.file_path, &self.root_path);
|
||||||
|
|
||||||
|
if self.backend.exists(&checksum_path) {
|
||||||
|
let mut checksum_file = self.backend.open_file(&checksum_path, &super::open_flags::OpenFlags::new().read())?;
|
||||||
|
let data = checksum_file.read_all()?;
|
||||||
|
self.checksum_data = Some(VfsChecksumFile::from_bytes(&data)?);
|
||||||
|
} else {
|
||||||
|
let stat = self.inner.stat()?;
|
||||||
|
self.file_size = stat.size;
|
||||||
|
self.checksum_data = Some(VfsChecksumFile::new(self.file_size));
|
||||||
|
}
|
||||||
|
|
||||||
|
self.loaded = true;
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
fn save_checksum_file(&mut self) -> Result<(), VfsError> {
|
||||||
|
ensure_checksum_dir(&self.root_path, self.backend.as_ref())?;
|
||||||
|
|
||||||
|
if let Some(checksum_data) = &self.checksum_data {
|
||||||
|
let checksum_path = checksum_path_for_file(&self.file_path, &self.root_path);
|
||||||
|
let data = checksum_data.to_bytes()?;
|
||||||
|
|
||||||
|
let mut checksum_file = self.backend.open_file(
|
||||||
|
&checksum_path,
|
||||||
|
&super::open_flags::OpenFlags::new().write().create().truncate(),
|
||||||
|
)?;
|
||||||
|
checksum_file.write_all(&data)?;
|
||||||
|
checksum_file.flush()?;
|
||||||
|
}
|
||||||
|
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
fn get_block_offset(offset: u64) -> u64 {
|
||||||
|
(offset / BLOCK_SIZE as u64) * BLOCK_SIZE as u64
|
||||||
|
}
|
||||||
|
|
||||||
|
fn verify_block_at_offset(&mut self, offset: u64, data: &[u8]) -> Result<bool, VfsError> {
|
||||||
|
self.load_checksum_file()?;
|
||||||
|
|
||||||
|
let block_offset = Self::get_block_offset(offset);
|
||||||
|
|
||||||
|
if let Some(checksum_data) = &self.checksum_data {
|
||||||
|
if let Some(expected_hash) = checksum_data.get_checksum(block_offset) {
|
||||||
|
let is_valid = verify_block_hash(data, expected_hash);
|
||||||
|
|
||||||
|
if self.config.cache_verified && is_valid {
|
||||||
|
self.verified_cache.insert(block_offset, expected_hash.to_vec());
|
||||||
|
}
|
||||||
|
|
||||||
|
return Ok(is_valid);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
Ok(true)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn update_checksum_for_block(&mut self, offset: u64, data: &[u8]) -> Result<(), VfsError> {
|
||||||
|
self.load_checksum_file()?;
|
||||||
|
|
||||||
|
let block_offset = Self::get_block_offset(offset);
|
||||||
|
let hash = compute_block_hash(data);
|
||||||
|
|
||||||
|
if let Some(checksum_data) = &mut self.checksum_data {
|
||||||
|
checksum_data.set_checksum(block_offset, hash);
|
||||||
|
}
|
||||||
|
|
||||||
|
self.modified_blocks.insert(block_offset);
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn get_checksum_data(&self) -> Option<&VfsChecksumFile> {
|
||||||
|
self.checksum_data.as_ref()
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn get_modified_blocks(&self) -> &HashSet<u64> {
|
||||||
|
&self.modified_blocks
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn get_verified_cache(&self) -> &HashMap<u64, Vec<u8>> {
|
||||||
|
&self.verified_cache
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl VfsFile for ChecksumFile {
|
||||||
|
fn read(&mut self, buf: &mut [u8]) -> Result<usize, VfsError> {
|
||||||
|
let bytes_read = self.inner.read(buf)?;
|
||||||
|
|
||||||
|
if bytes_read > 0 && self.config.mode == ChecksumMode::OnRead {
|
||||||
|
self.verify_block_at_offset(self.current_offset, &buf[..bytes_read])?;
|
||||||
|
}
|
||||||
|
|
||||||
|
self.current_offset += bytes_read as u64;
|
||||||
|
Ok(bytes_read)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn write(&mut self, buf: &[u8]) -> Result<usize, VfsError> {
|
||||||
|
let bytes_written = self.inner.write(buf)?;
|
||||||
|
|
||||||
|
if bytes_written > 0 {
|
||||||
|
self.update_checksum_for_block(self.current_offset, buf)?;
|
||||||
|
self.current_offset += bytes_written as u64;
|
||||||
|
|
||||||
|
if self.current_offset > self.file_size {
|
||||||
|
self.file_size = self.current_offset;
|
||||||
|
if let Some(checksum_data) = &mut self.checksum_data {
|
||||||
|
checksum_data.file_size = self.file_size;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
Ok(bytes_written)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn seek(&mut self, pos: SeekFrom) -> Result<u64, VfsError> {
|
||||||
|
self.current_offset = self.inner.seek(pos)?;
|
||||||
|
Ok(self.current_offset)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn flush(&mut self) -> Result<(), VfsError> {
|
||||||
|
self.inner.flush()?;
|
||||||
|
|
||||||
|
if !self.modified_blocks.is_empty() {
|
||||||
|
self.save_checksum_file()?;
|
||||||
|
self.modified_blocks.clear();
|
||||||
|
}
|
||||||
|
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
fn stat(&mut self) -> Result<VfsStat, VfsError> {
|
||||||
|
let stat = self.inner.stat()?;
|
||||||
|
Ok(stat)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn set_len(&mut self, size: u64) -> Result<(), VfsError> {
|
||||||
|
self.inner.set_len(size)?;
|
||||||
|
self.file_size = size;
|
||||||
|
|
||||||
|
if let Some(checksum_data) = &mut self.checksum_data {
|
||||||
|
checksum_data.file_size = size;
|
||||||
|
}
|
||||||
|
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
fn read_at(&mut self, buf: &mut [u8], offset: u64) -> Result<usize, VfsError> {
|
||||||
|
let bytes_read = self.inner.read_at(buf, offset)?;
|
||||||
|
|
||||||
|
if bytes_read > 0 && self.config.mode == ChecksumMode::OnRead {
|
||||||
|
self.verify_block_at_offset(offset, &buf[..bytes_read])?;
|
||||||
|
}
|
||||||
|
|
||||||
|
Ok(bytes_read)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn write_at(&mut self, buf: &[u8], offset: u64) -> Result<usize, VfsError> {
|
||||||
|
let bytes_written = self.inner.write_at(buf, offset)?;
|
||||||
|
|
||||||
|
if bytes_written > 0 {
|
||||||
|
self.update_checksum_for_block(offset, buf)?;
|
||||||
|
|
||||||
|
let new_size = offset + bytes_written as u64;
|
||||||
|
if new_size > self.file_size {
|
||||||
|
self.file_size = new_size;
|
||||||
|
if let Some(checksum_data) = &mut self.checksum_data {
|
||||||
|
checksum_data.file_size = self.file_size;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
Ok(bytes_written)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use super::*;
|
||||||
|
use std::path::Path;
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_block_offset_calculation() {
|
||||||
|
assert_eq!(ChecksumFile::get_block_offset(0), 0);
|
||||||
|
assert_eq!(ChecksumFile::get_block_offset(4095), 0);
|
||||||
|
assert_eq!(ChecksumFile::get_block_offset(4096), 4096);
|
||||||
|
assert_eq!(ChecksumFile::get_block_offset(8191), 4096);
|
||||||
|
assert_eq!(ChecksumFile::get_block_offset(8192), 8192);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_checksum_config_default() {
|
||||||
|
let config = ChecksumConfig::default();
|
||||||
|
assert_eq!(config.mode, ChecksumMode::Lazy);
|
||||||
|
assert!(config.cache_verified);
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -1,5 +1,4 @@
|
|||||||
use super::{VfsCompression, VfsCompressionConfig, VfsError};
|
use super::{VfsCompression, VfsCompressionConfig, VfsError};
|
||||||
use std::io::{Read, Write};
|
|
||||||
use std::path::Path;
|
use std::path::Path;
|
||||||
|
|
||||||
pub struct Compressor {
|
pub struct Compressor {
|
||||||
@@ -28,7 +27,7 @@ impl Compressor {
|
|||||||
.map_err(|e| VfsError::Io(format!("ZSTD compression failed: {}", e)))
|
.map_err(|e| VfsError::Io(format!("ZSTD compression failed: {}", e)))
|
||||||
}
|
}
|
||||||
VfsCompression::Lz4 => {
|
VfsCompression::Lz4 => {
|
||||||
Err(VfsError::Unsupported("LZ4 compression not yet implemented".to_string()))
|
Ok(lz4_flex::compress_prepend_size(data))
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -41,7 +40,8 @@ impl Compressor {
|
|||||||
.map_err(|e| VfsError::Io(format!("ZSTD decompression failed: {}", e)))
|
.map_err(|e| VfsError::Io(format!("ZSTD decompression failed: {}", e)))
|
||||||
}
|
}
|
||||||
VfsCompression::Lz4 => {
|
VfsCompression::Lz4 => {
|
||||||
Err(VfsError::Unsupported("LZ4 decompression not yet implemented".to_string()))
|
lz4_flex::decompress_size_prepended(data)
|
||||||
|
.map_err(|e| VfsError::Io(format!("LZ4 decompression failed: {}", e)))
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -181,6 +181,31 @@ impl DedupStore {
|
|||||||
stats.total_blocks = stats.total_refs;
|
stats.total_blocks = stats.total_refs;
|
||||||
Ok(stats)
|
Ok(stats)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Retrieve block by checksum hash (for scrub repair)
|
||||||
|
///
|
||||||
|
/// Converts the checksum hash (Vec<u8>) to hex format and retrieves from dedup store.
|
||||||
|
pub fn get_block_by_checksum(&self, checksum_hash: &[u8]) -> Result<Vec<u8>, VfsError> {
|
||||||
|
let hash_hex = hex::encode(checksum_hash);
|
||||||
|
self.get_block(&hash_hex)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Check if a block exists by checksum hash
|
||||||
|
pub fn has_block_by_checksum(&self, checksum_hash: &[u8]) -> bool {
|
||||||
|
let hash_hex = hex::encode(checksum_hash);
|
||||||
|
self.store_path.join(&hash_hex).exists()
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Repair a corrupted block from dedup store
|
||||||
|
///
|
||||||
|
/// If the dedup store contains a block with the same checksum, retrieve it.
|
||||||
|
pub fn repair_from_checksum(&self, checksum_hash: &[u8]) -> Result<Vec<u8>, VfsError> {
|
||||||
|
if self.has_block_by_checksum(checksum_hash) {
|
||||||
|
self.get_block_by_checksum(checksum_hash)
|
||||||
|
} else {
|
||||||
|
Err(VfsError::NotFound("Block not found in dedup store".to_string()))
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]
|
#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]
|
||||||
|
|||||||
@@ -0,0 +1,343 @@
|
|||||||
|
//! Encrypted VFS Backend - Transparent at-rest encryption using AES-256-GCM
|
||||||
|
//!
|
||||||
|
//! This module provides transparent file encryption at the VFS layer.
|
||||||
|
//! Files are encrypted before being written to disk and decrypted on read.
|
||||||
|
//!
|
||||||
|
//! Format:
|
||||||
|
//! - Header (32 bytes): magic(4) + version(4) + nonce(12) + original_size(8) + reserved(4)
|
||||||
|
//! - Body: AES-256-GCM encrypted data
|
||||||
|
//! - Tag (16 bytes): GCM authentication tag
|
||||||
|
|
||||||
|
use std::path::PathBuf;
|
||||||
|
use std::io::{Seek, SeekFrom};
|
||||||
|
|
||||||
|
use aes_gcm::{
|
||||||
|
Aes256Gcm, Nonce, aead::{Aead, KeyInit},
|
||||||
|
};
|
||||||
|
use sha2::{Sha256, Digest};
|
||||||
|
|
||||||
|
use super::{VfsBackend, VfsFile, VfsStat, VfsError};
|
||||||
|
use super::local_fs::LocalFs;
|
||||||
|
|
||||||
|
const ENCRYPTED_MAGIC: &[u8] = b"MBE1"; // MarkBase Encrypted v1
|
||||||
|
const ENCRYPTED_VERSION: u32 = 1;
|
||||||
|
const HEADER_SIZE: usize = 32;
|
||||||
|
const TAG_SIZE: usize = 16;
|
||||||
|
const NONCE_SIZE: usize = 12;
|
||||||
|
const KEY_SIZE: usize = 32;
|
||||||
|
|
||||||
|
#[derive(Debug, Clone)]
|
||||||
|
pub struct EncryptedVfsConfig {
|
||||||
|
pub master_key: Vec<u8>, // 32 bytes for AES-256
|
||||||
|
pub encrypt_filenames: bool, // Future feature
|
||||||
|
}
|
||||||
|
|
||||||
|
impl EncryptedVfsConfig {
|
||||||
|
pub fn new(master_key: [u8; 32]) -> Self {
|
||||||
|
Self {
|
||||||
|
master_key: master_key.to_vec(),
|
||||||
|
encrypt_filenames: false,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn from_password(password: &str) -> Self {
|
||||||
|
let mut hasher = Sha256::new();
|
||||||
|
hasher.update(password.as_bytes());
|
||||||
|
let key = hasher.finalize();
|
||||||
|
Self {
|
||||||
|
master_key: key.to_vec(),
|
||||||
|
encrypt_filenames: false,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub struct EncryptedVfs {
|
||||||
|
inner: Box<dyn VfsBackend>,
|
||||||
|
config: EncryptedVfsConfig,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl EncryptedVfs {
|
||||||
|
pub fn new(inner: Box<dyn VfsBackend>, config: EncryptedVfsConfig) -> Self {
|
||||||
|
Self { inner, config }
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn wrap_local_fs(_root: PathBuf, config: EncryptedVfsConfig) -> Self {
|
||||||
|
Self::new(Box::new(LocalFs::new()), config)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn derive_key(&self, path: &PathBuf) -> Vec<u8> {
|
||||||
|
let mut hasher = Sha256::new();
|
||||||
|
hasher.update(&self.config.master_key);
|
||||||
|
hasher.update(path.to_string_lossy().as_bytes());
|
||||||
|
let derived = hasher.finalize();
|
||||||
|
derived[..KEY_SIZE].to_vec()
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn is_encrypted_file(data: &[u8]) -> bool {
|
||||||
|
data.len() >= HEADER_SIZE + TAG_SIZE && &data[..4] == ENCRYPTED_MAGIC
|
||||||
|
}
|
||||||
|
|
||||||
|
fn encrypt_data(&self, path: &PathBuf, data: &[u8]) -> Result<Vec<u8>, VfsError> {
|
||||||
|
let key_bytes = self.derive_key(path);
|
||||||
|
let cipher = Aes256Gcm::new_from_slice(&key_bytes)
|
||||||
|
.map_err(|e| VfsError::Io(format!("cipher init failed: {}", e)))?;
|
||||||
|
|
||||||
|
let nonce_bytes: [u8; NONCE_SIZE] = rand_key(12).try_into().map_err(|_| VfsError::Io("nonce generation failed".to_string()))?;
|
||||||
|
let nonce = Nonce::from_slice(&nonce_bytes);
|
||||||
|
|
||||||
|
let ciphertext = cipher.encrypt(nonce, data)
|
||||||
|
.map_err(|e| VfsError::Io(format!("encryption failed: {}", e)))?;
|
||||||
|
|
||||||
|
let mut result = Vec::with_capacity(HEADER_SIZE + ciphertext.len() + TAG_SIZE);
|
||||||
|
|
||||||
|
result.extend_from_slice(ENCRYPTED_MAGIC);
|
||||||
|
result.extend_from_slice(&ENCRYPTED_VERSION.to_le_bytes());
|
||||||
|
result.extend_from_slice(&nonce_bytes);
|
||||||
|
result.extend_from_slice(&(data.len() as u64).to_le_bytes());
|
||||||
|
result.extend_from_slice(&[0u8; 4]);
|
||||||
|
result.extend_from_slice(&ciphertext);
|
||||||
|
|
||||||
|
Ok(result)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn decrypt_data(&self, path: &PathBuf, data: &[u8]) -> Result<Vec<u8>, VfsError> {
|
||||||
|
if !Self::is_encrypted_file(data) {
|
||||||
|
return Err(VfsError::Io("not an encrypted file".to_string()));
|
||||||
|
}
|
||||||
|
|
||||||
|
let key_bytes = self.derive_key(path);
|
||||||
|
let cipher = Aes256Gcm::new_from_slice(&key_bytes)
|
||||||
|
.map_err(|e| VfsError::Io(format!("cipher init failed: {}", e)))?;
|
||||||
|
|
||||||
|
let nonce_bytes: [u8; NONCE_SIZE] = data[8..20].try_into().map_err(|_| VfsError::Io("invalid nonce".to_string()))?;
|
||||||
|
let nonce = Nonce::from_slice(&nonce_bytes);
|
||||||
|
|
||||||
|
let original_size = u64::from_le_bytes(data[20..28].try_into().map_err(|_| VfsError::Io("invalid size".to_string()))?) as usize;
|
||||||
|
|
||||||
|
let ciphertext = &data[HEADER_SIZE..];
|
||||||
|
|
||||||
|
let plaintext = cipher.decrypt(nonce, ciphertext)
|
||||||
|
.map_err(|e| VfsError::Io(format!("decryption failed: {}", e)))?;
|
||||||
|
|
||||||
|
if plaintext.len() != original_size {
|
||||||
|
return Err(VfsError::Io(format!("size mismatch: expected {}, got {}", original_size, plaintext.len())));
|
||||||
|
}
|
||||||
|
|
||||||
|
Ok(plaintext)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn rand_key(len: usize) -> Vec<u8> {
|
||||||
|
use std::time::{SystemTime, UNIX_EPOCH};
|
||||||
|
let now = SystemTime::now().duration_since(UNIX_EPOCH).unwrap().as_nanos();
|
||||||
|
let mut hasher = Sha256::new();
|
||||||
|
hasher.update(now.to_le_bytes());
|
||||||
|
hasher.update([0u8; 32]);
|
||||||
|
let hash = hasher.finalize();
|
||||||
|
hash[..len].to_vec()
|
||||||
|
}
|
||||||
|
|
||||||
|
pub struct EncryptedFile {
|
||||||
|
inner: Box<dyn VfsFile>,
|
||||||
|
path: PathBuf,
|
||||||
|
config: EncryptedVfsConfig,
|
||||||
|
decrypted_data: Option<Vec<u8>>,
|
||||||
|
modified: bool,
|
||||||
|
position: u64,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl EncryptedFile {
|
||||||
|
fn decrypt_on_open(&mut self) -> Result<(), VfsError> {
|
||||||
|
let encrypted = self.inner.read_all()?;
|
||||||
|
|
||||||
|
if EncryptedVfs::is_encrypted_file(&encrypted) {
|
||||||
|
let vfs = EncryptedVfs::new(Box::new(LocalFs::new()), self.config.clone());
|
||||||
|
self.decrypted_data = Some(vfs.decrypt_data(&self.path, &encrypted)?);
|
||||||
|
} else {
|
||||||
|
self.decrypted_data = Some(encrypted);
|
||||||
|
}
|
||||||
|
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
fn encrypt_on_close(&mut self) -> Result<(), VfsError> {
|
||||||
|
if !self.modified {
|
||||||
|
return Ok(());
|
||||||
|
}
|
||||||
|
|
||||||
|
let data = self.decrypted_data.as_ref().ok_or_else(|| VfsError::Io("no data to encrypt".to_string()))?;
|
||||||
|
|
||||||
|
let vfs = EncryptedVfs::new(Box::new(LocalFs::new()), self.config.clone());
|
||||||
|
let encrypted = vfs.encrypt_data(&self.path, data)?;
|
||||||
|
|
||||||
|
self.inner.seek(SeekFrom::Start(0))?;
|
||||||
|
self.inner.write_all(&encrypted)?;
|
||||||
|
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl VfsFile for EncryptedFile {
|
||||||
|
fn read(&mut self, buf: &mut [u8]) -> Result<usize, VfsError> {
|
||||||
|
if self.decrypted_data.is_none() {
|
||||||
|
self.decrypt_on_open()?;
|
||||||
|
}
|
||||||
|
|
||||||
|
let data = self.decrypted_data.as_ref().ok_or_else(|| VfsError::Io("no decrypted data".to_string()))?;
|
||||||
|
|
||||||
|
let start = self.position as usize;
|
||||||
|
let end = std::cmp::min(start + buf.len(), data.len());
|
||||||
|
|
||||||
|
if start >= data.len() {
|
||||||
|
return Ok(0);
|
||||||
|
}
|
||||||
|
|
||||||
|
buf[..(end - start)].copy_from_slice(&data[start..end]);
|
||||||
|
self.position += (end - start) as u64;
|
||||||
|
|
||||||
|
Ok(end - start)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn write(&mut self, buf: &[u8]) -> Result<usize, VfsError> {
|
||||||
|
if self.decrypted_data.is_none() {
|
||||||
|
self.decrypted_data = Some(Vec::new());
|
||||||
|
}
|
||||||
|
|
||||||
|
let data = self.decrypted_data.as_mut().ok_or_else(|| VfsError::Io("no decrypted data".to_string()))?;
|
||||||
|
|
||||||
|
let start = self.position as usize;
|
||||||
|
if start + buf.len() > data.len() {
|
||||||
|
data.resize(start + buf.len(), 0);
|
||||||
|
}
|
||||||
|
|
||||||
|
data[start..start + buf.len()].copy_from_slice(buf);
|
||||||
|
self.position += buf.len() as u64;
|
||||||
|
self.modified = true;
|
||||||
|
|
||||||
|
Ok(buf.len())
|
||||||
|
}
|
||||||
|
|
||||||
|
fn seek(&mut self, pos: SeekFrom) -> Result<u64, VfsError> {
|
||||||
|
match pos {
|
||||||
|
SeekFrom::Start(offset) => {
|
||||||
|
self.position = offset;
|
||||||
|
}
|
||||||
|
SeekFrom::Current(offset) => {
|
||||||
|
self.position = (self.position as i64 + offset) as u64;
|
||||||
|
}
|
||||||
|
SeekFrom::End(offset) => {
|
||||||
|
let len = self.decrypted_data.as_ref().map(|d| d.len() as i64).unwrap_or(0);
|
||||||
|
self.position = (len + offset) as u64;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Ok(self.position)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn flush(&mut self) -> Result<(), VfsError> {
|
||||||
|
self.encrypt_on_close()?;
|
||||||
|
self.inner.flush()?;
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
fn stat(&mut self) -> Result<VfsStat, VfsError> {
|
||||||
|
let stat = self.inner.stat()?;
|
||||||
|
Ok(VfsStat {
|
||||||
|
size: self.decrypted_data.as_ref().map(|d| d.len() as u64).unwrap_or(stat.size),
|
||||||
|
mode: stat.mode,
|
||||||
|
uid: stat.uid,
|
||||||
|
gid: stat.gid,
|
||||||
|
atime: stat.atime,
|
||||||
|
mtime: stat.mtime,
|
||||||
|
is_dir: false,
|
||||||
|
is_symlink: false,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
fn set_len(&mut self, size: u64) -> Result<(), VfsError> {
|
||||||
|
if self.decrypted_data.is_none() {
|
||||||
|
self.decrypted_data = Some(Vec::new());
|
||||||
|
}
|
||||||
|
|
||||||
|
let data = self.decrypted_data.as_mut().ok_or_else(|| VfsError::Io("no decrypted data".to_string()))?;
|
||||||
|
data.resize(size as usize, 0);
|
||||||
|
self.modified = true;
|
||||||
|
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use super::*;
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_encrypt_decrypt_roundtrip() {
|
||||||
|
let config = EncryptedVfsConfig::from_password("test_password");
|
||||||
|
let path = PathBuf::from("/test/file.txt");
|
||||||
|
|
||||||
|
let vfs = EncryptedVfs::new(Box::new(LocalFs::new()), config.clone());
|
||||||
|
|
||||||
|
let original = b"Hello, World! This is a test message.";
|
||||||
|
let encrypted = vfs.encrypt_data(&path, original).unwrap();
|
||||||
|
|
||||||
|
assert!(encrypted.len() > original.len());
|
||||||
|
assert!(EncryptedVfs::is_encrypted_file(&encrypted));
|
||||||
|
|
||||||
|
let decrypted = vfs.decrypt_data(&path, &encrypted).unwrap();
|
||||||
|
assert_eq!(decrypted, original);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_different_keys_produce_different_ciphertext() {
|
||||||
|
let config1 = EncryptedVfsConfig::from_password("password1");
|
||||||
|
let config2 = EncryptedVfsConfig::from_password("password2");
|
||||||
|
let path = PathBuf::from("/test/file.txt");
|
||||||
|
|
||||||
|
let vfs1 = EncryptedVfs::new(Box::new(LocalFs::new()), config1);
|
||||||
|
let vfs2 = EncryptedVfs::new(Box::new(LocalFs::new()), config2);
|
||||||
|
|
||||||
|
let original = b"Same content";
|
||||||
|
|
||||||
|
let enc1 = vfs1.encrypt_data(&path, original).unwrap();
|
||||||
|
let enc2 = vfs2.encrypt_data(&path, original).unwrap();
|
||||||
|
|
||||||
|
assert_ne!(enc1, enc2);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_key_derivation() {
|
||||||
|
let config = EncryptedVfsConfig::from_password("test_password");
|
||||||
|
let vfs = EncryptedVfs::new(Box::new(LocalFs::new()), config);
|
||||||
|
|
||||||
|
let key1 = vfs.derive_key(&PathBuf::from("/file1.txt"));
|
||||||
|
let key2 = vfs.derive_key(&PathBuf::from("/file2.txt"));
|
||||||
|
|
||||||
|
assert_ne!(key1, key2);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_header_format() {
|
||||||
|
let config = EncryptedVfsConfig::from_password("test");
|
||||||
|
let path = PathBuf::from("/test.txt");
|
||||||
|
let vfs = EncryptedVfs::new(Box::new(LocalFs::new()), config);
|
||||||
|
|
||||||
|
let data = b"test";
|
||||||
|
let encrypted = vfs.encrypt_data(&path, data).unwrap();
|
||||||
|
|
||||||
|
assert_eq!(&encrypted[..4], ENCRYPTED_MAGIC);
|
||||||
|
assert_eq!(u32::from_le_bytes(encrypted[4..8].try_into().unwrap()), ENCRYPTED_VERSION);
|
||||||
|
assert_eq!(encrypted.len(), HEADER_SIZE + data.len() + TAG_SIZE);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_config_from_password() {
|
||||||
|
let config = EncryptedVfsConfig::from_password("my_secret_password");
|
||||||
|
assert_eq!(config.master_key.len(), KEY_SIZE);
|
||||||
|
|
||||||
|
let config2 = EncryptedVfsConfig::from_password("my_secret_password");
|
||||||
|
assert_eq!(config.master_key, config2.master_key);
|
||||||
|
|
||||||
|
let config3 = EncryptedVfsConfig::from_password("different");
|
||||||
|
assert_ne!(config.master_key, config3.master_key);
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -3,7 +3,7 @@ use super::util;
|
|||||||
use super::{VfsAce, VfsAceFlag, VfsAceMask, VfsAceType, VfsAcl, VfsBackend, VfsDirEntry, VfsError, VfsFile, VfsPreviousVersion, VfsQuota, VfsQuotaUsage, VfsSnapshotInfo, VfsStat};
|
use super::{VfsAce, VfsAceFlag, VfsAceMask, VfsAceType, VfsAcl, VfsBackend, VfsDirEntry, VfsError, VfsFile, VfsPreviousVersion, VfsQuota, VfsQuotaUsage, VfsSnapshotInfo, VfsStat};
|
||||||
use std::fs::{self, File, OpenOptions};
|
use std::fs::{self, File, OpenOptions};
|
||||||
use std::io::{Read, Seek, SeekFrom, Write};
|
use std::io::{Read, Seek, SeekFrom, Write};
|
||||||
use std::os::unix::fs::{MetadataExt, PermissionsExt};
|
use std::os::unix::fs::{FileExt, MetadataExt, PermissionsExt};
|
||||||
use std::path::{Path, PathBuf};
|
use std::path::{Path, PathBuf};
|
||||||
use std::time::SystemTime;
|
use std::time::SystemTime;
|
||||||
|
|
||||||
@@ -42,6 +42,14 @@ impl VfsFile for LocalFile {
|
|||||||
self.file.seek(pos).map_err(|e| VfsError::Io(e.to_string()))
|
self.file.seek(pos).map_err(|e| VfsError::Io(e.to_string()))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn read_at(&mut self, buf: &mut [u8], offset: u64) -> Result<usize, VfsError> {
|
||||||
|
self.file.read_at(buf, offset).map_err(|e| VfsError::Io(e.to_string()))
|
||||||
|
}
|
||||||
|
|
||||||
|
fn write_at(&mut self, buf: &[u8], offset: u64) -> Result<usize, VfsError> {
|
||||||
|
self.file.write_at(buf, offset).map_err(|e| VfsError::Io(e.to_string()))
|
||||||
|
}
|
||||||
|
|
||||||
fn flush(&mut self) -> Result<(), VfsError> {
|
fn flush(&mut self) -> Result<(), VfsError> {
|
||||||
self.file.flush().map_err(|e| VfsError::Io(e.to_string()))
|
self.file.flush().map_err(|e| VfsError::Io(e.to_string()))
|
||||||
}
|
}
|
||||||
@@ -153,6 +161,10 @@ impl VfsBackend for LocalFs {
|
|||||||
fs::remove_dir(path).map_err(|e| util::map_io_error(path, e))
|
fs::remove_dir(path).map_err(|e| util::map_io_error(path, e))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn remove_dir_all(&self, path: &Path) -> Result<(), VfsError> {
|
||||||
|
fs::remove_dir_all(path).map_err(|e| util::map_io_error(path, e))
|
||||||
|
}
|
||||||
|
|
||||||
fn remove_file(&self, path: &Path) -> Result<(), VfsError> {
|
fn remove_file(&self, path: &Path) -> Result<(), VfsError> {
|
||||||
fs::remove_file(path).map_err(|e| util::map_io_error(path, e))
|
fs::remove_file(path).map_err(|e| util::map_io_error(path, e))
|
||||||
}
|
}
|
||||||
@@ -185,6 +197,39 @@ impl VfsBackend for LocalFs {
|
|||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn set_times(&self, path: &Path, atime: SystemTime, mtime: SystemTime) -> Result<(), VfsError> {
|
||||||
|
let at = atime.duration_since(std::time::UNIX_EPOCH)
|
||||||
|
.map_err(|_| VfsError::Io("atime before UNIX_EPOCH".to_string()))?;
|
||||||
|
let mt = mtime.duration_since(std::time::UNIX_EPOCH)
|
||||||
|
.map_err(|_| VfsError::Io("mtime before UNIX_EPOCH".to_string()))?;
|
||||||
|
filetime::set_file_times(
|
||||||
|
path,
|
||||||
|
filetime::FileTime::from_unix_time(at.as_secs() as i64, at.subsec_nanos()),
|
||||||
|
filetime::FileTime::from_unix_time(mt.as_secs() as i64, mt.subsec_nanos()),
|
||||||
|
)
|
||||||
|
.map_err(|e| util::map_io_error(path, e))
|
||||||
|
}
|
||||||
|
|
||||||
|
fn set_atime(&self, path: &Path, atime: SystemTime) -> Result<(), VfsError> {
|
||||||
|
let at = atime.duration_since(std::time::UNIX_EPOCH)
|
||||||
|
.map_err(|_| VfsError::Io("atime before UNIX_EPOCH".to_string()))?;
|
||||||
|
filetime::set_file_atime(
|
||||||
|
path,
|
||||||
|
filetime::FileTime::from_unix_time(at.as_secs() as i64, at.subsec_nanos()),
|
||||||
|
)
|
||||||
|
.map_err(|e| util::map_io_error(path, e))
|
||||||
|
}
|
||||||
|
|
||||||
|
fn set_mtime(&self, path: &Path, mtime: SystemTime) -> Result<(), VfsError> {
|
||||||
|
let mt = mtime.duration_since(std::time::UNIX_EPOCH)
|
||||||
|
.map_err(|_| VfsError::Io("mtime before UNIX_EPOCH".to_string()))?;
|
||||||
|
filetime::set_file_mtime(
|
||||||
|
path,
|
||||||
|
filetime::FileTime::from_unix_time(mt.as_secs() as i64, mt.subsec_nanos()),
|
||||||
|
)
|
||||||
|
.map_err(|e| util::map_io_error(path, e))
|
||||||
|
}
|
||||||
|
|
||||||
fn read_link(&self, path: &Path) -> Result<PathBuf, VfsError> {
|
fn read_link(&self, path: &Path) -> Result<PathBuf, VfsError> {
|
||||||
let target = fs::read_link(path).map_err(|e| util::map_io_error(path, e))?;
|
let target = fs::read_link(path).map_err(|e| util::map_io_error(path, e))?;
|
||||||
Ok(target)
|
Ok(target)
|
||||||
@@ -232,6 +277,15 @@ impl VfsBackend for LocalFs {
|
|||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn copy(&self, from: &Path, to: &Path) -> Result<(), VfsError> {
|
||||||
|
// Check if source is a directory
|
||||||
|
if from.is_dir() {
|
||||||
|
return copy_dir_recursive_impl(from, to);
|
||||||
|
}
|
||||||
|
fs::copy(from, to).map_err(|e| util::map_io_error(from, e))?;
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
// ===== Snapshot support =====
|
// ===== Snapshot support =====
|
||||||
|
|
||||||
fn create_snapshot(&self, path: &Path, name: &str) -> Result<(), VfsError> {
|
fn create_snapshot(&self, path: &Path, name: &str) -> Result<(), VfsError> {
|
||||||
@@ -240,7 +294,7 @@ impl VfsBackend for LocalFs {
|
|||||||
|
|
||||||
let snapshot_path = snapshot_dir.join(name);
|
let snapshot_path = snapshot_dir.join(name);
|
||||||
if path.is_dir() {
|
if path.is_dir() {
|
||||||
self.copy_dir_recursive(path, &snapshot_path)?;
|
copy_dir_recursive_impl(path, &snapshot_path)?;
|
||||||
} else {
|
} else {
|
||||||
fs::copy(path, &snapshot_path).map_err(|e| util::map_io_error(path, e))?;
|
fs::copy(path, &snapshot_path).map_err(|e| util::map_io_error(path, e))?;
|
||||||
}
|
}
|
||||||
@@ -311,7 +365,7 @@ impl VfsBackend for LocalFs {
|
|||||||
}
|
}
|
||||||
|
|
||||||
if snapshot_path.is_dir() {
|
if snapshot_path.is_dir() {
|
||||||
self.copy_dir_recursive(&snapshot_path, path)?;
|
copy_dir_recursive_impl(&snapshot_path, path)?;
|
||||||
} else {
|
} else {
|
||||||
fs::copy(&snapshot_path, path).map_err(|e| util::map_io_error(&snapshot_path, e))?;
|
fs::copy(&snapshot_path, path).map_err(|e| util::map_io_error(&snapshot_path, e))?;
|
||||||
}
|
}
|
||||||
@@ -437,7 +491,7 @@ impl VfsBackend for LocalFs {
|
|||||||
for entry in fs::read_dir(&snapshots_dir)
|
for entry in fs::read_dir(&snapshots_dir)
|
||||||
.map_err(|e| util::map_io_error(&snapshots_dir, e))? {
|
.map_err(|e| util::map_io_error(&snapshots_dir, e))? {
|
||||||
let entry = entry.map_err(|e| VfsError::Io(e.to_string()))?;
|
let entry = entry.map_err(|e| VfsError::Io(e.to_string()))?;
|
||||||
let snapshot_name = entry.file_name().to_string_lossy().to_string();
|
let _snapshot_name = entry.file_name().to_string_lossy().to_string();
|
||||||
let snapshot_path = entry.path();
|
let snapshot_path = entry.path();
|
||||||
|
|
||||||
let meta_file = snapshot_path.join(".meta");
|
let meta_file = snapshot_path.join(".meta");
|
||||||
@@ -513,11 +567,10 @@ impl VfsBackend for LocalFs {
|
|||||||
let acl = self.get_acl(path)?;
|
let acl = self.get_acl(path)?;
|
||||||
|
|
||||||
for ace in &acl.aces {
|
for ace in &acl.aces {
|
||||||
if ace.principal == principal || ace.principal == "*" {
|
if (ace.principal == principal || ace.principal == "*")
|
||||||
if ace.mask.contains(&mask) {
|
&& ace.mask.contains(&mask) {
|
||||||
return Ok(ace.ace_type == VfsAceType::Allow);
|
return Ok(ace.ace_type == VfsAceType::Allow);
|
||||||
}
|
}
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
Ok(true)
|
Ok(true)
|
||||||
@@ -537,27 +590,68 @@ impl VfsBackend for LocalFs {
|
|||||||
acl.aces.remove(ace_index);
|
acl.aces.remove(ace_index);
|
||||||
self.set_acl(path, &acl)
|
self.set_acl(path, &acl)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// ===== Extended Attributes (xattr) support =====
|
||||||
|
|
||||||
|
fn get_xattr(&self, path: &Path, name: &str) -> Result<Vec<u8>, VfsError> {
|
||||||
|
#[cfg(unix)]
|
||||||
|
{
|
||||||
|
|
||||||
|
let _meta = path.metadata().map_err(|e| util::map_io_error(path, e))?;
|
||||||
|
xattr::get(path, name)
|
||||||
|
.map_err(|e| VfsError::Io(e.to_string()))?
|
||||||
|
.map(|v| v.to_vec())
|
||||||
|
.ok_or_else(|| VfsError::NotFound(format!("xattr {} not found", name)))
|
||||||
|
}
|
||||||
|
#[cfg(not(unix))]
|
||||||
|
{
|
||||||
|
Err(VfsError::Unsupported("get_xattr on non-Unix".to_string()))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn set_xattr(&self, path: &Path, name: &str, value: &[u8]) -> Result<(), VfsError> {
|
||||||
|
#[cfg(unix)]
|
||||||
|
{
|
||||||
|
xattr::set(path, name, value)
|
||||||
|
.map_err(|e| VfsError::Io(e.to_string()))
|
||||||
|
}
|
||||||
|
#[cfg(not(unix))]
|
||||||
|
{
|
||||||
|
Err(VfsError::Unsupported("set_xattr on non-Unix".to_string()))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn remove_xattr(&self, path: &Path, name: &str) -> Result<(), VfsError> {
|
||||||
|
#[cfg(unix)]
|
||||||
|
{
|
||||||
|
xattr::remove(path, name)
|
||||||
|
.map_err(|e| VfsError::Io(e.to_string()))
|
||||||
|
}
|
||||||
|
#[cfg(not(unix))]
|
||||||
|
{
|
||||||
|
Err(VfsError::Unsupported("remove_xattr on non-Unix".to_string()))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn list_xattrs(&self, path: &Path) -> Result<Vec<String>, VfsError> {
|
||||||
|
#[cfg(unix)]
|
||||||
|
{
|
||||||
|
xattr::list(path)
|
||||||
|
.map_err(|e| VfsError::Io(e.to_string()))?
|
||||||
|
.map(|s| s.to_string_lossy().into_owned())
|
||||||
|
.collect::<Vec<_>>()
|
||||||
|
.into_iter()
|
||||||
|
.map(Result::Ok)
|
||||||
|
.collect()
|
||||||
|
}
|
||||||
|
#[cfg(not(unix))]
|
||||||
|
{
|
||||||
|
Err(VfsError::Unsupported("list_xattrs on non-Unix".to_string()))
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
impl LocalFs {
|
impl LocalFs {
|
||||||
fn copy_dir_recursive(&self, src: &Path, dst: &Path) -> Result<(), VfsError> {
|
|
||||||
fs::create_dir_all(dst).map_err(|e| util::map_io_error(dst, e))?;
|
|
||||||
|
|
||||||
for entry in fs::read_dir(src).map_err(|e| util::map_io_error(src, e))? {
|
|
||||||
let entry = entry.map_err(|e| VfsError::Io(e.to_string()))?;
|
|
||||||
let src_path = entry.path();
|
|
||||||
let dst_path = dst.join(entry.file_name());
|
|
||||||
|
|
||||||
if src_path.is_dir() {
|
|
||||||
self.copy_dir_recursive(&src_path, &dst_path)?;
|
|
||||||
} else {
|
|
||||||
fs::copy(&src_path, &dst_path).map_err(|e| util::map_io_error(&src_path, e))?;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
Ok(())
|
|
||||||
}
|
|
||||||
|
|
||||||
fn calculate_size(&self, path: &Path) -> Result<u64, VfsError> {
|
fn calculate_size(&self, path: &Path) -> Result<u64, VfsError> {
|
||||||
if path.is_dir() {
|
if path.is_dir() {
|
||||||
let mut total = 0;
|
let mut total = 0;
|
||||||
@@ -772,6 +866,22 @@ impl VfsAclMeta {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Recursive directory copy helper (used by VfsBackend::copy)
|
||||||
|
fn copy_dir_recursive_impl(src: &Path, dst: &Path) -> Result<(), VfsError> {
|
||||||
|
fs::create_dir_all(dst).map_err(|e| util::map_io_error(dst, e))?;
|
||||||
|
for entry in fs::read_dir(src).map_err(|e| util::map_io_error(src, e))? {
|
||||||
|
let entry = entry.map_err(|e| util::map_io_error(src, e))?;
|
||||||
|
let src_entry = entry.path();
|
||||||
|
let dst_entry = dst.join(entry.file_name());
|
||||||
|
if src_entry.is_dir() {
|
||||||
|
copy_dir_recursive_impl(&src_entry, &dst_entry)?;
|
||||||
|
} else {
|
||||||
|
fs::copy(&src_entry, &dst_entry).map_err(|e| util::map_io_error(&src_entry, e))?;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
mod tests {
|
mod tests {
|
||||||
use super::*;
|
use super::*;
|
||||||
|
|||||||
@@ -1,13 +1,31 @@
|
|||||||
|
pub mod backup_manifest;
|
||||||
|
pub mod backup_scheduler;
|
||||||
|
pub mod cache;
|
||||||
|
pub mod checksum;
|
||||||
|
pub mod checksum_file;
|
||||||
pub mod compression;
|
pub mod compression;
|
||||||
pub mod dedup;
|
pub mod dedup;
|
||||||
|
pub mod encrypted_fs;
|
||||||
pub mod local_fs;
|
pub mod local_fs;
|
||||||
pub mod open_flags;
|
pub mod open_flags;
|
||||||
pub mod raid;
|
pub mod raid;
|
||||||
|
pub mod scrub_scheduler;
|
||||||
|
pub mod send_receive;
|
||||||
pub mod s3_fs;
|
pub mod s3_fs;
|
||||||
pub mod smb_fs;
|
pub mod smb_fs;
|
||||||
|
pub mod storage_stats;
|
||||||
#[cfg(feature = "smb-server")]
|
#[cfg(feature = "smb-server")]
|
||||||
pub mod smb_server_backend;
|
pub mod smb_server_backend;
|
||||||
pub mod util;
|
pub mod util;
|
||||||
|
pub mod virtual_fs;
|
||||||
|
#[cfg(feature = "async-vfs")]
|
||||||
|
pub mod async_fs;
|
||||||
|
#[cfg(feature = "async-vfs")]
|
||||||
|
pub mod async_s3_fs;
|
||||||
|
#[cfg(feature = "async-vfs")]
|
||||||
|
pub mod async_smb_fs;
|
||||||
|
#[cfg(feature = "nfs")]
|
||||||
|
pub mod nfs_server;
|
||||||
|
|
||||||
use std::path::{Path, PathBuf};
|
use std::path::{Path, PathBuf};
|
||||||
use std::time::SystemTime;
|
use std::time::SystemTime;
|
||||||
@@ -95,6 +113,20 @@ pub trait VfsFile: Send {
|
|||||||
fn stat(&mut self) -> Result<VfsStat, VfsError>;
|
fn stat(&mut self) -> Result<VfsStat, VfsError>;
|
||||||
fn set_len(&mut self, size: u64) -> Result<(), VfsError>;
|
fn set_len(&mut self, size: u64) -> Result<(), VfsError>;
|
||||||
|
|
||||||
|
/// Read at `offset` without changing the seek position (like pread).
|
||||||
|
/// Default implementation does seek + read.
|
||||||
|
fn read_at(&mut self, buf: &mut [u8], offset: u64) -> Result<usize, VfsError> {
|
||||||
|
self.seek(std::io::SeekFrom::Start(offset))?;
|
||||||
|
self.read(buf)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Write at `offset` without changing the seek position (like pwrite).
|
||||||
|
/// Default implementation does seek + write.
|
||||||
|
fn write_at(&mut self, buf: &[u8], offset: u64) -> Result<usize, VfsError> {
|
||||||
|
self.seek(std::io::SeekFrom::Start(offset))?;
|
||||||
|
self.write(buf)
|
||||||
|
}
|
||||||
|
|
||||||
/// Write all bytes (convenience, default loops write() until done)
|
/// Write all bytes (convenience, default loops write() until done)
|
||||||
fn write_all(&mut self, mut buf: &[u8]) -> Result<(), VfsError> {
|
fn write_all(&mut self, mut buf: &[u8]) -> Result<(), VfsError> {
|
||||||
while !buf.is_empty() {
|
while !buf.is_empty() {
|
||||||
@@ -118,6 +150,15 @@ pub trait VfsFile: Send {
|
|||||||
}
|
}
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Read all bytes (convenience, seeks to end first to get size)
|
||||||
|
fn read_all(&mut self) -> Result<Vec<u8>, VfsError> {
|
||||||
|
let size = self.seek(std::io::SeekFrom::End(0))?;
|
||||||
|
self.seek(std::io::SeekFrom::Start(0))?;
|
||||||
|
let mut buf = vec![0u8; size as usize];
|
||||||
|
self.read_exact(&mut buf)?;
|
||||||
|
Ok(buf)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/// VFS 后端 trait(所有文件系统操作)
|
/// VFS 后端 trait(所有文件系统操作)
|
||||||
@@ -148,6 +189,21 @@ pub trait VfsBackend: Send + Sync {
|
|||||||
/// 删除空目录
|
/// 删除空目录
|
||||||
fn remove_dir(&self, path: &Path) -> Result<(), VfsError>;
|
fn remove_dir(&self, path: &Path) -> Result<(), VfsError>;
|
||||||
|
|
||||||
|
/// 递归删除目录及其所有内容
|
||||||
|
fn remove_dir_all(&self, path: &Path) -> Result<(), VfsError> {
|
||||||
|
// Default: read entries and remove one by one
|
||||||
|
let entries = self.read_dir(path)?;
|
||||||
|
for entry in entries {
|
||||||
|
let child = path.join(&entry.name);
|
||||||
|
if entry.stat.is_dir {
|
||||||
|
self.remove_dir_all(&child)?;
|
||||||
|
} else {
|
||||||
|
self.remove_file(&child)?;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
self.remove_dir(path)
|
||||||
|
}
|
||||||
|
|
||||||
/// 删除文件
|
/// 删除文件
|
||||||
fn remove_file(&self, path: &Path) -> Result<(), VfsError>;
|
fn remove_file(&self, path: &Path) -> Result<(), VfsError>;
|
||||||
|
|
||||||
@@ -157,6 +213,28 @@ pub trait VfsBackend: Send + Sync {
|
|||||||
/// 设置文件属性
|
/// 设置文件属性
|
||||||
fn set_stat(&self, path: &Path, stat: &VfsStat) -> Result<(), VfsError>;
|
fn set_stat(&self, path: &Path, stat: &VfsStat) -> Result<(), VfsError>;
|
||||||
|
|
||||||
|
/// 原子性设置 atime 和 mtime(默认实现调用 stat + set_stat,有 race condition)
|
||||||
|
fn set_times(&self, path: &Path, atime: SystemTime, mtime: SystemTime) -> Result<(), VfsError> {
|
||||||
|
let mut stat = self.stat(path)?;
|
||||||
|
stat.atime = atime;
|
||||||
|
stat.mtime = mtime;
|
||||||
|
self.set_stat(path, &stat)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// 原子性设置 atime(默认实现调用 stat + set_stat,有 race condition)
|
||||||
|
fn set_atime(&self, path: &Path, atime: SystemTime) -> Result<(), VfsError> {
|
||||||
|
let mut stat = self.stat(path)?;
|
||||||
|
stat.atime = atime;
|
||||||
|
self.set_stat(path, &stat)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// 原子性设置 mtime(默认实现调用 stat + set_stat,有 race condition)
|
||||||
|
fn set_mtime(&self, path: &Path, mtime: SystemTime) -> Result<(), VfsError> {
|
||||||
|
let mut stat = self.stat(path)?;
|
||||||
|
stat.mtime = mtime;
|
||||||
|
self.set_stat(path, &stat)
|
||||||
|
}
|
||||||
|
|
||||||
/// 读取符号链接目标
|
/// 读取符号链接目标
|
||||||
fn read_link(&self, path: &Path) -> Result<PathBuf, VfsError>;
|
fn read_link(&self, path: &Path) -> Result<PathBuf, VfsError>;
|
||||||
|
|
||||||
@@ -172,6 +250,24 @@ pub trait VfsBackend: Send + Sync {
|
|||||||
/// 创建硬链接
|
/// 创建硬链接
|
||||||
fn hard_link(&self, original: &Path, link: &Path) -> Result<(), VfsError>;
|
fn hard_link(&self, original: &Path, link: &Path) -> Result<(), VfsError>;
|
||||||
|
|
||||||
|
/// 复制文件(高效实现,fallback 到 read+write)
|
||||||
|
fn copy(&self, from: &Path, to: &Path) -> Result<(), VfsError> {
|
||||||
|
let flags = open_flags::OpenFlags::new().read();
|
||||||
|
let mut src = self.open_file(from, &flags)?;
|
||||||
|
let write_flags = open_flags::OpenFlags::new().write().create().truncate().mode(0o644);
|
||||||
|
let mut dst = self.open_file(to, &write_flags)?;
|
||||||
|
let mut buf = vec![0u8; 65536];
|
||||||
|
loop {
|
||||||
|
match src.read(&mut buf) {
|
||||||
|
Ok(0) => break,
|
||||||
|
Ok(n) => dst.write_all(&buf[..n])?,
|
||||||
|
Err(e) => return Err(e),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
dst.flush()?;
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
// ===== Snapshot support (ZFS-style) =====
|
// ===== Snapshot support (ZFS-style) =====
|
||||||
|
|
||||||
/// 创建快照
|
/// 创建快照
|
||||||
@@ -264,6 +360,28 @@ pub trait VfsBackend: Send + Sync {
|
|||||||
fn remove_ace(&self, _path: &Path, _ace_index: usize) -> Result<(), VfsError> {
|
fn remove_ace(&self, _path: &Path, _ace_index: usize) -> Result<(), VfsError> {
|
||||||
Err(VfsError::Unsupported("remove_ace".to_string()))
|
Err(VfsError::Unsupported("remove_ace".to_string()))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// ===== Extended Attributes (xattr) support =====
|
||||||
|
|
||||||
|
/// 获取扩展属性
|
||||||
|
fn get_xattr(&self, _path: &Path, _name: &str) -> Result<Vec<u8>, VfsError> {
|
||||||
|
Err(VfsError::Unsupported("get_xattr".to_string()))
|
||||||
|
}
|
||||||
|
|
||||||
|
/// 设置扩展属性
|
||||||
|
fn set_xattr(&self, _path: &Path, _name: &str, _value: &[u8]) -> Result<(), VfsError> {
|
||||||
|
Err(VfsError::Unsupported("set_xattr".to_string()))
|
||||||
|
}
|
||||||
|
|
||||||
|
/// 删除扩展属性
|
||||||
|
fn remove_xattr(&self, _path: &Path, _name: &str) -> Result<(), VfsError> {
|
||||||
|
Err(VfsError::Unsupported("remove_xattr".to_string()))
|
||||||
|
}
|
||||||
|
|
||||||
|
/// 列出扩展属性名称
|
||||||
|
fn list_xattrs(&self, _path: &Path) -> Result<Vec<String>, VfsError> {
|
||||||
|
Err(VfsError::Unsupported("list_xattrs".to_string()))
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/// 快照信息
|
/// 快照信息
|
||||||
@@ -498,3 +616,109 @@ impl Default for VfsRaidConfig {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// ===== Async VfsBackend Design (Phase 1 - Framework) =====
|
||||||
|
|
||||||
|
/// Async VFS 文件 trait(用于异步操作)
|
||||||
|
///
|
||||||
|
/// 设计要点:
|
||||||
|
/// 1. 使用 `async fn` in traits (Rust 1.75+)
|
||||||
|
/// 2. 所有方法返回 `Pin<Box<dyn Future>>`
|
||||||
|
/// 3. 与 VfsFile 保持一致的接口
|
||||||
|
#[cfg(feature = "async-vfs")]
|
||||||
|
pub trait AsyncVfsFile: Send + Sync {
|
||||||
|
fn read<'a>(&'a mut self, buf: &'a mut [u8]) -> std::pin::Pin<Box<dyn std::future::Future<Output = Result<usize, VfsError>> + Send + 'a>>;
|
||||||
|
fn write<'a>(&'a mut self, buf: &'a [u8]) -> std::pin::Pin<Box<dyn std::future::Future<Output = Result<usize, VfsError>> + Send + 'a>>;
|
||||||
|
fn seek<'a>(&'a mut self, pos: std::io::SeekFrom) -> std::pin::Pin<Box<dyn std::future::Future<Output = Result<u64, VfsError>> + Send + 'a>>;
|
||||||
|
fn flush<'a>(&'a mut self) -> std::pin::Pin<Box<dyn std::future::Future<Output = Result<(), VfsError>> + Send + 'a>>;
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Async VFS 后端 trait(用于异步文件系统操作)
|
||||||
|
///
|
||||||
|
/// 设计要点:
|
||||||
|
/// 1. 使用 `async fn` in traits (Rust 1.75+)
|
||||||
|
/// 2. 所有方法返回 `Pin<Box<dyn Future>>`
|
||||||
|
/// 3. 与 VfsBackend 保持一致的接口
|
||||||
|
/// 4. 用于 WebDAV/SMB/SSH 异步处理
|
||||||
|
#[cfg(feature = "async-vfs")]
|
||||||
|
pub trait AsyncVfsBackend: Send + Sync {
|
||||||
|
fn clone_boxed(&self) -> Box<dyn AsyncVfsBackend>;
|
||||||
|
|
||||||
|
fn read_dir<'a>(&'a self, path: &'a Path) -> std::pin::Pin<Box<dyn std::future::Future<Output = Result<Vec<VfsDirEntry>, VfsError>> + Send + 'a>>;
|
||||||
|
fn open_file<'a>(&'a self, path: &'a Path, flags: &'a open_flags::OpenFlags) -> std::pin::Pin<Box<dyn std::future::Future<Output = Result<Box<dyn AsyncVfsFile>, VfsError>> + Send + 'a>>;
|
||||||
|
fn stat<'a>(&'a self, path: &'a Path) -> std::pin::Pin<Box<dyn std::future::Future<Output = Result<VfsStat, VfsError>> + Send + 'a>>;
|
||||||
|
fn create_dir<'a>(&'a self, path: &'a Path, mode: u32) -> std::pin::Pin<Box<dyn std::future::Future<Output = Result<(), VfsError>> + Send + 'a>>;
|
||||||
|
fn remove_dir<'a>(&'a self, path: &'a Path) -> std::pin::Pin<Box<dyn std::future::Future<Output = Result<(), VfsError>> + Send + 'a>>;
|
||||||
|
fn remove_file<'a>(&'a self, path: &'a Path) -> std::pin::Pin<Box<dyn std::future::Future<Output = Result<(), VfsError>> + Send + 'a>>;
|
||||||
|
fn rename<'a>(&'a self, from: &'a Path, to: &'a Path) -> std::pin::Pin<Box<dyn std::future::Future<Output = Result<(), VfsError>> + Send + 'a>>;
|
||||||
|
fn exists<'a>(&'a self, path: &'a Path) -> std::pin::Pin<Box<dyn std::future::Future<Output = bool> + Send + 'a>>;
|
||||||
|
}
|
||||||
|
|
||||||
|
// ===== Async VfsBackend Implementation Notes =====
|
||||||
|
//
|
||||||
|
// Phase 2: AsyncLocalFs (tokio::fs)
|
||||||
|
// - 使用 tokio::fs::File 替代 std::fs::File
|
||||||
|
// - 使用 tokio::fs::read_dir 替代 std::fs::read_dir
|
||||||
|
// - 使用 tokio::fs::create_dir 替代 std::fs::create_dir
|
||||||
|
//
|
||||||
|
// Phase 3: AsyncS3Vfs (ureq is blocking, need async client)
|
||||||
|
// - 使用 async-s3 或 rusoto
|
||||||
|
// - 或者使用 spawn_blocking 包装现有 ureq 调用
|
||||||
|
//
|
||||||
|
// Phase 4: AsyncSmbVfs
|
||||||
|
// - smb-server crate 使用 async internally
|
||||||
|
// - 需要 async wrapper
|
||||||
|
//
|
||||||
|
// Phase 5: WebDAV Integration
|
||||||
|
// - VfsDavFs 改为 AsyncVfsBackend
|
||||||
|
// - dav-server 已经是 async
|
||||||
|
// - 直接使用 async 方法
|
||||||
|
//
|
||||||
|
// 预估工作量:
|
||||||
|
// - AsyncVfsBackend trait: 1 hour
|
||||||
|
// - AsyncLocalFs: 3 hours
|
||||||
|
// - AsyncS3Vfs: 2 hours
|
||||||
|
// - AsyncSmbVfs: 2 hours
|
||||||
|
// - WebDAV integration: 3 hours
|
||||||
|
// - Tests: 2 hours
|
||||||
|
// Total: ~13 hours (multi-day project)
|
||||||
|
//
|
||||||
|
// ===== Phase 5 WebDAV Async Integration Design =====
|
||||||
|
//
|
||||||
|
// 现状分析:
|
||||||
|
// 1. dav-server DavFileSystem trait 方法返回 Pin<Box<dyn Future>>
|
||||||
|
// 2. 当前 VfsDavFs::open() 返回 Box::pin(ready(...))
|
||||||
|
// 3. 这是 "同步包装为 Future",不是真正的 async
|
||||||
|
// 4. DavFileSystem trait API 已变化(2026-06-21 session发现)
|
||||||
|
// - read_dir(path, ReadDirMeta) 而非 read_dir(path, depth)
|
||||||
|
// - have_props(path) 返回 Pin<Box<dyn Future>>
|
||||||
|
// - get_props/get_prop/patch_props 新方法
|
||||||
|
// - get_quota/set_accessed/set_modified 新方法
|
||||||
|
// - DavFile 需要 write_buf 方法
|
||||||
|
// - DavMetaData modified()/is_dir() 返回 Pin<Box<dyn Future>>
|
||||||
|
// - DavDirEntry name()/is_dir()/metadata() 返回 Pin<Box<dyn Future>>
|
||||||
|
//
|
||||||
|
// Phase 5 阻塞因素:
|
||||||
|
// 1. dav-server API 签名与预期不匹配(20+ 编译错误)
|
||||||
|
// 2. 需要 match 完整 DavFileSystem trait 所有方法(~30个)
|
||||||
|
// 3. AsyncVfsFile trait 方法签名需调整
|
||||||
|
// 4. 估算工作量:~8小时(而非原估计3小时)
|
||||||
|
//
|
||||||
|
// 实现方案选择:
|
||||||
|
// 方案A:spawn_blocking wrapper(推荐)
|
||||||
|
// - 创建 AsyncVfsDavFs 包装现有 VfsDavFs
|
||||||
|
// - 所有 DavFileSystem 方法使用 spawn_blocking 调用同步版本
|
||||||
|
// - 工作量:~2小时
|
||||||
|
// - 优点:快速实现,兼容现有 API
|
||||||
|
// - 缺点:仍为伪异步(阻塞线程池)
|
||||||
|
//
|
||||||
|
// 方案B:完整重写 DavFileSystem(长期)
|
||||||
|
// - 完全匹配 dav-server API
|
||||||
|
// - 使用真正的 AsyncVfsBackend async 方法
|
||||||
|
// - 工作量:~8小时
|
||||||
|
// - 优点:真正的异步
|
||||||
|
// - 缺点:需要完全理解 dav-server API
|
||||||
|
//
|
||||||
|
// 推荐方案A(spawn_blocking wrapper)
|
||||||
|
//
|
||||||
|
// 预估工作量:Phase 5 方案A ~2小时,方案B ~8小时
|
||||||
|
|||||||
@@ -0,0 +1,444 @@
|
|||||||
|
use crate::vfs::open_flags::OpenFlags;
|
||||||
|
use crate::vfs::{VfsBackend, VfsError, VfsStat};
|
||||||
|
use std::collections::HashMap;
|
||||||
|
use std::path::{Path, PathBuf};
|
||||||
|
use std::sync::{Arc, Mutex};
|
||||||
|
use std::time::{Duration, SystemTime};
|
||||||
|
|
||||||
|
use nfsserve::nfs;
|
||||||
|
use nfsserve::tcp::NFSTcp;
|
||||||
|
use nfsserve::vfs::{NFSFileSystem, ReadDirResult, VFSCapabilities};
|
||||||
|
use nfsserve::nfs::{fattr3, fileid3, filename3, nfsstat3, ftype3, sattr3, set_mode3, set_size3,
|
||||||
|
set_atime, set_mtime, nfstime3, specdata3, post_op_attr, nfspath3, fsinfo3};
|
||||||
|
|
||||||
|
/// Maps filesystem paths to stable 64-bit file IDs (NFS filehandle).
|
||||||
|
struct FileIdManager {
|
||||||
|
path_to_id: Mutex<HashMap<String, u64>>,
|
||||||
|
id_to_path: Mutex<HashMap<u64, String>>,
|
||||||
|
next_id: Mutex<u64>,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl FileIdManager {
|
||||||
|
fn new() -> Self {
|
||||||
|
Self {
|
||||||
|
path_to_id: Mutex::new(HashMap::new()),
|
||||||
|
id_to_path: Mutex::new(HashMap::new()),
|
||||||
|
next_id: Mutex::new(1), // 0 is reserved
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn get_or_create_id(&self, path: &str) -> u64 {
|
||||||
|
if let Some(id) = self.path_to_id.lock().unwrap().get(path) {
|
||||||
|
return *id;
|
||||||
|
}
|
||||||
|
let mut next = self.next_id.lock().unwrap();
|
||||||
|
let id = *next;
|
||||||
|
*next += 1;
|
||||||
|
self.path_to_id.lock().unwrap().insert(path.to_string(), id);
|
||||||
|
self.id_to_path.lock().unwrap().insert(id, path.to_string());
|
||||||
|
id
|
||||||
|
}
|
||||||
|
|
||||||
|
fn get_path(&self, id: u64) -> Option<String> {
|
||||||
|
self.id_to_path.lock().unwrap().get(&id).cloned()
|
||||||
|
}
|
||||||
|
|
||||||
|
fn get_id(&self, path: &str) -> Option<u64> {
|
||||||
|
self.path_to_id.lock().unwrap().get(path).copied()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// NFS server backed by our VfsBackend trait.
|
||||||
|
pub struct NfsVfsServer {
|
||||||
|
vfs: Arc<dyn VfsBackend>,
|
||||||
|
root: PathBuf,
|
||||||
|
port: u16,
|
||||||
|
fid_mgr: Arc<FileIdManager>,
|
||||||
|
export_name: String,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl NfsVfsServer {
|
||||||
|
pub fn new(vfs: Arc<dyn VfsBackend>, root: PathBuf) -> Self {
|
||||||
|
let fid_mgr = Arc::new(FileIdManager::new());
|
||||||
|
Self {
|
||||||
|
vfs,
|
||||||
|
root,
|
||||||
|
port: 2049,
|
||||||
|
fid_mgr,
|
||||||
|
export_name: "export".to_string(),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn with_port(mut self, port: u16) -> Self {
|
||||||
|
self.port = port;
|
||||||
|
self
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn with_export_name(mut self, name: &str) -> Self {
|
||||||
|
self.export_name = name.to_string();
|
||||||
|
self
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn root_dir(&self) -> u64 {
|
||||||
|
let root_s = self.root.to_string_lossy().to_string();
|
||||||
|
self.fid_mgr.get_or_create_id(&root_s)
|
||||||
|
}
|
||||||
|
|
||||||
|
pub async fn start(&self, port: u16) -> Result<(), VfsError> {
|
||||||
|
#[cfg(feature = "nfs")]
|
||||||
|
{
|
||||||
|
println!("NFS server starting on port {}", port);
|
||||||
|
println!("Export directory: {}", self.root.display());
|
||||||
|
println!("Export name: {}", self.export_name);
|
||||||
|
|
||||||
|
let ipstr = format!("0.0.0.0:{}", port);
|
||||||
|
let fs = NfsVfsFileSystem::new(
|
||||||
|
self.vfs.clone(),
|
||||||
|
self.root.clone(),
|
||||||
|
self.fid_mgr.clone(),
|
||||||
|
);
|
||||||
|
let listener = nfsserve::tcp::NFSTcpListener::bind(&ipstr, fs)
|
||||||
|
.await
|
||||||
|
.map_err(|e| VfsError::Io(format!("NFS bind failed: {}", e)))?;
|
||||||
|
|
||||||
|
// NFSTcpListener.with_export_name needs &mut self
|
||||||
|
// We'll skip this for now since default export name is /
|
||||||
|
|
||||||
|
println!("NFS server listening on port {}", listener.get_listen_port());
|
||||||
|
listener
|
||||||
|
.handle_forever()
|
||||||
|
.await
|
||||||
|
.map_err(|e| VfsError::Io(format!("NFS server error: {}", e)))
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(not(feature = "nfs"))]
|
||||||
|
{
|
||||||
|
let _ = port;
|
||||||
|
Err(VfsError::Unsupported(
|
||||||
|
"NFS server requires 'nfs' feature".to_string(),
|
||||||
|
))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn stat_to_fattr3(stat: &VfsStat, fileid: u64) -> fattr3 {
|
||||||
|
let sys_to_nfs = |t: SystemTime| -> nfstime3 {
|
||||||
|
let d = t.duration_since(SystemTime::UNIX_EPOCH).unwrap_or_default();
|
||||||
|
nfstime3 {
|
||||||
|
seconds: d.as_secs() as u32,
|
||||||
|
nseconds: d.subsec_nanos(),
|
||||||
|
}
|
||||||
|
};
|
||||||
|
fattr3 {
|
||||||
|
ftype: if stat.is_dir { ftype3::NF3DIR } else { ftype3::NF3REG },
|
||||||
|
mode: stat.mode,
|
||||||
|
nlink: if stat.is_dir { 2 } else { 1 },
|
||||||
|
uid: stat.uid,
|
||||||
|
gid: stat.gid,
|
||||||
|
size: stat.size,
|
||||||
|
used: stat.size,
|
||||||
|
rdev: specdata3 { specdata1: 0, specdata2: 0 },
|
||||||
|
fsid: 0,
|
||||||
|
fileid,
|
||||||
|
atime: sys_to_nfs(stat.atime),
|
||||||
|
mtime: sys_to_nfs(stat.mtime),
|
||||||
|
ctime: sys_to_nfs(stat.atime),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// NFSFileSystem implementation backed by VfsBackend.
|
||||||
|
struct NfsVfsFileSystem {
|
||||||
|
vfs: Arc<dyn VfsBackend>,
|
||||||
|
root: PathBuf,
|
||||||
|
fid_mgr: Arc<FileIdManager>,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl NfsVfsFileSystem {
|
||||||
|
fn new(vfs: Arc<dyn VfsBackend>, root: PathBuf, fid_mgr: Arc<FileIdManager>) -> Self {
|
||||||
|
Self { vfs, root, fid_mgr }
|
||||||
|
}
|
||||||
|
|
||||||
|
fn resolve_parent(&self, dirid: u64, filename: &[u8]) -> Result<PathBuf, nfsstat3> {
|
||||||
|
let dir_path = self
|
||||||
|
.fid_mgr
|
||||||
|
.get_path(dirid)
|
||||||
|
.ok_or(nfsstat3::NFS3ERR_NOENT)?;
|
||||||
|
let fname = String::from_utf8_lossy(filename);
|
||||||
|
Ok(PathBuf::from(dir_path).join(fname.as_ref()))
|
||||||
|
}
|
||||||
|
|
||||||
|
fn sattr3_to_vfs(&self, attr: &sattr3) -> Option<(Option<u32>, Option<u64>, Option<SystemTime>, Option<SystemTime>)> {
|
||||||
|
let mode = match &attr.mode {
|
||||||
|
set_mode3::mode(val) => Some(*val),
|
||||||
|
_ => None,
|
||||||
|
};
|
||||||
|
let size = match &attr.size {
|
||||||
|
set_size3::size(val) => Some(*val),
|
||||||
|
_ => None,
|
||||||
|
};
|
||||||
|
let atime = match attr.atime {
|
||||||
|
set_atime::SET_TO_SERVER_TIME => Some(SystemTime::now()),
|
||||||
|
set_atime::SET_TO_CLIENT_TIME(t) => Some(
|
||||||
|
SystemTime::UNIX_EPOCH + Duration::new(t.seconds as u64, t.nseconds),
|
||||||
|
),
|
||||||
|
_ => None,
|
||||||
|
};
|
||||||
|
let mtime = match attr.mtime {
|
||||||
|
set_mtime::SET_TO_SERVER_TIME => Some(SystemTime::now()),
|
||||||
|
set_mtime::SET_TO_CLIENT_TIME(t) => Some(
|
||||||
|
SystemTime::UNIX_EPOCH + Duration::new(t.seconds as u64, t.nseconds),
|
||||||
|
),
|
||||||
|
_ => None,
|
||||||
|
};
|
||||||
|
Some((mode, size, atime, mtime))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[async_trait::async_trait]
|
||||||
|
impl NFSFileSystem for NfsVfsFileSystem {
|
||||||
|
fn capabilities(&self) -> VFSCapabilities {
|
||||||
|
VFSCapabilities::ReadWrite
|
||||||
|
}
|
||||||
|
|
||||||
|
fn root_dir(&self) -> u64 {
|
||||||
|
let root_s = self.root.to_string_lossy().to_string();
|
||||||
|
self.fid_mgr.get_or_create_id(&root_s)
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn lookup(&self, dirid: u64, filename: &filename3) -> Result<u64, nfsstat3> {
|
||||||
|
let full = self.resolve_parent(dirid, filename.as_ref())?;
|
||||||
|
if !self.vfs.exists(&full) {
|
||||||
|
return Err(nfsstat3::NFS3ERR_NOENT);
|
||||||
|
}
|
||||||
|
let s = full.to_string_lossy().to_string();
|
||||||
|
Ok(self.fid_mgr.get_or_create_id(&s))
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn getattr(&self, id: u64) -> Result<fattr3, nfsstat3> {
|
||||||
|
let path = self
|
||||||
|
.fid_mgr
|
||||||
|
.get_path(id)
|
||||||
|
.ok_or(nfsstat3::NFS3ERR_NOENT)?;
|
||||||
|
let stat = self
|
||||||
|
.vfs
|
||||||
|
.stat(Path::new(&path))
|
||||||
|
.map_err(|_| nfsstat3::NFS3ERR_IO)?;
|
||||||
|
Ok(stat_to_fattr3(&stat, id))
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn setattr(&self, id: u64, setattr: sattr3) -> Result<fattr3, nfsstat3> {
|
||||||
|
let path = self
|
||||||
|
.fid_mgr
|
||||||
|
.get_path(id)
|
||||||
|
.ok_or(nfsstat3::NFS3ERR_NOENT)?;
|
||||||
|
|
||||||
|
if let Some((_mode, size, _atime, _mtime)) = self.sattr3_to_vfs(&setattr) {
|
||||||
|
if let Some(s) = size {
|
||||||
|
let mut vfs_stat = VfsStat::new();
|
||||||
|
vfs_stat.size = s;
|
||||||
|
self.vfs
|
||||||
|
.set_stat(Path::new(&path), &vfs_stat)
|
||||||
|
.map_err(|_| nfsstat3::NFS3ERR_IO)?;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
let stat = self
|
||||||
|
.vfs
|
||||||
|
.stat(Path::new(&path))
|
||||||
|
.map_err(|_| nfsstat3::NFS3ERR_IO)?;
|
||||||
|
Ok(stat_to_fattr3(&stat, id))
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn read(&self, id: u64, offset: u64, count: u32) -> Result<(Vec<u8>, bool), nfsstat3> {
|
||||||
|
let path = self
|
||||||
|
.fid_mgr
|
||||||
|
.get_path(id)
|
||||||
|
.ok_or(nfsstat3::NFS3ERR_NOENT)?;
|
||||||
|
let mut file = self
|
||||||
|
.vfs
|
||||||
|
.open_file(Path::new(&path), &OpenFlags::new().read())
|
||||||
|
.map_err(|_| nfsstat3::NFS3ERR_IO)?;
|
||||||
|
|
||||||
|
use std::io::{Read, Seek};
|
||||||
|
file.seek(std::io::SeekFrom::Start(offset))
|
||||||
|
.map_err(|_| nfsstat3::NFS3ERR_IO)?;
|
||||||
|
|
||||||
|
let mut buf = vec![0u8; count as usize];
|
||||||
|
let n = file
|
||||||
|
.read(&mut buf)
|
||||||
|
.map_err(|_| nfsstat3::NFS3ERR_IO)?;
|
||||||
|
buf.truncate(n);
|
||||||
|
|
||||||
|
let eof = n < count as usize;
|
||||||
|
Ok((buf, eof))
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn write(&self, id: u64, offset: u64, data: &[u8]) -> Result<fattr3, nfsstat3> {
|
||||||
|
let path = self
|
||||||
|
.fid_mgr
|
||||||
|
.get_path(id)
|
||||||
|
.ok_or(nfsstat3::NFS3ERR_NOENT)?;
|
||||||
|
let mut file = self
|
||||||
|
.vfs
|
||||||
|
.open_file(Path::new(&path), &OpenFlags::new().write())
|
||||||
|
.map_err(|_| nfsstat3::NFS3ERR_IO)?;
|
||||||
|
|
||||||
|
use std::io::{Seek, Write};
|
||||||
|
file.seek(std::io::SeekFrom::Start(offset))
|
||||||
|
.map_err(|_| nfsstat3::NFS3ERR_IO)?;
|
||||||
|
file.write_all(data)
|
||||||
|
.map_err(|_| nfsstat3::NFS3ERR_IO)?;
|
||||||
|
|
||||||
|
let stat = self
|
||||||
|
.vfs
|
||||||
|
.stat(Path::new(&path))
|
||||||
|
.map_err(|_| nfsstat3::NFS3ERR_IO)?;
|
||||||
|
Ok(stat_to_fattr3(&stat, id))
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn create(&self, dirid: u64, filename: &filename3, _attr: sattr3) -> Result<(u64, fattr3), nfsstat3> {
|
||||||
|
let full = self.resolve_parent(dirid, filename.as_ref())?;
|
||||||
|
let parent = full.parent().unwrap_or(&self.root);
|
||||||
|
|
||||||
|
let _ = self.vfs.create_dir(parent, 0o755); // ensure parent exists
|
||||||
|
let file = self
|
||||||
|
.vfs
|
||||||
|
.open_file(&full, &OpenFlags::new().write())
|
||||||
|
.map_err(|_| nfsstat3::NFS3ERR_IO)?;
|
||||||
|
drop(file);
|
||||||
|
|
||||||
|
let s = full.to_string_lossy().to_string();
|
||||||
|
let id = self.fid_mgr.get_or_create_id(&s);
|
||||||
|
let stat = self
|
||||||
|
.vfs
|
||||||
|
.stat(&full)
|
||||||
|
.map_err(|_| nfsstat3::NFS3ERR_IO)?;
|
||||||
|
Ok((id, stat_to_fattr3(&stat, id)))
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn create_exclusive(&self, dirid: u64, filename: &filename3) -> Result<u64, nfsstat3> {
|
||||||
|
let full = self.resolve_parent(dirid, filename.as_ref())?;
|
||||||
|
if self.vfs.exists(&full) {
|
||||||
|
return Err(nfsstat3::NFS3ERR_EXIST);
|
||||||
|
}
|
||||||
|
let file = self
|
||||||
|
.vfs
|
||||||
|
.open_file(&full, &OpenFlags::new().write())
|
||||||
|
.map_err(|_| nfsstat3::NFS3ERR_IO)?;
|
||||||
|
drop(file);
|
||||||
|
let s = full.to_string_lossy().to_string();
|
||||||
|
Ok(self.fid_mgr.get_or_create_id(&s))
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn mkdir(&self, dirid: u64, dirname: &filename3) -> Result<(u64, fattr3), nfsstat3> {
|
||||||
|
let full = self.resolve_parent(dirid, dirname.as_ref())?;
|
||||||
|
self.vfs
|
||||||
|
.create_dir_all(&full, 0o755)
|
||||||
|
.map_err(|_| nfsstat3::NFS3ERR_IO)?;
|
||||||
|
let s = full.to_string_lossy().to_string();
|
||||||
|
let id = self.fid_mgr.get_or_create_id(&s);
|
||||||
|
let stat = self
|
||||||
|
.vfs
|
||||||
|
.stat(&full)
|
||||||
|
.map_err(|_| nfsstat3::NFS3ERR_IO)?;
|
||||||
|
Ok((id, stat_to_fattr3(&stat, id)))
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn remove(&self, dirid: u64, filename: &filename3) -> Result<(), nfsstat3> {
|
||||||
|
let full = self.resolve_parent(dirid, filename.as_ref())?;
|
||||||
|
let is_dir = self.vfs.stat(&full).map(|s| s.is_dir).unwrap_or(false);
|
||||||
|
if is_dir {
|
||||||
|
self.vfs
|
||||||
|
.remove_dir(&full)
|
||||||
|
.map_err(|_| nfsstat3::NFS3ERR_IO)
|
||||||
|
} else {
|
||||||
|
self.vfs
|
||||||
|
.remove_file(&full)
|
||||||
|
.map_err(|_| nfsstat3::NFS3ERR_IO)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn rename(&self, from_dirid: u64, from_filename: &filename3, to_dirid: u64, to_filename: &filename3) -> Result<(), nfsstat3> {
|
||||||
|
let from = self.resolve_parent(from_dirid, from_filename.as_ref())?;
|
||||||
|
let to = self.resolve_parent(to_dirid, to_filename.as_ref())?;
|
||||||
|
self.vfs
|
||||||
|
.rename(&from, &to)
|
||||||
|
.map_err(|_| nfsstat3::NFS3ERR_IO)
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn readdir(&self, dirid: u64, start_after: u64, max_entries: usize) -> Result<ReadDirResult, nfsstat3> {
|
||||||
|
let dir_path = self
|
||||||
|
.fid_mgr
|
||||||
|
.get_path(dirid)
|
||||||
|
.ok_or(nfsstat3::NFS3ERR_NOENT)?;
|
||||||
|
let entries = self
|
||||||
|
.vfs
|
||||||
|
.read_dir(Path::new(&dir_path))
|
||||||
|
.map_err(|_| nfsstat3::NFS3ERR_IO)?;
|
||||||
|
|
||||||
|
let mut result = ReadDirResult {
|
||||||
|
entries: Vec::new(),
|
||||||
|
end: false,
|
||||||
|
};
|
||||||
|
|
||||||
|
for entry in entries {
|
||||||
|
let child_path = Path::new(&dir_path).join(&entry.name);
|
||||||
|
let child_s = child_path.to_string_lossy().to_string();
|
||||||
|
let child_id = self.fid_mgr.get_or_create_id(&child_s);
|
||||||
|
|
||||||
|
if child_id <= start_after {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
let stat = match self.vfs.stat(&child_path) {
|
||||||
|
Ok(s) => s,
|
||||||
|
Err(_) => continue,
|
||||||
|
};
|
||||||
|
|
||||||
|
result.entries.push(nfsserve::vfs::DirEntry {
|
||||||
|
fileid: child_id,
|
||||||
|
name: entry.name.as_bytes().to_vec().into(),
|
||||||
|
attr: stat_to_fattr3(&stat, child_id),
|
||||||
|
});
|
||||||
|
|
||||||
|
if result.entries.len() >= max_entries {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
result.end = true;
|
||||||
|
Ok(result)
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn symlink(&self, _dirid: u64, _linkname: &filename3, _symlink: &nfspath3, _attr: &sattr3) -> Result<(u64, fattr3), nfsstat3> {
|
||||||
|
Err(nfsstat3::NFS3ERR_ROFS)
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn readlink(&self, _id: u64) -> Result<nfspath3, nfsstat3> {
|
||||||
|
Err(nfsstat3::NFS3ERR_NOTSUPP)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub struct NfsConfig {
|
||||||
|
pub port: u16,
|
||||||
|
pub root: PathBuf,
|
||||||
|
pub vfs: Arc<dyn VfsBackend>,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Default for NfsConfig {
|
||||||
|
fn default() -> Self {
|
||||||
|
Self {
|
||||||
|
port: 2049,
|
||||||
|
root: PathBuf::from("/"),
|
||||||
|
vfs: Arc::new(crate::vfs::local_fs::LocalFs::new()),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl NfsConfig {
|
||||||
|
pub fn build(&self) -> NfsVfsServer {
|
||||||
|
NfsVfsServer::new(self.vfs.clone(), self.root.clone())
|
||||||
|
.with_port(self.port)
|
||||||
|
.with_export_name("export")
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -1,6 +1,5 @@
|
|||||||
use super::{VfsBackend, VfsDirEntry, VfsError, VfsFile, VfsQuota, VfsQuotaUsage, VfsStat, VfsRaidConfig, VfsRaidLevel};
|
use super::{VfsBackend, VfsDirEntry, VfsError, VfsFile, VfsStat, VfsRaidConfig, VfsRaidLevel};
|
||||||
use std::path::{Path, PathBuf};
|
use std::path::{Path, PathBuf};
|
||||||
use std::io::{Read, Seek, SeekFrom, Write};
|
|
||||||
|
|
||||||
pub struct VfsRaidBackend {
|
pub struct VfsRaidBackend {
|
||||||
config: VfsRaidConfig,
|
config: VfsRaidConfig,
|
||||||
@@ -48,6 +47,14 @@ impl VfsRaidBackend {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
pub fn level(&self) -> VfsRaidLevel {
|
||||||
|
self.config.level
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn backends(&self) -> &[Box<dyn VfsBackend>] {
|
||||||
|
&self.backends
|
||||||
|
}
|
||||||
|
|
||||||
fn calculate_parity_p(data: &[u8]) -> Vec<u8> {
|
fn calculate_parity_p(data: &[u8]) -> Vec<u8> {
|
||||||
data.iter().fold(vec![0u8; data.len()], |mut p, byte| {
|
data.iter().fold(vec![0u8; data.len()], |mut p, byte| {
|
||||||
for i in 0..p.len() {
|
for i in 0..p.len() {
|
||||||
@@ -115,12 +122,185 @@ impl VfsRaidBackend {
|
|||||||
return Err(VfsError::Io("Cannot rebuild single disk RAID".to_string()));
|
return Err(VfsError::Io("Cannot rebuild single disk RAID".to_string()));
|
||||||
}
|
}
|
||||||
|
|
||||||
for backend in &self.backends {
|
if failed_disk_index >= self.backends.len() {
|
||||||
backend.create_dir_all(&PathBuf::from("/"), 0o755)?;
|
return Err(VfsError::Io(format!("Invalid disk index {}", failed_disk_index)));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
let source_index = if self.backends.len() > 1 {
|
||||||
|
// Use backends[0] as source if failed_disk_index != 0, else use backends[1]
|
||||||
|
if failed_disk_index != 0 { 0 } else { 1 }
|
||||||
|
} else {
|
||||||
|
return Err(VfsError::Io("Not enough disks for rebuild".to_string()));
|
||||||
|
};
|
||||||
|
|
||||||
|
let target_backend = &self.backends[failed_disk_index];
|
||||||
|
let source_backend = &self.backends[source_index];
|
||||||
|
|
||||||
|
target_backend.create_dir_all(&PathBuf::from("/"), 0o755)?;
|
||||||
|
|
||||||
|
self.rebuild_recursive(source_backend, target_backend, &PathBuf::from("/"))?;
|
||||||
|
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn rebuild_recursive(
|
||||||
|
&self,
|
||||||
|
source: &Box<dyn VfsBackend>,
|
||||||
|
target: &Box<dyn VfsBackend>,
|
||||||
|
path: &Path,
|
||||||
|
) -> Result<(), VfsError> {
|
||||||
|
let entries = source.read_dir(path)?;
|
||||||
|
for entry in &entries {
|
||||||
|
let entry_path = path.join(&entry.name);
|
||||||
|
if entry.stat.is_dir {
|
||||||
|
target.create_dir_all(&entry_path, entry.stat.mode)?;
|
||||||
|
self.rebuild_recursive(source, target, &entry_path)?;
|
||||||
|
} else {
|
||||||
|
let mut src_file = source.open_file(&entry_path, &super::open_flags::OpenFlags::new().read())?;
|
||||||
|
let data = src_file.read_all()?;
|
||||||
|
let mut dst_file = target.open_file(
|
||||||
|
&entry_path,
|
||||||
|
&super::open_flags::OpenFlags::new().write().create().truncate(),
|
||||||
|
)?;
|
||||||
|
dst_file.write_all(&data)?;
|
||||||
|
if let Ok(stat) = source.stat(&entry_path) {
|
||||||
|
target.set_stat(&entry_path, &stat)?;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Repair a corrupted block from parity
|
||||||
|
///
|
||||||
|
/// This reads the block from surviving disks and reconstructs using parity.
|
||||||
|
/// Works for RAID-Z1/2/3 (requires parity disks).
|
||||||
|
pub fn repair_block_from_parity(
|
||||||
|
&self,
|
||||||
|
path: &Path,
|
||||||
|
offset: u64,
|
||||||
|
corrupted_disk_index: usize,
|
||||||
|
) -> Result<Vec<u8>, VfsError> {
|
||||||
|
if self.config.level == VfsRaidLevel::Single {
|
||||||
|
return Err(VfsError::Io("Cannot repair from single disk RAID".to_string()));
|
||||||
|
}
|
||||||
|
|
||||||
|
if corrupted_disk_index >= self.backends.len() {
|
||||||
|
return Err(VfsError::Io(format!("Invalid disk index {}", corrupted_disk_index)));
|
||||||
|
}
|
||||||
|
|
||||||
|
let block_size = self.stripe_size;
|
||||||
|
let mut data_blocks: Vec<Option<Vec<u8>>> = vec![None; self.backends.len()];
|
||||||
|
let mut parity_blocks: Vec<Vec<u8>> = vec![];
|
||||||
|
|
||||||
|
for (i, backend) in self.backends.iter().enumerate() {
|
||||||
|
if i == corrupted_disk_index {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
let mut file = backend.open_file(path, &super::open_flags::OpenFlags::new().read())?;
|
||||||
|
let mut buffer = vec![0u8; block_size];
|
||||||
|
let bytes_read = file.read_at(&mut buffer, offset)?;
|
||||||
|
|
||||||
|
if bytes_read > 0 {
|
||||||
|
if i < self.data_disks() {
|
||||||
|
data_blocks[i] = Some(buffer[..bytes_read].to_vec());
|
||||||
|
} else {
|
||||||
|
parity_blocks.push(buffer[..bytes_read].to_vec());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
match self.config.level {
|
||||||
|
VfsRaidLevel::RaidZ1 => {
|
||||||
|
if parity_blocks.is_empty() {
|
||||||
|
return Err(VfsError::Io("Not enough parity for RaidZ1 repair".to_string()));
|
||||||
|
}
|
||||||
|
let reconstructed = Self::reconstruct_from_p(
|
||||||
|
&data_blocks,
|
||||||
|
&parity_blocks[0],
|
||||||
|
corrupted_disk_index,
|
||||||
|
self.data_disks(),
|
||||||
|
);
|
||||||
|
Ok(reconstructed)
|
||||||
|
}
|
||||||
|
VfsRaidLevel::RaidZ2 => {
|
||||||
|
if parity_blocks.len() < 2 {
|
||||||
|
return Err(VfsError::Io("Not enough parity for RaidZ2 repair".to_string()));
|
||||||
|
}
|
||||||
|
let reconstructed = Self::reconstruct_from_pq(
|
||||||
|
&data_blocks,
|
||||||
|
&parity_blocks[0],
|
||||||
|
&parity_blocks[1],
|
||||||
|
corrupted_disk_index,
|
||||||
|
self.data_disks(),
|
||||||
|
);
|
||||||
|
Ok(reconstructed)
|
||||||
|
}
|
||||||
|
VfsRaidLevel::RaidZ3 => {
|
||||||
|
if parity_blocks.len() < 3 {
|
||||||
|
return Err(VfsError::Io("Not enough parity for RaidZ3 repair".to_string()));
|
||||||
|
}
|
||||||
|
let reconstructed = Self::reconstruct_from_pqr(
|
||||||
|
&data_blocks,
|
||||||
|
&parity_blocks[0],
|
||||||
|
&parity_blocks[1],
|
||||||
|
&parity_blocks[2],
|
||||||
|
corrupted_disk_index,
|
||||||
|
self.data_disks(),
|
||||||
|
);
|
||||||
|
Ok(reconstructed)
|
||||||
|
}
|
||||||
|
_ => Err(VfsError::Io("RAID level does not support block repair".to_string())),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn reconstruct_from_p(
|
||||||
|
data_blocks: &[Option<Vec<u8>>],
|
||||||
|
p_block: &[u8],
|
||||||
|
missing_index: usize,
|
||||||
|
data_disk_count: usize,
|
||||||
|
) -> Vec<u8> {
|
||||||
|
let size = p_block.len();
|
||||||
|
let mut reconstructed = vec![0u8; size];
|
||||||
|
|
||||||
|
for i in 0..data_disk_count {
|
||||||
|
if i != missing_index {
|
||||||
|
if let Some(data) = &data_blocks[i] {
|
||||||
|
for j in 0..size {
|
||||||
|
reconstructed[j] ^= data[j];
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
for j in 0..size {
|
||||||
|
reconstructed[j] ^= p_block[j];
|
||||||
|
}
|
||||||
|
|
||||||
|
reconstructed
|
||||||
|
}
|
||||||
|
|
||||||
|
fn reconstruct_from_pq(
|
||||||
|
data_blocks: &[Option<Vec<u8>>],
|
||||||
|
p_block: &[u8],
|
||||||
|
_q_block: &[u8],
|
||||||
|
missing_index: usize,
|
||||||
|
data_disk_count: usize,
|
||||||
|
) -> Vec<u8> {
|
||||||
|
Self::reconstruct_from_p(data_blocks, p_block, missing_index, data_disk_count)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn reconstruct_from_pqr(
|
||||||
|
data_blocks: &[Option<Vec<u8>>],
|
||||||
|
p_block: &[u8],
|
||||||
|
_q_block: &[u8],
|
||||||
|
_r_block: &[u8],
|
||||||
|
missing_index: usize,
|
||||||
|
data_disk_count: usize,
|
||||||
|
) -> Vec<u8> {
|
||||||
|
Self::reconstruct_from_p(data_blocks, p_block, missing_index, data_disk_count)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
impl VfsBackend for VfsRaidBackend {
|
impl VfsBackend for VfsRaidBackend {
|
||||||
|
|||||||
@@ -8,6 +8,7 @@ use std::time::{Duration, SystemTime};
|
|||||||
use url::Url;
|
use url::Url;
|
||||||
|
|
||||||
/// S3-compatible 文件系統後端
|
/// S3-compatible 文件系統後端
|
||||||
|
#[derive(Clone)]
|
||||||
pub struct S3Vfs {
|
pub struct S3Vfs {
|
||||||
bucket: Bucket,
|
bucket: Bucket,
|
||||||
credentials: Credentials,
|
credentials: Credentials,
|
||||||
@@ -417,6 +418,28 @@ impl VfsBackend for S3Vfs {
|
|||||||
let to_key = Self::path_to_key(link);
|
let to_key = Self::path_to_key(link);
|
||||||
self.copy_object(&from_key, &to_key)
|
self.copy_object(&from_key, &to_key)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn copy(&self, from: &Path, to: &Path) -> Result<(), VfsError> {
|
||||||
|
let from_key = Self::path_to_key(from);
|
||||||
|
let to_key = Self::path_to_key(to);
|
||||||
|
|
||||||
|
// Check if source is a directory marker
|
||||||
|
if from.ends_with("/") || from_key.ends_with('/') {
|
||||||
|
// Directory copy: create destination directory marker
|
||||||
|
let action = actions::PutObject::new(&self.bucket, Some(&self.credentials), &to_key);
|
||||||
|
let url = action.sign(Duration::from_secs(3600));
|
||||||
|
ureq::put(url.as_str())
|
||||||
|
.send_bytes(&[])
|
||||||
|
.map_err(|e| VfsError::Io(format!("S3 PutObject failed: {}", e)))?;
|
||||||
|
return Ok(());
|
||||||
|
}
|
||||||
|
|
||||||
|
// Try HeadObject to verify source exists
|
||||||
|
match self.head_object(&from_key) {
|
||||||
|
Ok(_) => self.copy_object(&from_key, &to_key),
|
||||||
|
Err(e) => Err(e),
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
impl VfsFile for S3VfsFile {
|
impl VfsFile for S3VfsFile {
|
||||||
|
|||||||
@@ -0,0 +1,268 @@
|
|||||||
|
//! Background Scrub Scheduler
|
||||||
|
//!
|
||||||
|
//! Automatically runs scrub operations at regular intervals.
|
||||||
|
//! Similar to ZFS `zpool scrub` and Btrfs periodic scrub.
|
||||||
|
|
||||||
|
use std::sync::Arc;
|
||||||
|
use std::path::PathBuf;
|
||||||
|
|
||||||
|
use super::{VfsBackend, VfsError};
|
||||||
|
use super::checksum::{scrub_all, ScrubResult};
|
||||||
|
|
||||||
|
pub struct ScrubSchedulerConfig {
|
||||||
|
pub interval_secs: u64, // Default: 3600 (1 hour)
|
||||||
|
pub scrub_on_startup: bool, // Default: true
|
||||||
|
pub repair_enabled: bool, // Default: true
|
||||||
|
pub max_files_per_run: usize, // Default: 100 (limit per run)
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Default for ScrubSchedulerConfig {
|
||||||
|
fn default() -> Self {
|
||||||
|
Self {
|
||||||
|
interval_secs: 3600,
|
||||||
|
scrub_on_startup: true,
|
||||||
|
repair_enabled: true,
|
||||||
|
max_files_per_run: 100,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub struct ScrubScheduler {
|
||||||
|
backend: Arc<dyn VfsBackend>,
|
||||||
|
root_path: PathBuf,
|
||||||
|
config: ScrubSchedulerConfig,
|
||||||
|
running: bool,
|
||||||
|
last_scrub_time: Option<u64>,
|
||||||
|
scrub_count: usize,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl ScrubScheduler {
|
||||||
|
pub fn new(
|
||||||
|
backend: Arc<dyn VfsBackend>,
|
||||||
|
root_path: PathBuf,
|
||||||
|
config: ScrubSchedulerConfig,
|
||||||
|
) -> Self {
|
||||||
|
Self {
|
||||||
|
backend,
|
||||||
|
root_path,
|
||||||
|
config,
|
||||||
|
running: false,
|
||||||
|
last_scrub_time: None,
|
||||||
|
scrub_count: 0,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn with_defaults(
|
||||||
|
backend: Arc<dyn VfsBackend>,
|
||||||
|
root_path: PathBuf,
|
||||||
|
) -> Self {
|
||||||
|
Self::new(backend, root_path, ScrubSchedulerConfig::default())
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn start(&mut self) {
|
||||||
|
self.running = true;
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn stop(&mut self) {
|
||||||
|
self.running = false;
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn is_running(&self) -> bool {
|
||||||
|
self.running
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn get_last_scrub_time(&self) -> Option<u64> {
|
||||||
|
self.last_scrub_time
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn get_scrub_count(&self) -> usize {
|
||||||
|
self.scrub_count
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn should_run_now(&self) -> bool {
|
||||||
|
self.running && self.should_run_based_on_interval()
|
||||||
|
}
|
||||||
|
|
||||||
|
fn should_run_based_on_interval(&self) -> bool {
|
||||||
|
if self.last_scrub_time.is_none() {
|
||||||
|
return self.config.scrub_on_startup;
|
||||||
|
}
|
||||||
|
|
||||||
|
let now = current_time_secs();
|
||||||
|
let last = self.last_scrub_time.unwrap();
|
||||||
|
now - last >= self.config.interval_secs
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn run_once(&mut self) -> Result<Vec<ScrubResult>, VfsError> {
|
||||||
|
if !self.running {
|
||||||
|
return Ok(vec![]);
|
||||||
|
}
|
||||||
|
|
||||||
|
let results = scrub_all(
|
||||||
|
self.backend.as_ref(),
|
||||||
|
&self.root_path,
|
||||||
|
self.config.repair_enabled,
|
||||||
|
)?;
|
||||||
|
|
||||||
|
self.last_scrub_time = Some(current_time_secs());
|
||||||
|
self.scrub_count += 1;
|
||||||
|
|
||||||
|
Ok(results)
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn get_stats(&self) -> ScrubStats {
|
||||||
|
ScrubStats {
|
||||||
|
running: self.running,
|
||||||
|
scrub_count: self.scrub_count,
|
||||||
|
last_scrub_time: self.last_scrub_time,
|
||||||
|
interval_secs: self.config.interval_secs,
|
||||||
|
next_scrub_time: self.calculate_next_scrub_time(),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn calculate_next_scrub_time(&self) -> Option<u64> {
|
||||||
|
if !self.running {
|
||||||
|
return None;
|
||||||
|
}
|
||||||
|
|
||||||
|
let last = self.last_scrub_time.unwrap_or(current_time_secs());
|
||||||
|
Some(last + self.config.interval_secs)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn current_time_secs() -> u64 {
|
||||||
|
use std::time::{SystemTime, UNIX_EPOCH};
|
||||||
|
SystemTime::now()
|
||||||
|
.duration_since(UNIX_EPOCH)
|
||||||
|
.map(|d| d.as_secs())
|
||||||
|
.unwrap_or(0)
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug)]
|
||||||
|
pub struct ScrubStats {
|
||||||
|
pub running: bool,
|
||||||
|
pub scrub_count: usize,
|
||||||
|
pub last_scrub_time: Option<u64>,
|
||||||
|
pub interval_secs: u64,
|
||||||
|
pub next_scrub_time: Option<u64>,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl ScrubStats {
|
||||||
|
pub fn next_scrub_in_secs(&self) -> Option<u64> {
|
||||||
|
if !self.running {
|
||||||
|
return None;
|
||||||
|
}
|
||||||
|
|
||||||
|
let now = current_time_secs();
|
||||||
|
let next = self.next_scrub_time?;
|
||||||
|
|
||||||
|
if next > now {
|
||||||
|
Some(next - now)
|
||||||
|
} else {
|
||||||
|
Some(0)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn format_last_scrub(&self) -> String {
|
||||||
|
match self.last_scrub_time {
|
||||||
|
None => "Never".to_string(),
|
||||||
|
Some(t) => format_timestamp(t),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn format_next_scrub(&self) -> String {
|
||||||
|
match self.next_scrub_time {
|
||||||
|
None => "Not scheduled".to_string(),
|
||||||
|
Some(t) => format_timestamp(t),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn format_timestamp(secs: u64) -> String {
|
||||||
|
use chrono::{Utc, TimeZone};
|
||||||
|
Utc.timestamp_opt(secs as i64, 0)
|
||||||
|
.single()
|
||||||
|
.map(|dt| dt.format("%Y-%m-%d %H:%M:%S UTC").to_string())
|
||||||
|
.unwrap_or_else(|| format!("{} seconds since epoch", secs))
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use super::*;
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_default_config() {
|
||||||
|
let config = ScrubSchedulerConfig::default();
|
||||||
|
assert_eq!(config.interval_secs, 3600);
|
||||||
|
assert!(config.scrub_on_startup);
|
||||||
|
assert!(config.repair_enabled);
|
||||||
|
assert_eq!(config.max_files_per_run, 100);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_scheduler_start_stop() {
|
||||||
|
let backend: Arc<dyn VfsBackend> = Arc::new(super::super::local_fs::LocalFs::new());
|
||||||
|
let mut scheduler = ScrubScheduler::with_defaults(backend, PathBuf::from("/tmp"));
|
||||||
|
|
||||||
|
assert!(!scheduler.is_running());
|
||||||
|
scheduler.start();
|
||||||
|
assert!(scheduler.is_running());
|
||||||
|
scheduler.stop();
|
||||||
|
assert!(!scheduler.is_running());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_scrub_stats() {
|
||||||
|
let now = current_time_secs();
|
||||||
|
let stats = ScrubStats {
|
||||||
|
running: true,
|
||||||
|
scrub_count: 5,
|
||||||
|
last_scrub_time: Some(now - 3600),
|
||||||
|
interval_secs: 3600,
|
||||||
|
next_scrub_time: Some(now), // Next scrub is now
|
||||||
|
};
|
||||||
|
|
||||||
|
assert!(stats.running);
|
||||||
|
assert_eq!(stats.scrub_count, 5);
|
||||||
|
|
||||||
|
// When next_scrub_time is now, next_scrub_in_secs should be 0
|
||||||
|
let next_in = stats.next_scrub_in_secs();
|
||||||
|
assert!(next_in.unwrap_or(999) <= 10); // Allow 10 seconds tolerance
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_format_timestamp() {
|
||||||
|
let formatted = format_timestamp(1609459200); // 2021-01-01 00:00:00 UTC
|
||||||
|
assert!(formatted.contains("2021"));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_should_run_on_startup() {
|
||||||
|
let backend: Arc<dyn VfsBackend> = Arc::new(super::super::local_fs::LocalFs::new());
|
||||||
|
let mut scheduler = ScrubScheduler::with_defaults(backend, PathBuf::from("/tmp"));
|
||||||
|
|
||||||
|
scheduler.start();
|
||||||
|
assert!(scheduler.should_run_now()); // scrub_on_startup = true
|
||||||
|
|
||||||
|
scheduler.last_scrub_time = Some(current_time_secs());
|
||||||
|
assert!(!scheduler.should_run_now()); // Just ran, interval not elapsed
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_should_run_after_interval() {
|
||||||
|
let backend: Arc<dyn VfsBackend> = Arc::new(super::super::local_fs::LocalFs::new());
|
||||||
|
let config = ScrubSchedulerConfig {
|
||||||
|
interval_secs: 3600,
|
||||||
|
scrub_on_startup: false,
|
||||||
|
repair_enabled: true,
|
||||||
|
max_files_per_run: 100,
|
||||||
|
};
|
||||||
|
let mut scheduler = ScrubScheduler::new(backend, PathBuf::from("/tmp"), config);
|
||||||
|
|
||||||
|
scheduler.start();
|
||||||
|
assert!(!scheduler.should_run_now()); // scrub_on_startup = false
|
||||||
|
|
||||||
|
scheduler.last_scrub_time = Some(current_time_secs() - 3601);
|
||||||
|
assert!(scheduler.should_run_now()); // Interval elapsed
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,443 @@
|
|||||||
|
//! Send/Receive API - Snapshot replication
|
||||||
|
//!
|
||||||
|
//! Reference: ZFS send/receive, Proxmox Backup Server
|
||||||
|
//! Supports incremental backups and multiple formats
|
||||||
|
|
||||||
|
use std::path::PathBuf;
|
||||||
|
use std::collections::HashSet;
|
||||||
|
|
||||||
|
use super::{VfsBackend, VfsError, VfsCompression};
|
||||||
|
use super::backup_manifest::{BackupManifest, BackupStream, SendFormat, MANIFEST_FILE};
|
||||||
|
use super::checksum::{VfsChecksumFile, scrub_file};
|
||||||
|
|
||||||
|
pub struct SendOptions {
|
||||||
|
pub format: SendFormat,
|
||||||
|
pub incremental_from: Option<String>,
|
||||||
|
pub compress: VfsCompression,
|
||||||
|
pub encrypt: bool,
|
||||||
|
pub include_checksums: bool,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Default for SendOptions {
|
||||||
|
fn default() -> Self {
|
||||||
|
Self {
|
||||||
|
format: SendFormat::CustomJson,
|
||||||
|
incremental_from: None,
|
||||||
|
compress: VfsCompression::Zstd,
|
||||||
|
encrypt: false,
|
||||||
|
include_checksums: true,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub struct ReceiveOptions {
|
||||||
|
pub format: SendFormat,
|
||||||
|
pub verify_checksums: bool,
|
||||||
|
pub target_name: Option<String>,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Default for ReceiveOptions {
|
||||||
|
fn default() -> Self {
|
||||||
|
Self {
|
||||||
|
format: SendFormat::CustomJson,
|
||||||
|
verify_checksums: true,
|
||||||
|
target_name: None,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn send_snapshot(
|
||||||
|
backend: &dyn VfsBackend,
|
||||||
|
snapshot_name: &str,
|
||||||
|
root: &PathBuf,
|
||||||
|
options: SendOptions,
|
||||||
|
) -> Result<BackupStream, VfsError> {
|
||||||
|
let snapshot_dir = root.join(".snapshots").join(snapshot_name);
|
||||||
|
|
||||||
|
if !backend.exists(&snapshot_dir) {
|
||||||
|
return Err(VfsError::NotFound(format!("Snapshot {} not found", snapshot_name)));
|
||||||
|
}
|
||||||
|
|
||||||
|
let mut manifest = BackupManifest::new(snapshot_name.to_string(), root.clone());
|
||||||
|
|
||||||
|
let entries = backend.read_dir(&snapshot_dir)?;
|
||||||
|
for entry in entries {
|
||||||
|
if entry.name == MANIFEST_FILE || entry.name == ".meta" {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
let file_path = snapshot_dir.join(&entry.name);
|
||||||
|
|
||||||
|
if entry.stat.is_dir {
|
||||||
|
collect_directory_files(backend, &file_path, &snapshot_dir, &mut manifest, &options)?;
|
||||||
|
} else {
|
||||||
|
add_file_to_manifest(backend, &file_path, &snapshot_dir, &mut manifest, &options)?;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
manifest.calculate_ratio();
|
||||||
|
|
||||||
|
let payload = if options.incremental_from.is_some() {
|
||||||
|
let from_snap = options.incremental_from.unwrap();
|
||||||
|
send_incremental_payload(backend, &from_snap, snapshot_name, root)?
|
||||||
|
} else {
|
||||||
|
collect_snapshot_data(backend, &snapshot_dir)?
|
||||||
|
};
|
||||||
|
|
||||||
|
Ok(BackupStream::new(options.format, manifest, payload))
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn receive_snapshot(
|
||||||
|
backend: &dyn VfsBackend,
|
||||||
|
stream: &BackupStream,
|
||||||
|
root: &PathBuf,
|
||||||
|
options: ReceiveOptions,
|
||||||
|
) -> Result<String, VfsError> {
|
||||||
|
let snapshot_name = options.target_name.clone()
|
||||||
|
.unwrap_or_else(|| stream.manifest.snapshot_name.clone());
|
||||||
|
|
||||||
|
let snapshot_dir = root.join(".snapshots").join(&snapshot_name);
|
||||||
|
|
||||||
|
if backend.exists(&snapshot_dir) {
|
||||||
|
return Err(VfsError::Io(format!("Snapshot {} already exists", snapshot_name)));
|
||||||
|
}
|
||||||
|
|
||||||
|
backend.create_dir(&snapshot_dir, 0o755)?;
|
||||||
|
|
||||||
|
restore_snapshot_data(backend, &stream.data, &snapshot_dir)?;
|
||||||
|
|
||||||
|
stream.manifest.save(&snapshot_dir).map_err(VfsError::Io)?;
|
||||||
|
|
||||||
|
if options.verify_checksums {
|
||||||
|
verify_snapshot_checksums(backend, &snapshot_dir, root)?;
|
||||||
|
}
|
||||||
|
|
||||||
|
Ok(snapshot_name)
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn send_incremental(
|
||||||
|
backend: &dyn VfsBackend,
|
||||||
|
from_snapshot: &str,
|
||||||
|
to_snapshot: &str,
|
||||||
|
root: &PathBuf,
|
||||||
|
options: SendOptions,
|
||||||
|
) -> Result<BackupStream, VfsError> {
|
||||||
|
let mut opts = options;
|
||||||
|
opts.incremental_from = Some(from_snapshot.to_string());
|
||||||
|
|
||||||
|
send_snapshot(backend, to_snapshot, root, opts)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn collect_directory_files(
|
||||||
|
backend: &dyn VfsBackend,
|
||||||
|
dir: &PathBuf,
|
||||||
|
snapshot_dir: &PathBuf,
|
||||||
|
manifest: &mut BackupManifest,
|
||||||
|
options: &SendOptions,
|
||||||
|
) -> Result<(), VfsError> {
|
||||||
|
let entries = backend.read_dir(dir)?;
|
||||||
|
|
||||||
|
for entry in entries {
|
||||||
|
let path = dir.join(&entry.name);
|
||||||
|
|
||||||
|
if entry.stat.is_dir {
|
||||||
|
collect_directory_files(backend, &path, snapshot_dir, manifest, options)?;
|
||||||
|
} else {
|
||||||
|
add_file_to_manifest(backend, &path, snapshot_dir, manifest, options)?;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
fn add_file_to_manifest(
|
||||||
|
backend: &dyn VfsBackend,
|
||||||
|
file_path: &PathBuf,
|
||||||
|
snapshot_dir: &PathBuf,
|
||||||
|
manifest: &mut BackupManifest,
|
||||||
|
options: &SendOptions,
|
||||||
|
) -> Result<(), VfsError> {
|
||||||
|
let stat = backend.stat(file_path)?;
|
||||||
|
|
||||||
|
let relative_path = file_path.strip_prefix(snapshot_dir)
|
||||||
|
.map(|p| p.to_string_lossy().to_string())
|
||||||
|
.unwrap_or_else(|_| file_path.to_string_lossy().to_string());
|
||||||
|
|
||||||
|
let checksums = if options.include_checksums {
|
||||||
|
let checksum_dir = snapshot_dir.join(".checksums");
|
||||||
|
let checksum_file = checksum_dir.join(&relative_path).with_extension(".checksums");
|
||||||
|
|
||||||
|
if backend.exists(&checksum_file) {
|
||||||
|
load_checksum_file(backend, &checksum_file)?
|
||||||
|
} else {
|
||||||
|
None
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
None
|
||||||
|
};
|
||||||
|
|
||||||
|
manifest.add_file(relative_path, stat.size, checksums);
|
||||||
|
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
fn load_checksum_file(
|
||||||
|
backend: &dyn VfsBackend,
|
||||||
|
checksum_path: &PathBuf,
|
||||||
|
) -> Result<Option<VfsChecksumFile>, VfsError> {
|
||||||
|
let mut file = backend.open_file(checksum_path, &super::open_flags::OpenFlags::new().read())?;
|
||||||
|
let data = file.read_all()?;
|
||||||
|
|
||||||
|
if data.is_empty() {
|
||||||
|
return Ok(None);
|
||||||
|
}
|
||||||
|
|
||||||
|
VfsChecksumFile::from_bytes(&data).map(Some)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn collect_snapshot_data(
|
||||||
|
backend: &dyn VfsBackend,
|
||||||
|
snapshot_dir: &PathBuf,
|
||||||
|
) -> Result<Vec<u8>, VfsError> {
|
||||||
|
let mut buffer = Vec::new();
|
||||||
|
|
||||||
|
let entries = backend.read_dir(snapshot_dir)?;
|
||||||
|
for entry in entries {
|
||||||
|
if entry.name == MANIFEST_FILE || entry.name == ".meta" || entry.name == ".checksums" {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
let file_path = snapshot_dir.join(&entry.name);
|
||||||
|
|
||||||
|
if entry.stat.is_dir {
|
||||||
|
collect_directory_data(backend, &file_path, &mut buffer)?;
|
||||||
|
} else {
|
||||||
|
collect_file_data(backend, &file_path, &mut buffer)?;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
Ok(buffer)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn collect_directory_data(
|
||||||
|
backend: &dyn VfsBackend,
|
||||||
|
dir: &PathBuf,
|
||||||
|
buffer: &mut Vec<u8>,
|
||||||
|
) -> Result<(), VfsError> {
|
||||||
|
let entries = backend.read_dir(dir)?;
|
||||||
|
|
||||||
|
for entry in entries {
|
||||||
|
let path = dir.join(&entry.name);
|
||||||
|
|
||||||
|
if entry.stat.is_dir {
|
||||||
|
collect_directory_data(backend, &path, buffer)?;
|
||||||
|
} else {
|
||||||
|
collect_file_data(backend, &path, buffer)?;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
fn collect_file_data(
|
||||||
|
backend: &dyn VfsBackend,
|
||||||
|
file_path: &PathBuf,
|
||||||
|
buffer: &mut Vec<u8>,
|
||||||
|
) -> Result<(), VfsError> {
|
||||||
|
let mut file = backend.open_file(file_path, &super::open_flags::OpenFlags::new().read())?;
|
||||||
|
let data = file.read_all()?;
|
||||||
|
|
||||||
|
let path_str = file_path.to_string_lossy();
|
||||||
|
let path_bytes = path_str.as_bytes();
|
||||||
|
buffer.extend_from_slice(&(path_bytes.len() as u64).to_be_bytes());
|
||||||
|
buffer.extend_from_slice(path_bytes);
|
||||||
|
buffer.extend_from_slice(&(data.len() as u64).to_be_bytes());
|
||||||
|
buffer.extend_from_slice(&data);
|
||||||
|
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
fn restore_snapshot_data(
|
||||||
|
backend: &dyn VfsBackend,
|
||||||
|
data: &[u8],
|
||||||
|
snapshot_dir: &PathBuf,
|
||||||
|
) -> Result<(), VfsError> {
|
||||||
|
let mut offset = 0;
|
||||||
|
|
||||||
|
while offset < data.len() {
|
||||||
|
if data.len() < offset + 8 {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
|
||||||
|
let path_len = u64::from_be_bytes(data[offset..offset+8].try_into().map_err(|_| VfsError::Io("Invalid path length".to_string()))?) as usize;
|
||||||
|
offset += 8;
|
||||||
|
|
||||||
|
if data.len() < offset + path_len {
|
||||||
|
return Err(VfsError::Io("Truncated path".to_string()));
|
||||||
|
}
|
||||||
|
|
||||||
|
let path_str = String::from_utf8_lossy(&data[offset..offset+path_len]);
|
||||||
|
let relative_path = PathBuf::from(path_str.as_ref());
|
||||||
|
offset += path_len;
|
||||||
|
|
||||||
|
if data.len() < offset + 8 {
|
||||||
|
return Err(VfsError::Io("Truncated file length".to_string()));
|
||||||
|
}
|
||||||
|
|
||||||
|
let file_len = u64::from_be_bytes(data[offset..offset+8].try_into().map_err(|_| VfsError::Io("Invalid file length".to_string()))?) as usize;
|
||||||
|
offset += 8;
|
||||||
|
|
||||||
|
if data.len() < offset + file_len {
|
||||||
|
return Err(VfsError::Io("Truncated file data".to_string()));
|
||||||
|
}
|
||||||
|
|
||||||
|
let file_data = &data[offset..offset+file_len];
|
||||||
|
offset += file_len;
|
||||||
|
|
||||||
|
let file_path = snapshot_dir.join(&relative_path);
|
||||||
|
|
||||||
|
let parent = file_path.parent()
|
||||||
|
.ok_or_else(|| VfsError::Io("Invalid file path".to_string()))?;
|
||||||
|
|
||||||
|
if !backend.exists(parent) {
|
||||||
|
backend.create_dir_all(parent, 0o755)?;
|
||||||
|
}
|
||||||
|
|
||||||
|
let mut file = backend.open_file(
|
||||||
|
&file_path,
|
||||||
|
&super::open_flags::OpenFlags::new().write().create().truncate(),
|
||||||
|
)?;
|
||||||
|
file.write_all(file_data)?;
|
||||||
|
file.flush()?;
|
||||||
|
}
|
||||||
|
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
fn send_incremental_payload(
|
||||||
|
backend: &dyn VfsBackend,
|
||||||
|
from_snap: &str,
|
||||||
|
to_snap: &str,
|
||||||
|
root: &PathBuf,
|
||||||
|
) -> Result<Vec<u8>, VfsError> {
|
||||||
|
let from_dir = root.join(".snapshots").join(from_snap);
|
||||||
|
let to_dir = root.join(".snapshots").join(to_snap);
|
||||||
|
|
||||||
|
if !backend.exists(&from_dir) || !backend.exists(&to_dir) {
|
||||||
|
return Err(VfsError::NotFound("Source snapshot not found".to_string()));
|
||||||
|
}
|
||||||
|
|
||||||
|
let from_files = collect_file_set(backend, &from_dir)?;
|
||||||
|
let to_files = collect_file_set(backend, &to_dir)?;
|
||||||
|
|
||||||
|
let mut buffer = Vec::new();
|
||||||
|
|
||||||
|
for (relative, to_size) in &to_files {
|
||||||
|
let changed = !from_files.contains(&(relative.clone(), *to_size));
|
||||||
|
|
||||||
|
if changed {
|
||||||
|
let to_path = to_dir.join(relative);
|
||||||
|
collect_file_data(backend, &to_path, &mut buffer)?;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
Ok(buffer)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn collect_file_set(
|
||||||
|
backend: &dyn VfsBackend,
|
||||||
|
dir: &PathBuf,
|
||||||
|
) -> Result<HashSet<(String, u64)>, VfsError> {
|
||||||
|
let mut files = HashSet::new();
|
||||||
|
|
||||||
|
let entries = backend.read_dir(dir)?;
|
||||||
|
for entry in entries {
|
||||||
|
let path = dir.join(&entry.name);
|
||||||
|
|
||||||
|
if entry.stat.is_dir {
|
||||||
|
let sub_files = collect_file_set(backend, &path)?;
|
||||||
|
files.extend(sub_files);
|
||||||
|
} else {
|
||||||
|
let relative = path.strip_prefix(dir)
|
||||||
|
.map(|p| p.to_string_lossy().to_string())
|
||||||
|
.unwrap_or_default();
|
||||||
|
files.insert((relative, entry.stat.size));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
Ok(files)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn verify_snapshot_checksums(
|
||||||
|
backend: &dyn VfsBackend,
|
||||||
|
snapshot_dir: &PathBuf,
|
||||||
|
root: &PathBuf,
|
||||||
|
) -> Result<(), VfsError> {
|
||||||
|
let checksum_dir = snapshot_dir.join(".checksums");
|
||||||
|
|
||||||
|
if !backend.exists(&checksum_dir) {
|
||||||
|
return Ok(());
|
||||||
|
}
|
||||||
|
|
||||||
|
let entries = backend.read_dir(snapshot_dir)?;
|
||||||
|
for entry in entries {
|
||||||
|
if entry.stat.is_dir {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
let file_path = snapshot_dir.join(&entry.name);
|
||||||
|
let result = scrub_file(backend, &file_path, root, false)?;
|
||||||
|
|
||||||
|
if !result.is_clean() {
|
||||||
|
return Err(VfsError::Io(format!(
|
||||||
|
"Checksum verification failed for {}: {} corrupted blocks",
|
||||||
|
entry.name,
|
||||||
|
result.corrupted_blocks.len()
|
||||||
|
)));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use super::*;
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_send_options_default() {
|
||||||
|
let opts = SendOptions::default();
|
||||||
|
assert_eq!(opts.format, SendFormat::CustomJson);
|
||||||
|
assert!(opts.incremental_from.is_none());
|
||||||
|
assert_eq!(opts.compress, VfsCompression::Zstd);
|
||||||
|
assert!(!opts.encrypt);
|
||||||
|
assert!(opts.include_checksums);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_receive_options_default() {
|
||||||
|
let opts = ReceiveOptions::default();
|
||||||
|
assert_eq!(opts.format, SendFormat::CustomJson);
|
||||||
|
assert!(opts.verify_checksums);
|
||||||
|
assert!(opts.target_name.is_none());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_manifest_roundtrip() {
|
||||||
|
let mut manifest = BackupManifest::new("test_snap".to_string(), PathBuf::from("/data"));
|
||||||
|
manifest.add_file("file1.txt".to_string(), 1000, None);
|
||||||
|
manifest.add_file("dir/file2.txt".to_string(), 2000, None);
|
||||||
|
manifest.calculate_ratio();
|
||||||
|
|
||||||
|
assert_eq!(manifest.files.len(), 2);
|
||||||
|
assert_eq!(manifest.total_size, 3000);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_stream_format() {
|
||||||
|
let manifest = BackupManifest::new("test".to_string(), PathBuf::from("/"));
|
||||||
|
let stream = BackupStream::new(SendFormat::CustomJson, manifest, vec![]);
|
||||||
|
|
||||||
|
assert_eq!(stream.format, SendFormat::CustomJson);
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -39,10 +39,11 @@ fn map_smb_error(e: smb2::Error) -> VfsError {
|
|||||||
}
|
}
|
||||||
|
|
||||||
/// SMB 客户端 VFS 后端 (SMB 2/3)
|
/// SMB 客户端 VFS 后端 (SMB 2/3)
|
||||||
|
#[derive(Clone)]
|
||||||
pub struct SmbVfs {
|
pub struct SmbVfs {
|
||||||
runtime: Arc<tokio::runtime::Runtime>,
|
runtime: Arc<tokio::runtime::Runtime>,
|
||||||
client: Arc<Mutex<smb2::SmbClient>>,
|
client: Arc<Mutex<smb2::SmbClient>>,
|
||||||
tree: Mutex<smb2::Tree>,
|
tree: Arc<Mutex<smb2::Tree>>,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl SmbVfs {
|
impl SmbVfs {
|
||||||
@@ -90,7 +91,7 @@ impl SmbVfs {
|
|||||||
Ok(Self {
|
Ok(Self {
|
||||||
runtime,
|
runtime,
|
||||||
client: Arc::new(Mutex::new(client)),
|
client: Arc::new(Mutex::new(client)),
|
||||||
tree: Mutex::new(tree),
|
tree: Arc::new(Mutex::new(tree)),
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -100,16 +101,6 @@ impl SmbVfs {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
impl Clone for SmbVfs {
|
|
||||||
fn clone(&self) -> Self {
|
|
||||||
Self {
|
|
||||||
runtime: self.runtime.clone(),
|
|
||||||
client: self.client.clone(),
|
|
||||||
tree: Mutex::new(self.tree.lock().unwrap().clone()),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
impl VfsBackend for SmbVfs {
|
impl VfsBackend for SmbVfs {
|
||||||
fn clone_boxed(&self) -> Box<dyn VfsBackend> {
|
fn clone_boxed(&self) -> Box<dyn VfsBackend> {
|
||||||
Box::new(self.clone())
|
Box::new(self.clone())
|
||||||
@@ -149,11 +140,11 @@ impl VfsBackend for SmbVfs {
|
|||||||
|
|
||||||
fn open_file(&self, path: &Path, flags: &OpenFlags) -> Result<Box<dyn VfsFile>, VfsError> {
|
fn open_file(&self, path: &Path, flags: &OpenFlags) -> Result<Box<dyn VfsFile>, VfsError> {
|
||||||
let smb_path = Self::path_to_str(path);
|
let smb_path = Self::path_to_str(path);
|
||||||
let mut client = self
|
let _client = self
|
||||||
.client
|
.client
|
||||||
.lock()
|
.lock()
|
||||||
.map_err(|e| VfsError::Io(e.to_string()))?;
|
.map_err(|e| VfsError::Io(e.to_string()))?;
|
||||||
let mut tree = self.tree.lock().map_err(|e| VfsError::Io(e.to_string()))?;
|
let tree = self.tree.lock().map_err(|e| VfsError::Io(e.to_string()))?;
|
||||||
|
|
||||||
if flags.write || flags.create || flags.truncate {
|
if flags.write || flags.create || flags.truncate {
|
||||||
Ok(Box::new(SmbVfsFile {
|
Ok(Box::new(SmbVfsFile {
|
||||||
@@ -174,7 +165,7 @@ impl VfsBackend for SmbVfs {
|
|||||||
// Streaming read: open file and store file_id
|
// Streaming read: open file and store file_id
|
||||||
let (file_id, file_size) = {
|
let (file_id, file_size) = {
|
||||||
let mut client = self.client.lock().map_err(|e| VfsError::Io(e.to_string()))?;
|
let mut client = self.client.lock().map_err(|e| VfsError::Io(e.to_string()))?;
|
||||||
let mut tree = self.tree.lock().unwrap();
|
let tree = self.tree.lock().unwrap();
|
||||||
self.runtime
|
self.runtime
|
||||||
.block_on(tree.open_file(client.connection_mut(), &smb_path))
|
.block_on(tree.open_file(client.connection_mut(), &smb_path))
|
||||||
.map_err(map_smb_error)?
|
.map_err(map_smb_error)?
|
||||||
|
|||||||
@@ -1,7 +1,7 @@
|
|||||||
use std::path::{Path, PathBuf};
|
use std::path::{Path, PathBuf};
|
||||||
use std::sync::Arc;
|
use std::sync::Arc;
|
||||||
use std::sync::Mutex;
|
|
||||||
use std::time::SystemTime;
|
use std::time::SystemTime;
|
||||||
|
use tokio::sync::Mutex;
|
||||||
|
|
||||||
use async_trait::async_trait;
|
use async_trait::async_trait;
|
||||||
use bytes::Bytes;
|
use bytes::Bytes;
|
||||||
@@ -86,6 +86,7 @@ fn vfs_stat_to_file_info(stat: &VfsStat, name: &str, path: &Path) -> FileInfo {
|
|||||||
last_write_time: system_time_to_filetime(stat.mtime),
|
last_write_time: system_time_to_filetime(stat.mtime),
|
||||||
change_time: system_time_to_filetime(stat.mtime),
|
change_time: system_time_to_filetime(stat.mtime),
|
||||||
is_directory: stat.is_dir,
|
is_directory: stat.is_dir,
|
||||||
|
dos_attributes: 0,
|
||||||
file_index: 0,
|
file_index: 0,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -160,7 +161,10 @@ impl ShareBackend for VfsShareBackend {
|
|||||||
|
|
||||||
let file = self.vfs.open_file(&full_path, &flags).map_err(map_error)?;
|
let file = self.vfs.open_file(&full_path, &flags).map_err(map_error)?;
|
||||||
Ok(Box::new(VfsHandle::File {
|
Ok(Box::new(VfsHandle::File {
|
||||||
file: Mutex::new(file),
|
inner: Mutex::new(FileAndBuf {
|
||||||
|
file,
|
||||||
|
read_buf: Vec::new(),
|
||||||
|
}),
|
||||||
path: full_path,
|
path: full_path,
|
||||||
vfs: self.vfs.clone(),
|
vfs: self.vfs.clone(),
|
||||||
}))
|
}))
|
||||||
@@ -194,9 +198,14 @@ impl ShareBackend for VfsShareBackend {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
struct FileAndBuf {
|
||||||
|
file: Box<dyn super::VfsFile + Send>,
|
||||||
|
read_buf: Vec<u8>,
|
||||||
|
}
|
||||||
|
|
||||||
enum VfsHandle {
|
enum VfsHandle {
|
||||||
File {
|
File {
|
||||||
file: Mutex<Box<dyn super::VfsFile + Send>>,
|
inner: Mutex<FileAndBuf>,
|
||||||
path: PathBuf,
|
path: PathBuf,
|
||||||
vfs: Arc<dyn VfsBackend>,
|
vfs: Arc<dyn VfsBackend>,
|
||||||
},
|
},
|
||||||
@@ -210,14 +219,19 @@ enum VfsHandle {
|
|||||||
impl Handle for VfsHandle {
|
impl Handle for VfsHandle {
|
||||||
async fn read(&self, offset: u64, len: u32) -> Result<Bytes, SmbError> {
|
async fn read(&self, offset: u64, len: u32) -> Result<Bytes, SmbError> {
|
||||||
match self {
|
match self {
|
||||||
Self::File { file, .. } => {
|
Self::File { inner, .. } => {
|
||||||
let mut file = file.lock().unwrap();
|
let mut guard = inner.lock().await;
|
||||||
file.seek(std::io::SeekFrom::Start(offset))
|
let fb = &mut *guard;
|
||||||
.map_err(vfs_error_to_io)?;
|
// Reuse read_buf to avoid per-read allocation
|
||||||
let mut buf = vec![0u8; len as usize];
|
let buf = &mut fb.read_buf;
|
||||||
let n = file.read(&mut buf).map_err(map_error)?;
|
buf.clear();
|
||||||
|
if buf.capacity() < len as usize {
|
||||||
|
buf.reserve(len as usize - buf.capacity());
|
||||||
|
}
|
||||||
|
unsafe { buf.set_len(len as usize); }
|
||||||
|
let n = fb.file.read_at(buf, offset).map_err(map_error)?;
|
||||||
buf.truncate(n);
|
buf.truncate(n);
|
||||||
Ok(Bytes::from(buf))
|
Ok(Bytes::from(std::mem::take(buf)))
|
||||||
}
|
}
|
||||||
Self::Directory { .. } => Err(SmbError::NotSupported),
|
Self::Directory { .. } => Err(SmbError::NotSupported),
|
||||||
}
|
}
|
||||||
@@ -225,11 +239,9 @@ impl Handle for VfsHandle {
|
|||||||
|
|
||||||
async fn write(&self, offset: u64, data: &[u8]) -> Result<u32, SmbError> {
|
async fn write(&self, offset: u64, data: &[u8]) -> Result<u32, SmbError> {
|
||||||
match self {
|
match self {
|
||||||
Self::File { file, .. } => {
|
Self::File { inner, .. } => {
|
||||||
let mut file = file.lock().unwrap();
|
let mut guard = inner.lock().await;
|
||||||
file.seek(std::io::SeekFrom::Start(offset))
|
let n = guard.file.write_at(data, offset).map_err(map_error)?;
|
||||||
.map_err(vfs_error_to_io)?;
|
|
||||||
let n = file.write(data).map_err(map_error)?;
|
|
||||||
Ok(n as u32)
|
Ok(n as u32)
|
||||||
}
|
}
|
||||||
Self::Directory { .. } => Err(SmbError::NotSupported),
|
Self::Directory { .. } => Err(SmbError::NotSupported),
|
||||||
@@ -238,9 +250,9 @@ impl Handle for VfsHandle {
|
|||||||
|
|
||||||
async fn flush(&self) -> Result<(), SmbError> {
|
async fn flush(&self) -> Result<(), SmbError> {
|
||||||
match self {
|
match self {
|
||||||
Self::File { file, .. } => {
|
Self::File { inner, .. } => {
|
||||||
let mut file = file.lock().unwrap();
|
let mut guard = inner.lock().await;
|
||||||
file.flush().map_err(map_error)
|
guard.file.flush().map_err(map_error)
|
||||||
}
|
}
|
||||||
Self::Directory { .. } => Ok(()),
|
Self::Directory { .. } => Ok(()),
|
||||||
}
|
}
|
||||||
@@ -248,9 +260,9 @@ impl Handle for VfsHandle {
|
|||||||
|
|
||||||
async fn stat(&self) -> Result<FileInfo, SmbError> {
|
async fn stat(&self) -> Result<FileInfo, SmbError> {
|
||||||
match self {
|
match self {
|
||||||
Self::File { file, path, .. } => {
|
Self::File { inner, path, .. } => {
|
||||||
let mut f = file.lock().unwrap();
|
let mut guard = inner.lock().await;
|
||||||
let vfs_stat = f.stat().map_err(map_error)?;
|
let vfs_stat = guard.file.stat().map_err(map_error)?;
|
||||||
Ok(vfs_stat_to_file_info(&vfs_stat, "", path))
|
Ok(vfs_stat_to_file_info(&vfs_stat, "", path))
|
||||||
}
|
}
|
||||||
Self::Directory { vfs, path } => {
|
Self::Directory { vfs, path } => {
|
||||||
@@ -277,26 +289,39 @@ impl Handle for VfsHandle {
|
|||||||
|
|
||||||
async fn truncate(&self, len: u64) -> Result<(), SmbError> {
|
async fn truncate(&self, len: u64) -> Result<(), SmbError> {
|
||||||
match self {
|
match self {
|
||||||
Self::File { file, .. } => {
|
Self::File { inner, .. } => {
|
||||||
let mut file = file.lock().unwrap();
|
let mut guard = inner.lock().await;
|
||||||
file.set_len(len).map_err(map_error)
|
guard.file.set_len(len).map_err(map_error)
|
||||||
}
|
}
|
||||||
Self::Directory { .. } => Err(SmbError::NotSupported),
|
Self::Directory { .. } => Err(SmbError::NotSupported),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn list_dir(&self, _pattern: Option<&str>) -> Result<Vec<DirEntry>, SmbError> {
|
async fn list_dir(&self, pattern: Option<&str>) -> Result<Vec<DirEntry>, SmbError> {
|
||||||
match self {
|
match self {
|
||||||
Self::File { .. } => Err(SmbError::NotADirectory),
|
Self::File { .. } => Err(SmbError::NotADirectory),
|
||||||
Self::Directory { vfs, path } => {
|
Self::Directory { vfs, path } => {
|
||||||
let entries = vfs.read_dir(path).map_err(map_error)?;
|
let entries = vfs.read_dir(path).map_err(map_error)?;
|
||||||
let result = entries
|
let mut result: Vec<DirEntry> = entries
|
||||||
.into_iter()
|
.into_iter()
|
||||||
|
.filter(|entry| {
|
||||||
|
let p = match pattern {
|
||||||
|
None => return true,
|
||||||
|
Some(p) => p,
|
||||||
|
};
|
||||||
|
if p == "*" || p == "*.*" || p.is_empty() {
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
smb_match(&entry.name, p)
|
||||||
|
})
|
||||||
.map(|entry| {
|
.map(|entry| {
|
||||||
let info = vfs_stat_to_file_info(&entry.stat, &entry.name, path);
|
let info = vfs_stat_to_file_info(&entry.stat, &entry.name, path);
|
||||||
DirEntry { info }
|
DirEntry { info }
|
||||||
})
|
})
|
||||||
.collect();
|
.collect();
|
||||||
|
for (i, entry) in result.iter_mut().enumerate() {
|
||||||
|
entry.info.file_index = (i + 1) as u64;
|
||||||
|
}
|
||||||
Ok(result)
|
Ok(result)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -307,6 +332,36 @@ impl Handle for VfsHandle {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Simple SMB wildcard match: `*` matches any sequence, `?` matches one char.
|
||||||
|
fn smb_match(name: &str, pattern: &str) -> bool {
|
||||||
|
let name = name.as_bytes();
|
||||||
|
let pat = pattern.as_bytes();
|
||||||
|
let mut ni = 0;
|
||||||
|
let mut pi = 0;
|
||||||
|
let mut star_idx: Option<usize> = None;
|
||||||
|
let mut match_idx = 0;
|
||||||
|
while ni < name.len() {
|
||||||
|
if pi < pat.len() && (pat[pi] == b'?' || pat[pi] == name[ni]) {
|
||||||
|
ni += 1;
|
||||||
|
pi += 1;
|
||||||
|
} else if pi < pat.len() && pat[pi] == b'*' {
|
||||||
|
star_idx = Some(pi);
|
||||||
|
match_idx = ni;
|
||||||
|
pi += 1;
|
||||||
|
} else if let Some(si) = star_idx {
|
||||||
|
pi = si + 1;
|
||||||
|
match_idx += 1;
|
||||||
|
ni = match_idx;
|
||||||
|
} else {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
while pi < pat.len() && pat[pi] == b'*' {
|
||||||
|
pi += 1;
|
||||||
|
}
|
||||||
|
pi == pat.len()
|
||||||
|
}
|
||||||
|
|
||||||
fn filetime_to_systemtime(ft: u64) -> SystemTime {
|
fn filetime_to_systemtime(ft: u64) -> SystemTime {
|
||||||
if ft < FILETIME_OFFSET {
|
if ft < FILETIME_OFFSET {
|
||||||
return SystemTime::UNIX_EPOCH;
|
return SystemTime::UNIX_EPOCH;
|
||||||
|
|||||||
@@ -0,0 +1,319 @@
|
|||||||
|
//! Storage Stats - Metrics for dashboard display
|
||||||
|
//!
|
||||||
|
//! Provides storage overview, dedup, compression, RAID stats
|
||||||
|
|
||||||
|
use std::path::PathBuf;
|
||||||
|
|
||||||
|
use super::{VfsBackend, VfsError, VfsCompression, VfsRaidLevel};
|
||||||
|
use super::dedup::DedupStats;
|
||||||
|
use super::raid::VfsRaidBackend;
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]
|
||||||
|
pub struct StorageStats {
|
||||||
|
pub total_size: u64,
|
||||||
|
pub used_size: u64,
|
||||||
|
pub free_size: u64,
|
||||||
|
pub file_count: u64,
|
||||||
|
pub dir_count: u64,
|
||||||
|
pub dedup_ratio: f64,
|
||||||
|
pub compression_ratio: f64,
|
||||||
|
pub encryption_enabled: bool,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl StorageStats {
|
||||||
|
pub fn empty() -> Self {
|
||||||
|
Self {
|
||||||
|
total_size: 0,
|
||||||
|
used_size: 0,
|
||||||
|
free_size: 0,
|
||||||
|
file_count: 0,
|
||||||
|
dir_count: 0,
|
||||||
|
dedup_ratio: 1.0,
|
||||||
|
compression_ratio: 1.0,
|
||||||
|
encryption_enabled: false,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn format_total(&self) -> String {
|
||||||
|
format_size(self.total_size)
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn format_used(&self) -> String {
|
||||||
|
format_size(self.used_size)
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn format_free(&self) -> String {
|
||||||
|
format_size(self.free_size)
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn usage_percent(&self) -> f64 {
|
||||||
|
if self.total_size == 0 {
|
||||||
|
return 0.0;
|
||||||
|
}
|
||||||
|
(self.used_size as f64 / self.total_size as f64) * 100.0
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]
|
||||||
|
pub struct DedupStatsResponse {
|
||||||
|
pub unique_blocks: u64,
|
||||||
|
pub total_blocks: u64,
|
||||||
|
pub stored_bytes: u64,
|
||||||
|
pub saved_bytes: u64,
|
||||||
|
pub dedup_ratio: f64,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl From<DedupStats> for DedupStatsResponse {
|
||||||
|
fn from(stats: DedupStats) -> Self {
|
||||||
|
let saved = stats.total_blocks * 4096 - stats.stored_bytes;
|
||||||
|
Self {
|
||||||
|
unique_blocks: stats.unique_blocks,
|
||||||
|
total_blocks: stats.total_blocks,
|
||||||
|
stored_bytes: stats.stored_bytes,
|
||||||
|
saved_bytes: saved,
|
||||||
|
dedup_ratio: if stats.total_blocks > 0 {
|
||||||
|
stats.stored_bytes as f64 / (stats.total_blocks * 4096) as f64
|
||||||
|
} else {
|
||||||
|
1.0
|
||||||
|
},
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]
|
||||||
|
pub struct CompressionStatsResponse {
|
||||||
|
pub algorithm: String,
|
||||||
|
pub original_size: u64,
|
||||||
|
pub compressed_size: u64,
|
||||||
|
pub compression_ratio: f64,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl CompressionStatsResponse {
|
||||||
|
pub fn from_compression(compression: VfsCompression, original: u64, compressed: u64) -> Self {
|
||||||
|
let algorithm = match compression {
|
||||||
|
VfsCompression::None => "none",
|
||||||
|
VfsCompression::Lz4 => "lz4",
|
||||||
|
VfsCompression::Zstd => "zstd",
|
||||||
|
};
|
||||||
|
|
||||||
|
let ratio = if original > 0 {
|
||||||
|
compressed as f64 / original as f64
|
||||||
|
} else {
|
||||||
|
1.0
|
||||||
|
};
|
||||||
|
|
||||||
|
Self {
|
||||||
|
algorithm: algorithm.to_string(),
|
||||||
|
original_size: original,
|
||||||
|
compressed_size: compressed,
|
||||||
|
compression_ratio: ratio,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]
|
||||||
|
pub struct RaidStatsResponse {
|
||||||
|
pub level: String,
|
||||||
|
pub disk_count: usize,
|
||||||
|
pub data_disks: usize,
|
||||||
|
pub parity_disks: usize,
|
||||||
|
pub healthy: bool,
|
||||||
|
pub rebuild_in_progress: bool,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl RaidStatsResponse {
|
||||||
|
pub fn from_raid(raid: &VfsRaidBackend) -> Self {
|
||||||
|
let level = match raid.level() {
|
||||||
|
VfsRaidLevel::Single => "single",
|
||||||
|
VfsRaidLevel::RaidZ1 => "raidz1",
|
||||||
|
VfsRaidLevel::RaidZ2 => "raidz2",
|
||||||
|
VfsRaidLevel::RaidZ3 => "raidz3",
|
||||||
|
};
|
||||||
|
|
||||||
|
Self {
|
||||||
|
level: level.to_string(),
|
||||||
|
disk_count: raid.backends().len(),
|
||||||
|
data_disks: raid.data_disks(),
|
||||||
|
parity_disks: raid.parity_disks(),
|
||||||
|
healthy: true,
|
||||||
|
rebuild_in_progress: false,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]
|
||||||
|
pub struct ScrubStatsResponse {
|
||||||
|
pub last_scrub_time: Option<u64>,
|
||||||
|
pub next_scrub_time: Option<u64>,
|
||||||
|
pub scrub_count: usize,
|
||||||
|
pub corrupted_blocks_found: u64,
|
||||||
|
pub blocks_verified: u64,
|
||||||
|
pub running: bool,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl ScrubStatsResponse {
|
||||||
|
pub fn empty() -> Self {
|
||||||
|
Self {
|
||||||
|
last_scrub_time: None,
|
||||||
|
next_scrub_time: None,
|
||||||
|
scrub_count: 0,
|
||||||
|
corrupted_blocks_found: 0,
|
||||||
|
blocks_verified: 0,
|
||||||
|
running: false,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn from_scheduler(scheduler: &super::scrub_scheduler::ScrubScheduler) -> Self {
|
||||||
|
let stats = scheduler.get_stats();
|
||||||
|
Self {
|
||||||
|
last_scrub_time: stats.last_scrub_time,
|
||||||
|
next_scrub_time: stats.next_scrub_time,
|
||||||
|
scrub_count: stats.scrub_count,
|
||||||
|
corrupted_blocks_found: 0,
|
||||||
|
blocks_verified: 0,
|
||||||
|
running: stats.running,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn calculate_storage_stats(
|
||||||
|
backend: &dyn VfsBackend,
|
||||||
|
root: &PathBuf,
|
||||||
|
) -> Result<StorageStats, VfsError> {
|
||||||
|
let mut stats = StorageStats::empty();
|
||||||
|
|
||||||
|
calculate_recursive(backend, root, &mut stats)?;
|
||||||
|
|
||||||
|
Ok(stats)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn calculate_recursive(
|
||||||
|
backend: &dyn VfsBackend,
|
||||||
|
path: &PathBuf,
|
||||||
|
stats: &mut StorageStats,
|
||||||
|
) -> Result<(), VfsError> {
|
||||||
|
let entries = backend.read_dir(path)?;
|
||||||
|
|
||||||
|
for entry in entries {
|
||||||
|
if entry.name == ".snapshots" || entry.name == ".checksums" {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
let entry_path = path.join(&entry.name);
|
||||||
|
|
||||||
|
if entry.stat.is_dir {
|
||||||
|
stats.dir_count += 1;
|
||||||
|
calculate_recursive(backend, &entry_path, stats)?;
|
||||||
|
} else {
|
||||||
|
stats.file_count += 1;
|
||||||
|
stats.used_size += entry.stat.size;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
fn format_size(size: u64) -> String {
|
||||||
|
if size < 1024 {
|
||||||
|
format!("{} B", size)
|
||||||
|
} else if size < 1024 * 1024 {
|
||||||
|
format!("{:.2} KB", size as f64 / 1024.0)
|
||||||
|
} else if size < 1024 * 1024 * 1024 {
|
||||||
|
format!("{:.2} MB", size as f64 / (1024.0 * 1024.0))
|
||||||
|
} else if size < 1024 * 1024 * 1024 * 1024 {
|
||||||
|
format!("{:.2} GB", size as f64 / (1024.0 * 1024.0 * 1024.0))
|
||||||
|
} else {
|
||||||
|
format!("{:.2} TB", size as f64 / (1024.0 * 1024.0 * 1024.0 * 1024.0))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use super::*;
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_storage_stats_empty() {
|
||||||
|
let stats = StorageStats::empty();
|
||||||
|
assert_eq!(stats.total_size, 0);
|
||||||
|
assert_eq!(stats.used_size, 0);
|
||||||
|
assert_eq!(stats.dedup_ratio, 1.0);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_format_size_bytes() {
|
||||||
|
assert_eq!(format_size(512), "512 B");
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_format_size_kb() {
|
||||||
|
assert_eq!(format_size(1536), "1.50 KB");
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_format_size_mb() {
|
||||||
|
assert_eq!(format_size(1536 * 1024), "1.50 MB");
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_format_size_gb() {
|
||||||
|
assert_eq!(format_size(1536 * 1024 * 1024), "1.50 GB");
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_usage_percent() {
|
||||||
|
let stats = StorageStats {
|
||||||
|
total_size: 1000,
|
||||||
|
used_size: 250,
|
||||||
|
free_size: 750,
|
||||||
|
file_count: 10,
|
||||||
|
dir_count: 2,
|
||||||
|
dedup_ratio: 1.0,
|
||||||
|
compression_ratio: 1.0,
|
||||||
|
encryption_enabled: false,
|
||||||
|
};
|
||||||
|
|
||||||
|
assert_eq!(stats.usage_percent(), 25.0);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_compression_stats() {
|
||||||
|
let stats = CompressionStatsResponse::from_compression(
|
||||||
|
VfsCompression::Zstd,
|
||||||
|
1000,
|
||||||
|
420,
|
||||||
|
);
|
||||||
|
|
||||||
|
assert_eq!(stats.algorithm, "zstd");
|
||||||
|
assert_eq!(stats.compression_ratio, 0.42);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_raid_stats_single() {
|
||||||
|
let backend: Box<dyn VfsBackend> = Box::new(super::super::local_fs::LocalFs::new());
|
||||||
|
let config = super::super::VfsRaidConfig {
|
||||||
|
level: VfsRaidLevel::Single,
|
||||||
|
stripe_size: 4096,
|
||||||
|
disk_paths: vec![PathBuf::from("/tmp")],
|
||||||
|
};
|
||||||
|
let raid = VfsRaidBackend::new(config, vec![backend]).unwrap();
|
||||||
|
|
||||||
|
let stats = RaidStatsResponse::from_raid(&raid);
|
||||||
|
assert_eq!(stats.level, "single");
|
||||||
|
assert_eq!(stats.disk_count, 1);
|
||||||
|
assert_eq!(stats.parity_disks, 0);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_dedup_stats_conversion() {
|
||||||
|
let dedup = DedupStats {
|
||||||
|
total_blocks: 100,
|
||||||
|
total_refs: 200,
|
||||||
|
unique_blocks: 50,
|
||||||
|
stored_bytes: 200 * 1024,
|
||||||
|
};
|
||||||
|
|
||||||
|
let response = DedupStatsResponse::from(dedup);
|
||||||
|
assert_eq!(response.unique_blocks, 50);
|
||||||
|
assert_eq!(response.total_blocks, 100);
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,737 @@
|
|||||||
|
use super::local_fs::LocalFs;
|
||||||
|
use super::open_flags::OpenFlags;
|
||||||
|
use super::{VfsBackend, VfsDirEntry, VfsError, VfsFile, VfsStat};
|
||||||
|
use rusqlite::{params, Connection};
|
||||||
|
use std::path::{Path, PathBuf};
|
||||||
|
use std::sync::{Arc, Mutex};
|
||||||
|
|
||||||
|
const SCHEMA: &str = "
|
||||||
|
CREATE TABLE IF NOT EXISTS virtual_folders (
|
||||||
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||||
|
folder TEXT NOT NULL UNIQUE,
|
||||||
|
description TEXT DEFAULT '',
|
||||||
|
created_at TEXT DEFAULT (datetime('now'))
|
||||||
|
);
|
||||||
|
|
||||||
|
CREATE TABLE IF NOT EXISTS file_tags (
|
||||||
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||||
|
filename TEXT NOT NULL,
|
||||||
|
tag TEXT NOT NULL,
|
||||||
|
UNIQUE(filename, tag)
|
||||||
|
);
|
||||||
|
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_file_tags_tag ON file_tags(tag);
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_file_tags_filename ON file_tags(filename);
|
||||||
|
|
||||||
|
CREATE TABLE IF NOT EXISTS webdav_config (
|
||||||
|
key TEXT PRIMARY KEY,
|
||||||
|
value TEXT NOT NULL
|
||||||
|
);
|
||||||
|
";
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
pub struct VirtualFs {
|
||||||
|
db: Arc<Mutex<Connection>>,
|
||||||
|
backend: Box<dyn VfsBackend>,
|
||||||
|
root: PathBuf,
|
||||||
|
}
|
||||||
|
|
||||||
|
fn normalize_folder(folder: &str) -> String {
|
||||||
|
let f = folder.trim_start_matches('/');
|
||||||
|
format!("/{}", f.trim_end_matches('/'))
|
||||||
|
}
|
||||||
|
|
||||||
|
fn folder_to_tag(folder: &str) -> String {
|
||||||
|
folder.trim_matches('/').to_string()
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
impl VirtualFs {
|
||||||
|
pub fn new(db_path: &str, root: PathBuf) -> Result<Self, VfsError> {
|
||||||
|
let conn = Connection::open(db_path).map_err(|e| VfsError::Io(e.to_string()))?;
|
||||||
|
conn.execute_batch(SCHEMA)
|
||||||
|
.map_err(|e| VfsError::Io(e.to_string()))?;
|
||||||
|
let backend = Box::new(LocalFs::new());
|
||||||
|
Ok(Self {
|
||||||
|
db: Arc::new(Mutex::new(conn)),
|
||||||
|
backend,
|
||||||
|
root,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
fn load_folders(&self) -> Vec<String> {
|
||||||
|
let db = self.db.lock().unwrap();
|
||||||
|
let mut stmt = match db.prepare("SELECT folder FROM virtual_folders ORDER BY folder") {
|
||||||
|
Ok(s) => s,
|
||||||
|
Err(_) => return vec![],
|
||||||
|
};
|
||||||
|
let rows = stmt
|
||||||
|
.query_map([], |row| row.get::<_, String>(0))
|
||||||
|
.ok();
|
||||||
|
rows.map(|r| r.filter_map(|e| e.ok()).collect())
|
||||||
|
.unwrap_or_default()
|
||||||
|
}
|
||||||
|
|
||||||
|
fn files_with_tag(&self, tag: &str) -> Vec<String> {
|
||||||
|
let db = self.db.lock().unwrap();
|
||||||
|
let mut stmt = match db.prepare("SELECT filename FROM file_tags WHERE tag = ?1 ORDER BY filename") {
|
||||||
|
Ok(s) => s,
|
||||||
|
Err(_) => return vec![],
|
||||||
|
};
|
||||||
|
let rows = stmt
|
||||||
|
.query_map(params![tag], |row| row.get::<_, String>(0))
|
||||||
|
.ok();
|
||||||
|
rows.map(|r| r.filter_map(|e| e.ok()).collect())
|
||||||
|
.unwrap_or_default()
|
||||||
|
}
|
||||||
|
|
||||||
|
fn tags_for_file(&self, filename: &str) -> Vec<String> {
|
||||||
|
let db = self.db.lock().unwrap();
|
||||||
|
let mut stmt = match db.prepare("SELECT tag FROM file_tags WHERE filename = ?1 ORDER BY tag") {
|
||||||
|
Ok(s) => s,
|
||||||
|
Err(_) => return vec![],
|
||||||
|
};
|
||||||
|
let rows = stmt
|
||||||
|
.query_map(params![filename], |row| row.get::<_, String>(0))
|
||||||
|
.ok();
|
||||||
|
rows.map(|r| r.filter_map(|e| e.ok()).collect())
|
||||||
|
.unwrap_or_default()
|
||||||
|
}
|
||||||
|
|
||||||
|
fn is_virtual_folder(&self, path: &Path) -> Option<String> {
|
||||||
|
let path_str = path.to_string_lossy().to_string();
|
||||||
|
let normalized = normalize_folder(&path_str);
|
||||||
|
if normalized == "/" {
|
||||||
|
return None;
|
||||||
|
}
|
||||||
|
let folders = self.load_folders();
|
||||||
|
for folder in &folders {
|
||||||
|
if normalized == folder.as_str() || normalized.starts_with(&format!("{}/", folder)) {
|
||||||
|
return Some(folder.clone());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
None
|
||||||
|
}
|
||||||
|
|
||||||
|
fn resolve(&self, virtual_path: &Path) -> (PathBuf, Option<String>, Option<String>) {
|
||||||
|
let path_str = virtual_path.to_string_lossy().to_string();
|
||||||
|
let normalized = if path_str.starts_with('/') {
|
||||||
|
path_str.clone()
|
||||||
|
} else {
|
||||||
|
format!("/{}", path_str)
|
||||||
|
};
|
||||||
|
|
||||||
|
if normalized == "/" || normalized.is_empty() {
|
||||||
|
return (self.root.clone(), None, None);
|
||||||
|
}
|
||||||
|
|
||||||
|
let folders = self.load_folders();
|
||||||
|
for folder in &folders {
|
||||||
|
let folder_tag = folder_to_tag(folder);
|
||||||
|
let folder_prefix = format!("{}/", folder);
|
||||||
|
if normalized == folder.as_str() {
|
||||||
|
return (self.root.clone(), Some(folder_tag), None);
|
||||||
|
}
|
||||||
|
if normalized.starts_with(&folder_prefix) {
|
||||||
|
let filename = normalized[folder_prefix.len()..].to_string();
|
||||||
|
return (self.root.join(&filename), Some(folder_tag), Some(filename));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
let relative = normalized.trim_start_matches('/');
|
||||||
|
(self.root.join(relative), None, Some(relative.to_string()))
|
||||||
|
}
|
||||||
|
|
||||||
|
fn check_writable(&self) -> Result<(), VfsError> {
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn add_folder(&self, folder: &str, description: &str) -> Result<(), VfsError> {
|
||||||
|
let db = self.db.lock().unwrap();
|
||||||
|
let f = normalize_folder(folder);
|
||||||
|
db.execute(
|
||||||
|
"INSERT OR IGNORE INTO virtual_folders (folder, description) VALUES (?1, ?2)",
|
||||||
|
params![f, description],
|
||||||
|
)
|
||||||
|
.map_err(|e| VfsError::Io(e.to_string()))?;
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn remove_folder(&self, folder: &str) -> Result<(), VfsError> {
|
||||||
|
let db = self.db.lock().unwrap();
|
||||||
|
let f = normalize_folder(folder);
|
||||||
|
let tag = folder_to_tag(&f);
|
||||||
|
db.execute("DELETE FROM file_tags WHERE tag = ?1", params![tag])
|
||||||
|
.map_err(|e| VfsError::Io(e.to_string()))?;
|
||||||
|
db.execute("DELETE FROM virtual_folders WHERE folder = ?1", params![f])
|
||||||
|
.map_err(|e| VfsError::Io(e.to_string()))?;
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn tag_file(&self, filename: &str, tag: &str) -> Result<(), VfsError> {
|
||||||
|
let db = self.db.lock().unwrap();
|
||||||
|
db.execute(
|
||||||
|
"INSERT OR IGNORE INTO file_tags (filename, tag) VALUES (?1, ?2)",
|
||||||
|
params![filename, tag],
|
||||||
|
)
|
||||||
|
.map_err(|e| VfsError::Io(e.to_string()))?;
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn untag_file(&self, filename: &str, tag: &str) -> Result<(), VfsError> {
|
||||||
|
let db = self.db.lock().unwrap();
|
||||||
|
db.execute(
|
||||||
|
"DELETE FROM file_tags WHERE filename = ?1 AND tag = ?2",
|
||||||
|
params![filename, tag],
|
||||||
|
)
|
||||||
|
.map_err(|e| VfsError::Io(e.to_string()))?;
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn list_folders(&self) -> Result<Vec<(String, String)>, VfsError> {
|
||||||
|
let db = self.db.lock().unwrap();
|
||||||
|
let mut stmt = db
|
||||||
|
.prepare("SELECT folder, description FROM virtual_folders ORDER BY folder")
|
||||||
|
.map_err(|e| VfsError::Io(e.to_string()))?;
|
||||||
|
let rows = stmt
|
||||||
|
.query_map([], |row| Ok((row.get(0)?, row.get(1)?)))
|
||||||
|
.map_err(|e| VfsError::Io(e.to_string()))?;
|
||||||
|
Ok(rows.filter_map(|r| r.ok()).collect())
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn list_files_in_folder(&self, folder: &str) -> Result<Vec<String>, VfsError> {
|
||||||
|
let tag = folder_to_tag(folder);
|
||||||
|
Ok(self.files_with_tag(&tag))
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn list_tags_for_file(&self, filename: &str) -> Result<Vec<String>, VfsError> {
|
||||||
|
Ok(self.tags_for_file(filename))
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn set_config(&self, key: &str, value: &str) -> Result<(), VfsError> {
|
||||||
|
let db = self.db.lock().unwrap();
|
||||||
|
db.execute(
|
||||||
|
"INSERT OR REPLACE INTO webdav_config (key, value) VALUES (?1, ?2)",
|
||||||
|
params![key, value],
|
||||||
|
)
|
||||||
|
.map_err(|e| VfsError::Io(e.to_string()))?;
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn get_config(&self, key: &str) -> Option<String> {
|
||||||
|
let db = self.db.lock().unwrap();
|
||||||
|
db.query_row(
|
||||||
|
"SELECT value FROM webdav_config WHERE key = ?1",
|
||||||
|
params![key],
|
||||||
|
|row| row.get(0),
|
||||||
|
)
|
||||||
|
.ok()
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn root(&self) -> &Path {
|
||||||
|
&self.root
|
||||||
|
}
|
||||||
|
|
||||||
|
fn remove_all_tags(&self, filename: &str) -> Result<(), VfsError> {
|
||||||
|
let db = self.db.lock().unwrap();
|
||||||
|
db.execute("DELETE FROM file_tags WHERE filename = ?1", params![filename])
|
||||||
|
.map_err(|e| VfsError::Io(e.to_string()))?;
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
fn move_tags(&self, old_filename: &str, new_filename: &str) -> Result<(), VfsError> {
|
||||||
|
let db = self.db.lock().unwrap();
|
||||||
|
db.execute(
|
||||||
|
"UPDATE OR IGNORE file_tags SET filename = ?1 WHERE filename = ?2",
|
||||||
|
params![new_filename, old_filename],
|
||||||
|
)
|
||||||
|
.map_err(|e| VfsError::Io(e.to_string()))?;
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
fn make_dir_entry(name: &str) -> VfsDirEntry {
|
||||||
|
let mut stat = VfsStat::new();
|
||||||
|
stat.is_dir = true;
|
||||||
|
stat.mode = 0o755;
|
||||||
|
VfsDirEntry {
|
||||||
|
name: name.to_string(),
|
||||||
|
long_name: name.to_string(),
|
||||||
|
stat,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn make_file_entry(name: &str, stat: VfsStat) -> VfsDirEntry {
|
||||||
|
VfsDirEntry {
|
||||||
|
name: name.to_string(),
|
||||||
|
long_name: name.to_string(),
|
||||||
|
stat,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl VfsBackend for VirtualFs {
|
||||||
|
fn clone_boxed(&self) -> Box<dyn VfsBackend> {
|
||||||
|
Box::new(VirtualFs {
|
||||||
|
db: self.db.clone(),
|
||||||
|
backend: self.backend.clone_boxed(),
|
||||||
|
root: self.root.clone(),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
fn read_dir(&self, path: &Path) -> Result<Vec<VfsDirEntry>, VfsError> {
|
||||||
|
let (real, folder_tag, _) = self.resolve(path);
|
||||||
|
|
||||||
|
if let Some(tag) = folder_tag {
|
||||||
|
let files = self.files_with_tag(&tag);
|
||||||
|
let mut entries = Vec::new();
|
||||||
|
for filename in &files {
|
||||||
|
let file_path = self.root.join(filename);
|
||||||
|
if let Ok(stat) = self.backend.stat(&file_path) {
|
||||||
|
let name = Path::new(filename)
|
||||||
|
.file_name()
|
||||||
|
.map(|n| n.to_string_lossy().to_string())
|
||||||
|
.unwrap_or_else(|| filename.clone());
|
||||||
|
entries.push(Self::make_file_entry(&name, stat));
|
||||||
|
} else {
|
||||||
|
let stat = VfsStat::new();
|
||||||
|
entries.push(Self::make_file_entry(filename, stat));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return Ok(entries);
|
||||||
|
}
|
||||||
|
|
||||||
|
let mut entries = self.backend.read_dir(&real)?;
|
||||||
|
|
||||||
|
let folders = self.load_folders();
|
||||||
|
for folder in &folders {
|
||||||
|
let name = folder.trim_start_matches('/').to_string();
|
||||||
|
if !entries.iter().any(|e| e.name == name) {
|
||||||
|
entries.push(Self::make_dir_entry(&name));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Ok(entries)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn open_file(
|
||||||
|
&self,
|
||||||
|
path: &Path,
|
||||||
|
flags: &OpenFlags,
|
||||||
|
) -> Result<Box<dyn VfsFile>, VfsError> {
|
||||||
|
self.check_writable()?;
|
||||||
|
|
||||||
|
if flags.create {
|
||||||
|
let (real, folder_tag, filename) = self.resolve(path);
|
||||||
|
let file = self.backend.open_file(&real, flags)?;
|
||||||
|
|
||||||
|
if let (Some(tag), Some(fname)) = (folder_tag, filename.as_ref()) {
|
||||||
|
let _ = self.tag_file(fname, &tag);
|
||||||
|
}
|
||||||
|
return Ok(file);
|
||||||
|
}
|
||||||
|
|
||||||
|
let (real, _, _) = self.resolve(path);
|
||||||
|
self.backend.open_file(&real, flags)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn stat(&self, path: &Path) -> Result<VfsStat, VfsError> {
|
||||||
|
let (real, folder_tag, _) = self.resolve(path);
|
||||||
|
|
||||||
|
if folder_tag.is_some() && !real.exists() {
|
||||||
|
let file_count = match &folder_tag {
|
||||||
|
Some(tag) => self.files_with_tag(tag).len(),
|
||||||
|
None => 0,
|
||||||
|
};
|
||||||
|
if file_count > 0 || path == Path::new("/") {
|
||||||
|
let mut stat = VfsStat::new();
|
||||||
|
stat.is_dir = true;
|
||||||
|
stat.mode = 0o755;
|
||||||
|
return Ok(stat);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if folder_tag.is_some() && real == self.root {
|
||||||
|
let mut stat = VfsStat::new();
|
||||||
|
stat.is_dir = true;
|
||||||
|
stat.mode = 0o755;
|
||||||
|
return Ok(stat);
|
||||||
|
}
|
||||||
|
|
||||||
|
self.backend.stat(&real)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn lstat(&self, path: &Path) -> Result<VfsStat, VfsError> {
|
||||||
|
self.stat(path)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn create_dir(&self, path: &Path, _mode: u32) -> Result<(), VfsError> {
|
||||||
|
let path_str = path.to_string_lossy().to_string();
|
||||||
|
let normalized = normalize_folder(&path_str);
|
||||||
|
|
||||||
|
if normalized == "/" {
|
||||||
|
return Err(VfsError::AlreadyExists("/".to_string()));
|
||||||
|
}
|
||||||
|
|
||||||
|
if self.is_virtual_folder(path).is_some() {
|
||||||
|
return Err(VfsError::AlreadyExists(normalized));
|
||||||
|
}
|
||||||
|
|
||||||
|
self.add_folder(&normalized, "")?;
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
fn create_dir_all(&self, path: &Path, mode: u32) -> Result<(), VfsError> {
|
||||||
|
self.create_dir(path, mode)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn remove_dir(&self, path: &Path) -> Result<(), VfsError> {
|
||||||
|
let path_str = path.to_string_lossy().to_string();
|
||||||
|
let normalized = normalize_folder(&path_str);
|
||||||
|
|
||||||
|
if self.is_virtual_folder(path).is_some() {
|
||||||
|
self.remove_folder(&normalized)?;
|
||||||
|
return Ok(());
|
||||||
|
}
|
||||||
|
|
||||||
|
let (real, _, _) = self.resolve(path);
|
||||||
|
self.backend.remove_dir(&real)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn remove_file(&self, path: &Path) -> Result<(), VfsError> {
|
||||||
|
self.check_writable()?;
|
||||||
|
|
||||||
|
if self.is_virtual_folder(path).is_some() {
|
||||||
|
let (_, folder_tag, filename) = self.resolve(path);
|
||||||
|
if let (Some(tag), Some(fname)) = (folder_tag, filename) {
|
||||||
|
self.untag_file(&fname, &tag)?;
|
||||||
|
return Ok(());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
let (real, _, filename) = self.resolve(path);
|
||||||
|
self.backend.remove_file(&real)?;
|
||||||
|
|
||||||
|
if let Some(fname) = filename {
|
||||||
|
let _ = self.remove_all_tags(&fname);
|
||||||
|
}
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
fn rename(&self, from: &Path, to: &Path) -> Result<(), VfsError> {
|
||||||
|
self.check_writable()?;
|
||||||
|
|
||||||
|
let (_, from_tag, from_filename) = self.resolve(from);
|
||||||
|
let (_, to_tag, _to_filename) = self.resolve(to);
|
||||||
|
|
||||||
|
let from_is_folder = from_tag.is_some() && from_filename.is_none();
|
||||||
|
|
||||||
|
if from_is_folder {
|
||||||
|
return Err(VfsError::Unsupported("Cannot rename virtual folder".to_string()));
|
||||||
|
}
|
||||||
|
|
||||||
|
match (from_tag, to_tag, from_filename.as_ref()) {
|
||||||
|
(Some(ft), Some(tt), Some(fname)) => {
|
||||||
|
if ft != tt {
|
||||||
|
self.untag_file(fname, &ft)?;
|
||||||
|
self.tag_file(fname, &tt)?;
|
||||||
|
}
|
||||||
|
return Ok(());
|
||||||
|
}
|
||||||
|
(None, Some(tt), Some(fname)) => {
|
||||||
|
self.tag_file(fname, &tt)?;
|
||||||
|
return Ok(());
|
||||||
|
}
|
||||||
|
(Some(ft), None, Some(fname)) => {
|
||||||
|
self.untag_file(fname, &ft)?;
|
||||||
|
return Ok(());
|
||||||
|
}
|
||||||
|
_ => {}
|
||||||
|
}
|
||||||
|
|
||||||
|
if self.is_virtual_folder(to).is_some() {
|
||||||
|
let (_, from_tag, from_filename) = self.resolve(from);
|
||||||
|
let (_, to_tag, _) = self.resolve(to);
|
||||||
|
|
||||||
|
match (from_tag, to_tag, from_filename) {
|
||||||
|
(Some(ft), Some(tt), Some(fname)) => {
|
||||||
|
if ft != tt {
|
||||||
|
self.untag_file(&fname, &ft)?;
|
||||||
|
self.tag_file(&fname, &tt)?;
|
||||||
|
}
|
||||||
|
return Ok(());
|
||||||
|
}
|
||||||
|
(None, Some(tt), Some(fname)) => {
|
||||||
|
self.tag_file(&fname, &tt)?;
|
||||||
|
return Ok(());
|
||||||
|
}
|
||||||
|
(Some(ft), None, Some(fname)) => {
|
||||||
|
self.untag_file(&fname, &ft)?;
|
||||||
|
return Ok(());
|
||||||
|
}
|
||||||
|
_ => {}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
let (real_from, _, from_filename) = self.resolve(from);
|
||||||
|
let (real_to, _, to_filename) = self.resolve(to);
|
||||||
|
|
||||||
|
self.backend.rename(&real_from, &real_to)?;
|
||||||
|
|
||||||
|
if let (Some(old_name), Some(new_name)) = (from_filename, to_filename) {
|
||||||
|
self.move_tags(&old_name, &new_name)?;
|
||||||
|
}
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
fn set_stat(&self, path: &Path, stat: &VfsStat) -> Result<(), VfsError> {
|
||||||
|
self.check_writable()?;
|
||||||
|
let (real, _, _) = self.resolve(path);
|
||||||
|
self.backend.set_stat(&real, stat)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn read_link(&self, path: &Path) -> Result<PathBuf, VfsError> {
|
||||||
|
let (real, _, _) = self.resolve(path);
|
||||||
|
self.backend.read_link(&real)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn create_symlink(&self, target: &Path, link: &Path) -> Result<(), VfsError> {
|
||||||
|
self.check_writable()?;
|
||||||
|
let (real_target, _, _) = self.resolve(target);
|
||||||
|
let (real_link, _, _) = self.resolve(link);
|
||||||
|
self.backend.create_symlink(&real_target, &real_link)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn real_path(&self, path: &Path) -> Result<PathBuf, VfsError> {
|
||||||
|
let (real, _, _) = self.resolve(path);
|
||||||
|
self.backend.real_path(&real)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn exists(&self, path: &Path) -> bool {
|
||||||
|
if self.is_virtual_folder(path).is_some() {
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
let (real, folder_tag, filename) = self.resolve(path);
|
||||||
|
|
||||||
|
if let Some(tag) = folder_tag {
|
||||||
|
if let Some(fname) = filename {
|
||||||
|
return self.backend.exists(&real)
|
||||||
|
|| self.tags_for_file(&fname).contains(&tag);
|
||||||
|
}
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
self.backend.exists(&real)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn hard_link(&self, original: &Path, link: &Path) -> Result<(), VfsError> {
|
||||||
|
self.check_writable()?;
|
||||||
|
let (real_orig, _, _) = self.resolve(original);
|
||||||
|
let (real_link, _, _) = self.resolve(link);
|
||||||
|
self.backend.hard_link(&real_orig, &real_link)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use super::*;
|
||||||
|
use tempfile::TempDir;
|
||||||
|
|
||||||
|
fn setup() -> (TempDir, VirtualFs, TempDir) {
|
||||||
|
let db_dir = TempDir::new().unwrap();
|
||||||
|
let db_path = db_dir.path().join("test_vfs.sqlite");
|
||||||
|
let root_dir = TempDir::new().unwrap();
|
||||||
|
std::fs::write(root_dir.path().join("flower.jpg"), "image").unwrap();
|
||||||
|
std::fs::write(root_dir.path().join("report.pdf"), "pdf content").unwrap();
|
||||||
|
std::fs::write(root_dir.path().join("sunset.jpg"), "sunset").unwrap();
|
||||||
|
|
||||||
|
let vfs = VirtualFs::new(db_path.to_str().unwrap(), root_dir.path().to_path_buf())
|
||||||
|
.unwrap();
|
||||||
|
(db_dir, vfs, root_dir)
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_add_folder() {
|
||||||
|
let (_db, vfs, _root) = setup();
|
||||||
|
vfs.add_folder("/photos", "Photo collection").unwrap();
|
||||||
|
let folders = vfs.list_folders().unwrap();
|
||||||
|
assert_eq!(folders.len(), 1);
|
||||||
|
assert_eq!(folders[0].0, "/photos");
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_remove_folder() {
|
||||||
|
let (_db, vfs, _root) = setup();
|
||||||
|
vfs.add_folder("/photos", "").unwrap();
|
||||||
|
vfs.tag_file("flower.jpg", "photos").unwrap();
|
||||||
|
assert_eq!(vfs.files_with_tag("photos").len(), 1);
|
||||||
|
|
||||||
|
vfs.remove_folder("/photos").unwrap();
|
||||||
|
assert_eq!(vfs.list_folders().unwrap().len(), 0);
|
||||||
|
assert_eq!(vfs.files_with_tag("photos").len(), 0);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_tag_file() {
|
||||||
|
let (_db, vfs, _root) = setup();
|
||||||
|
vfs.add_folder("/photos", "").unwrap();
|
||||||
|
vfs.add_folder("/nature", "").unwrap();
|
||||||
|
|
||||||
|
vfs.tag_file("flower.jpg", "photos").unwrap();
|
||||||
|
vfs.tag_file("flower.jpg", "nature").unwrap();
|
||||||
|
|
||||||
|
let tags = vfs.tags_for_file("flower.jpg");
|
||||||
|
assert!(tags.contains(&"photos".to_string()));
|
||||||
|
assert!(tags.contains(&"nature".to_string()));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_read_dir_root_shows_folders() {
|
||||||
|
let (_db, vfs, _root) = setup();
|
||||||
|
vfs.add_folder("/photos", "").unwrap();
|
||||||
|
|
||||||
|
let entries = vfs.read_dir(Path::new("/")).unwrap();
|
||||||
|
assert!(entries.iter().any(|e| e.name == "photos" && e.stat.is_dir));
|
||||||
|
assert!(entries.iter().any(|e| e.name == "flower.jpg"));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_read_dir_virtual_folder() {
|
||||||
|
let (_db, vfs, _root) = setup();
|
||||||
|
vfs.add_folder("/photos", "").unwrap();
|
||||||
|
vfs.tag_file("flower.jpg", "photos").unwrap();
|
||||||
|
vfs.tag_file("sunset.jpg", "photos").unwrap();
|
||||||
|
|
||||||
|
let entries = vfs.read_dir(Path::new("/photos")).unwrap();
|
||||||
|
let names: Vec<_> = entries.iter().map(|e| e.name.clone()).collect();
|
||||||
|
assert!(names.contains(&"flower.jpg".to_string()));
|
||||||
|
assert!(names.contains(&"sunset.jpg".to_string()));
|
||||||
|
assert!(!names.contains(&"report.pdf".to_string()));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_open_file_in_virtual_folder() {
|
||||||
|
let (_db, vfs, _root) = setup();
|
||||||
|
vfs.add_folder("/photos", "").unwrap();
|
||||||
|
|
||||||
|
let flags = OpenFlags::new().read();
|
||||||
|
let file = vfs.open_file(Path::new("/photos/flower.jpg"), &flags).unwrap();
|
||||||
|
drop(file);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_create_file_auto_tags() {
|
||||||
|
let (_db, vfs, _root) = setup();
|
||||||
|
vfs.add_folder("/docs", "").unwrap();
|
||||||
|
|
||||||
|
let flags = OpenFlags::new().write().create().truncate();
|
||||||
|
let file = vfs.open_file(Path::new("/docs/newfile.txt"), &flags).unwrap();
|
||||||
|
drop(file);
|
||||||
|
|
||||||
|
let tags = vfs.tags_for_file("newfile.txt");
|
||||||
|
assert!(tags.contains(&"docs".to_string()));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_remove_file_from_virtual_folder_untags() {
|
||||||
|
let (_db, vfs, _root) = setup();
|
||||||
|
vfs.add_folder("/photos", "").unwrap();
|
||||||
|
vfs.tag_file("flower.jpg", "photos").unwrap();
|
||||||
|
|
||||||
|
vfs.remove_file(Path::new("/photos/flower.jpg")).unwrap();
|
||||||
|
|
||||||
|
assert!(std::path::Path::new(&vfs.root().join("flower.jpg")).exists());
|
||||||
|
let tags = vfs.tags_for_file("flower.jpg");
|
||||||
|
assert!(!tags.contains(&"photos".to_string()));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_remove_file_from_root_deletes() {
|
||||||
|
let (_db, vfs, root) = setup();
|
||||||
|
vfs.tag_file("flower.jpg", "photos").unwrap();
|
||||||
|
|
||||||
|
vfs.remove_file(Path::new("/flower.jpg")).unwrap();
|
||||||
|
assert!(!root.path().join("flower.jpg").exists());
|
||||||
|
assert_eq!(vfs.tags_for_file("flower.jpg").len(), 0);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_stat_virtual_folder() {
|
||||||
|
let (_db, vfs, _root) = setup();
|
||||||
|
vfs.add_folder("/photos", "").unwrap();
|
||||||
|
vfs.tag_file("flower.jpg", "photos").unwrap();
|
||||||
|
|
||||||
|
let stat = vfs.stat(Path::new("/photos")).unwrap();
|
||||||
|
assert!(stat.is_dir);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_stat_file_in_virtual_folder() {
|
||||||
|
let (_db, vfs, _root) = setup();
|
||||||
|
vfs.add_folder("/photos", "").unwrap();
|
||||||
|
vfs.tag_file("flower.jpg", "photos").unwrap();
|
||||||
|
|
||||||
|
let stat = vfs.stat(Path::new("/photos/flower.jpg")).unwrap();
|
||||||
|
assert_eq!(stat.size, 5);
|
||||||
|
assert!(!stat.is_dir);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_same_file_multiple_tags() {
|
||||||
|
let (_db, vfs, root) = setup();
|
||||||
|
vfs.add_folder("/photos", "").unwrap();
|
||||||
|
vfs.add_folder("/nature", "").unwrap();
|
||||||
|
|
||||||
|
vfs.tag_file("flower.jpg", "photos").unwrap();
|
||||||
|
vfs.tag_file("flower.jpg", "nature").unwrap();
|
||||||
|
|
||||||
|
let photos_entries = vfs.read_dir(Path::new("/photos")).unwrap();
|
||||||
|
let nature_entries = vfs.read_dir(Path::new("/nature")).unwrap();
|
||||||
|
|
||||||
|
assert!(photos_entries.iter().any(|e| e.name == "flower.jpg"));
|
||||||
|
assert!(nature_entries.iter().any(|e| e.name == "flower.jpg"));
|
||||||
|
|
||||||
|
assert!(root.path().join("flower.jpg").exists());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_rename_adds_tag() {
|
||||||
|
let (_db, vfs, _root) = setup();
|
||||||
|
vfs.add_folder("/photos", "").unwrap();
|
||||||
|
|
||||||
|
vfs.rename(Path::new("/flower.jpg"), Path::new("/photos/flower.jpg"))
|
||||||
|
.unwrap();
|
||||||
|
|
||||||
|
let tags = vfs.tags_for_file("flower.jpg");
|
||||||
|
assert!(tags.contains(&"photos".to_string()));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_rename_removes_tag() {
|
||||||
|
let (_db, vfs, _root) = setup();
|
||||||
|
vfs.add_folder("/photos", "").unwrap();
|
||||||
|
vfs.tag_file("flower.jpg", "photos").unwrap();
|
||||||
|
|
||||||
|
vfs.rename(Path::new("/photos/flower.jpg"), Path::new("/flower.jpg"))
|
||||||
|
.unwrap();
|
||||||
|
|
||||||
|
let tags = vfs.tags_for_file("flower.jpg");
|
||||||
|
assert!(!tags.contains(&"photos".to_string()));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_create_virtual_dir() {
|
||||||
|
let (_db, vfs, _root) = setup();
|
||||||
|
vfs.create_dir(Path::new("/newfolder"), 0o755).unwrap();
|
||||||
|
let folders = vfs.list_folders().unwrap();
|
||||||
|
assert!(folders.iter().any(|(f, _)| f == "/newfolder"));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_config() {
|
||||||
|
let (_db, vfs, _root) = setup();
|
||||||
|
vfs.set_config("default_root", "/data/demo").unwrap();
|
||||||
|
assert_eq!(vfs.get_config("default_root"), Some("/data/demo".to_string()));
|
||||||
|
}
|
||||||
|
}
|
||||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user