diff --git a/data/auth.sqlite b/data/auth.sqlite index 1633175..ec07aa6 100644 Binary files a/data/auth.sqlite and b/data/auth.sqlite differ diff --git a/data/webdav_versions/07ea0e78-2d67-4fdc-8ddb-ea41c4ca091a b/data/webdav_versions/07ea0e78-2d67-4fdc-8ddb-ea41c4ca091a new file mode 100644 index 0000000..9baa2d9 --- /dev/null +++ b/data/webdav_versions/07ea0e78-2d67-4fdc-8ddb-ea41c4ca091a @@ -0,0 +1 @@ +Small test content diff --git a/data/webdav_versions/fa52dfd7-54af-4f37-8b2c-add0c371f4df b/data/webdav_versions/fa52dfd7-54af-4f37-8b2c-add0c371f4df new file mode 100644 index 0000000..9baa2d9 --- /dev/null +++ b/data/webdav_versions/fa52dfd7-54af-4f37-8b2c-add0c371f4df @@ -0,0 +1 @@ +Small test content diff --git a/data/webdav_versions/version_index.json b/data/webdav_versions/version_index.json index ead1d75..f725c2a 100644 --- a/data/webdav_versions/version_index.json +++ b/data/webdav_versions/version_index.json @@ -1 +1 @@ -{"history:/Users/accusys/momentry/var/sftpgo/data/demo/test_put_fixed.txt:info":[123,34,102,105,108,101,95,112,97,116,104,34,58,34,47,85,115,101,114,115,47,97,99,99,117,115,121,115,47,109,111,109,101,110,116,114,121,47,118,97,114,47,115,102,116,112,103,111,47,100,97,116,97,47,100,101,109,111,47,116,101,115,116,95,112,117,116,95,102,105,120,101,100,46,116,120,116,34,44,34,118,101,114,115,105,111,110,115,34,58,91,123,34,118,101,114,115,105,111,110,95,105,100,34,58,34,52,99,101,51,97,57,50,49,45,98,51,98,50,45,52,53,99,48,45,98,52,57,100,45,98,48,57,53,100,98,53,100,50,49,49,51,34,44,34,102,105,108,101,95,112,97,116,104,34,58,34,47,85,115,101,114,115,47,97,99,99,117,115,121,115,47,109,111,109,101,110,116,114,121,47,118,97,114,47,115,102,116,112,103,111,47,100,97,116,97,47,100,101,109,111,47,116,101,115,116,95,112,117,116,95,102,105,120,101,100,46,116,120,116,34,44,34,99,114,101,97,116,101,100,95,97,116,34,58,123,34,115,101,99,115,95,115,105,110,99,101,95,101,112,111,99,104,34,58,49,55,56,50,55,54,53,52,55,55,44,34,110,97,110,111,115,95,115,105,110,99,101,95,101,112,111,99,104,34,58,56,52,53,55,57,48,48,48,125,44,34,115,105,122,101,34,58,51,49,44,34,99,104,101,99,107,115,117,109,34,58,34,51,50,101,98,53,98,55,102,100,57,97,56,97,52,55,52,102,52,98,99,98,57,101,100,57,99,53,100,98,99,102,54,100,100,51,97,51,51,48,50,52,52,56,97,49,57,53,100,56,98,51,53,57,48,100,97,98,56,48,57,101,98,48,50,34,44,34,97,117,116,104,111,114,34,58,110,117,108,108,44,34,99,111,109,109,101,110,116,34,58,110,117,108,108,44,34,105,115,95,99,117,114,114,101,110,116,34,58,116,114,117,101,125,93,44,34,99,117,114,114,101,110,116,95,118,101,114,115,105,111,110,34,58,34,52,99,101,51,97,57,50,49,45,98,51,98,50,45,52,53,99,48,45,98,52,57,100,45,98,48,57,53,100,98,53,100,50,49,49,51,34,44,34,116,111,116,97,108,95,118,101,114,115,105,111,110,115,34,58,49,125],"version:/Users/accusys/momentry/var/sftpgo/data/demo/test_put_fixed.txt:4ce3a921-b3b2-45c0-b49d-b095db5d2113":[123,34,118,101,114,115,105,111,110,95,105,100,34,58,34,52,99,101,51,97,57,50,49,45,98,51,98,50,45,52,53,99,48,45,98,52,57,100,45,98,48,57,53,100,98,53,100,50,49,49,51,34,44,34,102,105,108,101,95,112,97,116,104,34,58,34,47,85,115,101,114,115,47,97,99,99,117,115,121,115,47,109,111,109,101,110,116,114,121,47,118,97,114,47,115,102,116,112,103,111,47,100,97,116,97,47,100,101,109,111,47,116,101,115,116,95,112,117,116,95,102,105,120,101,100,46,116,120,116,34,44,34,99,114,101,97,116,101,100,95,97,116,34,58,123,34,115,101,99,115,95,115,105,110,99,101,95,101,112,111,99,104,34,58,49,55,56,50,55,54,53,52,55,55,44,34,110,97,110,111,115,95,115,105,110,99,101,95,101,112,111,99,104,34,58,56,52,53,55,57,48,48,48,125,44,34,115,105,122,101,34,58,51,49,44,34,99,104,101,99,107,115,117,109,34,58,34,51,50,101,98,53,98,55,102,100,57,97,56,97,52,55,52,102,52,98,99,98,57,101,100,57,99,53,100,98,99,102,54,100,100,51,97,51,51,48,50,52,52,56,97,49,57,53,100,56,98,51,53,57,48,100,97,98,56,48,57,101,98,48,50,34,44,34,97,117,116,104,111,114,34,58,110,117,108,108,44,34,99,111,109,109,101,110,116,34,58,110,117,108,108,44,34,105,115,95,99,117,114,114,101,110,116,34,58,116,114,117,101,125]} \ No newline at end of file +{"version:/Users/accusys/momentry/var/sftpgo/data/demo/test_put_fixed.txt:4ce3a921-b3b2-45c0-b49d-b095db5d2113":[123,34,118,101,114,115,105,111,110,95,105,100,34,58,34,52,99,101,51,97,57,50,49,45,98,51,98,50,45,52,53,99,48,45,98,52,57,100,45,98,48,57,53,100,98,53,100,50,49,49,51,34,44,34,102,105,108,101,95,112,97,116,104,34,58,34,47,85,115,101,114,115,47,97,99,99,117,115,121,115,47,109,111,109,101,110,116,114,121,47,118,97,114,47,115,102,116,112,103,111,47,100,97,116,97,47,100,101,109,111,47,116,101,115,116,95,112,117,116,95,102,105,120,101,100,46,116,120,116,34,44,34,99,114,101,97,116,101,100,95,97,116,34,58,123,34,115,101,99,115,95,115,105,110,99,101,95,101,112,111,99,104,34,58,49,55,56,50,55,54,53,52,55,55,44,34,110,97,110,111,115,95,115,105,110,99,101,95,101,112,111,99,104,34,58,56,52,53,55,57,48,48,48,125,44,34,115,105,122,101,34,58,51,49,44,34,99,104,101,99,107,115,117,109,34,58,34,51,50,101,98,53,98,55,102,100,57,97,56,97,52,55,52,102,52,98,99,98,57,101,100,57,99,53,100,98,99,102,54,100,100,51,97,51,51,48,50,52,52,56,97,49,57,53,100,56,98,51,53,57,48,100,97,98,56,48,57,101,98,48,50,34,44,34,97,117,116,104,111,114,34,58,110,117,108,108,44,34,99,111,109,109,101,110,116,34,58,110,117,108,108,44,34,105,115,95,99,117,114,114,101,110,116,34,58,116,114,117,101,125],"history:/Users/accusys/momentry/var/sftpgo/data/demo/test_thread.txt:info":[123,34,102,105,108,101,95,112,97,116,104,34,58,34,47,85,115,101,114,115,47,97,99,99,117,115,121,115,47,109,111,109,101,110,116,114,121,47,118,97,114,47,115,102,116,112,103,111,47,100,97,116,97,47,100,101,109,111,47,116,101,115,116,95,116,104,114,101,97,100,46,116,120,116,34,44,34,118,101,114,115,105,111,110,115,34,58,91,123,34,118,101,114,115,105,111,110,95,105,100,34,58,34,48,55,101,97,48,101,55,56,45,50,100,54,55,45,52,102,100,99,45,56,100,100,98,45,101,97,52,49,99,52,99,97,48,57,49,97,34,44,34,102,105,108,101,95,112,97,116,104,34,58,34,47,85,115,101,114,115,47,97,99,99,117,115,121,115,47,109,111,109,101,110,116,114,121,47,118,97,114,47,115,102,116,112,103,111,47,100,97,116,97,47,100,101,109,111,47,116,101,115,116,95,116,104,114,101,97,100,46,116,120,116,34,44,34,99,114,101,97,116,101,100,95,97,116,34,58,123,34,115,101,99,115,95,115,105,110,99,101,95,101,112,111,99,104,34,58,49,55,56,50,55,54,56,52,52,54,44,34,110,97,110,111,115,95,115,105,110,99,101,95,101,112,111,99,104,34,58,54,57,53,48,49,52,48,48,48,125,44,34,115,105,122,101,34,58,49,57,44,34,99,104,101,99,107,115,117,109,34,58,34,102,53,97,55,56,100,98,50,57,102,97,55,50,57,102,53,98,50,99,98,52,99,97,101,55,51,101,56,100,55,99,55,49,57,50,48,49,55,54,97,97,54,49,57,54,100,51,51,48,99,99,101,51,100,101,52,98,49,97,97,54,101,102,53,34,44,34,97,117,116,104,111,114,34,58,110,117,108,108,44,34,99,111,109,109,101,110,116,34,58,110,117,108,108,44,34,105,115,95,99,117,114,114,101,110,116,34,58,116,114,117,101,125,93,44,34,99,117,114,114,101,110,116,95,118,101,114,115,105,111,110,34,58,34,48,55,101,97,48,101,55,56,45,50,100,54,55,45,52,102,100,99,45,56,100,100,98,45,101,97,52,49,99,52,99,97,48,57,49,97,34,44,34,116,111,116,97,108,95,118,101,114,115,105,111,110,115,34,58,49,125],"history:/Users/accusys/momentry/var/sftpgo/data/demo/test_put_fixed.txt:info":[123,34,102,105,108,101,95,112,97,116,104,34,58,34,47,85,115,101,114,115,47,97,99,99,117,115,121,115,47,109,111,109,101,110,116,114,121,47,118,97,114,47,115,102,116,112,103,111,47,100,97,116,97,47,100,101,109,111,47,116,101,115,116,95,112,117,116,95,102,105,120,101,100,46,116,120,116,34,44,34,118,101,114,115,105,111,110,115,34,58,91,123,34,118,101,114,115,105,111,110,95,105,100,34,58,34,52,99,101,51,97,57,50,49,45,98,51,98,50,45,52,53,99,48,45,98,52,57,100,45,98,48,57,53,100,98,53,100,50,49,49,51,34,44,34,102,105,108,101,95,112,97,116,104,34,58,34,47,85,115,101,114,115,47,97,99,99,117,115,121,115,47,109,111,109,101,110,116,114,121,47,118,97,114,47,115,102,116,112,103,111,47,100,97,116,97,47,100,101,109,111,47,116,101,115,116,95,112,117,116,95,102,105,120,101,100,46,116,120,116,34,44,34,99,114,101,97,116,101,100,95,97,116,34,58,123,34,115,101,99,115,95,115,105,110,99,101,95,101,112,111,99,104,34,58,49,55,56,50,55,54,53,52,55,55,44,34,110,97,110,111,115,95,115,105,110,99,101,95,101,112,111,99,104,34,58,56,52,53,55,57,48,48,48,125,44,34,115,105,122,101,34,58,51,49,44,34,99,104,101,99,107,115,117,109,34,58,34,51,50,101,98,53,98,55,102,100,57,97,56,97,52,55,52,102,52,98,99,98,57,101,100,57,99,53,100,98,99,102,54,100,100,51,97,51,51,48,50,52,52,56,97,49,57,53,100,56,98,51,53,57,48,100,97,98,56,48,57,101,98,48,50,34,44,34,97,117,116,104,111,114,34,58,110,117,108,108,44,34,99,111,109,109,101,110,116,34,58,110,117,108,108,44,34,105,115,95,99,117,114,114,101,110,116,34,58,116,114,117,101,125,93,44,34,99,117,114,114,101,110,116,95,118,101,114,115,105,111,110,34,58,34,52,99,101,51,97,57,50,49,45,98,51,98,50,45,52,53,99,48,45,98,52,57,100,45,98,48,57,53,100,98,53,100,50,49,49,51,34,44,34,116,111,116,97,108,95,118,101,114,115,105,111,110,115,34,58,49,125],"version:/Users/accusys/momentry/var/sftpgo/data/demo/test_flush_debug2.txt:fa52dfd7-54af-4f37-8b2c-add0c371f4df":[123,34,118,101,114,115,105,111,110,95,105,100,34,58,34,102,97,53,50,100,102,100,55,45,53,52,97,102,45,52,102,51,55,45,56,98,50,99,45,97,100,100,48,99,51,55,49,102,52,100,102,34,44,34,102,105,108,101,95,112,97,116,104,34,58,34,47,85,115,101,114,115,47,97,99,99,117,115,121,115,47,109,111,109,101,110,116,114,121,47,118,97,114,47,115,102,116,112,103,111,47,100,97,116,97,47,100,101,109,111,47,116,101,115,116,95,102,108,117,115,104,95,100,101,98,117,103,50,46,116,120,116,34,44,34,99,114,101,97,116,101,100,95,97,116,34,58,123,34,115,101,99,115,95,115,105,110,99,101,95,101,112,111,99,104,34,58,49,55,56,50,55,54,56,51,49,55,44,34,110,97,110,111,115,95,115,105,110,99,101,95,101,112,111,99,104,34,58,57,54,51,50,52,51,48,48,48,125,44,34,115,105,122,101,34,58,49,57,44,34,99,104,101,99,107,115,117,109,34,58,34,102,53,97,55,56,100,98,50,57,102,97,55,50,57,102,53,98,50,99,98,52,99,97,101,55,51,101,56,100,55,99,55,49,57,50,48,49,55,54,97,97,54,49,57,54,100,51,51,48,99,99,101,51,100,101,52,98,49,97,97,54,101,102,53,34,44,34,97,117,116,104,111,114,34,58,110,117,108,108,44,34,99,111,109,109,101,110,116,34,58,110,117,108,108,44,34,105,115,95,99,117,114,114,101,110,116,34,58,116,114,117,101,125],"history:/Users/accusys/momentry/var/sftpgo/data/demo/test_flush_debug2.txt:info":[123,34,102,105,108,101,95,112,97,116,104,34,58,34,47,85,115,101,114,115,47,97,99,99,117,115,121,115,47,109,111,109,101,110,116,114,121,47,118,97,114,47,115,102,116,112,103,111,47,100,97,116,97,47,100,101,109,111,47,116,101,115,116,95,102,108,117,115,104,95,100,101,98,117,103,50,46,116,120,116,34,44,34,118,101,114,115,105,111,110,115,34,58,91,123,34,118,101,114,115,105,111,110,95,105,100,34,58,34,102,97,53,50,100,102,100,55,45,53,52,97,102,45,52,102,51,55,45,56,98,50,99,45,97,100,100,48,99,51,55,49,102,52,100,102,34,44,34,102,105,108,101,95,112,97,116,104,34,58,34,47,85,115,101,114,115,47,97,99,99,117,115,121,115,47,109,111,109,101,110,116,114,121,47,118,97,114,47,115,102,116,112,103,111,47,100,97,116,97,47,100,101,109,111,47,116,101,115,116,95,102,108,117,115,104,95,100,101,98,117,103,50,46,116,120,116,34,44,34,99,114,101,97,116,101,100,95,97,116,34,58,123,34,115,101,99,115,95,115,105,110,99,101,95,101,112,111,99,104,34,58,49,55,56,50,55,54,56,51,49,55,44,34,110,97,110,111,115,95,115,105,110,99,101,95,101,112,111,99,104,34,58,57,54,51,50,52,51,48,48,48,125,44,34,115,105,122,101,34,58,49,57,44,34,99,104,101,99,107,115,117,109,34,58,34,102,53,97,55,56,100,98,50,57,102,97,55,50,57,102,53,98,50,99,98,52,99,97,101,55,51,101,56,100,55,99,55,49,57,50,48,49,55,54,97,97,54,49,57,54,100,51,51,48,99,99,101,51,100,101,52,98,49,97,97,54,101,102,53,34,44,34,97,117,116,104,111,114,34,58,110,117,108,108,44,34,99,111,109,109,101,110,116,34,58,110,117,108,108,44,34,105,115,95,99,117,114,114,101,110,116,34,58,116,114,117,101,125,93,44,34,99,117,114,114,101,110,116,95,118,101,114,115,105,111,110,34,58,34,102,97,53,50,100,102,100,55,45,53,52,97,102,45,52,102,51,55,45,56,98,50,99,45,97,100,100,48,99,51,55,49,102,52,100,102,34,44,34,116,111,116,97,108,95,118,101,114,115,105,111,110,115,34,58,49,125],"version:/Users/accusys/momentry/var/sftpgo/data/demo/test_thread.txt:07ea0e78-2d67-4fdc-8ddb-ea41c4ca091a":[123,34,118,101,114,115,105,111,110,95,105,100,34,58,34,48,55,101,97,48,101,55,56,45,50,100,54,55,45,52,102,100,99,45,56,100,100,98,45,101,97,52,49,99,52,99,97,48,57,49,97,34,44,34,102,105,108,101,95,112,97,116,104,34,58,34,47,85,115,101,114,115,47,97,99,99,117,115,121,115,47,109,111,109,101,110,116,114,121,47,118,97,114,47,115,102,116,112,103,111,47,100,97,116,97,47,100,101,109,111,47,116,101,115,116,95,116,104,114,101,97,100,46,116,120,116,34,44,34,99,114,101,97,116,101,100,95,97,116,34,58,123,34,115,101,99,115,95,115,105,110,99,101,95,101,112,111,99,104,34,58,49,55,56,50,55,54,56,52,52,54,44,34,110,97,110,111,115,95,115,105,110,99,101,95,101,112,111,99,104,34,58,54,57,53,48,49,52,48,48,48,125,44,34,115,105,122,101,34,58,49,57,44,34,99,104,101,99,107,115,117,109,34,58,34,102,53,97,55,56,100,98,50,57,102,97,55,50,57,102,53,98,50,99,98,52,99,97,101,55,51,101,56,100,55,99,55,49,57,50,48,49,55,54,97,97,54,49,57,54,100,51,51,48,99,99,101,51,100,101,52,98,49,97,97,54,101,102,53,34,44,34,97,117,116,104,111,114,34,58,110,117,108,108,44,34,99,111,109,109,101,110,116,34,58,110,117,108,108,44,34,105,115,95,99,117,114,114,101,110,116,34,58,116,114,117,101,125]} \ No newline at end of file diff --git a/markbase-core/src/server.rs b/markbase-core/src/server.rs index f730fb3..495f95f 100644 --- a/markbase-core/src/server.rs +++ b/markbase-core/src/server.rs @@ -167,6 +167,20 @@ pub async fn run(port: u16, file: Option) -> anyhow::Result<()> { Arc::new(crate::webdav_version::WebDavVersioning::new(vs)) }; + // Background thread to periodically flush version index (every 60 seconds) + let versioning_flush = webdav_versioning.clone(); + log::info!("Starting version index flush thread (interval=60s)"); + std::thread::spawn(move || { + log::info!("Version flush thread started"); + loop { + std::thread::sleep(std::time::Duration::from_secs(60)); + log::info!("Version flush thread waking up"); + if let Err(e) = versioning_flush.flush() { + log::warn!("Failed to flush version index: {:?}", e); + } + } + }); + log::info!( "WebDAV configured: parent={}, versioning={}, upload_hook={}, use_s3={}", webdav_parent.display(), @@ -2544,7 +2558,7 @@ fn create_handler_for_user( user_root, Some(upload_hook.clone()), username.to_string(), - None, // Disabled versioning to fix PUT timeout (save_index() blocks) + Some(versioning.clone()), // Re-enabled with incremental save locks_file, ) } diff --git a/markbase-core/src/webdav_version.rs b/markbase-core/src/webdav_version.rs index 0bc35b2..556c7fa 100644 --- a/markbase-core/src/webdav_version.rs +++ b/markbase-core/src/webdav_version.rs @@ -39,24 +39,31 @@ pub struct WebDavVersioning { db: Arc>>>, version_storage: PathBuf, index_path: PathBuf, + dirty: Arc>, // Track if index needs saving } impl WebDavVersioning { pub fn new(version_storage: PathBuf) -> Self { let index_path = version_storage.join("version_index.json"); let db = Arc::new(RwLock::new(HashMap::new())); + let dirty = Arc::new(RwLock::new(false)); - // Load persisted index from disk - // TEMPORARILY DISABLED for performance testing (index loading causes 10+ second delay) - // if index_path.exists() { - // if let Ok(json) = std::fs::read_to_string(&index_path) { - // if let Ok(map) = serde_json::from_str::>>(&json) { - // *recover_rwlock(db.write()) = map; - // } - // } - // } + // Load index asynchronously to avoid blocking OPTIONS/PROPFIND + let db_clone = db.clone(); + let index_path_clone = index_path.clone(); + std::thread::spawn(move || { + if index_path_clone.exists() { + if let Ok(json) = std::fs::read_to_string(&index_path_clone) { + if let Ok(map) = serde_json::from_str::>>(&json) { + let len = map.len(); + *recover_rwlock(db_clone.write()) = map; + log::info!("Loaded {} version entries from index", len); + } + } + } + }); - Self { db, version_storage, index_path } + Self { db, version_storage, index_path, dirty } } fn save_index(&self) -> Result<(), VersionError> { @@ -66,6 +73,18 @@ impl WebDavVersioning { Ok(()) } + /// Flush dirty index to disk (call periodically or on shutdown) + pub fn flush(&self) -> Result<(), VersionError> { + let dirty_flag = *recover_rwlock(self.dirty.read()); + log::info!("flush() called, dirty={}", dirty_flag); + if dirty_flag { + self.save_index()?; + *recover_rwlock(self.dirty.write()) = false; + log::info!("Flushed version index to disk"); + } + Ok(()) + } + pub fn create_version( &self, file_path: &str, @@ -104,7 +123,8 @@ impl WebDavVersioning { self.update_version_history(file_path, &version_id)?; - self.save_index()?; + // Mark dirty instead of immediate save (incremental save strategy) + *recover_rwlock(self.dirty.write()) = true; Ok(version_info) }