VFS/DataProvider/Config refactoring + SSH public key authentication

Phase 1-6 of refactoring plan:
- VFS abstraction (VfsBackend trait + LocalFs + OpenFlags builder)
- DataProvider trait (SqliteProvider + PgProvider, SFTPGo-compatible)
- Config refactoring (AppConfig unified sections, env overrides)
- SSH handlers (sftp/scp/rsync) migrated to VFS + DataProvider
- SSH public key authentication (Ed25519 signature verification)
- SSH stderr → CHANNEL_EXTENDED_DATA support
- Web auth uses DataProvider instead of direct SQL
- User home directory from provider (per-user isolation)
- PostgreSQL auth provider for SFTPGo compatibility

markbase-core/src/config/mod.rs

@@ -0,0 +1,233 @@
+pub mod web;
+
+use anyhow::Result;
+use serde::{Deserialize, Serialize};
+
+// Re-export web config for backward compatibility
+pub use web::*;
+
+/// Unified application configuration
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct AppConfig {
+    #[serde(default)]
+    pub web: WebSection,
+    #[serde(default)]
+    pub s3: S3Section,
+    #[serde(default)]
+    pub sftp: SftpSection,
+    #[serde(default)]
+    pub ssh: SshSection,
+}
+
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct WebSection {
+    pub host: String,
+    pub port: u16,
+    pub log_level: String,
+    pub auth_db_path: String,
+    pub users_db_dir: String,
+}
+
+impl Default for WebSection {
+    fn default() -> Self {
+        Self {
+            host: "127.0.0.1".to_string(),
+            port: 11438,
+            log_level: "info".to_string(),
+            auth_db_path: "data/auth.sqlite".to_string(),
+            users_db_dir: "data/users".to_string(),
+        }
+    }
+}
+
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct S3Section {
+    pub enabled: bool,
+    pub endpoint: String,
+    pub region: String,
+    pub require_auth: bool,
+    pub default_access_key: String,
+    pub default_secret_key: String,
+    pub keys_db_path: String,
+}
+
+impl Default for S3Section {
+    fn default() -> Self {
+        Self {
+            enabled: true,
+            endpoint: "http://localhost:11438/s3".to_string(),
+            region: "us-east-1".to_string(),
+            require_auth: false,
+            default_access_key: "markbase_access_key_001".to_string(),
+            default_secret_key: "markbase_secret_key_xyz123".to_string(),
+            keys_db_path: "data/s3_keys.json".to_string(),
+        }
+    }
+}
+
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct SftpSection {
+    pub enabled: bool,
+    pub port: u16,
+    pub base_path: String,
+    pub auth_db_path: String,
+    pub max_connections: usize,
+    pub chunk_size: usize,
+    pub require_path_validation: bool,
+    pub audit_logging: bool,
+    pub path_traversal_protection: bool,
+    pub symlink_check: bool,
+}
+
+impl Default for SftpSection {
+    fn default() -> Self {
+        Self {
+            enabled: true,
+            port: 2023,
+            base_path: "/Users/accusys/momentry/var/sftpgo/data".to_string(),
+            auth_db_path: "data/auth.sqlite".to_string(),
+            max_connections: 100,
+            chunk_size: 65536,
+            require_path_validation: true,
+            audit_logging: true,
+            path_traversal_protection: true,
+            symlink_check: true,
+        }
+    }
+}
+
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct SshSection {
+    pub enabled: bool,
+    pub port: u16,
+    pub bind_address: String,
+    pub security_config_path: String,
+}
+
+impl Default for SshSection {
+    fn default() -> Self {
+        Self {
+            enabled: true,
+            port: 2024,
+            bind_address: "127.0.0.1".to_string(),
+            security_config_path: "data/ssh_config.json".to_string(),
+        }
+    }
+}
+
+impl AppConfig {
+    pub fn load(path: &str) -> Result<Self> {
+        let config_path = std::path::PathBuf::from(path);
+
+        if !config_path.exists() {
+            log::warn!("Config file not found: {}, using defaults", path);
+            return Ok(Self::default());
+        }
+
+        let content = std::fs::read_to_string(&config_path)?;
+        let config: AppConfig = toml::from_str(&content)?;
+        log::info!("App config loaded from: {}", path);
+        Ok(config)
+    }
+
+    pub fn load_default() -> Result<Self> {
+        Self::load("config/app.toml")
+    }
+
+    pub fn save(&self, path: &str) -> Result<()> {
+        let config_path = std::path::PathBuf::from(path);
+
+        if config_path.exists() {
+            let backup_path = config_path.with_extension("toml.bak");
+            std::fs::copy(&config_path, &backup_path)?;
+            log::info!("Backup created: {}", backup_path.display());
+        }
+
+        let content = toml::to_string_pretty(self)?;
+        std::fs::write(&config_path, content)?;
+        log::info!("App config saved to: {}", path);
+        Ok(())
+    }
+
+    pub fn merge_env(&mut self) {
+        if let Ok(v) = std::env::var("MB_WEB_HOST") {
+            self.web.host = v;
+        }
+        if let Ok(v) = std::env::var("MB_WEB_PORT") {
+            if let Ok(p) = v.parse() { self.web.port = p; }
+        }
+        if let Ok(v) = std::env::var("MB_SSH_PORT") {
+            if let Ok(p) = v.parse() { self.ssh.port = p; }
+        }
+        if let Ok(v) = std::env::var("MB_SFTP_PORT") {
+            if let Ok(p) = v.parse() { self.sftp.port = p; }
+        }
+        if let Ok(v) = std::env::var("MB_S3_ENABLED") {
+            self.s3.enabled = v == "true" || v == "1";
+        }
+        if let Ok(v) = std::env::var("MB_AUTH_DB") {
+            self.web.auth_db_path = v.clone();
+            self.sftp.auth_db_path = v;
+        }
+    }
+}
+
+impl Default for AppConfig {
+    fn default() -> Self {
+        Self {
+            web: WebSection::default(),
+            s3: S3Section::default(),
+            sftp: SftpSection::default(),
+            ssh: SshSection::default(),
+        }
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use tempfile::TempDir;
+
+    #[test]
+    fn test_default_config() {
+        let config = AppConfig::default();
+        assert_eq!(config.web.port, 11438);
+        assert_eq!(config.ssh.port, 2024);
+        assert_eq!(config.sftp.port, 2023);
+        assert_eq!(config.s3.region, "us-east-1");
+    }
+
+    #[test]
+    fn test_load_missing() {
+        let config = AppConfig::load("/tmp/nonexistent/config.toml").unwrap();
+        assert_eq!(config.web.port, 11438);
+    }
+
+    #[test]
+    fn test_merge_env() {
+        std::env::set_var("MB_WEB_PORT", "9090");
+        std::env::set_var("MB_SSH_PORT", "2222");
+
+        let mut config = AppConfig::default();
+        config.merge_env();
+
+        assert_eq!(config.web.port, 9090);
+        assert_eq!(config.ssh.port, 2222);
+
+        std::env::remove_var("MB_WEB_PORT");
+        std::env::remove_var("MB_SSH_PORT");
+    }
+
+    #[test]
+    fn test_save_and_load() {
+        let dir = TempDir::new().unwrap();
+        let path = dir.path().join("test.toml");
+        let path_str = path.to_string_lossy().to_string();
+
+        let config = AppConfig::default();
+        config.save(&path_str).unwrap();
+
+        let loaded = AppConfig::load(&path_str).unwrap();
+        assert_eq!(loaded.web.port, 11438);
+    }
+}

markbase-core/src/config/web.rs

@@ -0,0 +1,358 @@
+use anyhow::Result;
+use serde::{Deserialize, Serialize};
+use std::path::Path;
+
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct MarkBaseConfig {
+    pub server: ServerConfig,
+    pub postgresql: PostgreSQLConfig,
+    pub authentication: AuthenticationConfig,
+    pub test: TestConfig,
+    pub logging: LoggingConfig,
+}
+
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct ServerConfig {
+    pub host: String,
+    pub port: u16,
+    pub log_level: String,
+    pub auth_db_path: String,
+    pub users_db_dir: String,
+}
+
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct PostgreSQLConfig {
+    pub host: String,
+    pub port: u16,
+    pub user: String,
+    pub password: String,
+    pub database: String,
+    pub connection_pool_size: u8,
+}
+
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct AuthenticationConfig {
+    pub bcrypt_cost: u32,
+    pub token_validity_hours: u8,
+    pub session_storage: String,
+    pub max_sessions_per_user: u8,
+    pub default_user: String,
+    pub default_password: String,
+}
+
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct TestConfig {
+    pub users: Vec<String>,
+    pub password: String,
+    pub login_test_iterations: u16,
+    pub verify_test_iterations: u16,
+    pub api_test_iterations: u16,
+    pub performance_report: bool,
+    pub output_format: String,
+}
+
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct LoggingConfig {
+    pub level: String,
+    pub file_path: String,
+    pub console_output: bool,
+    pub structured_logging: bool,
+}
+
+impl MarkBaseConfig {
+    pub fn load(path: &Path) -> Result<Self> {
+        let content = std::fs::read_to_string(path)?;
+        let config: MarkBaseConfig = toml::from_str(&content)?;
+        Ok(config)
+    }
+
+    pub fn save(&self, path: &Path) -> Result<()> {
+        if path.exists() {
+            let backup_path = path.with_extension("toml.bak");
+            std::fs::copy(path, &backup_path)?;
+            log::info!("Backup created: {}", backup_path.display());
+        }
+
+        let content = toml::to_string_pretty(self)?;
+        std::fs::write(path, content)?;
+        log::info!("Configuration saved to: {}", path.display());
+        Ok(())
+    }
+
+    pub fn default_config() -> Self {
+        Self {
+            server: ServerConfig {
+                host: "127.0.0.1".to_string(),
+                port: 11438,
+                log_level: "info".to_string(),
+                auth_db_path: "data/auth.sqlite".to_string(),
+                users_db_dir: "data/users".to_string(),
+            },
+            postgresql: PostgreSQLConfig {
+                host: "127.0.0.1".to_string(),
+                port: 5432,
+                user: "sftpgo".to_string(),
+                password: "sftpgo_pass_2026".to_string(),
+                database: "sftpgo".to_string(),
+                connection_pool_size: 5,
+            },
+            authentication: AuthenticationConfig {
+                bcrypt_cost: 10,
+                token_validity_hours: 24,
+                session_storage: "memory".to_string(),
+                max_sessions_per_user: 5,
+                default_user: "demo".to_string(),
+                default_password: "demo123".to_string(),
+            },
+            test: TestConfig {
+                users: vec![
+                    "warren".to_string(),
+                    "momentry".to_string(),
+                    "demo".to_string(),
+                ],
+                password: "demo123".to_string(),
+                login_test_iterations: 10,
+                verify_test_iterations: 100,
+                api_test_iterations: 50,
+                performance_report: true,
+                output_format: "markdown".to_string(),
+            },
+            logging: LoggingConfig {
+                level: "info".to_string(),
+                file_path: "logs/markbase.log".to_string(),
+                console_output: true,
+                structured_logging: false,
+            },
+        }
+    }
+
+    pub fn merge_env(&mut self) {
+        if let Ok(host) = std::env::var("MB_HOST") {
+            self.server.host = host;
+        }
+        if let Ok(port) = std::env::var("MB_PORT") {
+            if let Ok(p) = port.parse() {
+                self.server.port = p;
+            }
+        }
+        if let Ok(log_level) = std::env::var("MB_LOG_LEVEL") {
+            self.server.log_level = log_level;
+        }
+
+        if let Ok(pg_host) = std::env::var("PG_HOST") {
+            self.postgresql.host = pg_host;
+        }
+        if let Ok(pg_port) = std::env::var("PG_PORT") {
+            if let Ok(p) = pg_port.parse() {
+                self.postgresql.port = p;
+            }
+        }
+        if let Ok(pg_user) = std::env::var("PG_USER") {
+            self.postgresql.user = pg_user;
+        }
+        if let Ok(pg_password) = std::env::var("PG_PASSWORD") {
+            self.postgresql.password = pg_password;
+        }
+        if let Ok(pg_database) = std::env::var("PG_DATABASE") {
+            self.postgresql.database = pg_database;
+        }
+
+        if let Ok(bcrypt_cost) = std::env::var("MB_BCRYPT_COST") {
+            if let Ok(c) = bcrypt_cost.parse() {
+                self.authentication.bcrypt_cost = c;
+            }
+        }
+        if let Ok(token_hours) = std::env::var("MB_TOKEN_VALIDITY_HOURS") {
+            if let Ok(h) = token_hours.parse() {
+                self.authentication.token_validity_hours = h;
+            }
+        }
+    }
+
+    pub fn get(&self, key: &str) -> Option<String> {
+        match key {
+            "server.host" => Some(self.server.host.clone()),
+            "server.port" => Some(self.server.port.to_string()),
+            "server.log_level" => Some(self.server.log_level.clone()),
+            "server.auth_db_path" => Some(self.server.auth_db_path.clone()),
+            "server.users_db_dir" => Some(self.server.users_db_dir.clone()),
+
+            "postgresql.host" => Some(self.postgresql.host.clone()),
+            "postgresql.port" => Some(self.postgresql.port.to_string()),
+            "postgresql.user" => Some(self.postgresql.user.clone()),
+            "postgresql.password" => Some(self.postgresql.password.clone()),
+            "postgresql.database" => Some(self.postgresql.database.clone()),
+            "postgresql.connection_pool_size" => {
+                Some(self.postgresql.connection_pool_size.to_string())
+            }
+
+            "authentication.bcrypt_cost" => Some(self.authentication.bcrypt_cost.to_string()),
+            "authentication.token_validity_hours" => {
+                Some(self.authentication.token_validity_hours.to_string())
+            }
+            "authentication.session_storage" => Some(self.authentication.session_storage.clone()),
+            "authentication.max_sessions_per_user" => {
+                Some(self.authentication.max_sessions_per_user.to_string())
+            }
+            "authentication.default_user" => Some(self.authentication.default_user.clone()),
+            "authentication.default_password" => Some(self.authentication.default_password.clone()),
+
+            "test.users" => Some(serde_json::to_string(&self.test.users).unwrap_or_default()),
+            "test.password" => Some(self.test.password.clone()),
+            "test.login_test_iterations" => Some(self.test.login_test_iterations.to_string()),
+            "test.verify_test_iterations" => Some(self.test.verify_test_iterations.to_string()),
+            "test.api_test_iterations" => Some(self.test.api_test_iterations.to_string()),
+            "test.performance_report" => Some(self.test.performance_report.to_string()),
+            "test.output_format" => Some(self.test.output_format.clone()),
+
+            "logging.level" => Some(self.logging.level.clone()),
+            "logging.file_path" => Some(self.logging.file_path.clone()),
+            "logging.console_output" => Some(self.logging.console_output.to_string()),
+            "logging.structured_logging" => Some(self.logging.structured_logging.to_string()),
+
+            _ => None,
+        }
+    }
+
+    pub fn set(&mut self, key: &str, value: &str) -> Result<()> {
+        match key {
+            "server.host" => self.server.host = value.to_string(),
+            "server.port" => self.server.port = value.parse()?,
+            "server.log_level" => self.server.log_level = value.to_string(),
+            "server.auth_db_path" => self.server.auth_db_path = value.to_string(),
+            "server.users_db_dir" => self.server.users_db_dir = value.to_string(),
+
+            "postgresql.host" => self.postgresql.host = value.to_string(),
+            "postgresql.port" => self.postgresql.port = value.parse()?,
+            "postgresql.user" => self.postgresql.user = value.to_string(),
+            "postgresql.password" => self.postgresql.password = value.to_string(),
+            "postgresql.database" => self.postgresql.database = value.to_string(),
+            "postgresql.connection_pool_size" => {
+                self.postgresql.connection_pool_size = value.parse()?
+            }
+
+            "authentication.bcrypt_cost" => self.authentication.bcrypt_cost = value.parse()?,
+            "authentication.token_validity_hours" => {
+                self.authentication.token_validity_hours = value.parse()?
+            }
+            "authentication.session_storage" => {
+                self.authentication.session_storage = value.to_string()
+            }
+            "authentication.max_sessions_per_user" => {
+                self.authentication.max_sessions_per_user = value.parse()?
+            }
+            "authentication.default_user" => self.authentication.default_user = value.to_string(),
+            "authentication.default_password" => {
+                self.authentication.default_password = value.to_string()
+            }
+
+            "test.password" => self.test.password = value.to_string(),
+            "test.login_test_iterations" => self.test.login_test_iterations = value.parse()?,
+            "test.verify_test_iterations" => self.test.verify_test_iterations = value.parse()?,
+            "test.api_test_iterations" => self.test.api_test_iterations = value.parse()?,
+            "test.performance_report" => self.test.performance_report = value.parse()?,
+            "test.output_format" => self.test.output_format = value.to_string(),
+
+            "logging.level" => self.logging.level = value.to_string(),
+            "logging.file_path" => self.logging.file_path = value.to_string(),
+            "logging.console_output" => self.logging.console_output = value.parse()?,
+            "logging.structured_logging" => self.logging.structured_logging = value.parse()?,
+
+            _ => return Err(anyhow::anyhow!("Invalid config key: {}", key)),
+        }
+        Ok(())
+    }
+
+    pub fn validate(&self) -> Result<()> {
+        if self.server.port < 1024 {
+            return Err(anyhow::anyhow!(
+                "Invalid server port: {}. Must be >= 1024",
+                self.server.port
+            ));
+        }
+
+        if self.server.host.is_empty() {
+            return Err(anyhow::anyhow!("server.host cannot be empty"));
+        }
+
+        if self.server.auth_db_path.is_empty() {
+            return Err(anyhow::anyhow!("server.auth_db_path cannot be empty"));
+        }
+
+        if self.server.users_db_dir.is_empty() {
+            return Err(anyhow::anyhow!("server.users_db_dir cannot be empty"));
+        }
+
+        if self.postgresql.port == 0 {
+            return Err(anyhow::anyhow!(
+                "Invalid PostgreSQL port: {}",
+                self.postgresql.port
+            ));
+        }
+
+        if self.postgresql.host.is_empty() {
+            return Err(anyhow::anyhow!("postgresql.host cannot be empty"));
+        }
+
+        if self.postgresql.user.is_empty() {
+            return Err(anyhow::anyhow!("postgresql.user cannot be empty"));
+        }
+
+        if self.postgresql.database.is_empty() {
+            return Err(anyhow::anyhow!("postgresql.database cannot be empty"));
+        }
+
+        if self.postgresql.connection_pool_size == 0 {
+            return Err(anyhow::anyhow!(
+                "postgresql.connection_pool_size must be >= 1"
+            ));
+        }
+
+        if self.authentication.bcrypt_cost < 4 || self.authentication.bcrypt_cost > 31 {
+            return Err(anyhow::anyhow!(
+                "Invalid bcrypt_cost: {}. Must be 4-31",
+                self.authentication.bcrypt_cost
+            ));
+        }
+
+        if self.authentication.token_validity_hours == 0 {
+            return Err(anyhow::anyhow!(
+                "Invalid token_validity_hours: {}. Must be >= 1",
+                self.authentication.token_validity_hours
+            ));
+        }
+
+        if self.authentication.default_user.is_empty() {
+            return Err(anyhow::anyhow!("authentication.default_user cannot be empty"));
+        }
+
+        if self.authentication.default_password.is_empty() {
+            return Err(anyhow::anyhow!("authentication.default_password cannot be empty"));
+        }
+
+        if self.authentication.max_sessions_per_user == 0 {
+            return Err(anyhow::anyhow!(
+                "authentication.max_sessions_per_user must be >= 1"
+            ));
+        }
+
+        if self.test.users.is_empty() {
+            return Err(anyhow::anyhow!("test.users must not be empty"));
+        }
+
+        if self.logging.level.is_empty() {
+            return Err(anyhow::anyhow!("logging.level cannot be empty"));
+        }
+
+        let valid_log_levels = ["trace", "debug", "info", "warn", "error", "off"];
+        if !valid_log_levels.contains(&self.logging.level.as_str()) {
+            return Err(anyhow::anyhow!(
+                "Invalid logging.level: {}. Must be one of: {}",
+                self.logging.level,
+                valid_log_levels.join(", ")
+            ));
+        }
+
+        Ok(())
+    }
+}