Files
markbase/src/main.rs
Warren 05f89ea1ac feat: Add file scan and async hash system
Features:
1. scan command - Fast import without hash (skip_hash=true)
   - Scans directory structure
   - Generates deterministic UUIDs (SHA256(path|name|mac|mtime))
   - Stores full path in aliases.json
   - Inserts nodes in batches
   - Performance: 14243 nodes/sec (11857 files in 0.89s)

2. hash command - Async hash calculation
   - Multi-threaded (default: 4 threads)
   - Reads paths from aliases.json
   - Updates database with SHA256 hashes
   - Performance: 28 files/sec (11857 files in 417.58s)

Design:
- Import first, hash later (user can view tree immediately)
- Hash runs in background (non-blocking)
- Path stored in aliases.json (temporary solution)
- Deterministic UUIDs (same file = same UUID)

Performance breakdown:
- Scanning: 0.10s (11%)
- ID generation: 0.57s (64%)
- DB insertion: 0.21s (24%)
- Hash: 417.58s (async, background)

Files:
- src/scan.rs (new, 499 lines)
- src/main.rs (scan/hash commands)
- src/lib.rs (scan module)

Test result:
- warren user: 12658 nodes imported
- 11857 hashes calculated successfully
2026-05-17 03:20:35 +08:00

211 lines
6.8 KiB
Rust

use clap::{Parser, Subcommand};
use std::path::Path;
#[derive(Parser)]
#[command(name = "markbase", about = "Momentry Display Engine")]
struct Cli {
#[command(subcommand)]
command: Commands,
}
#[derive(Subcommand)]
enum Commands {
/// Start display server
Display {
#[arg(short, long, default_value = "11438")]
port: u16,
/// Optional initial markdown file
file: Option<String>,
},
/// Render markdown to HTML (stdout)
Render {
file: String,
#[arg(short, long)]
output: Option<String>,
},
/// Configuration management
Config {
#[command(subcommand)]
action: ConfigCommands,
},
/// Scan and import files from directory
Scan {
/// User ID
#[arg(short, long)]
user: String,
/// Directory to scan
#[arg(short, long)]
dir: String,
/// Batch size for database insertion
#[arg(short, long, default_value = "100")]
batch: usize,
/// Skip SHA256 hash calculation (faster import)
#[arg(short, long, default_value = "true")]
skip_hash: bool,
/// Number of threads for hash calculation (if skip_hash=false)
#[arg(short, long, default_value = "4")]
threads: usize,
},
/// Compute SHA256 hashes for imported files
Hash {
/// User ID
#[arg(short, long)]
user: String,
/// Number of threads for parallel hash calculation
#[arg(short, long, default_value = "4")]
threads: usize,
},
}
#[derive(Subcommand)]
enum ConfigCommands {
/// Initialize default configuration file
Init {
#[arg(short, long)]
force: bool,
},
/// Show current configuration
Show {
#[arg(short, long)]
section: Option<String>,
},
/// Edit configuration
Edit {
#[arg(short, long)]
key: String,
#[arg(short, long)]
value: String,
},
/// Validate configuration
Validate,
}
#[tokio::main]
async fn main() -> anyhow::Result<()> {
let cli = Cli::parse();
match cli.command {
Commands::Display { port, file } => {
markbase::server::run(port, file).await?;
}
Commands::Render { file, output } => {
let md = std::fs::read_to_string(&file)?;
let html = markbase::render::md_to_html(&md);
if let Some(path) = &output {
std::fs::write(path, html)?;
} else {
println!("{html}");
}
}
Commands::Config { action } => {
handle_config_command(action)?;
}
Commands::Scan { user, dir, batch, skip_hash, threads } => {
use markbase::scan::ScanOptions;
let options = ScanOptions {
skip_hash,
threads,
};
markbase::scan::scan_directory(&user, &dir, batch, options)?;
}
Commands::Hash { user, threads } => {
markbase::scan::compute_hashes(&user, threads)?;
}
}
Ok(())
}
fn handle_config_command(action: ConfigCommands) -> anyhow::Result<()> {
match action {
ConfigCommands::Init { force } => {
let config_path = Path::new("config/markbase.toml");
if config_path.exists() && !force {
println!("Configuration file already exists at config/markbase.toml");
println!("Use --force to overwrite");
return Ok(());
}
let config = markbase::config::MarkBaseConfig::default_config();
config.save(config_path)?;
println!("✓ Configuration file created: config/markbase.toml");
println!("Default values:");
println!(" Server port: {}", config.server.port);
println!(" PostgreSQL host: {}", config.postgresql.host);
println!(" Test users: {}", config.test.users.join(", "));
}
ConfigCommands::Show { section } => {
let config_path = Path::new("config/markbase.toml");
if !config_path.exists() {
println!("Configuration file not found. Run 'markbase config init' first.");
return Ok(());
}
let config = markbase::config::MarkBaseConfig::load(config_path)?;
if let Some(s) = section {
show_section(&config, &s);
} else {
println!("{}", toml::to_string_pretty(&config)?);
}
}
ConfigCommands::Edit { key, value } => {
let config_path = Path::new("config/markbase.toml");
if !config_path.exists() {
println!("Configuration file not found. Run 'markbase config init' first.");
return Ok(());
}
let mut config = markbase::config::MarkBaseConfig::load(config_path)?;
match config.get(&key) {
Some(old_value) => {
config.set(&key, &value)?;
config.validate()?;
config.save(config_path)?;
println!("✓ Updated {}: {}{}", key, old_value, value);
}
None => {
println!("Invalid config key: {}", key);
println!("Valid keys: server.*, postgresql.*, authentication.*, test.*, logging.*");
}
}
}
ConfigCommands::Validate => {
let config_path = Path::new("config/markbase.toml");
if !config_path.exists() {
println!("Configuration file not found. Run 'markbase config init' first.");
return Ok(());
}
let config = markbase::config::MarkBaseConfig::load(config_path)?;
match config.validate() {
Ok(_) => {
println!("✓ Configuration is valid");
}
Err(e) => {
println!("✗ Configuration validation failed: {}", e);
}
}
}
}
Ok(())
}
fn show_section(config: &markbase::config::MarkBaseConfig, section: &str) {
match section {
"server" => println!("{}", toml::to_string_pretty(&config.server).unwrap()),
"postgresql" => println!("{}", toml::to_string_pretty(&config.postgresql).unwrap()),
"authentication" => println!("{}", toml::to_string_pretty(&config.authentication).unwrap()),
"test" => println!("{}", toml::to_string_pretty(&config.test).unwrap()),
"logging" => println!("{}", toml::to_string_pretty(&config.logging).unwrap()),
_ => println!("Invalid section: {}. Valid sections: server, postgresql, authentication, test, logging", section),
}
}