MarkBase架构升级:Multi-Volume Virtual Tree + Dual-View Management + Git Remote修正
核心功能: - ✅ Categories/Series双视图管理(category_view.rs + import_markdown.rs) - ✅ FUSE Multi-Volume支持(tree_type参数) - ✅ SSH/SFTP/SCP/rsync协议完整实现(4042行) - ✅ NFS/SMB Module Phase 1-3完成 - ✅ Archive Module Phase 1-4完成(2916行) - ✅ Download Center API完整实现 - ✅ S3兼容API实现(560行) Git配置修正: - ✅ 删除错误origin(gitea.momentry.ddns.net) - ✅ 删除m5max128(指向机器名) - ✅ 设置origin = m5max128gitea.momentry.ddns.net/admin/markbase - ✅ 设置m4minigitea = m4minigitea.momentry.ddns.net/warren/markbase 数据清理: - ✅ 删除38个临时SQLite(保留accusys.sqlite、demo.sqlite) - ✅ 删除.bak、test_*.bin、调试脚本等临时文件 - ✅ 删除临时目录(build/、download files/、raid_test/等) - ✅ 更新.gitignore排除临时文件 架构优化: - 52个文件修改,2434行新增,4739行删除 - Workspace成员整合(16个crate) - 数据库状态:accusys.sqlite保留(主demo测试) 远程同步: - ✅ 准备推送到m5max128gitea(远程Gitea) - ✅ 准备推送到m4minigitea(本地Gitea)
This commit is contained in:
425
markbase-core/src/import_markdown.rs
Normal file
425
markbase-core/src/import_markdown.rs
Normal file
@@ -0,0 +1,425 @@
|
||||
use anyhow::Result;
|
||||
use std::fs;
|
||||
use std::path::Path;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use pulldown_cmark::{Parser, Event, Tag, HeadingLevel, TagEnd};
|
||||
use regex::Regex;
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct MarkdownFile {
|
||||
pub filename: String,
|
||||
pub size: Option<String>,
|
||||
pub download_url: String,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct CategorySection {
|
||||
pub product: String,
|
||||
pub files: Vec<MarkdownFile>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct SeriesSection {
|
||||
pub category: String,
|
||||
pub files: Vec<MarkdownFile>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct CategoryMarkdown {
|
||||
pub category: String,
|
||||
pub sections: Vec<CategorySection>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct SeriesMarkdown {
|
||||
pub series: String,
|
||||
pub sections: Vec<SeriesSection>,
|
||||
}
|
||||
|
||||
pub fn parse_category_markdown(content: &str) -> Result<CategoryMarkdown> {
|
||||
let mut category = String::new();
|
||||
let mut sections: Vec<CategorySection> = Vec::new();
|
||||
|
||||
let lines: Vec<&str> = content.lines().collect();
|
||||
let mut current_product = String::new();
|
||||
let mut current_files: Vec<MarkdownFile> = Vec::new();
|
||||
let mut pending_file: Option<(String, String)> = None;
|
||||
|
||||
for i in 0..lines.len() {
|
||||
let line = lines[i].trim();
|
||||
|
||||
if line.contains("**Category**:") {
|
||||
category = line.replace("**Category**:", "").replace("**", "").trim().to_string();
|
||||
} else if line.starts_with("## ") {
|
||||
if !current_product.is_empty() && !current_files.is_empty() {
|
||||
sections.push(CategorySection {
|
||||
product: current_product.clone(),
|
||||
files: current_files.clone(),
|
||||
});
|
||||
current_files.clear();
|
||||
}
|
||||
current_product = line.replace("## ", "").trim().to_string();
|
||||
} else if line.starts_with("**") && line.contains("** (") {
|
||||
let clean = line.replace("**", "");
|
||||
let parts: Vec<&str> = clean.splitn(2, '(').collect();
|
||||
if parts.len() == 2 {
|
||||
let filename = parts[0].trim().to_string();
|
||||
let size = parts[1].trim_end_matches(')').trim().to_string();
|
||||
pending_file = Some((filename, size));
|
||||
}
|
||||
} else if line.contains("https://download.accusys.ddns.net/api/v2/download") {
|
||||
if let Some((filename, size)) = pending_file.clone() {
|
||||
current_files.push(MarkdownFile {
|
||||
filename,
|
||||
size: Some(size),
|
||||
download_url: line.trim_start_matches('`').trim_end_matches('`').trim().to_string(),
|
||||
});
|
||||
pending_file = None;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if !current_product.is_empty() && !current_files.is_empty() {
|
||||
sections.push(CategorySection {
|
||||
product: current_product.clone(),
|
||||
files: current_files.clone(),
|
||||
});
|
||||
}
|
||||
|
||||
Ok(CategoryMarkdown { category, sections })
|
||||
}
|
||||
|
||||
pub fn parse_series_markdown(content: &str) -> Result<SeriesMarkdown> {
|
||||
let mut series = String::new();
|
||||
let mut sections: Vec<SeriesSection> = Vec::new();
|
||||
|
||||
let lines: Vec<&str> = content.lines().collect();
|
||||
let mut current_category = String::new();
|
||||
let mut current_files: Vec<MarkdownFile> = Vec::new();
|
||||
let mut pending_file: Option<(String, String)> = None;
|
||||
|
||||
for i in 0..lines.len() {
|
||||
let line = lines[i].trim();
|
||||
|
||||
if line.starts_with("# ") && line.contains("Download Links") {
|
||||
series = line.replace("# ", "").replace(" Download Links", "").trim().to_string();
|
||||
} else if line.starts_with("## ") {
|
||||
if !current_category.is_empty() && !current_files.is_empty() {
|
||||
sections.push(SeriesSection {
|
||||
category: current_category.clone(),
|
||||
files: current_files.clone(),
|
||||
});
|
||||
current_files.clear();
|
||||
}
|
||||
current_category = line.replace("## ", "").trim().to_string();
|
||||
} else if line.starts_with("**") && line.contains("(") {
|
||||
let clean = line.replace("**", "");
|
||||
let parts: Vec<&str> = clean.splitn(2, '(').collect();
|
||||
if parts.len() == 2 {
|
||||
let filename = parts[0].trim().to_string();
|
||||
let size = parts[1].trim_end_matches(')').trim().to_string();
|
||||
pending_file = Some((filename, size));
|
||||
}
|
||||
} else if line.contains("https://download.accusys.ddns.net/api/v2/download") {
|
||||
if let Some((filename, size)) = pending_file.clone() {
|
||||
current_files.push(MarkdownFile {
|
||||
filename,
|
||||
size: Some(size),
|
||||
download_url: line.trim_start_matches('`').trim_end_matches('`').trim().to_string(),
|
||||
});
|
||||
pending_file = None;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if !current_category.is_empty() && !current_files.is_empty() {
|
||||
sections.push(SeriesSection {
|
||||
category: current_category.clone(),
|
||||
files: current_files.clone(),
|
||||
});
|
||||
}
|
||||
|
||||
Ok(SeriesMarkdown { series, sections })
|
||||
}
|
||||
|
||||
pub fn read_category_files(dir: &Path) -> Result<Vec<(String, String)>> {
|
||||
let mut files = Vec::new();
|
||||
|
||||
for entry in fs::read_dir(dir)? {
|
||||
let entry = entry?;
|
||||
let path = entry.path();
|
||||
|
||||
if path.extension().map_or(false, |ext| ext == "md") && path.file_name() != Some(std::ffi::OsStr::new("README.md")) {
|
||||
let filename = path.file_name().unwrap().to_string_lossy().to_string();
|
||||
let content = fs::read_to_string(&path)?;
|
||||
files.push((filename, content));
|
||||
}
|
||||
}
|
||||
|
||||
Ok(files)
|
||||
}
|
||||
|
||||
pub fn read_series_files(dir: &Path) -> Result<Vec<(String, String)>> {
|
||||
let mut files = Vec::new();
|
||||
|
||||
for entry in fs::read_dir(dir)? {
|
||||
let entry = entry?;
|
||||
let path = entry.path();
|
||||
|
||||
if path.extension().map_or(false, |ext| ext == "md") && path.file_name() != Some(std::ffi::OsStr::new("README.md")) {
|
||||
let filename = path.file_name().unwrap().to_string_lossy().to_string();
|
||||
let content = fs::read_to_string(&path)?;
|
||||
files.push((filename, content));
|
||||
}
|
||||
}
|
||||
|
||||
Ok(files)
|
||||
}
|
||||
|
||||
pub fn import_categories_to_db(conn: &rusqlite::Connection, user_id: &str, tree_type: &str) -> Result<()> {
|
||||
use crate::FileTree;
|
||||
use filetree::node::{FileNode, Aliases, NodeType};
|
||||
use uuid::Uuid;
|
||||
use std::collections::HashMap;
|
||||
|
||||
let category_dir = Path::new("/Users/accusys/markbase/data/downloads/by_category");
|
||||
let files = read_category_files(category_dir)?;
|
||||
|
||||
println!("Found {} Markdown files", files.len());
|
||||
|
||||
let mut tree = FileTree::load(conn, user_id, tree_type)?;
|
||||
|
||||
for (_filename, content) in files {
|
||||
let parsed = parse_category_markdown(&content)?;
|
||||
|
||||
println!("Parsed category: '{}', sections: {}", parsed.category, parsed.sections.len());
|
||||
|
||||
if parsed.category.is_empty() {
|
||||
println!("Warning: category is empty, skipping");
|
||||
continue;
|
||||
}
|
||||
|
||||
let category_node_id = Uuid::new_v4().to_string();
|
||||
let mut aliases_map = HashMap::new();
|
||||
aliases_map.insert("category_type".to_string(), "category".to_string());
|
||||
|
||||
let category_node = FileNode {
|
||||
node_id: category_node_id.clone(),
|
||||
label: parsed.category.clone(),
|
||||
aliases: Aliases { map: aliases_map },
|
||||
file_uuid: None,
|
||||
sha256: None,
|
||||
parent_id: None,
|
||||
children: Vec::new(),
|
||||
node_type: NodeType::Folder,
|
||||
icon: Some("📁".to_string()),
|
||||
color: None,
|
||||
bg_color: None,
|
||||
file_size: None,
|
||||
registered_at: None,
|
||||
created_at: chrono::Utc::now().to_rfc3339(),
|
||||
updated_at: chrono::Utc::now().to_rfc3339(),
|
||||
sort_order: 0,
|
||||
};
|
||||
|
||||
println!("Inserting category node: {} (id: {})", category_node.label, category_node_id);
|
||||
|
||||
tree.insert_node(conn, &category_node)?;
|
||||
|
||||
println!("Category node inserted successfully");
|
||||
|
||||
for section in parsed.sections {
|
||||
println!("Processing section: {} with {} files", section.product, section.files.len());
|
||||
|
||||
let product_node_id = Uuid::new_v4().to_string();
|
||||
let mut aliases_map = HashMap::new();
|
||||
aliases_map.insert("product".to_string(), section.product.clone());
|
||||
|
||||
let product_node = FileNode {
|
||||
node_id: product_node_id.clone(),
|
||||
label: section.product.clone(),
|
||||
aliases: Aliases { map: aliases_map },
|
||||
file_uuid: None,
|
||||
sha256: None,
|
||||
parent_id: Some(category_node_id.clone()),
|
||||
children: Vec::new(),
|
||||
node_type: NodeType::Folder,
|
||||
icon: Some("📁".to_string()),
|
||||
color: None,
|
||||
bg_color: None,
|
||||
file_size: None,
|
||||
registered_at: None,
|
||||
created_at: chrono::Utc::now().to_rfc3339(),
|
||||
updated_at: chrono::Utc::now().to_rfc3339(),
|
||||
sort_order: 0,
|
||||
};
|
||||
|
||||
tree.insert_node(conn, &product_node)?;
|
||||
|
||||
for file in section.files {
|
||||
let file_node_id = Uuid::new_v4().to_string();
|
||||
let mut aliases_map = HashMap::new();
|
||||
aliases_map.insert("download_url".to_string(), file.download_url.clone());
|
||||
aliases_map.insert("file_size_display".to_string(), file.size.clone().unwrap_or_else(|| "Unknown".to_string()));
|
||||
|
||||
let file_node = FileNode {
|
||||
node_id: file_node_id.clone(),
|
||||
label: file.filename.clone(),
|
||||
aliases: Aliases { map: aliases_map },
|
||||
file_uuid: None,
|
||||
sha256: None,
|
||||
parent_id: Some(product_node_id.clone()),
|
||||
children: Vec::new(),
|
||||
node_type: NodeType::File,
|
||||
icon: Some("📄".to_string()),
|
||||
color: None,
|
||||
bg_color: None,
|
||||
file_size: None,
|
||||
registered_at: None,
|
||||
created_at: chrono::Utc::now().to_rfc3339(),
|
||||
updated_at: chrono::Utc::now().to_rfc3339(),
|
||||
sort_order: 0,
|
||||
};
|
||||
|
||||
tree.insert_node(conn, &file_node)?;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn import_series_to_db(conn: &rusqlite::Connection, user_id: &str, tree_type: &str) -> Result<()> {
|
||||
use crate::FileTree;
|
||||
use filetree::node::{FileNode, Aliases, NodeType};
|
||||
use uuid::Uuid;
|
||||
use std::collections::HashMap;
|
||||
|
||||
let series_dir = Path::new("/Users/accusys/markbase/data/downloads/by_series");
|
||||
let files = read_series_files(series_dir)?;
|
||||
|
||||
println!("Found {} Markdown files for series", files.len());
|
||||
|
||||
let mut tree = FileTree::load(conn, user_id, tree_type)?;
|
||||
|
||||
for (_filename, content) in files {
|
||||
let parsed = parse_series_markdown(&content)?;
|
||||
|
||||
println!("Parsed series: '{}', sections: {}", parsed.series, parsed.sections.len());
|
||||
|
||||
if parsed.series.is_empty() {
|
||||
println!("Warning: series is empty, skipping");
|
||||
continue;
|
||||
}
|
||||
|
||||
let series_node_id = Uuid::new_v4().to_string();
|
||||
let mut aliases_map = HashMap::new();
|
||||
aliases_map.insert("series_type".to_string(), "series".to_string());
|
||||
|
||||
let series_node = FileNode {
|
||||
node_id: series_node_id.clone(),
|
||||
label: parsed.series.clone(),
|
||||
aliases: Aliases { map: aliases_map },
|
||||
file_uuid: None,
|
||||
sha256: None,
|
||||
parent_id: None,
|
||||
children: Vec::new(),
|
||||
node_type: NodeType::Folder,
|
||||
icon: Some("📁".to_string()),
|
||||
color: None,
|
||||
bg_color: None,
|
||||
file_size: None,
|
||||
registered_at: None,
|
||||
created_at: chrono::Utc::now().to_rfc3339(),
|
||||
updated_at: chrono::Utc::now().to_rfc3339(),
|
||||
sort_order: 0,
|
||||
};
|
||||
|
||||
tree.insert_node(conn, &series_node)?;
|
||||
|
||||
println!("Series node inserted successfully");
|
||||
|
||||
for section in parsed.sections {
|
||||
println!("Processing section: {} with {} files", section.category, section.files.len());
|
||||
|
||||
let category_node_id = Uuid::new_v4().to_string();
|
||||
let mut aliases_map = HashMap::new();
|
||||
aliases_map.insert("category".to_string(), section.category.clone());
|
||||
|
||||
let category_node = FileNode {
|
||||
node_id: category_node_id.clone(),
|
||||
label: section.category.clone(),
|
||||
aliases: Aliases { map: aliases_map },
|
||||
file_uuid: None,
|
||||
sha256: None,
|
||||
parent_id: Some(series_node_id.clone()),
|
||||
children: Vec::new(),
|
||||
node_type: NodeType::Folder,
|
||||
icon: Some("📁".to_string()),
|
||||
color: None,
|
||||
bg_color: None,
|
||||
file_size: None,
|
||||
registered_at: None,
|
||||
created_at: chrono::Utc::now().to_rfc3339(),
|
||||
updated_at: chrono::Utc::now().to_rfc3339(),
|
||||
sort_order: 0,
|
||||
};
|
||||
|
||||
tree.insert_node(conn, &category_node)?;
|
||||
|
||||
for file in section.files {
|
||||
let file_node_id = Uuid::new_v4().to_string();
|
||||
let mut aliases_map = HashMap::new();
|
||||
aliases_map.insert("download_url".to_string(), file.download_url.clone());
|
||||
aliases_map.insert("file_size_display".to_string(), file.size.clone().unwrap_or_else(|| "Unknown".to_string()));
|
||||
|
||||
let file_node = FileNode {
|
||||
node_id: file_node_id.clone(),
|
||||
label: file.filename.clone(),
|
||||
aliases: Aliases { map: aliases_map },
|
||||
file_uuid: None,
|
||||
sha256: None,
|
||||
parent_id: Some(category_node_id.clone()),
|
||||
children: Vec::new(),
|
||||
node_type: NodeType::File,
|
||||
icon: Some("📄".to_string()),
|
||||
color: None,
|
||||
bg_color: None,
|
||||
file_size: None,
|
||||
registered_at: None,
|
||||
created_at: chrono::Utc::now().to_rfc3339(),
|
||||
updated_at: chrono::Utc::now().to_rfc3339(),
|
||||
sort_order: 0,
|
||||
};
|
||||
|
||||
tree.insert_node(conn, &file_node)?;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn test_parse_category_markdown() {
|
||||
let content = r#"# GUI Download Links
|
||||
|
||||
**Category**: GUI
|
||||
|
||||
---
|
||||
|
||||
## ExaSAN-DAS
|
||||
|
||||
**C2M-QIG20170906.zip** (353.7KB)
|
||||
```https://download.accusys.ddns.net/api/v2/download/products/ExaSAN-DAS/C1M_C2M/User%20Guide/C2M-QIG20170906.zip
|
||||
```
|
||||
"#;
|
||||
|
||||
let result = parse_category_markdown(content).unwrap();
|
||||
assert_eq!(result.category, "GUI");
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user