From 862ed1c08afbc74d4e1a40bb88f305cea02637ea Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1=C5=A1=20Ml=C3=A1dek?= Date: Sun, 5 Nov 2023 14:19:04 +0100 Subject: [PATCH] refactor: tree mode -> (new) blob mode --- cli/src/main.rs | 2 +- cli/src/routes.rs | 8 +++---- db/src/lib.rs | 43 ++++++++++++++++++++---------------- db/src/stores/fs/mod.rs | 35 +++++++++++++++-------------- db/src/stores/mod.rs | 4 ++-- webui/src/views/Home.svelte | 2 +- webui/src/views/Setup.svelte | 6 ++--- 7 files changed, 53 insertions(+), 47 deletions(-) diff --git a/cli/src/main.rs b/cli/src/main.rs index b2ef389..a334a2a 100644 --- a/cli/src/main.rs +++ b/cli/src/main.rs @@ -455,7 +455,7 @@ async fn main() -> Result<()> { initial: false, tree_mode: connection .get_vault_options()? - .tree_mode + .blob_mode .unwrap_or_default(), }, ); diff --git a/cli/src/routes.rs b/cli/src/routes.rs index 584353f..dbf3d57 100644 --- a/cli/src/routes.rs +++ b/cli/src/routes.rs @@ -40,9 +40,9 @@ use upend_db::hierarchies::{list_roots, resolve_path, UHierPath}; use upend_db::jobs; use upend_db::stores::UpdateOptions; use upend_db::stores::{Blob, UpStore}; +use upend_db::BlobMode; use upend_db::UpEndDatabase; use upend_db::VaultOptions; -use upend_db::VaultRescanMode; use url::Url; #[cfg(feature = "desktop")] @@ -768,7 +768,7 @@ pub async fn list_hier_roots(state: web::Data) -> Result, - tree_mode: Option, + tree_mode: Option, } #[post("/api/refresh")] @@ -790,7 +790,7 @@ pub async fn api_refresh( tree_mode: query.tree_mode.unwrap_or( connection .get_vault_options()? - .tree_mode + .blob_mode .unwrap_or_default(), ), }, @@ -1156,7 +1156,7 @@ mod tests { job_container.clone(), UpdateOptions { initial: true, - tree_mode: upend_db::VaultRescanMode::default(), + tree_mode: upend_db::BlobMode::default(), }, ) .unwrap(); diff --git a/db/src/lib.rs b/db/src/lib.rs index d5481bd..fd3efc6 100644 --- a/db/src/lib.rs +++ b/db/src/lib.rs @@ -239,28 +239,30 @@ impl UpEndConnection { } pub fn set_vault_options(&self, options: VaultOptions) -> Result<()> { - if let Some(tree_mode) = options.tree_mode { - let tree_mode = match tree_mode { - VaultRescanMode::Flat => "FLAT".to_string(), - VaultRescanMode::DepthFirst => "DEPTH_FIRST".to_string(), - VaultRescanMode::Mirror => "MIRROR".to_string(), - VaultRescanMode::Incoming(None) => "INCOMING".to_string(), - VaultRescanMode::Incoming(Some(group)) => format!("INCOMING:{}", group), + if let Some(blob_mode) = options.blob_mode { + let tree_mode = match blob_mode { + BlobMode::Flat => "FLAT".to_string(), + BlobMode::DepthFirst => "DEPTH_FIRST".to_string(), + BlobMode::Mirror => "MIRROR".to_string(), + BlobMode::Incoming(None) => "INCOMING".to_string(), + BlobMode::Incoming(Some(group)) => format!("INCOMING:{}", group), + BlobMode::StoreOnly => "STORE_ONLY".to_string(), }; - self.set_meta("VAULT_TREE_MODE", tree_mode)?; + self.set_meta("VAULT_BLOB_MODE", tree_mode)?; } Ok(()) } pub fn get_vault_options(&self) -> Result { - let tree_mode = match self.get_meta("VAULT_TREE_MODE")? { + let blob_mode = match self.get_meta("VAULT_BLOB_MODE")? { Some(mode) => match mode.as_str() { - "FLAT" => Some(VaultRescanMode::Flat), - "DEPTH_FIRST" => Some(VaultRescanMode::DepthFirst), - "MIRROR" => Some(VaultRescanMode::Mirror), - "INCOMING" => Some(VaultRescanMode::Incoming(None)), + "FLAT" => Some(BlobMode::Flat), + "DEPTH_FIRST" => Some(BlobMode::DepthFirst), + "MIRROR" => Some(BlobMode::Mirror), + "INCOMING" => Some(BlobMode::Incoming(None)), + "STORE_ONLY" => Some(BlobMode::StoreOnly), mode if mode.starts_with("INCOMING:") => { - Some(VaultRescanMode::Incoming(Some(mode[9..].to_string()))) + Some(BlobMode::Incoming(Some(mode[9..].to_string()))) } _ => { warn!("Unknown vault tree mode: {}", mode); @@ -270,7 +272,7 @@ impl UpEndConnection { None => None, }; - Ok(VaultOptions { tree_mode }) + Ok(VaultOptions { blob_mode }) } pub fn retrieve_entry(&self, hash: &UpMultihash) -> Result> { @@ -603,18 +605,21 @@ mod test { #[derive(Debug, Serialize, Deserialize)] pub struct VaultOptions { - pub tree_mode: Option, + pub blob_mode: Option, } +/// Specifies how to store new blobs #[derive(Debug, Clone, Serialize, Deserialize, Default)] -pub enum VaultRescanMode { +pub enum BlobMode { #[default] - /// Mirror the filesystem tree + /// Mirror the original tree Mirror, /// Like Mirror, but chooses the shortest path DepthFirst, - /// Use only the last level of the filesystem tree as groups + /// Use only the last level of the tree as a group Flat, /// Place all files in a single group Incoming(Option), + /// Only store files, don't place them anywhere + StoreOnly, } diff --git a/db/src/stores/fs/mod.rs b/db/src/stores/fs/mod.rs index f7a7ad8..b7f9c18 100644 --- a/db/src/stores/fs/mod.rs +++ b/db/src/stores/fs/mod.rs @@ -5,8 +5,7 @@ use crate::hierarchies::{resolve_path, resolve_path_cached, ResolveCache, UHierP use crate::jobs::{JobContainer, JobHandle}; use crate::util::hash_at_path; use crate::{ - ConnectionOptions, LoggingHandler, UpEndConnection, UpEndDatabase, VaultRescanMode, - UPEND_SUBDIR, + BlobMode, ConnectionOptions, LoggingHandler, UpEndConnection, UpEndDatabase, UPEND_SUBDIR, }; use anyhow::{anyhow, Error, Result}; use chrono::prelude::*; @@ -135,7 +134,7 @@ impl FsStore { let mut upaths: HashMap = HashMap::new(); match options.tree_mode { - VaultRescanMode::Flat => { + BlobMode::Flat => { for pb in &pathbufs { let normalized_path = self.normalize_path(pb).unwrap(); let dirname = normalized_path.parent().and_then(|p| p.components().last()); @@ -152,7 +151,7 @@ impl FsStore { upaths.insert(pb.clone(), upath); } } - VaultRescanMode::DepthFirst => { + BlobMode::DepthFirst => { let mut shallowest: HashMap = HashMap::new(); for path in &pathbufs { let normalized_path = self.normalize_path(path).unwrap(); @@ -185,7 +184,7 @@ impl FsStore { } } } - VaultRescanMode::Mirror => { + BlobMode::Mirror => { for pb in &pathbufs { let normalized_path = self.normalize_path(&pb).unwrap(); let path = normalized_path.parent().unwrap(); @@ -199,7 +198,7 @@ impl FsStore { upaths.insert(pb.clone(), UHierPath(upath)); } } - VaultRescanMode::Incoming(group) => { + BlobMode::Incoming(group) => { let upath = UHierPath(vec![group .unwrap_or("INCOMING".to_string()) .parse() @@ -208,6 +207,7 @@ impl FsStore { upaths.insert(pb.clone(), upath.clone()); } } + BlobMode::StoreOnly => {} }; let path_entries = pathbufs @@ -227,6 +227,7 @@ impl FsStore { Arc::new(Mutex::new(LruCache::new(256))); let total = path_entries.len() as f32; let shared_job_handle = Arc::new(Mutex::new(job_handle)); + let path_outcomes: Vec = path_entries .into_par_iter() .map(|(path, upath)| { @@ -870,7 +871,7 @@ mod test { job_container, UpdateOptions { initial: true, - tree_mode: VaultRescanMode::default(), + tree_mode: BlobMode::default(), }, ); assert!(rescan_result.is_ok()); @@ -916,7 +917,7 @@ mod test { job, UpdateOptions { initial: quick, - tree_mode: VaultRescanMode::default(), + tree_mode: BlobMode::default(), }, ); @@ -936,7 +937,7 @@ mod test { job, UpdateOptions { initial: quick, - tree_mode: VaultRescanMode::default(), + tree_mode: BlobMode::default(), }, ); @@ -959,7 +960,7 @@ mod test { job, UpdateOptions { initial: quick, - tree_mode: VaultRescanMode::default(), + tree_mode: BlobMode::default(), }, ); @@ -1004,7 +1005,7 @@ mod test { /// │   └── bar.txt /// └── in_root.txt /// ``` - fn _prepare_hier_vault(tree_mode: VaultRescanMode) -> (UpEndConnection, TempDir) { + fn _prepare_hier_vault(tree_mode: BlobMode) -> (UpEndConnection, TempDir) { // Prepare temporary filesystem structure let temp_dir = TempDir::new().unwrap(); let temp_dir_path = temp_dir.path().canonicalize().unwrap(); @@ -1068,7 +1069,7 @@ mod test { }); } - fn test_initial_scan(mode: VaultRescanMode, expected_paths: Vec<&str>) { + fn test_initial_scan(mode: BlobMode, expected_paths: Vec<&str>) { let (connection, _vault_dir) = _prepare_hier_vault(mode); assert_paths(expected_paths, &connection); } @@ -1076,7 +1077,7 @@ mod test { #[test] fn test_mirror_mode() { test_initial_scan( - VaultRescanMode::Mirror, + BlobMode::Mirror, vec![ "NATIVE", "NATIVE/nested_directory/nested_two/nested_three/foo.txt", @@ -1090,7 +1091,7 @@ mod test { #[test] fn test_flat_mode() { test_initial_scan( - VaultRescanMode::Flat, + BlobMode::Flat, vec![ "NATIVE", "NATIVE/nested_three/foo.txt", @@ -1104,7 +1105,7 @@ mod test { #[test] fn test_depth_mode() { test_initial_scan( - VaultRescanMode::DepthFirst, + BlobMode::DepthFirst, vec![ "NATIVE", "NATIVE/nested_directory/nested_two/nested_four/baz.txt", @@ -1118,7 +1119,7 @@ mod test { #[test] fn test_incoming_mode() { test_initial_scan( - VaultRescanMode::Incoming(None), + BlobMode::Incoming(None), vec![ "INCOMING/foo.txt", "INCOMING/baz.txt", @@ -1128,7 +1129,7 @@ mod test { ); test_initial_scan( - VaultRescanMode::Incoming(Some("new files".to_string())), + BlobMode::Incoming(Some("new files".to_string())), vec![ "new files/foo.txt", "new files/baz.txt", diff --git a/db/src/stores/mod.rs b/db/src/stores/mod.rs index 370a067..edf3f29 100644 --- a/db/src/stores/mod.rs +++ b/db/src/stores/mod.rs @@ -1,7 +1,7 @@ use std::path::{Path, PathBuf}; use super::{UpEndConnection, UpEndDatabase}; -use crate::{jobs::JobContainer, VaultRescanMode}; +use crate::{jobs::JobContainer, BlobMode}; use upend_base::hash::UpMultihash; pub mod fs; @@ -73,5 +73,5 @@ pub trait UpStore { #[derive(Debug, Clone)] pub struct UpdateOptions { pub initial: bool, - pub tree_mode: VaultRescanMode, + pub tree_mode: BlobMode, } diff --git a/webui/src/views/Home.svelte b/webui/src/views/Home.svelte index 83d60c7..7348639 100644 --- a/webui/src/views/Home.svelte +++ b/webui/src/views/Home.svelte @@ -156,7 +156,7 @@ fetch("/api/options") .then((res) => res.json()) .then((options) => { - if (!options.tree_mode) { + if (!options.blob_mode) { navigate("/setup"); } }); diff --git a/webui/src/views/Setup.svelte b/webui/src/views/Setup.svelte index 93de16d..1d945fb 100644 --- a/webui/src/views/Setup.svelte +++ b/webui/src/views/Setup.svelte @@ -9,15 +9,15 @@ let mode: "Flat" | "DepthFirst" | "Mirror" | "Incoming" = undefined; async function submitOptions() { - const tree_mode = {}; - tree_mode[mode] = null; + const blob_mode = {}; + blob_mode[mode] = null; const optionResponse = await fetch("/api/options", { method: "PUT", headers: { "Content-Type": "application/json", }, - body: JSON.stringify({ tree_mode }), + body: JSON.stringify({ blob_mode }), }); if (!optionResponse.ok) { throw new Error("Failed to set options");