thm
/
upend
1
0
Fork 0
Code Issues 73 Pull Requests Packages Projects Releases 5 Wiki Activity

refactor: tree mode -> (new) blob mode

refactor/errors
Tomáš Mládek 2023-11-05 14:19:04 +01:00
parent 659ed571b6
commit 862ed1c08a
7 changed files with 53 additions and 47 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
cli/src/main.rs
View File

@ -455,7 +455,7 @@ async fn main() -> Result<()> {
initial: false,
tree_mode: connection
.get_vault_options()?
.tree_mode
.blob_mode
.unwrap_or_default(),
},
);

8
cli/src/routes.rs
View File

@ -40,9 +40,9 @@ use upend_db::hierarchies::{list_roots, resolve_path, UHierPath};
use upend_db::jobs;
use upend_db::stores::UpdateOptions;
use upend_db::stores::{Blob, UpStore};
use upend_db::BlobMode;
use upend_db::UpEndDatabase;
use upend_db::VaultOptions;
use upend_db::VaultRescanMode;
use url::Url;
#[cfg(feature = "desktop")]
@ -768,7 +768,7 @@ pub async fn list_hier_roots(state: web::Data<State>) -> Result<HttpResponse, Er
#[derive(Deserialize)]
pub struct RescanRequest {
initial: Option<bool>,
tree_mode: Option<VaultRescanMode>,
tree_mode: Option<BlobMode>,
}
#[post("/api/refresh")]
@ -790,7 +790,7 @@ pub async fn api_refresh(
tree_mode: query.tree_mode.unwrap_or(
connection
.get_vault_options()?
.tree_mode
.blob_mode
.unwrap_or_default(),
),
},
@ -1156,7 +1156,7 @@ mod tests {
job_container.clone(),
UpdateOptions {
initial: true,
tree_mode: upend_db::VaultRescanMode::default(),
tree_mode: upend_db::BlobMode::default(),
},
)
.unwrap();

43
db/src/lib.rs
View File

@ -239,28 +239,30 @@ impl UpEndConnection {
}
pub fn set_vault_options(&self, options: VaultOptions) -> Result<()> {
if let Some(tree_mode) = options.tree_mode {
let tree_mode = match tree_mode {
VaultRescanMode::Flat => "FLAT".to_string(),
VaultRescanMode::DepthFirst => "DEPTH_FIRST".to_string(),
VaultRescanMode::Mirror => "MIRROR".to_string(),
VaultRescanMode::Incoming(None) => "INCOMING".to_string(),
VaultRescanMode::Incoming(Some(group)) => format!("INCOMING:{}", group),
if let Some(blob_mode) = options.blob_mode {
let tree_mode = match blob_mode {
BlobMode::Flat => "FLAT".to_string(),
BlobMode::DepthFirst => "DEPTH_FIRST".to_string(),
BlobMode::Mirror => "MIRROR".to_string(),
BlobMode::Incoming(None) => "INCOMING".to_string(),
BlobMode::Incoming(Some(group)) => format!("INCOMING:{}", group),
BlobMode::StoreOnly => "STORE_ONLY".to_string(),
};
self.set_meta("VAULT_TREE_MODE", tree_mode)?;
self.set_meta("VAULT_BLOB_MODE", tree_mode)?;
}
Ok(())
}
pub fn get_vault_options(&self) -> Result<VaultOptions> {
let tree_mode = match self.get_meta("VAULT_TREE_MODE")? {
let blob_mode = match self.get_meta("VAULT_BLOB_MODE")? {
Some(mode) => match mode.as_str() {
"FLAT" => Some(VaultRescanMode::Flat),
"DEPTH_FIRST" => Some(VaultRescanMode::DepthFirst),
"MIRROR" => Some(VaultRescanMode::Mirror),
"INCOMING" => Some(VaultRescanMode::Incoming(None)),
"FLAT" => Some(BlobMode::Flat),
"DEPTH_FIRST" => Some(BlobMode::DepthFirst),
"MIRROR" => Some(BlobMode::Mirror),
"INCOMING" => Some(BlobMode::Incoming(None)),
"STORE_ONLY" => Some(BlobMode::StoreOnly),
mode if mode.starts_with("INCOMING:") => {
Some(VaultRescanMode::Incoming(Some(mode[9..].to_string())))
Some(BlobMode::Incoming(Some(mode[9..].to_string())))
}
_ => {
warn!("Unknown vault tree mode: {}", mode);
@ -270,7 +272,7 @@ impl UpEndConnection {
None => None,
};
Ok(VaultOptions { tree_mode })
Ok(VaultOptions { blob_mode })
}
pub fn retrieve_entry(&self, hash: &UpMultihash) -> Result<Option<Entry>> {
@ -603,18 +605,21 @@ mod test {
#[derive(Debug, Serialize, Deserialize)]
pub struct VaultOptions {
pub tree_mode: Option<VaultRescanMode>,
pub blob_mode: Option<BlobMode>,
}
/// Specifies how to store new blobs
#[derive(Debug, Clone, Serialize, Deserialize, Default)]
pub enum VaultRescanMode {
pub enum BlobMode {
#[default]
/// Mirror the filesystem tree
/// Mirror the original tree
Mirror,
/// Like Mirror, but chooses the shortest path
DepthFirst,
/// Use only the last level of the filesystem tree as groups
/// Use only the last level of the tree as a group
Flat,
/// Place all files in a single group
Incoming(Option<String>),
/// Only store files, don't place them anywhere
StoreOnly,
}

35
db/src/stores/fs/mod.rs
View File

@ -5,8 +5,7 @@ use crate::hierarchies::{resolve_path, resolve_path_cached, ResolveCache, UHierP
use crate::jobs::{JobContainer, JobHandle};
use crate::util::hash_at_path;
use crate::{
ConnectionOptions, LoggingHandler, UpEndConnection, UpEndDatabase, VaultRescanMode,
UPEND_SUBDIR,
BlobMode, ConnectionOptions, LoggingHandler, UpEndConnection, UpEndDatabase, UPEND_SUBDIR,
};
use anyhow::{anyhow, Error, Result};
use chrono::prelude::*;
@ -135,7 +134,7 @@ impl FsStore {
let mut upaths: HashMap<PathBuf, UHierPath> = HashMap::new();
match options.tree_mode {
VaultRescanMode::Flat => {
BlobMode::Flat => {
for pb in &pathbufs {
let normalized_path = self.normalize_path(pb).unwrap();
let dirname = normalized_path.parent().and_then(|p| p.components().last());
@ -152,7 +151,7 @@ impl FsStore {
upaths.insert(pb.clone(), upath);
}
}
VaultRescanMode::DepthFirst => {
BlobMode::DepthFirst => {
let mut shallowest: HashMap<String, PathBuf> = HashMap::new();
for path in &pathbufs {
let normalized_path = self.normalize_path(path).unwrap();
@ -185,7 +184,7 @@ impl FsStore {
}
}
}
VaultRescanMode::Mirror => {
BlobMode::Mirror => {
for pb in &pathbufs {
let normalized_path = self.normalize_path(&pb).unwrap();
let path = normalized_path.parent().unwrap();
@ -199,7 +198,7 @@ impl FsStore {
upaths.insert(pb.clone(), UHierPath(upath));
}
}
VaultRescanMode::Incoming(group) => {
BlobMode::Incoming(group) => {
let upath = UHierPath(vec![group
.unwrap_or("INCOMING".to_string())
.parse()
@ -208,6 +207,7 @@ impl FsStore {
upaths.insert(pb.clone(), upath.clone());
}
}
BlobMode::StoreOnly => {}
};
let path_entries = pathbufs
@ -227,6 +227,7 @@ impl FsStore {
Arc::new(Mutex::new(LruCache::new(256)));
let total = path_entries.len() as f32;
let shared_job_handle = Arc::new(Mutex::new(job_handle));
let path_outcomes: Vec<UpdatePathOutcome> = path_entries
.into_par_iter()
.map(|(path, upath)| {
@ -870,7 +871,7 @@ mod test {
job_container,
UpdateOptions {
initial: true,
tree_mode: VaultRescanMode::default(),
tree_mode: BlobMode::default(),
},
);
assert!(rescan_result.is_ok());
@ -916,7 +917,7 @@ mod test {
job,
UpdateOptions {
initial: quick,
tree_mode: VaultRescanMode::default(),
tree_mode: BlobMode::default(),
},
);
@ -936,7 +937,7 @@ mod test {
job,
UpdateOptions {
initial: quick,
tree_mode: VaultRescanMode::default(),
tree_mode: BlobMode::default(),
},
);
@ -959,7 +960,7 @@ mod test {
job,
UpdateOptions {
initial: quick,
tree_mode: VaultRescanMode::default(),
tree_mode: BlobMode::default(),
},
);
@ -1004,7 +1005,7 @@ mod test {
/// │   └── bar.txt
/// └── in_root.txt
/// ```
fn _prepare_hier_vault(tree_mode: VaultRescanMode) -> (UpEndConnection, TempDir) {
fn _prepare_hier_vault(tree_mode: BlobMode) -> (UpEndConnection, TempDir) {
// Prepare temporary filesystem structure
let temp_dir = TempDir::new().unwrap();
let temp_dir_path = temp_dir.path().canonicalize().unwrap();
@ -1068,7 +1069,7 @@ mod test {
});
}
fn test_initial_scan(mode: VaultRescanMode, expected_paths: Vec<&str>) {
fn test_initial_scan(mode: BlobMode, expected_paths: Vec<&str>) {
let (connection, _vault_dir) = _prepare_hier_vault(mode);
assert_paths(expected_paths, &connection);
}
@ -1076,7 +1077,7 @@ mod test {
#[test]
fn test_mirror_mode() {
test_initial_scan(
VaultRescanMode::Mirror,
BlobMode::Mirror,
vec![
"NATIVE",
"NATIVE/nested_directory/nested_two/nested_three/foo.txt",
@ -1090,7 +1091,7 @@ mod test {
#[test]
fn test_flat_mode() {
test_initial_scan(
VaultRescanMode::Flat,
BlobMode::Flat,
vec![
"NATIVE",
"NATIVE/nested_three/foo.txt",
@ -1104,7 +1105,7 @@ mod test {
#[test]
fn test_depth_mode() {
test_initial_scan(
VaultRescanMode::DepthFirst,
BlobMode::DepthFirst,
vec![
"NATIVE",
"NATIVE/nested_directory/nested_two/nested_four/baz.txt",
@ -1118,7 +1119,7 @@ mod test {
#[test]
fn test_incoming_mode() {
test_initial_scan(
VaultRescanMode::Incoming(None),
BlobMode::Incoming(None),
vec![
"INCOMING/foo.txt",
"INCOMING/baz.txt",
@ -1128,7 +1129,7 @@ mod test {
);
test_initial_scan(
VaultRescanMode::Incoming(Some("new files".to_string())),
BlobMode::Incoming(Some("new files".to_string())),
vec![
"new files/foo.txt",
"new files/baz.txt",

4
db/src/stores/mod.rs
View File

@ -1,7 +1,7 @@
use std::path::{Path, PathBuf};
use super::{UpEndConnection, UpEndDatabase};
use crate::{jobs::JobContainer, VaultRescanMode};
use crate::{jobs::JobContainer, BlobMode};
use upend_base::hash::UpMultihash;
pub mod fs;
@ -73,5 +73,5 @@ pub trait UpStore {
#[derive(Debug, Clone)]
pub struct UpdateOptions {
pub initial: bool,
pub tree_mode: VaultRescanMode,
pub tree_mode: BlobMode,
}

2
webui/src/views/Home.svelte
View File

@ -156,7 +156,7 @@
fetch("/api/options")
.then((res) => res.json())
.then((options) => {
if (!options.tree_mode) {
if (!options.blob_mode) {
navigate("/setup");
}
});

6
webui/src/views/Setup.svelte
View File

@ -9,15 +9,15 @@
let mode: "Flat" | "DepthFirst" | "Mirror" | "Incoming" = undefined;
async function submitOptions() {
const tree_mode = {};
tree_mode[mode] = null;
const blob_mode = {};
blob_mode[mode] = null;
const optionResponse = await fetch("/api/options", {
method: "PUT",
headers: {
"Content-Type": "application/json",
},
body: JSON.stringify({ tree_mode }),
body: JSON.stringify({ blob_mode }),
});
if (!optionResponse.ok) {
throw new Error("Failed to set options");