feat(db): add an "INCOMING" rescan mode
ci/woodpecker/push/woodpecker Pipeline failed
Details
ci/woodpecker/push/woodpecker Pipeline failed
Details
parent
203b105b15
commit
d10b28621e
|
@ -42,6 +42,7 @@ use upend_db::stores::UpdateOptions;
|
|||
use upend_db::stores::{Blob, UpStore};
|
||||
use upend_db::UpEndDatabase;
|
||||
use upend_db::VaultOptions;
|
||||
use upend_db::VaultRescanMode;
|
||||
use url::Url;
|
||||
|
||||
#[cfg(feature = "desktop")]
|
||||
|
@ -767,6 +768,7 @@ pub async fn list_hier_roots(state: web::Data<State>) -> Result<HttpResponse, Er
|
|||
#[derive(Deserialize)]
|
||||
pub struct RescanRequest {
|
||||
initial: Option<bool>,
|
||||
tree_mode: Option<VaultRescanMode>,
|
||||
}
|
||||
|
||||
#[post("/api/refresh")]
|
||||
|
@ -785,10 +787,12 @@ pub async fn api_refresh(
|
|||
state.job_container.clone(),
|
||||
UpdateOptions {
|
||||
initial: query.initial.unwrap_or(false),
|
||||
tree_mode: connection
|
||||
.get_vault_options()?
|
||||
.tree_mode
|
||||
.unwrap_or_default(),
|
||||
tree_mode: query.tree_mode.unwrap_or(
|
||||
connection
|
||||
.get_vault_options()?
|
||||
.tree_mode
|
||||
.unwrap_or_default(),
|
||||
),
|
||||
},
|
||||
);
|
||||
let _ = crate::extractors::extract_all(
|
||||
|
@ -1152,7 +1156,7 @@ mod tests {
|
|||
job_container.clone(),
|
||||
UpdateOptions {
|
||||
initial: true,
|
||||
tree_mode: upend_db::VaultTreeMode::default(),
|
||||
tree_mode: upend_db::VaultRescanMode::default(),
|
||||
},
|
||||
)
|
||||
.unwrap();
|
||||
|
|
|
@ -241,9 +241,11 @@ impl UpEndConnection {
|
|||
pub fn set_vault_options(&self, options: VaultOptions) -> Result<()> {
|
||||
if let Some(tree_mode) = options.tree_mode {
|
||||
let tree_mode = match tree_mode {
|
||||
VaultTreeMode::Flat => "FLAT",
|
||||
VaultTreeMode::DepthFirst => "DEPTH_FIRST",
|
||||
VaultTreeMode::Mirror => "MIRROR",
|
||||
VaultRescanMode::Flat => "FLAT".to_string(),
|
||||
VaultRescanMode::DepthFirst => "DEPTH_FIRST".to_string(),
|
||||
VaultRescanMode::Mirror => "MIRROR".to_string(),
|
||||
VaultRescanMode::Incoming(None) => "INCOMING".to_string(),
|
||||
VaultRescanMode::Incoming(Some(group)) => format!("INCOMING:{}", group),
|
||||
};
|
||||
self.set_meta("VAULT_TREE_MODE", tree_mode)?;
|
||||
}
|
||||
|
@ -253,9 +255,13 @@ impl UpEndConnection {
|
|||
pub fn get_vault_options(&self) -> Result<VaultOptions> {
|
||||
let tree_mode = match self.get_meta("VAULT_TREE_MODE")? {
|
||||
Some(mode) => match mode.as_str() {
|
||||
"FLAT" => Some(VaultTreeMode::Flat),
|
||||
"DEPTH_FIRST" => Some(VaultTreeMode::DepthFirst),
|
||||
"MIRROR" => Some(VaultTreeMode::Mirror),
|
||||
"FLAT" => Some(VaultRescanMode::Flat),
|
||||
"DEPTH_FIRST" => Some(VaultRescanMode::DepthFirst),
|
||||
"MIRROR" => Some(VaultRescanMode::Mirror),
|
||||
"INCOMING" => Some(VaultRescanMode::Incoming(None)),
|
||||
mode if mode.starts_with("INCOMING:") => {
|
||||
Some(VaultRescanMode::Incoming(Some(mode[9..].to_string())))
|
||||
}
|
||||
_ => {
|
||||
warn!("Unknown vault tree mode: {}", mode);
|
||||
None
|
||||
|
@ -597,13 +603,18 @@ mod test {
|
|||
|
||||
#[derive(Debug, Serialize, Deserialize)]
|
||||
pub struct VaultOptions {
|
||||
pub tree_mode: Option<VaultTreeMode>,
|
||||
pub tree_mode: Option<VaultRescanMode>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize, Default)]
|
||||
pub enum VaultTreeMode {
|
||||
Flat,
|
||||
DepthFirst,
|
||||
pub enum VaultRescanMode {
|
||||
#[default]
|
||||
/// Mirror the filesystem tree
|
||||
Mirror,
|
||||
/// Like Mirror, but chooses the shortest path
|
||||
DepthFirst,
|
||||
/// Use only the last level of the filesystem tree as groups
|
||||
Flat,
|
||||
/// Place all files in a single group
|
||||
Incoming(Option<String>),
|
||||
}
|
||||
|
|
|
@ -5,7 +5,8 @@ use crate::hierarchies::{resolve_path, resolve_path_cached, ResolveCache, UHierP
|
|||
use crate::jobs::{JobContainer, JobHandle};
|
||||
use crate::util::hash_at_path;
|
||||
use crate::{
|
||||
ConnectionOptions, LoggingHandler, UpEndConnection, UpEndDatabase, VaultTreeMode, UPEND_SUBDIR,
|
||||
ConnectionOptions, LoggingHandler, UpEndConnection, UpEndDatabase, VaultRescanMode,
|
||||
UPEND_SUBDIR,
|
||||
};
|
||||
use anyhow::{anyhow, Error, Result};
|
||||
use chrono::prelude::*;
|
||||
|
@ -134,7 +135,7 @@ impl FsStore {
|
|||
|
||||
let mut upaths: HashMap<PathBuf, UHierPath> = HashMap::new();
|
||||
match options.tree_mode {
|
||||
VaultTreeMode::Flat => {
|
||||
VaultRescanMode::Flat => {
|
||||
for pb in &pathbufs {
|
||||
let normalized_path = self.normalize_path(pb).unwrap();
|
||||
let dirname = normalized_path.parent().and_then(|p| p.components().last());
|
||||
|
@ -151,7 +152,7 @@ impl FsStore {
|
|||
upaths.insert(pb.clone(), upath);
|
||||
}
|
||||
}
|
||||
VaultTreeMode::DepthFirst => {
|
||||
VaultRescanMode::DepthFirst => {
|
||||
let mut shallowest: HashMap<String, PathBuf> = HashMap::new();
|
||||
for path in &pathbufs {
|
||||
let normalized_path = self.normalize_path(path).unwrap();
|
||||
|
@ -184,7 +185,7 @@ impl FsStore {
|
|||
}
|
||||
}
|
||||
}
|
||||
VaultTreeMode::Mirror => {
|
||||
VaultRescanMode::Mirror => {
|
||||
for pb in &pathbufs {
|
||||
let normalized_path = self.normalize_path(&pb).unwrap();
|
||||
let path = normalized_path.parent().unwrap();
|
||||
|
@ -198,6 +199,15 @@ impl FsStore {
|
|||
upaths.insert(pb.clone(), UHierPath(upath));
|
||||
}
|
||||
}
|
||||
VaultRescanMode::Incoming(group) => {
|
||||
let upath = UHierPath(vec![group
|
||||
.unwrap_or("INCOMING".to_string())
|
||||
.parse()
|
||||
.unwrap()]);
|
||||
for pb in &pathbufs {
|
||||
upaths.insert(pb.clone(), upath.clone());
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
let path_entries = pathbufs
|
||||
|
@ -860,7 +870,7 @@ mod test {
|
|||
job_container,
|
||||
UpdateOptions {
|
||||
initial: true,
|
||||
tree_mode: VaultTreeMode::default(),
|
||||
tree_mode: VaultRescanMode::default(),
|
||||
},
|
||||
);
|
||||
assert!(rescan_result.is_ok());
|
||||
|
@ -906,7 +916,7 @@ mod test {
|
|||
job,
|
||||
UpdateOptions {
|
||||
initial: quick,
|
||||
tree_mode: VaultTreeMode::default(),
|
||||
tree_mode: VaultRescanMode::default(),
|
||||
},
|
||||
);
|
||||
|
||||
|
@ -926,7 +936,7 @@ mod test {
|
|||
job,
|
||||
UpdateOptions {
|
||||
initial: quick,
|
||||
tree_mode: VaultTreeMode::default(),
|
||||
tree_mode: VaultRescanMode::default(),
|
||||
},
|
||||
);
|
||||
|
||||
|
@ -949,7 +959,7 @@ mod test {
|
|||
job,
|
||||
UpdateOptions {
|
||||
initial: quick,
|
||||
tree_mode: VaultTreeMode::default(),
|
||||
tree_mode: VaultRescanMode::default(),
|
||||
},
|
||||
);
|
||||
|
||||
|
@ -994,7 +1004,7 @@ mod test {
|
|||
/// │ └── bar.txt
|
||||
/// └── in_root.txt
|
||||
/// ```
|
||||
fn _prepare_hier_vault(tree_mode: VaultTreeMode) -> (UpEndConnection, TempDir) {
|
||||
fn _prepare_hier_vault(tree_mode: VaultRescanMode) -> (UpEndConnection, TempDir) {
|
||||
// Prepare temporary filesystem structure
|
||||
let temp_dir = TempDir::new().unwrap();
|
||||
let temp_dir_path = temp_dir.path().canonicalize().unwrap();
|
||||
|
@ -1058,7 +1068,7 @@ mod test {
|
|||
});
|
||||
}
|
||||
|
||||
fn test_initial_scan(mode: VaultTreeMode, expected_paths: Vec<&str>) {
|
||||
fn test_initial_scan(mode: VaultRescanMode, expected_paths: Vec<&str>) {
|
||||
let (connection, _vault_dir) = _prepare_hier_vault(mode);
|
||||
assert_paths(expected_paths, &connection);
|
||||
}
|
||||
|
@ -1066,7 +1076,7 @@ mod test {
|
|||
#[test]
|
||||
fn test_mirror_mode() {
|
||||
test_initial_scan(
|
||||
VaultTreeMode::Mirror,
|
||||
VaultRescanMode::Mirror,
|
||||
vec![
|
||||
"NATIVE",
|
||||
"NATIVE/nested_directory/nested_two/nested_three/foo.txt",
|
||||
|
@ -1080,7 +1090,7 @@ mod test {
|
|||
#[test]
|
||||
fn test_flat_mode() {
|
||||
test_initial_scan(
|
||||
VaultTreeMode::Flat,
|
||||
VaultRescanMode::Flat,
|
||||
vec![
|
||||
"NATIVE",
|
||||
"NATIVE/nested_three/foo.txt",
|
||||
|
@ -1094,7 +1104,7 @@ mod test {
|
|||
#[test]
|
||||
fn test_depth_mode() {
|
||||
test_initial_scan(
|
||||
VaultTreeMode::DepthFirst,
|
||||
VaultRescanMode::DepthFirst,
|
||||
vec![
|
||||
"NATIVE",
|
||||
"NATIVE/nested_directory/nested_two/nested_four/baz.txt",
|
||||
|
@ -1104,4 +1114,27 @@ mod test {
|
|||
],
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_incoming_mode() {
|
||||
test_initial_scan(
|
||||
VaultRescanMode::Incoming(None),
|
||||
vec![
|
||||
"INCOMING/foo.txt",
|
||||
"INCOMING/baz.txt",
|
||||
"INCOMING/bar.txt",
|
||||
"INCOMING/in_root.txt",
|
||||
],
|
||||
);
|
||||
|
||||
test_initial_scan(
|
||||
VaultRescanMode::Incoming(Some("new files".to_string())),
|
||||
vec![
|
||||
"new files/foo.txt",
|
||||
"new files/baz.txt",
|
||||
"new files/bar.txt",
|
||||
"new files/in_root.txt",
|
||||
],
|
||||
);
|
||||
}
|
||||
}
|
||||
|
|
|
@ -1,7 +1,7 @@
|
|||
use std::path::{Path, PathBuf};
|
||||
|
||||
use super::{UpEndConnection, UpEndDatabase};
|
||||
use crate::{jobs::JobContainer, VaultTreeMode};
|
||||
use crate::{jobs::JobContainer, VaultRescanMode};
|
||||
use upend_base::hash::UpMultihash;
|
||||
|
||||
pub mod fs;
|
||||
|
@ -73,5 +73,5 @@ pub trait UpStore {
|
|||
#[derive(Debug, Clone)]
|
||||
pub struct UpdateOptions {
|
||||
pub initial: bool,
|
||||
pub tree_mode: VaultTreeMode,
|
||||
pub tree_mode: VaultRescanMode,
|
||||
}
|
||||
|
|
Loading…
Reference in New Issue