feat(db): add an "INCOMING" rescan mode

cli/src/routes.rs

@@ -42,6 +42,7 @@ use upend_db::stores::UpdateOptions;
 use upend_db::stores::{Blob, UpStore};
 use upend_db::UpEndDatabase;
 use upend_db::VaultOptions;
+use upend_db::VaultRescanMode;
 use url::Url;
 
 #[cfg(feature = "desktop")]
@@ -767,6 +768,7 @@ pub async fn list_hier_roots(state: web::Data<State>) -> Result<HttpResponse, Er
 #[derive(Deserialize)]
 pub struct RescanRequest {
     initial: Option<bool>,
+    tree_mode: Option<VaultRescanMode>,
 }
 
 #[post("/api/refresh")]
@@ -785,10 +787,12 @@ pub async fn api_refresh(
             state.job_container.clone(),
             UpdateOptions {
                 initial: query.initial.unwrap_or(false),
-                tree_mode: connection
+                tree_mode: query.tree_mode.unwrap_or(
-                    .get_vault_options()?
+                    connection
-                    .tree_mode
+                        .get_vault_options()?
-                    .unwrap_or_default(),
+                        .tree_mode
+                        .unwrap_or_default(),
+                ),
             },
         );
         let _ = crate::extractors::extract_all(
@@ -1152,7 +1156,7 @@ mod tests {
                 job_container.clone(),
                 UpdateOptions {
                     initial: true,
-                    tree_mode: upend_db::VaultTreeMode::default(),
+                    tree_mode: upend_db::VaultRescanMode::default(),
                 },
             )
             .unwrap();

db/src/lib.rs

@@ -241,9 +241,11 @@ impl UpEndConnection {
     pub fn set_vault_options(&self, options: VaultOptions) -> Result<()> {
         if let Some(tree_mode) = options.tree_mode {
             let tree_mode = match tree_mode {
-                VaultTreeMode::Flat => "FLAT",
+                VaultRescanMode::Flat => "FLAT".to_string(),
-                VaultTreeMode::DepthFirst => "DEPTH_FIRST",
+                VaultRescanMode::DepthFirst => "DEPTH_FIRST".to_string(),
-                VaultTreeMode::Mirror => "MIRROR",
+                VaultRescanMode::Mirror => "MIRROR".to_string(),
+                VaultRescanMode::Incoming(None) => "INCOMING".to_string(),
+                VaultRescanMode::Incoming(Some(group)) => format!("INCOMING:{}", group),
             };
             self.set_meta("VAULT_TREE_MODE", tree_mode)?;
         }
@@ -253,9 +255,13 @@ impl UpEndConnection {
     pub fn get_vault_options(&self) -> Result<VaultOptions> {
         let tree_mode = match self.get_meta("VAULT_TREE_MODE")? {
             Some(mode) => match mode.as_str() {
-                "FLAT" => Some(VaultTreeMode::Flat),
+                "FLAT" => Some(VaultRescanMode::Flat),
-                "DEPTH_FIRST" => Some(VaultTreeMode::DepthFirst),
+                "DEPTH_FIRST" => Some(VaultRescanMode::DepthFirst),
-                "MIRROR" => Some(VaultTreeMode::Mirror),
+                "MIRROR" => Some(VaultRescanMode::Mirror),
+                "INCOMING" => Some(VaultRescanMode::Incoming(None)),
+                mode if mode.starts_with("INCOMING:") => {
+                    Some(VaultRescanMode::Incoming(Some(mode[9..].to_string())))
+                }
                 _ => {
                     warn!("Unknown vault tree mode: {}", mode);
                     None
@@ -597,13 +603,18 @@ mod test {
 
 #[derive(Debug, Serialize, Deserialize)]
 pub struct VaultOptions {
-    pub tree_mode: Option<VaultTreeMode>,
+    pub tree_mode: Option<VaultRescanMode>,
 }
 
 #[derive(Debug, Clone, Serialize, Deserialize, Default)]
-pub enum VaultTreeMode {
+pub enum VaultRescanMode {
-    Flat,
-    DepthFirst,
     #[default]
+    /// Mirror the filesystem tree
     Mirror,
+    /// Like Mirror, but chooses the shortest path
+    DepthFirst,
+    /// Use only the last level of the filesystem tree as groups
+    Flat,
+    /// Place all files in a single group
+    Incoming(Option<String>),
 }

db/src/stores/fs/mod.rs

@@ -5,7 +5,8 @@ use crate::hierarchies::{resolve_path, resolve_path_cached, ResolveCache, UHierP
 use crate::jobs::{JobContainer, JobHandle};
 use crate::util::hash_at_path;
 use crate::{
-    ConnectionOptions, LoggingHandler, UpEndConnection, UpEndDatabase, VaultTreeMode, UPEND_SUBDIR,
+    ConnectionOptions, LoggingHandler, UpEndConnection, UpEndDatabase, VaultRescanMode,
+    UPEND_SUBDIR,
 };
 use anyhow::{anyhow, Error, Result};
 use chrono::prelude::*;
@@ -134,7 +135,7 @@ impl FsStore {
 
         let mut upaths: HashMap<PathBuf, UHierPath> = HashMap::new();
         match options.tree_mode {
-            VaultTreeMode::Flat => {
+            VaultRescanMode::Flat => {
                 for pb in &pathbufs {
                     let normalized_path = self.normalize_path(pb).unwrap();
                     let dirname = normalized_path.parent().and_then(|p| p.components().last());
@@ -151,7 +152,7 @@ impl FsStore {
                     upaths.insert(pb.clone(), upath);
                 }
             }
-            VaultTreeMode::DepthFirst => {
+            VaultRescanMode::DepthFirst => {
                 let mut shallowest: HashMap<String, PathBuf> = HashMap::new();
                 for path in &pathbufs {
                     let normalized_path = self.normalize_path(path).unwrap();
@@ -184,7 +185,7 @@ impl FsStore {
                     }
                 }
             }
-            VaultTreeMode::Mirror => {
+            VaultRescanMode::Mirror => {
                 for pb in &pathbufs {
                     let normalized_path = self.normalize_path(&pb).unwrap();
                     let path = normalized_path.parent().unwrap();
@@ -198,6 +199,15 @@ impl FsStore {
                     upaths.insert(pb.clone(), UHierPath(upath));
                 }
             }
+            VaultRescanMode::Incoming(group) => {
+                let upath = UHierPath(vec![group
+                    .unwrap_or("INCOMING".to_string())
+                    .parse()
+                    .unwrap()]);
+                for pb in &pathbufs {
+                    upaths.insert(pb.clone(), upath.clone());
+                }
+            }
         };
 
         let path_entries = pathbufs
@@ -860,7 +870,7 @@ mod test {
             job_container,
             UpdateOptions {
                 initial: true,
-                tree_mode: VaultTreeMode::default(),
+                tree_mode: VaultRescanMode::default(),
             },
         );
         assert!(rescan_result.is_ok());
@@ -906,7 +916,7 @@ mod test {
             job,
             UpdateOptions {
                 initial: quick,
-                tree_mode: VaultTreeMode::default(),
+                tree_mode: VaultRescanMode::default(),
             },
         );
 
@@ -926,7 +936,7 @@ mod test {
             job,
             UpdateOptions {
                 initial: quick,
-                tree_mode: VaultTreeMode::default(),
+                tree_mode: VaultRescanMode::default(),
             },
         );
 
@@ -949,7 +959,7 @@ mod test {
             job,
             UpdateOptions {
                 initial: quick,
-                tree_mode: VaultTreeMode::default(),
+                tree_mode: VaultRescanMode::default(),
             },
         );
 
@@ -994,7 +1004,7 @@ mod test {
     /// │       └── bar.txt
     /// └── in_root.txt
     /// ```
-    fn _prepare_hier_vault(tree_mode: VaultTreeMode) -> (UpEndConnection, TempDir) {
+    fn _prepare_hier_vault(tree_mode: VaultRescanMode) -> (UpEndConnection, TempDir) {
         // Prepare temporary filesystem structure
         let temp_dir = TempDir::new().unwrap();
         let temp_dir_path = temp_dir.path().canonicalize().unwrap();
@@ -1058,7 +1068,7 @@ mod test {
         });
     }
 
-    fn test_initial_scan(mode: VaultTreeMode, expected_paths: Vec<&str>) {
+    fn test_initial_scan(mode: VaultRescanMode, expected_paths: Vec<&str>) {
         let (connection, _vault_dir) = _prepare_hier_vault(mode);
         assert_paths(expected_paths, &connection);
     }
@@ -1066,7 +1076,7 @@ mod test {
     #[test]
     fn test_mirror_mode() {
         test_initial_scan(
-            VaultTreeMode::Mirror,
+            VaultRescanMode::Mirror,
             vec![
                 "NATIVE",
                 "NATIVE/nested_directory/nested_two/nested_three/foo.txt",
@@ -1080,7 +1090,7 @@ mod test {
     #[test]
     fn test_flat_mode() {
         test_initial_scan(
-            VaultTreeMode::Flat,
+            VaultRescanMode::Flat,
             vec![
                 "NATIVE",
                 "NATIVE/nested_three/foo.txt",
@@ -1094,7 +1104,7 @@ mod test {
     #[test]
     fn test_depth_mode() {
         test_initial_scan(
-            VaultTreeMode::DepthFirst,
+            VaultRescanMode::DepthFirst,
             vec![
                 "NATIVE",
                 "NATIVE/nested_directory/nested_two/nested_four/baz.txt",
@@ -1104,4 +1114,27 @@ mod test {
             ],
         );
     }
+
+    #[test]
+    fn test_incoming_mode() {
+        test_initial_scan(
+            VaultRescanMode::Incoming(None),
+            vec![
+                "INCOMING/foo.txt",
+                "INCOMING/baz.txt",
+                "INCOMING/bar.txt",
+                "INCOMING/in_root.txt",
+            ],
+        );
+
+        test_initial_scan(
+            VaultRescanMode::Incoming(Some("new files".to_string())),
+            vec![
+                "new files/foo.txt",
+                "new files/baz.txt",
+                "new files/bar.txt",
+                "new files/in_root.txt",
+            ],
+        );
+    }
 }

db/src/stores/mod.rs

@@ -1,7 +1,7 @@
 use std::path::{Path, PathBuf};
 
 use super::{UpEndConnection, UpEndDatabase};
-use crate::{jobs::JobContainer, VaultTreeMode};
+use crate::{jobs::JobContainer, VaultRescanMode};
 use upend_base::hash::UpMultihash;
 
 pub mod fs;
@@ -73,5 +73,5 @@ pub trait UpStore {
 #[derive(Debug, Clone)]
 pub struct UpdateOptions {
     pub initial: bool,
-    pub tree_mode: VaultTreeMode,
+    pub tree_mode: VaultRescanMode,
 }