From ef590908595a0bbe958d5217234f347dd7bad548 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1=C5=A1=20Ml=C3=A1dek?= Date: Sun, 23 Jan 2022 13:00:59 +0100 Subject: [PATCH] separate adding file into its own fn, refactor private fn names --- src/database/mod.rs | 6 ++-- src/filesystem.rs | 85 ++++++++++++++++++++++++++++++++------------- 2 files changed, 63 insertions(+), 28 deletions(-) diff --git a/src/database/mod.rs b/src/database/mod.rs index cb275d8..7b40292 100644 --- a/src/database/mod.rs +++ b/src/database/mod.rs @@ -337,7 +337,7 @@ impl UpEndConnection { pub fn insert_entry(&self, entry: Entry) -> Result
{ debug!("Inserting: {}", entry); let db_entry = models::Entry::try_from(&entry)?; - self._insert_entry(db_entry)?; + self.insert_model_entry(db_entry)?; entry.address() } @@ -345,11 +345,11 @@ impl UpEndConnection { debug!("Inserting immutably: {}", entry); let address = entry.address()?; let db_entry = models::Entry::try_from(&ImmutableEntry(entry))?; - self._insert_entry(db_entry)?; + self.insert_model_entry(db_entry)?; Ok(address) } - fn _insert_entry(&self, entry: models::Entry) -> Result { + fn insert_model_entry(&self, entry: models::Entry) -> Result { let result = diesel::insert_into(data::table) .values(&entry) .execute(&self.conn); diff --git a/src/filesystem.rs b/src/filesystem.rs index b31cab6..fcc27a3 100644 --- a/src/filesystem.rs +++ b/src/filesystem.rs @@ -10,12 +10,14 @@ use crate::database::constants::{ HIER_HAS_ATTR, IS_OF_TYPE_ATTR, TYPE_ADDR, TYPE_BASE_ATTR, TYPE_HAS_ATTR, }; use crate::database::entry::{Entry, EntryValue, InvariantEntry}; -use crate::database::hierarchies::{resolve_path_cached, ResolveCache, UHierPath, UNode}; +use crate::database::hierarchies::{ + resolve_path, resolve_path_cached, ResolveCache, UHierPath, UNode, +}; use crate::database::inner::models; use crate::database::{UpEndConnection, UpEndDatabase, UPEND_SUBDIR}; use crate::util::hash::{Hash, Hashable}; use crate::util::jobs::{Job, JobContainer, JobId, State}; -use anyhow::{Error, Result}; +use anyhow::{anyhow, Error, Result}; use chrono::prelude::*; use log::{debug, error, info, warn}; use lru::LruCache; @@ -60,9 +62,10 @@ pub async fn rescan_vault( .unwrap(); let job_container_rescan = job_container.clone(); - let result = - actix_web::web::block(move || _rescan_vault(db, job_container_rescan, job_id, initial)) - .await; + let result = actix_web::web::block(move || { + rescan_vault_inner(db, job_container_rescan, job_id, initial) + }) + .await; if result.is_err() { let err = result.err().unwrap(); @@ -100,7 +103,7 @@ enum UpdatePathOutcome { Failed(PathBuf, Error), } -fn _rescan_vault>( +fn rescan_vault_inner>( db: D, job_container: Arc>, job_id: JobId, @@ -146,7 +149,7 @@ fn _rescan_vault>( let path_outcomes: Vec = path_entries .into_par_iter() .map(|path| { - let result = _process_directory_entry( + let result = process_directory_entry( db.connection().unwrap(), &resolve_cache, path.clone(), @@ -237,7 +240,7 @@ fn _rescan_vault>( Ok(all_outcomes) } -fn _process_directory_entry>( +fn process_directory_entry>( connection: UpEndConnection, resolve_cache: &Arc>, path: PathBuf, @@ -317,36 +320,63 @@ fn _process_directory_entry>( if file_hash.is_none() { file_hash = Some(path.hash()?); } - let file_hash = file_hash.unwrap(); + let mime_type = tree_magic_mini::from_filepath(&path).map(|s| s.to_string()); + insert_file_with_metadata( + &connection, + normalized_path, + file_hash.unwrap(), + size, + mtime, + mime_type, + Some(resolve_cache), + ) + .map(|_| { + info!("Added: {:?}", path); + UpdatePathOutcome::Added(path.clone()) + }) +} + +fn insert_file_with_metadata( + connection: &UpEndConnection, + normalized_path: &Path, + hash: Hash, + size: i64, + mtime: Option, + mime_type: Option, + resolve_cache: Option<&Arc>>, +) -> Result<()> { let new_file = models::NewFile { - path: normalized_path_str.to_string(), - hash: (file_hash.clone()).0, + path: normalized_path + .to_str() + .ok_or(anyhow!("Path not UTF-8?!"))? + .to_string(), + hash: (hash.clone()).0, added: NaiveDateTime::from_timestamp(Utc::now().timestamp(), 0), size, mtime, }; - // Insert metadata + // Metadata let type_entry = Entry { - entity: Address::Hash(file_hash.clone()), + entity: Address::Hash(hash.clone()), attribute: String::from(IS_OF_TYPE_ATTR), value: EntryValue::Address(BLOB_TYPE_ADDR.clone()), }; let size_entry = Entry { - entity: Address::Hash(file_hash.clone()), + entity: Address::Hash(hash.clone()), attribute: FILE_SIZE_KEY.to_string(), value: EntryValue::Value(Value::from(size)), }; - let mime_entry = tree_magic_mini::from_filepath(&path).map(|mime_type| Entry { - entity: Address::Hash(file_hash.clone()), + let mime_entry = mime_type.map(|mime_type| Entry { + entity: Address::Hash(hash.clone()), attribute: FILE_MIME_KEY.to_string(), - value: EntryValue::Value(Value::String(mime_type.to_string())), + value: EntryValue::Value(Value::String(mime_type)), }); - // Finally, add the appropriate entries w/r/t virtual filesystem location + // Add the appropriate entries w/r/t virtual filesystem location let components = normalized_path.components().collect::>(); let (filename, dir_path) = components.split_last().unwrap(); @@ -357,9 +387,13 @@ fn _process_directory_entry>( })) .collect(), ); - let resolved_path = resolve_path_cached(&connection, &upath, true, resolve_cache)?; + let resolved_path = match resolve_cache { + Some(cache) => resolve_path_cached(connection, &upath, true, cache)?, + None => resolve_path(connection, &upath, true)?, + }; let parent_dir = resolved_path.last().unwrap(); + // Insert all connection.transaction::<_, Error, _>(|| { connection.insert_file(new_file)?; @@ -372,7 +406,7 @@ fn _process_directory_entry>( let dir_has_entry = Entry { entity: parent_dir.clone(), attribute: HIER_HAS_ATTR.to_string(), - value: EntryValue::Address(Address::Hash(file_hash.clone())), + value: EntryValue::Address(Address::Hash(hash.clone())), }; let dir_has_entry_addr = connection.insert_entry(dir_has_entry)?; @@ -385,8 +419,7 @@ fn _process_directory_entry>( }; connection.insert_entry(name_entry)?; - info!("Added: {:?}", path); - Ok(UpdatePathOutcome::Added(path.clone())) + Ok(()) }) } @@ -427,7 +460,8 @@ mod test { .unwrap(); // Initial scan - let rescan_result = _rescan_vault(&open_result.db, job_container.clone(), job_id, true); + let rescan_result = + rescan_vault_inner(&open_result.db, job_container.clone(), job_id, true); assert!(rescan_result.is_ok()); let rescan_result = rescan_result.unwrap(); @@ -438,7 +472,8 @@ mod test { // Modification-less rescan - let rescan_result = _rescan_vault(&open_result.db, job_container.clone(), job_id, false); + let rescan_result = + rescan_vault_inner(&open_result.db, job_container.clone(), job_id, false); assert!(rescan_result.is_ok()); let rescan_result = rescan_result.unwrap(); @@ -451,7 +486,7 @@ mod test { std::fs::remove_file(temp_dir.path().join("hello-world.txt")).unwrap(); - let rescan_result = _rescan_vault(&open_result.db, job_container, job_id, false); + let rescan_result = rescan_vault_inner(&open_result.db, job_container, job_id, false); assert!(rescan_result.is_ok()); let rescan_result = rescan_result.unwrap();