thm
/
upend
1
0
Fork 0
Code Issues 73 Pull Requests Packages Projects Releases 5 Wiki Activity

separate adding file into its own fn, refactor private fn names

feat/vaults
Tomáš Mládek 2022-01-23 13:00:59 +01:00
parent dec2e4daf1
commit ef59090859
No known key found for this signature in database
GPG Key ID: ED21612889E75EC5
2 changed files with 63 additions and 28 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
src/database/mod.rs
View File

@ -337,7 +337,7 @@ impl UpEndConnection {
pub fn insert_entry(&self, entry: Entry) -> Result<Address> {
debug!("Inserting: {}", entry);
let db_entry = models::Entry::try_from(&entry)?;
self._insert_entry(db_entry)?;
self.insert_model_entry(db_entry)?;
entry.address()
}
@ -345,11 +345,11 @@ impl UpEndConnection {
debug!("Inserting immutably: {}", entry);
let address = entry.address()?;
let db_entry = models::Entry::try_from(&ImmutableEntry(entry))?;
self._insert_entry(db_entry)?;
self.insert_model_entry(db_entry)?;
Ok(address)
}
fn _insert_entry(&self, entry: models::Entry) -> Result<usize> {
fn insert_model_entry(&self, entry: models::Entry) -> Result<usize> {
let result = diesel::insert_into(data::table)
.values(&entry)
.execute(&self.conn);

85
src/filesystem.rs
View File

@ -10,12 +10,14 @@ use crate::database::constants::{
HIER_HAS_ATTR, IS_OF_TYPE_ATTR, TYPE_ADDR, TYPE_BASE_ATTR, TYPE_HAS_ATTR,
};
use crate::database::entry::{Entry, EntryValue, InvariantEntry};
use crate::database::hierarchies::{resolve_path_cached, ResolveCache, UHierPath, UNode};
use crate::database::hierarchies::{
resolve_path, resolve_path_cached, ResolveCache, UHierPath, UNode,
};
use crate::database::inner::models;
use crate::database::{UpEndConnection, UpEndDatabase, UPEND_SUBDIR};
use crate::util::hash::{Hash, Hashable};
use crate::util::jobs::{Job, JobContainer, JobId, State};
use anyhow::{Error, Result};
use anyhow::{anyhow, Error, Result};
use chrono::prelude::*;
use log::{debug, error, info, warn};
use lru::LruCache;
@ -60,9 +62,10 @@ pub async fn rescan_vault(
.unwrap();
let job_container_rescan = job_container.clone();
let result =
actix_web::web::block(move || _rescan_vault(db, job_container_rescan, job_id, initial))
.await;
let result = actix_web::web::block(move || {
rescan_vault_inner(db, job_container_rescan, job_id, initial)
})
.await;
if result.is_err() {
let err = result.err().unwrap();
@ -100,7 +103,7 @@ enum UpdatePathOutcome {
Failed(PathBuf, Error),
}
fn _rescan_vault<D: Borrow<UpEndDatabase>>(
fn rescan_vault_inner<D: Borrow<UpEndDatabase>>(
db: D,
job_container: Arc<RwLock<JobContainer>>,
job_id: JobId,
@ -146,7 +149,7 @@ fn _rescan_vault<D: Borrow<UpEndDatabase>>(
let path_outcomes: Vec<UpdatePathOutcome> = path_entries
.into_par_iter()
.map(|path| {
let result = _process_directory_entry(
let result = process_directory_entry(
db.connection().unwrap(),
&resolve_cache,
path.clone(),
@ -237,7 +240,7 @@ fn _rescan_vault<D: Borrow<UpEndDatabase>>(
Ok(all_outcomes)
}
fn _process_directory_entry<P: AsRef<Path>>(
fn process_directory_entry<P: AsRef<Path>>(
connection: UpEndConnection,
resolve_cache: &Arc<Mutex<ResolveCache>>,
path: PathBuf,
@ -317,36 +320,63 @@ fn _process_directory_entry<P: AsRef<Path>>(
if file_hash.is_none() {
file_hash = Some(path.hash()?);
}
let file_hash = file_hash.unwrap();
let mime_type = tree_magic_mini::from_filepath(&path).map(|s| s.to_string());
insert_file_with_metadata(
&connection,
normalized_path,
file_hash.unwrap(),
size,
mtime,
mime_type,
Some(resolve_cache),
)
.map(|_| {
info!("Added: {:?}", path);
UpdatePathOutcome::Added(path.clone())
})
}
fn insert_file_with_metadata(
connection: &UpEndConnection,
normalized_path: &Path,
hash: Hash,
size: i64,
mtime: Option<NaiveDateTime>,
mime_type: Option<String>,
resolve_cache: Option<&Arc<Mutex<ResolveCache>>>,
) -> Result<()> {
let new_file = models::NewFile {
path: normalized_path_str.to_string(),
hash: (file_hash.clone()).0,
path: normalized_path
.to_str()
.ok_or(anyhow!("Path not UTF-8?!"))?
.to_string(),
hash: (hash.clone()).0,
added: NaiveDateTime::from_timestamp(Utc::now().timestamp(), 0),
size,
mtime,
};
// Insert metadata
// Metadata
let type_entry = Entry {
entity: Address::Hash(file_hash.clone()),
entity: Address::Hash(hash.clone()),
attribute: String::from(IS_OF_TYPE_ATTR),
value: EntryValue::Address(BLOB_TYPE_ADDR.clone()),
};
let size_entry = Entry {
entity: Address::Hash(file_hash.clone()),
entity: Address::Hash(hash.clone()),
attribute: FILE_SIZE_KEY.to_string(),
value: EntryValue::Value(Value::from(size)),
};
let mime_entry = tree_magic_mini::from_filepath(&path).map(|mime_type| Entry {
entity: Address::Hash(file_hash.clone()),
let mime_entry = mime_type.map(|mime_type| Entry {
entity: Address::Hash(hash.clone()),
attribute: FILE_MIME_KEY.to_string(),
value: EntryValue::Value(Value::String(mime_type.to_string())),
value: EntryValue::Value(Value::String(mime_type)),
});
// Finally, add the appropriate entries w/r/t virtual filesystem location
// Add the appropriate entries w/r/t virtual filesystem location
let components = normalized_path.components().collect::<Vec<Component>>();
let (filename, dir_path) = components.split_last().unwrap();
@ -357,9 +387,13 @@ fn _process_directory_entry<P: AsRef<Path>>(
}))
.collect(),
);
let resolved_path = resolve_path_cached(&connection, &upath, true, resolve_cache)?;
let resolved_path = match resolve_cache {
Some(cache) => resolve_path_cached(connection, &upath, true, cache)?,
None => resolve_path(connection, &upath, true)?,
};
let parent_dir = resolved_path.last().unwrap();
// Insert all
connection.transaction::<_, Error, _>(|| {
connection.insert_file(new_file)?;
@ -372,7 +406,7 @@ fn _process_directory_entry<P: AsRef<Path>>(
let dir_has_entry = Entry {
entity: parent_dir.clone(),
attribute: HIER_HAS_ATTR.to_string(),
value: EntryValue::Address(Address::Hash(file_hash.clone())),
value: EntryValue::Address(Address::Hash(hash.clone())),
};
let dir_has_entry_addr = connection.insert_entry(dir_has_entry)?;
@ -385,8 +419,7 @@ fn _process_directory_entry<P: AsRef<Path>>(
};
connection.insert_entry(name_entry)?;
info!("Added: {:?}", path);
Ok(UpdatePathOutcome::Added(path.clone()))
Ok(())
})
}
@ -427,7 +460,8 @@ mod test {
.unwrap();
// Initial scan
let rescan_result = _rescan_vault(&open_result.db, job_container.clone(), job_id, true);
let rescan_result =
rescan_vault_inner(&open_result.db, job_container.clone(), job_id, true);
assert!(rescan_result.is_ok());
let rescan_result = rescan_result.unwrap();
@ -438,7 +472,8 @@ mod test {
// Modification-less rescan
let rescan_result = _rescan_vault(&open_result.db, job_container.clone(), job_id, false);
let rescan_result =
rescan_vault_inner(&open_result.db, job_container.clone(), job_id, false);
assert!(rescan_result.is_ok());
let rescan_result = rescan_result.unwrap();
@ -451,7 +486,7 @@ mod test {
std::fs::remove_file(temp_dir.path().join("hello-world.txt")).unwrap();
let rescan_result = _rescan_vault(&open_result.db, job_container, job_id, false);
let rescan_result = rescan_vault_inner(&open_result.db, job_container, job_id, false);
assert!(rescan_result.is_ok());
let rescan_result = rescan_result.unwrap();