thm
/
upend
1
0
Fork 0
Code Issues 73 Pull Requests Packages Projects Releases 5 Wiki Activity

add basic types

feat/vaults
Tomáš Mládek 2021-03-14 22:16:28 +01:00
parent 6f31ef5e15
commit ce36c97f2a
2 changed files with 95 additions and 15 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

42
src/database.rs
View File

@ -25,6 +25,10 @@ use std::path::{Path, PathBuf};
use std::str::FromStr;
use std::time::Duration;
pub const TYPE_ATTR: &str = "TYPE";
pub const TYPE_HAS_ATTR: &str = "TYPE_HAS";
pub const IS_TYPE_ATTR: &str = "IS";
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct Entry {
pub entity: Address,
@ -52,6 +56,19 @@ impl TryFrom<&models::Entry> for Entry {
}
}
impl TryFrom<&Entry> for models::Entry {
type Error = anyhow::Error;
fn try_from(e: &Entry) -> Result<Self, Self::Error> {
Ok(models::Entry {
identity: e.address()?.encode()?,
entity: e.entity.encode()?,
attribute: e.attribute.clone(),
value: e.value.to_string()?,
})
}
}
impl std::fmt::Display for Entry {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
write!(f, "{} | {} | {}", self.entity, self.attribute, self.value)
@ -581,12 +598,7 @@ pub fn insert_entry<C: Connection<Backend = Sqlite>>(
) -> Result<Address> {
debug!("Inserting: {}", entry);
let insert_entry = models::Entry {
identity: entry.address()?.encode()?,
entity: entry.entity.encode()?,
attribute: entry.attribute,
value: entry.value.to_string()?,
};
let insert_entry = models::Entry::try_from(&entry)?;
let entry = Entry::try_from(&insert_entry)?;
@ -604,6 +616,24 @@ pub fn insert_entry<C: Connection<Backend = Sqlite>>(
Ok(Address::Hash(entry.hash()?))
}
pub fn ensure_invariant<C: Connection<Backend = Sqlite>>(
connection: &C,
attribute: String,
value: EntryValue,
) -> Result<Address> {
let mut entity = Cursor::new(vec![0u8; 0]);
entity.write_all(attribute.as_bytes())?;
entity.write_all(value.to_string()?.as_bytes())?;
let entry = Entry {
entity: Address::Hash(Hash(entity.into_inner())),
attribute,
value,
};
insert_entry(connection, entry)
}
#[derive(Debug)]
pub struct ConnectionOptions {
pub enable_foreign_keys: bool,

68
src/filesystem.rs
View File

@ -1,7 +1,8 @@
use crate::addressing::Address;
use crate::database::{
bulk_retrieve_objects, file_set_valid, insert_entry, insert_file, query, retrieve_all_files,
DbPool, Entry, EntryQuery, EntryValue, Query, QueryComponent, QueryPart, DATABASE_FILENAME,
bulk_retrieve_objects, ensure_invariant, file_set_valid, insert_entry, insert_file, query,
retrieve_all_files, DbPool, Entry, EntryQuery, EntryValue, Query, QueryComponent, QueryPart,
DATABASE_FILENAME, IS_TYPE_ATTR, TYPE_ATTR, TYPE_HAS_ATTR,
};
use crate::hash::Hashable;
use crate::jobs::{Job, JobContainer, JobId};
@ -21,9 +22,11 @@ use std::{fs, iter};
use uuid::Uuid;
use walkdir::WalkDir;
const DIR_TYPE: &str = "FS_DIR";
const DIR_KEY: &str = "DIR";
const DIR_HAS_KEY: &str = "DIR_HAS";
const FILE_TYPE: &str = "FS_FILE";
const FILE_IDENTITY_KEY: &str = "FILE_IS";
const FILENAME_KEY: &str = "FILE_NAME";
@ -234,12 +237,19 @@ pub fn fetch_or_create_dir<C: Connection<Backend = Sqlite>>(
0 => {
if create {
let new_directory_address = Address::UUID(Uuid::new_v4());
let type_entry = Entry {
entity: new_directory_address.clone(),
attribute: String::from(IS_TYPE_ATTR),
value: EntryValue::Value(Value::from(DIR_TYPE)),
};
insert_entry(connection, type_entry)?;
let directory_entry = Entry {
entity: new_directory_address.clone(),
attribute: String::from(DIR_KEY),
value: dir_value,
};
let _ = insert_entry(connection, directory_entry)?;
insert_entry(connection, directory_entry)?;
if let Some(parent_addr) = parent {
let has_entry = Entry {
@ -247,7 +257,7 @@ pub fn fetch_or_create_dir<C: Connection<Backend = Sqlite>>(
attribute: String::from(DIR_HAS_KEY),
value: EntryValue::Address(new_directory_address.clone()),
};
let _ = insert_entry(connection, has_entry)?;
insert_entry(connection, has_entry)?;
}
Ok(new_directory_address)
@ -319,6 +329,38 @@ fn _rescan_vault<T: AsRef<Path>>(
) -> Result<Vec<UpdatePathResult>> {
let start = Instant::now();
// Initialize types, etc...
let file_type = ensure_invariant(
&pool.get()?,
String::from(TYPE_ATTR),
EntryValue::Value(Value::from(FILE_TYPE)),
)?;
for attr in &[FILE_IDENTITY_KEY, FILENAME_KEY] {
insert_entry(
&pool.get()?,
Entry {
entity: file_type.clone(),
attribute: String::from(TYPE_HAS_ATTR),
value: EntryValue::Value(Value::from(*attr)),
},
)?;
}
let dir_type = ensure_invariant(
&pool.get()?,
String::from(TYPE_ATTR),
EntryValue::Value(Value::from(DIR_TYPE)),
)?;
insert_entry(
&pool.get()?,
Entry {
entity: dir_type,
attribute: String::from(TYPE_HAS_ATTR),
value: EntryValue::Value(Value::from(DIR_KEY)),
},
)?;
// Walk through the vault, find all paths
let path_entries: Vec<PathBuf> = WalkDir::new(&directory)
.into_iter()
.filter_map(|e| e.ok())
@ -326,10 +368,12 @@ fn _rescan_vault<T: AsRef<Path>>(
.map(|e| fs::canonicalize(e.into_path()).unwrap())
.collect();
// Prepare for processing
let rw_pool = Arc::new(RwLock::new(pool.clone()));
let absolute_path = fs::canonicalize(&directory)?;
let existing_files = Arc::new(RwLock::new(retrieve_all_files(&pool.get()?)?));
// Actual processing
let count = RwLock::new(0_usize);
let total = path_entries.len() as f32;
let path_results: Vec<UpdatePathResult> = path_entries
@ -358,7 +402,7 @@ fn _rescan_vault<T: AsRef<Path>>(
.map(|file| {
let connection = pool.get()?;
connection.transaction::<_, Error, _>(|| {
let _ = file_set_valid(&connection, file.id, false)?;
file_set_valid(&connection, file.id, false)?;
// remove_object(&connection, )?
Ok(UpdatePathOutcome::Removed(PathBuf::from(file.path.clone())))
})
@ -433,7 +477,7 @@ fn _process_directory_entry<P: AsRef<Path>>(
mtime,
};
let _ = insert_file(&db_pool.write().unwrap().get()?, new_file)?;
insert_file(&db_pool.write().unwrap().get()?, new_file)?;
let components = normalized_path.components().collect::<Vec<Component>>();
let (filename, dir_path) = components.split_last().unwrap();
@ -454,6 +498,12 @@ fn _process_directory_entry<P: AsRef<Path>>(
let connection = _pool.get()?;
connection.transaction::<_, Error, _>(|| {
let file_address = Address::UUID(Uuid::new_v4());
let type_entry = Entry {
entity: file_address.clone(),
attribute: String::from(IS_TYPE_ATTR),
value: EntryValue::Value(Value::from(FILE_TYPE)),
};
insert_entry(&connection, type_entry)?;
let name_entry = Entry {
entity: file_address.clone(),
@ -462,7 +512,7 @@ fn _process_directory_entry<P: AsRef<Path>>(
filename.as_os_str().to_string_lossy().to_string(),
)),
};
let _ = insert_entry(&connection, name_entry)?;
insert_entry(&connection, name_entry)?;
let identity_entry = Entry {
entity: file_address.clone(),
@ -470,14 +520,14 @@ fn _process_directory_entry<P: AsRef<Path>>(
value: EntryValue::Address(Address::Hash(digest.clone())),
};
let _ = insert_entry(&connection, identity_entry)?;
insert_entry(&connection, identity_entry)?;
let dir_has_entry = Entry {
entity: parent_dir.clone(),
attribute: DIR_HAS_KEY.to_string(),
value: EntryValue::Address(file_address),
};
let _ = insert_entry(&connection, dir_has_entry)?;
insert_entry(&connection, dir_has_entry)?;
Ok(UpdatePathOutcome::Added(path.clone()))
})