2020-09-12 14:27:45 +02:00
|
|
|
use crate::addressing::Address;
|
|
|
|
use crate::database::{Entry, EntryValue, InnerEntry, InsertEntry, QueryEntries};
|
|
|
|
use crate::hash::{ComputeHash, Hash, HasherWorker};
|
|
|
|
use crate::models;
|
|
|
|
use anyhow::{anyhow, Result};
|
|
|
|
use log::{info, warn};
|
|
|
|
use serde::export::Formatter;
|
|
|
|
use serde_json::Value;
|
2020-08-27 00:11:50 +02:00
|
|
|
use std::fs;
|
|
|
|
use std::path::{Path, PathBuf};
|
2020-09-12 14:27:45 +02:00
|
|
|
use walkdir::WalkDir;
|
2020-08-27 00:11:50 +02:00
|
|
|
|
|
|
|
use actix::prelude::*;
|
2020-09-06 12:32:17 +02:00
|
|
|
use chrono::prelude::*;
|
2020-09-12 14:27:45 +02:00
|
|
|
use uuid::Uuid;
|
|
|
|
|
|
|
|
const DIR_KEY: &str = "DIR";
|
|
|
|
const DIR_HAS_KEY: &str = "DIR_HAS";
|
2020-08-27 00:11:50 +02:00
|
|
|
|
2020-09-12 14:27:45 +02:00
|
|
|
#[derive(Debug, PartialEq)]
|
|
|
|
pub struct UDirectory {
|
|
|
|
name: String,
|
|
|
|
}
|
2020-08-27 00:11:50 +02:00
|
|
|
|
2020-09-12 14:27:45 +02:00
|
|
|
#[derive(Debug, PartialEq)]
|
|
|
|
pub struct UPath(Vec<UDirectory>);
|
|
|
|
|
|
|
|
const TOP_SEPARATOR: &str = "//";
|
|
|
|
|
|
|
|
impl std::str::FromStr for UPath {
|
|
|
|
type Err = anyhow::Error;
|
|
|
|
|
|
|
|
fn from_str(s: &str) -> Result<Self, Self::Err> {
|
|
|
|
if s.len() == 0 {
|
|
|
|
Ok(UPath(vec![]))
|
|
|
|
} else {
|
|
|
|
match s.find(TOP_SEPARATOR) {
|
|
|
|
Some(head_idx) => {
|
|
|
|
let (head, rest) = s.split_at(head_idx);
|
|
|
|
let mut result: Vec<UDirectory> = Vec::new();
|
|
|
|
result.push(UDirectory {
|
|
|
|
name: String::from(head),
|
|
|
|
});
|
|
|
|
result.append(
|
|
|
|
rest[TOP_SEPARATOR.len()..rest.len()]
|
|
|
|
.split("/")
|
|
|
|
.map(|part| UDirectory {
|
|
|
|
name: String::from(part),
|
|
|
|
})
|
|
|
|
.collect::<Vec<UDirectory>>()
|
|
|
|
.as_mut(),
|
|
|
|
);
|
|
|
|
Ok(UPath(result))
|
|
|
|
}
|
|
|
|
None => Ok(UPath(
|
|
|
|
s.split("/")
|
|
|
|
.map(|part| UDirectory {
|
|
|
|
name: String::from(part),
|
|
|
|
})
|
|
|
|
.collect(),
|
|
|
|
)),
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
impl std::fmt::Display for UDirectory {
|
|
|
|
fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
|
|
|
|
write!(f, "{}", self.name)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
impl std::fmt::Display for UPath {
|
|
|
|
fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
|
|
|
|
match self.0.len() {
|
|
|
|
0 => write!(f, ""),
|
|
|
|
1 => write!(f, "{}", self.0.first().unwrap().name),
|
|
|
|
_ => {
|
|
|
|
let (head, tail) = self.0.split_first().unwrap();
|
|
|
|
write!(
|
|
|
|
f,
|
|
|
|
"{}//{}",
|
|
|
|
head.name,
|
|
|
|
tail.iter()
|
|
|
|
.map(|udir| udir.name.clone())
|
|
|
|
.collect::<Vec<String>>()
|
|
|
|
.join("/")
|
|
|
|
)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
pub async fn fetch_or_create_dir(
|
|
|
|
db_executor: &Addr<crate::database::DbExecutor>,
|
|
|
|
parent: Option<Address>,
|
|
|
|
directory: UDirectory,
|
|
|
|
) -> Result<Address> {
|
|
|
|
let dir_value = EntryValue::Value(Value::String(directory.name));
|
|
|
|
let directories: Vec<Address> = db_executor
|
|
|
|
.send(QueryEntries {
|
|
|
|
target: None,
|
|
|
|
key: Some(String::from(DIR_KEY)),
|
|
|
|
value: Some(dir_value.clone()),
|
|
|
|
})
|
|
|
|
.await??
|
|
|
|
.into_iter()
|
|
|
|
.map(|e: Entry| e.target)
|
|
|
|
.collect();
|
|
|
|
|
|
|
|
let valid_directories: Vec<Address> = match parent.clone() {
|
|
|
|
Some(address) => {
|
|
|
|
let parent_has: Vec<Address> = db_executor
|
|
|
|
.send(QueryEntries {
|
|
|
|
target: Some(address),
|
|
|
|
key: Some(String::from(DIR_HAS_KEY)),
|
|
|
|
value: None,
|
|
|
|
})
|
|
|
|
.await??
|
|
|
|
.into_iter()
|
|
|
|
.map(|e: Entry| e.target)
|
|
|
|
.collect();
|
|
|
|
|
|
|
|
directories
|
|
|
|
.into_iter()
|
|
|
|
.filter(|a| parent_has.contains(a))
|
|
|
|
.collect()
|
|
|
|
}
|
|
|
|
None => directories,
|
|
|
|
};
|
|
|
|
|
|
|
|
match valid_directories.len() {
|
|
|
|
0 => {
|
|
|
|
let new_directory_address = Address::UUID(Uuid::new_v4());
|
|
|
|
let directory_entry = InnerEntry {
|
|
|
|
target: new_directory_address.clone(),
|
|
|
|
key: String::from(DIR_KEY),
|
|
|
|
value: dir_value,
|
|
|
|
};
|
|
|
|
let _ = db_executor
|
|
|
|
.send(InsertEntry {
|
|
|
|
entry: directory_entry,
|
|
|
|
})
|
|
|
|
.await??;
|
|
|
|
|
|
|
|
if parent.is_some() {
|
|
|
|
let has_entry = InnerEntry {
|
|
|
|
target: parent.unwrap(),
|
|
|
|
key: String::from(DIR_HAS_KEY),
|
|
|
|
value: EntryValue::Address(new_directory_address.clone()),
|
|
|
|
};
|
|
|
|
let _ = db_executor.send(InsertEntry { entry: has_entry }).await??;
|
|
|
|
}
|
|
|
|
|
|
|
|
Ok(new_directory_address)
|
|
|
|
}
|
|
|
|
1 => Ok(valid_directories[0].clone()),
|
|
|
|
_ => Err(anyhow!(
|
|
|
|
"Invalid database state - more than one directory matches the query!"
|
|
|
|
)),
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
pub fn fetch_path(path: &UPath) -> Vec<Entry> {
|
|
|
|
unimplemented!();
|
|
|
|
}
|
|
|
|
|
|
|
|
async fn _reimport_directory<T: AsRef<Path>>(
|
2020-08-27 00:11:50 +02:00
|
|
|
directory: T,
|
|
|
|
db_executor: &Addr<crate::database::DbExecutor>,
|
|
|
|
hasher_worker: &Addr<HasherWorker>,
|
|
|
|
) -> Result<()> {
|
|
|
|
for entry in WalkDir::new(&directory)
|
|
|
|
.into_iter()
|
|
|
|
.filter_map(|e| e.ok())
|
|
|
|
.filter(|e| e.path().is_file())
|
|
|
|
{
|
|
|
|
info!("Processing: {}", entry.path().display());
|
|
|
|
|
|
|
|
let metadata = fs::metadata(entry.path())?;
|
|
|
|
let size = metadata.len() as i64;
|
|
|
|
if size < 0 {
|
|
|
|
panic!("File {} too large?!", entry.path().display());
|
|
|
|
}
|
|
|
|
|
|
|
|
let msg = ComputeHash {
|
|
|
|
path: entry.path().to_path_buf(),
|
|
|
|
};
|
|
|
|
|
2020-09-07 21:21:54 +02:00
|
|
|
let digest: Result<Result<Hash>, MailboxError> = hasher_worker.send(msg).await;
|
2020-08-27 00:11:50 +02:00
|
|
|
|
2020-09-12 14:27:45 +02:00
|
|
|
let new_file = models::NewFile {
|
2020-08-27 00:11:50 +02:00
|
|
|
path: entry
|
|
|
|
.path()
|
|
|
|
.to_str()
|
|
|
|
.expect("path not valid unicode?!")
|
|
|
|
.to_string(),
|
2020-09-12 14:27:45 +02:00
|
|
|
hash: digest??.0,
|
2020-08-28 13:51:22 +02:00
|
|
|
size,
|
2020-09-06 12:32:17 +02:00
|
|
|
created: NaiveDateTime::from_timestamp(Utc::now().timestamp(), 0),
|
2020-08-27 00:11:50 +02:00
|
|
|
};
|
|
|
|
|
|
|
|
let _insert_result = db_executor
|
|
|
|
.send(crate::database::InsertFile { file: new_file })
|
|
|
|
.await?;
|
|
|
|
}
|
|
|
|
info!("Finished updating {}.", directory.as_ref().display());
|
|
|
|
|
|
|
|
Ok(())
|
|
|
|
}
|
|
|
|
|
2020-09-12 14:27:45 +02:00
|
|
|
pub async fn reimport_directory(
|
2020-08-30 22:11:32 +02:00
|
|
|
directory: PathBuf,
|
|
|
|
db_executor: Addr<crate::database::DbExecutor>,
|
|
|
|
hasher_worker: Addr<HasherWorker>,
|
|
|
|
) {
|
2020-09-12 14:27:45 +02:00
|
|
|
let result = _reimport_directory(directory, &db_executor, &hasher_worker).await;
|
2020-08-30 22:14:24 +02:00
|
|
|
if result.is_err() {
|
2020-08-30 22:11:32 +02:00
|
|
|
warn!("Update did not succeed!");
|
|
|
|
}
|
|
|
|
}
|
2020-09-12 14:27:45 +02:00
|
|
|
|
|
|
|
#[cfg(test)]
|
|
|
|
mod tests {
|
|
|
|
use crate::filesystem::{UDirectory, UPath};
|
|
|
|
use anyhow::Result;
|
|
|
|
|
|
|
|
#[test]
|
|
|
|
fn test_path_codec() {
|
|
|
|
let path = UPath(vec![
|
|
|
|
UDirectory {
|
|
|
|
name: "top".to_string(),
|
|
|
|
},
|
|
|
|
UDirectory {
|
|
|
|
name: "foo".to_string(),
|
|
|
|
},
|
|
|
|
UDirectory {
|
|
|
|
name: "bar".to_string(),
|
|
|
|
},
|
|
|
|
UDirectory {
|
|
|
|
name: "baz".to_string(),
|
|
|
|
},
|
|
|
|
]);
|
|
|
|
|
|
|
|
let str_path = path.to_string();
|
|
|
|
assert!(str_path.len() > 0);
|
|
|
|
|
|
|
|
let decoded_path: Result<UPath> = str_path.parse();
|
|
|
|
assert!(decoded_path.is_ok());
|
|
|
|
|
|
|
|
assert_eq!(path, decoded_path.unwrap());
|
|
|
|
}
|
|
|
|
}
|