thm
/
upend
1
0
Fork 0
Code Issues 73 Pull Requests Packages Projects Releases 5 Wiki Activity

wip: split upend_base and upend_db

feat/type-attributes
Tomáš Mládek 2023-06-25 15:29:52 +02:00
parent e18986b400
commit 2e348a9b29
45 changed files with 533 additions and 424 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

55
Cargo.lock generated
View File

@ -3187,36 +3187,22 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a156c684c91ea7d62626509bce3cb4e1d9ed5c4d978f7b4352658f96a4c26b4a"
[[package]]
name = "upend"
version = "0.0.71"
name = "upend-base"
version = "0.0.1"
dependencies = [
"anyhow",
"chrono",
"diesel",
"diesel_migrations",
"filebuffer",
"lazy_static",
"lexpr",
"libsqlite3-sys",
"log",
"lru",
"multibase",
"multihash",
"nonempty",
"num_cpus",
"once_cell",
"rayon",
"regex",
"serde",
"serde_json",
"shadow-rs",
"tempfile",
"tracing",
"tracing-subscriber",
"tree_magic_mini",
"url",
"uuid",
"walkdir",
]
[[package]]
@ -3269,7 +3255,8 @@ dependencies = [
"tracing",
"tracing-subscriber",
"tree_magic_mini",
"upend",
"upend-base",
"upend-db",
"url",
"uuid",
"walkdir",
@ -3278,6 +3265,40 @@ dependencies = [
"webpage",
]
[[package]]
name = "upend-db"
version = "0.0.1"
dependencies = [
"anyhow",
"chrono",
"diesel",
"diesel_migrations",
"filebuffer",
"lazy_static",
"lexpr",
"libsqlite3-sys",
"log",
"lru",
"multibase",
"multihash",
"nonempty",
"num_cpus",
"once_cell",
"rayon",
"regex",
"serde",
"serde_json",
"shadow-rs",
"tempfile",
"tracing",
"tracing-subscriber",
"tree_magic_mini",
"upend-base",
"url",
"uuid",
"walkdir",
]
[[package]]
name = "url"
version = "2.3.1"

64
Cargo.toml
View File

@ -1,64 +1,2 @@
[package]
name = "upend"
description = "A user-oriented all-purpose graph database."
version = "0.0.71"
homepage = "https://upend.dev/"
repository = "https://git.thm.place/thm/upend"
authors = ["Tomáš Mládek <t@mldk.cz>"]
license = "AGPL-3.0-or-later"
edition = "2018"
build = "build.rs"
[workspace]
members = ["cli"]
[dependencies]
log = "0.4"
tracing = "0.1"
tracing-subscriber = { version = "0.3", features = ["env-filter"] }
anyhow = "1.0"
rayon = "1.4.0"
num_cpus = "1.13"
lazy_static = "1.4.0"
once_cell = "1.7.2"
lru = "0.7.0"
diesel = { version = "1.4", features = [
"sqlite",
"r2d2",
"chrono",
"serde_json",
] }
diesel_migrations = "1.4"
libsqlite3-sys = { version = "^0", features = ["bundled"] }
chrono = { version = "0.4", features = ["serde"] }
serde = { version = "1.0", features = ["derive"] }
serde_json = "1.0"
lexpr = "0.2.6"
regex = "1"
multibase = "0.9"
multihash = { version = "*", default-features = false, features = [
"alloc",
"multihash-impl",
"sha2",
"identity",
] }
uuid = { version = "0.8", features = ["v4"] }
url = { version = "2", features = ["serde"] }
filebuffer = "0.4.0"
tempfile = "^3.2.0"
walkdir = "2"
tree_magic_mini = {version = "3.0.2", features=["with-gpl-data"] }
nonempty = "0.6.0"
shadow-rs = "0.17"
[build-dependencies]
shadow-rs = "0.17"
members = ["base", "db", "cli"]

39
base/Cargo.toml Normal file
View File

@ -0,0 +1,39 @@
[package]
name = "upend-base"
version = "0.0.1"
homepage = "https://upend.dev/"
repository = "https://git.thm.place/thm/upend"
authors = ["Tomáš Mládek <t@mldk.cz>"]
license = "AGPL-3.0-or-later"
edition = "2018"
[features]
diesel = []
[dependencies]
log = "0.4"
anyhow = "1.0"
lazy_static = "1.4.0"
diesel = { version = "1.4", features = ["sqlite"] }
chrono = { version = "0.4", features = ["serde"] }
serde = { version = "1.0", features = ["derive"] }
lexpr = "0.2.6"
multibase = "0.9"
multihash = { version = "*", default-features = false, features = [
"alloc",
"multihash-impl",
"sha2",
"identity",
] }
uuid = { version = "0.8", features = ["v4"] }
url = { version = "2", features = ["serde"] }
nonempty = "0.6.0"
[build-dependencies]
shadow-rs = "0.17"

17
src/addressing.rs → base/src/addressing.rs
View File

@ -1,15 +1,16 @@
use crate::util::hash::{b58_decode, b58_encode, Hash, Hashable};
use crate::hash::{b58_decode, b58_encode, AsDigest, Digest};
use anyhow::{anyhow, Result};
use serde::de::Visitor;
use serde::{de, ser, Deserialize, Deserializer, Serialize, Serializer};
use std::fmt;
use std::hash::Hash;
use std::str::FromStr;
use url::Url;
use uuid::Uuid;
#[derive(Clone, Eq, PartialEq, Hash)]
pub enum Address {
Hash(Hash),
Hash(Digest),
Uuid(Uuid),
Attribute(String),
Url(Url),
@ -20,7 +21,7 @@ const SHA2_256: u64 = 0x12;
// multihash identity
const IDENTITY: u64 = 0x00;
type LargeMultihash = multihash::MultihashGeneric<256>;
pub type LargeMultihash = multihash::MultihashGeneric<256>;
impl Address {
pub fn encode(&self) -> Result<Vec<u8>> {
@ -50,7 +51,7 @@ impl Address {
.map_err(|err| anyhow!("Error decoding address: {}", err))?;
match multihash.code() {
SHA2_256 => Ok(Self::Hash(Hash(multihash.digest().to_vec()))),
SHA2_256 => Ok(Self::Hash(Digest(multihash.digest().to_vec()))),
IDENTITY => {
let digest = multihash.digest().to_owned();
let digest_content: Vec<u8> = digest.clone().into_iter().skip(1).collect();
@ -146,9 +147,9 @@ impl std::fmt::Debug for Address {
}
}
pub trait Addressable: Hashable {
pub trait Addressable: AsDigest {
fn address(&self) -> Result<Address> {
Ok(Address::Hash(self.hash()?))
Ok(Address::Hash(self.as_digest()?))
}
}
@ -159,11 +160,11 @@ mod tests {
use uuid::Uuid;
use crate::addressing::Address;
use crate::util::hash::Hash;
use crate::hash::Digest;
#[test]
fn test_hash_codec() -> Result<()> {
let addr = Address::Hash(Hash(vec![1, 2, 3, 4, 5]));
let addr = Address::Hash(Digest(vec![1, 2, 3, 4, 5]));
let encoded = addr.encode()?;
let decoded = Address::decode(&encoded)?;
assert_eq!(addr, decoded);

4
src/database/constants.rs → base/src/constants.rs
View File

@ -1,5 +1,5 @@
use crate::addressing::Address;
use crate::database::entry::InvariantEntry;
use crate::entry::InvariantEntry;
/// Attribute denoting (hierarchical) relation, in the "upwards" direction. For example, a file `IN` a group, an image `IN` photos, etc.
pub const ATTR_IN: &str = "IN";
@ -22,7 +22,7 @@ lazy_static! {
value: "HIER_ROOT".into(),
};
pub static ref HIER_ROOT_ADDR: Address = HIER_ROOT_INVARIANT.entity().unwrap();
pub static ref TYPE_HASH_ADDRESS: Address = Address::Hash(crate::util::hash::Hash(vec![]));
pub static ref TYPE_HASH_ADDRESS: Address = Address::Hash(crate::hash::Digest(vec![]));
pub static ref TYPE_UUID_ADDRESS: Address = Address::Uuid(uuid::Uuid::nil());
pub static ref TYPE_ATTRIBUTE_ADDRESS: Address = Address::Attribute("".to_string());
pub static ref TYPE_URL_ADDRESS: Address = Address::Url(url::Url::parse("up:").unwrap());

130
src/database/entry.rs → base/src/entry.rs
View File

@ -1,6 +1,5 @@
use crate::addressing::{Address, Addressable};
use crate::database::inner::models;
use crate::util::hash::{b58_decode, hash, Hash, Hashable};
use crate::hash::{b58_decode, sha256hash, AsDigest, AsDigestError, Digest};
use anyhow::{anyhow, Result};
use chrono::NaiveDateTime;
use serde::{Deserialize, Serialize};
@ -36,87 +35,6 @@ pub enum EntryValue {
Invalid,
}
impl TryFrom<&models::Entry> for Entry {
type Error = anyhow::Error;
fn try_from(e: &models::Entry) -> Result<Self, Self::Error> {
if let Some(value_str) = &e.value_str {
Ok(Entry {
entity: Address::decode(&e.entity)?,
attribute: e.attribute.clone(),
value: value_str.parse()?,
provenance: e.provenance.clone(),
timestamp: e.timestamp,
})
} else if let Some(value_num) = e.value_num {
Ok(Entry {
entity: Address::decode(&e.entity)?,
attribute: e.attribute.clone(),
value: EntryValue::Number(value_num),
provenance: e.provenance.clone(),
timestamp: e.timestamp,
})
} else {
Ok(Entry {
entity: Address::decode(&e.entity)?,
attribute: e.attribute.clone(),
value: EntryValue::Number(f64::NAN),
provenance: e.provenance.clone(),
timestamp: e.timestamp,
})
}
}
}
impl TryFrom<&Entry> for models::Entry {
type Error = anyhow::Error;
fn try_from(e: &Entry) -> Result<Self, Self::Error> {
if e.attribute.is_empty() {
return Err(anyhow!("Attribute cannot be empty."));
}
let base_entry = models::Entry {
identity: e.address()?.encode()?,
entity_searchable: match &e.entity {
Address::Attribute(attr) => Some(attr.clone()),
Address::Url(url) => Some(url.to_string()),
_ => None,
},
entity: e.entity.encode()?,
attribute: e.attribute.clone(),
value_str: None,
value_num: None,
immutable: false,
provenance: e.provenance.clone(),
timestamp: e.timestamp,
};
match e.value {
EntryValue::Number(n) => Ok(models::Entry {
value_str: None,
value_num: Some(n),
..base_entry
}),
_ => Ok(models::Entry {
value_str: Some(e.value.to_string()?),
value_num: None,
..base_entry
}),
}
}
}
impl TryFrom<&ImmutableEntry> for models::Entry {
type Error = anyhow::Error;
fn try_from(e: &ImmutableEntry) -> Result<Self, Self::Error> {
Ok(models::Entry {
immutable: true,
..models::Entry::try_from(&e.0)?
})
}
}
impl TryFrom<&InvariantEntry> for Entry {
type Error = anyhow::Error;
@ -136,7 +54,7 @@ impl InvariantEntry {
let mut entity = Cursor::new(vec![0u8; 0]);
entity.write_all(self.attribute.as_bytes())?;
entity.write_all(self.value.to_string()?.as_bytes())?;
Ok(Address::Hash(hash(entity.into_inner())))
Ok(Address::Hash(sha256hash(entity.into_inner())))
}
}
@ -146,19 +64,47 @@ impl std::fmt::Display for Entry {
}
}
impl Hashable for Entry {
fn hash(self: &Entry) -> Result<Hash> {
// impl Hashable for Entry {
// fn hash(self: &Entry) -> Result<Hash> {
// let mut result = Cursor::new(vec![0u8; 0]);
// result.write_all(self.entity.encode()?.as_slice())?;
// result.write_all(self.attribute.as_bytes())?;
// result.write_all(self.value.to_string()?.as_bytes())?;
// Ok(hash(result.get_ref()))
// }
// }
// impl Hashable for InvariantEntry {
// fn hash(&self) -> Result<Hash> {
// Entry::try_from(self)?.hash()
// }
// }
impl AsDigest for Entry {
fn as_digest(&self) -> Result<Digest, AsDigestError> {
let mut result = Cursor::new(vec![0u8; 0]);
result.write_all(self.entity.encode()?.as_slice())?;
result.write_all(
self.entity
.encode()
.map_err(|e| AsDigestError(e.to_string()))?
.as_slice(),
)?;
result.write_all(self.attribute.as_bytes())?;
result.write_all(self.value.to_string()?.as_bytes())?;
Ok(hash(result.get_ref()))
result.write_all(
self.value
.to_string()
.map_err(|e| AsDigestError(e.to_string()))?
.as_bytes(),
)?;
Ok(sha256hash(result.get_ref()))
}
}
impl Hashable for InvariantEntry {
fn hash(&self) -> Result<Hash> {
Entry::try_from(self)?.hash()
impl AsDigest for InvariantEntry {
fn as_digest(&self) -> Result<Digest, AsDigestError> {
Entry::try_from(self)
.map_err(|e| AsDigestError(e.to_string()))?
.as_digest()
}
}

66
src/util/hash.rs → base/src/hash.rs
View File

@ -1,31 +1,20 @@
use crate::addressing::Address;
use anyhow::Result;
use diesel::backend::Backend;
use diesel::deserialize::FromSql;
use diesel::sqlite::Sqlite;
use diesel::{deserialize, sql_types};
use filebuffer::FileBuffer;
use multihash::Hasher;
use serde::{ser, Serialize, Serializer};
use std::path::{Path};
use tracing::trace;
#[derive(Debug, Clone, Eq, PartialEq, FromSqlRow, Hash)]
pub struct Hash(pub Vec<u8>);
#[derive(Debug, Clone, Eq, PartialEq, Hash)]
#[cfg_attr(feature = "diesel", derive(diesel::FromSqlRow))]
impl AsRef<[u8]> for Hash {
pub struct Digest(pub Vec<u8>);
impl AsRef<[u8]> for Digest {
fn as_ref(&self) -> &[u8] {
self.0.as_ref()
}
}
impl FromSql<sql_types::Binary, Sqlite> for Hash {
fn from_sql(bytes: Option<&<Sqlite as Backend>::RawValue>) -> deserialize::Result<Self> {
Ok(Hash(Vec::from(not_none!(bytes).read_blob())))
}
}
impl Serialize for Hash {
impl Serialize for Digest {
fn serialize<S>(&self, serializer: S) -> Result<<S as Serializer>::Ok, <S as Serializer>::Error>
where
S: Serializer,
@ -41,23 +30,19 @@ impl Serialize for Hash {
}
}
pub trait Hashable {
fn hash(&self) -> Result<Hash>;
}
impl Hashable for Path {
fn hash(self: &Path) -> Result<Hash> {
trace!("Hashing {:?}...", self);
let fbuffer = FileBuffer::open(self)?;
trace!("Finished hashing {:?}...", self);
Ok(hash(&fbuffer))
#[cfg(feature = "diesel")]
impl diesel::types::FromSql<diesel::sql_types::Binary, diesel::sqlite::Sqlite> for Digest {
fn from_sql(
bytes: Option<&<diesel::sqlite::Sqlite as diesel::backend::Backend>::RawValue>,
) -> diesel::deserialize::Result<Self> {
Ok(Digest(Vec::from(diesel::not_none!(bytes).read_blob())))
}
}
pub fn hash<T: AsRef<[u8]>>(input: T) -> Hash {
pub fn sha256hash<T: AsRef<[u8]>>(input: T) -> Digest {
let mut hasher = multihash::Sha2_256::default();
hasher.update(input.as_ref());
Hash(Vec::from(hasher.finalize()))
Digest(Vec::from(hasher.finalize()))
}
pub fn b58_encode<T: AsRef<[u8]>>(vec: T) -> String {
@ -70,9 +55,30 @@ pub fn b58_decode<T: AsRef<str>>(input: T) -> Result<Vec<u8>> {
Ok(data)
}
#[derive(Debug, Clone)]
pub struct AsDigestError(pub String);
impl std::fmt::Display for AsDigestError {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
write!(f, "{}", self.0)
}
}
impl std::error::Error for AsDigestError {}
impl From<std::io::Error> for AsDigestError {
fn from(err: std::io::Error) -> Self {
AsDigestError(err.to_string())
}
}
pub trait AsDigest {
fn as_digest(&self) -> Result<Digest, AsDigestError>;
}
#[cfg(test)]
mod tests {
use crate::util::hash::{b58_decode, b58_encode};
use crate::hash::{b58_decode, b58_encode};
#[test]
fn test_encode_decode() {

28
src/database/lang.rs → base/src/lang.rs
View File

@ -1,5 +1,5 @@
use crate::addressing::Address;
use crate::database::entry::EntryValue;
use crate::entry::EntryValue;
use nonempty::NonEmpty;
use std::borrow::Borrow;
use std::convert::TryFrom;
@ -55,19 +55,6 @@ impl TryFrom<lexpr::Value> for Address {
}
}
impl TryFrom<lexpr::Value> for Attribute {
type Error = QueryParseError;
fn try_from(value: lexpr::Value) -> Result<Self, Self::Error> {
match value {
lexpr::Value::String(str) => Ok(Attribute(str.to_string())),
_ => Err(QueryParseError(
"Can only convert to attribute from string.".into(),
)),
}
}
}
impl TryFrom<lexpr::Value> for EntryValue {
type Error = QueryParseError;
@ -88,6 +75,19 @@ impl TryFrom<lexpr::Value> for EntryValue {
}
}
impl TryFrom<lexpr::Value> for Attribute {
type Error = QueryParseError;
fn try_from(value: lexpr::Value) -> Result<Self, Self::Error> {
match value {
lexpr::Value::String(str) => Ok(Attribute(str.to_string())),
_ => Err(QueryParseError(
"Can only convert to attribute from string.".into(),
)),
}
}
}
#[derive(Debug, Clone, PartialEq)]
pub enum QueryPart {
Matches(PatternQuery),

8
base/src/lib.rs Normal file
View File

@ -0,0 +1,8 @@
#[macro_use]
extern crate lazy_static;
pub mod addressing;
pub mod constants;
pub mod entry;
pub mod hash;
pub mod lang;

17
cli/Cargo.toml
View File

@ -5,7 +5,8 @@ version = "0.1.0"
edition = "2021"
[dependencies]
upend = { path = "../" }
upend-base = { path = "../base" }
upend-db = { path = "../db" }
clap = { version = "4.2.4", features = ["derive", "env", "color"] }
log = "0.4"
@ -63,7 +64,7 @@ walkdir = "2"
rand = "0.8"
mime = "^0.3.16"
tree_magic_mini = {version = "3.0.2", features=["with-gpl-data"] }
tree_magic_mini = { version = "3.0.2", features = ["with-gpl-data"] }
opener = { version = "^0.5.0", optional = true }
is_executable = { version = "1.0.1", optional = true }
@ -74,7 +75,7 @@ nonempty = "0.6.0"
image = { version = "0.23.14", optional = true }
webp = { version = "0.2.0", optional = true }
webpage = { version = "1.5.0", optional = true, default-features = false}
webpage = { version = "1.5.0", optional = true, default-features = false }
id3 = { version = "1.0.2", optional = true }
kamadak-exif = { version = "0.5.4", optional = true }
@ -89,7 +90,15 @@ signal-hook = "0.3.15"
shadow-rs = "0.17"
[features]
default = ["desktop", "previews", "previews-image", "extractors-web", "extractors-audio", "extractors-photo", "extractors-media"]
default = [
"desktop",
"previews",
"previews-image",
"extractors-web",
"extractors-audio",
"extractors-photo",
"extractors-media",
]
desktop = ["webbrowser", "opener", "is_executable"]
previews = []
previews-image = ["image", "webp", "kamadak-exif"]

3
cli/build.rs Normal file
View File

@ -0,0 +1,3 @@
fn main() -> shadow_rs::SdResult<()> {
shadow_rs::new()
}

4
cli/src/addressing.rs
View File

@ -1,4 +1,3 @@
use crate::util::hash::{b58_decode, b58_encode, Hash, Hashable};
use anyhow::{anyhow, Result};
use multihash::{Code, Multihash, MultihashDigest};
use serde::de::Visitor;
@ -6,6 +5,7 @@ use serde::{de, ser, Deserialize, Deserializer, Serialize, Serializer};
use std::fmt;
use std::str::FromStr;
use thiserror::private::DisplayAsDisplay;
use upend_base::hash::{b58_decode, b58_encode, Hash, Hashable};
use uuid::Uuid;
#[derive(Clone, Eq, PartialEq, Hash)]
@ -150,7 +150,7 @@ mod tests {
use uuid::Uuid;
use crate::addressing::Address;
use crate::util::hash::Hash;
use upend_base::hash::Hash;
#[test]
fn test_hash_codec() -> Result<()> {

11
cli/src/common.rs
View File

@ -1,6 +1,8 @@
use anyhow::{anyhow, Result};
use lazy_static::lazy_static;
use shadow_rs::is_debug;
use shadow_rs::{is_debug, shadow};
shadow!(build);
pub fn get_static_dir<S: AsRef<str>>(dir: S) -> Result<std::path::PathBuf> {
let cwd = std::env::current_exe()?.parent().unwrap().to_path_buf();
@ -18,16 +20,11 @@ pub fn get_static_dir<S: AsRef<str>>(dir: S) -> Result<std::path::PathBuf> {
}
lazy_static! {
static ref APP_USER_AGENT: String = format!(
"{} / {}",
upend::common::build::PROJECT_NAME,
upend::common::build::PKG_VERSION
);
static ref APP_USER_AGENT: String = format!("upend / {}", build::PKG_VERSION);
pub static ref REQWEST_CLIENT: reqwest::blocking::Client = reqwest::blocking::Client::builder()
.user_agent(APP_USER_AGENT.as_str())
.build()
.unwrap();
pub static ref REQWEST_ASYNC_CLIENT: reqwest::Client = reqwest::Client::builder()
.user_agent(APP_USER_AGENT.as_str())
.build()

18
cli/src/extractors/audio.rs
View File

@ -2,15 +2,15 @@ use std::sync::Arc;
use super::Extractor;
use anyhow::{anyhow, Result};
use upend::{
use upend_base::{
addressing::Address,
database::{
constants,
entry::{Entry, EntryValue},
stores::{fs::FILE_MIME_KEY, UpStore},
UpEndConnection,
},
util::jobs::{JobContainer, JobState},
constants::ATTR_LABEL,
entry::{Entry, EntryValue},
};
use upend_db::{
jobs::{JobContainer, JobState},
stores::{fs::FILE_MIME_KEY, UpStore},
UpEndConnection,
};
pub struct ID3Extractor;
@ -59,7 +59,7 @@ impl Extractor for ID3Extractor {
},
Entry {
entity: Address::Attribute(format!("ID3_{}", frame.id())),
attribute: constants::ATTR_LABEL.into(),
attribute: ATTR_LABEL.into(),
value: format!("ID3: {}", frame.name()).into(),
provenance: "SYSTEM EXTRACTOR".to_string(),
timestamp: chrono::Utc::now().naive_utc(),

16
cli/src/extractors/media.rs
View File

@ -2,15 +2,15 @@ use std::{process::Command, sync::Arc};
use super::Extractor;
use anyhow::{anyhow, Result};
use upend::{
use upend_base::{
addressing::Address,
database::{
constants::ATTR_LABEL,
entry::{Entry, EntryValue},
stores::{fs::FILE_MIME_KEY, UpStore},
UpEndConnection,
},
util::jobs::{JobContainer, JobState},
constants::ATTR_LABEL,
entry::{Entry, EntryValue},
};
use upend_db::{
jobs::{JobContainer, JobState},
stores::{fs::FILE_MIME_KEY, UpStore},
UpEndConnection,
};
const DURATION_KEY: &str = "MEDIA_DURATION";

7
cli/src/extractors/mod.rs
View File

@ -1,8 +1,3 @@
use upend::{
addressing::Address,
database::{entry::Entry, stores::UpStore, UpEndConnection, UpEndDatabase},
util::jobs::JobContainer,
};
use anyhow::Result;
use rayon::iter::{IntoParallelRefIterator, ParallelIterator};
use std::{
@ -10,6 +5,8 @@ use std::{
sync::{Arc, Mutex, RwLock},
};
use tracing::{debug, info, trace};
use upend_base::{addressing::Address, entry::Entry};
use upend_db::{jobs::JobContainer, stores::UpStore, UpEndConnection, UpEndDatabase};
#[cfg(feature = "extractors-web")]
pub mod web;

18
cli/src/extractors/photo.rs
View File

@ -2,15 +2,15 @@ use std::sync::Arc;
use super::Extractor;
use anyhow::{anyhow, Result};
use upend::{
use upend_base::{
addressing::Address,
database::{
constants,
entry::{Entry, EntryValue},
stores::{fs::FILE_MIME_KEY, UpStore},
UpEndConnection,
},
util::jobs::{JobContainer, JobState},
constants::ATTR_LABEL,
entry::{Entry, EntryValue},
};
use upend_db::{
jobs::{JobContainer, JobState},
stores::{fs::FILE_MIME_KEY, UpStore},
UpEndConnection,
};
pub struct ExifExtractor;
@ -74,7 +74,7 @@ impl Extractor for ExifExtractor {
},
Entry {
entity: Address::Attribute(attribute),
attribute: constants::ATTR_LABEL.into(),
attribute: ATTR_LABEL.into(),
value: format!("EXIF: {}", tag_description).into(),
provenance: "SYSTEM EXTRACTOR".to_string(),
timestamp: chrono::Utc::now().naive_utc(),

19
cli/src/extractors/web.rs
View File

@ -4,13 +4,14 @@ use super::Extractor;
use crate::common::REQWEST_CLIENT;
use anyhow::anyhow;
use anyhow::Result;
use upend::database::constants::ATTR_LABEL;
use upend::{
addressing::Address,
database::{entry::Entry, stores::UpStore, UpEndConnection},
util::jobs::{JobContainer, JobState},
};
use upend_base::addressing::Address;
use upend_base::constants::ATTR_LABEL;
use upend_base::entry::Entry;
use upend_db::jobs::JobContainer;
use upend_db::jobs::JobState;
use upend_db::stores::UpStore;
use upend_db::UpEndConnection;
use webpage::HTML;
pub struct WebExtractor;
@ -109,8 +110,8 @@ impl Extractor for WebExtractor {
#[cfg(test)]
mod test {
use upend::database::stores::fs::FsStore;
use upend::util::jobs::JobContainer;
use upend_db::jobs::JobContainer;
use upend_db::stores::fs::FsStore;
use url::Url;
use super::*;
@ -121,7 +122,7 @@ mod test {
#[test]
fn test_extract() -> Result<()> {
let temp_dir = TempDir::new().unwrap();
let open_result = upend::database::UpEndDatabase::open(&temp_dir, true)?;
let open_result = upend_db::UpEndDatabase::open(&temp_dir, true)?;
let connection = open_result.db.connection()?;
let store =
Arc::new(Box::new(FsStore::from_path(&temp_dir)?) as Box<dyn UpStore + Sync + Send>);

31
cli/src/main.rs
View File

@ -1,6 +1,8 @@
#[macro_use]
extern crate upend;
extern crate upend_db;
use crate::common::{get_static_dir, REQWEST_ASYNC_CLIENT};
use crate::config::UpEndConfig;
use actix_web::HttpServer;
use anyhow::Result;
use clap::{Args, Parser, Subcommand, ValueEnum};
@ -18,23 +20,18 @@ use std::sync::Arc;
use tracing::trace;
use tracing::{debug, error, info, warn};
use tracing_subscriber::filter::{EnvFilter, LevelFilter};
use upend::addressing::Address;
use upend::database::entry::EntryValue;
use upend::util::hash::hash;
use upend::{
common::build,
config::UpEndConfig,
database::{
stores::{fs::FsStore, UpStore},
UpEndDatabase,
},
util::jobs::JobContainer,
};
use upend_base::addressing::Address;
use upend_base::entry::EntryValue;
use upend_base::hash::sha256hash;
use upend_db::jobs::JobContainer;
use upend_db::stores::fs::FsStore;
use upend_db::stores::UpStore;
use upend_db::UpEndDatabase;
use crate::util::exec::block_background;
mod common;
mod config;
mod routes;
mod serve;
mod util;
@ -310,7 +307,7 @@ async fn main() -> Result<()> {
AddressType::File => hash_path(&input)?,
AddressType::Sha256sum => {
let digest = multibase::Base::Base16Lower.decode(input)?;
Address::Hash(upend::util::hash::Hash(digest))
Address::Hash(upend_base::hash::Digest(digest))
}
};
@ -320,7 +317,7 @@ async fn main() -> Result<()> {
}
}
Commands::Serve(args) => {
info!("Starting UpEnd {}...", build::PKG_VERSION);
info!("Starting UpEnd {}...", common::build::PKG_VERSION);
let term_now = Arc::new(std::sync::atomic::AtomicBool::new(false));
for sig in signal_hook::consts::TERM_SIGNALS {
@ -528,7 +525,7 @@ fn hash_path<P: AsRef<Path>>(filepath: P) -> Result<Address> {
let filepath = filepath.as_ref();
debug!("Hashing {:?}...", filepath);
let fbuffer = FileBuffer::open(filepath)?;
let digest = hash(&fbuffer);
let digest = sha256hash(&fbuffer);
trace!("Finished hashing {:?}...", filepath);
Ok(Address::Hash(digest))
}

15
cli/src/previews/mod.rs
View File

@ -1,9 +1,8 @@
use upend::database::stores::UpStore;
use upend::util::hash::b58_encode;
use upend::util::hash::Hash;
use upend::util::jobs::{JobContainer, JobState};
use anyhow::{anyhow, Result};
use tracing::{debug, trace};
use upend_base::hash::{b58_encode, Digest};
use upend_db::jobs::{JobContainer, JobState};
use upend_db::stores::UpStore;
use std::{
collections::HashMap,
@ -27,12 +26,12 @@ pub trait Previewable {
fn get_thumbnail(&self, options: HashMap<String, String>) -> Result<Option<Vec<u8>>>;
}
type HashWithOptions = (Hash, String);
type DigestWithOptions = (Digest, String);
pub struct PreviewStore {
path: PathBuf,
store: Arc<Box<dyn UpStore + Send + Sync>>,
locks: Mutex<HashMap<HashWithOptions, Arc<Mutex<PathBuf>>>>,
locks: Mutex<HashMap<DigestWithOptions, Arc<Mutex<PathBuf>>>>,
}
#[cfg(feature = "previews")]
@ -45,7 +44,7 @@ impl PreviewStore {
}
}
fn get_path(&self, hash: &Hash, options: &HashMap<String, String>) -> Arc<Mutex<PathBuf>> {
fn get_path(&self, hash: &Digest, options: &HashMap<String, String>) -> Arc<Mutex<PathBuf>> {
let mut locks = self.locks.lock().unwrap();
let mut options_strs = options
.iter()
@ -74,7 +73,7 @@ impl PreviewStore {
pub fn get(
&self,
hash: Hash,
hash: Digest,
options: HashMap<String, String>,
mut job_container: JobContainer,
) -> Result<Option<PathBuf>> {

37
cli/src/routes.rs
View File

@ -1,4 +1,6 @@
use crate::common::build;
use crate::common::REQWEST_CLIENT;
use crate::config::UpEndConfig;
use crate::extractors;
use crate::previews::PreviewStore;
use crate::util::exec::block_background;
@ -28,17 +30,15 @@ use std::sync::Arc;
use std::time::{SystemTime, UNIX_EPOCH};
use tempfile::NamedTempFile;
use tracing::{debug, info, trace};
use upend::addressing::{Address, Addressable};
use upend::common::build;
use upend::config::UpEndConfig;
use upend::database::constants::{ATTR_ADDED, ATTR_LABEL};
use upend::database::entry::{Entry, EntryValue, InvariantEntry};
use upend::database::hierarchies::{list_roots, resolve_path, UHierPath};
use upend::database::lang::Query;
use upend::database::stores::{Blob, UpStore};
use upend::database::UpEndDatabase;
use upend::util::hash::{b58_decode, b58_encode, hash};
use upend::util::jobs;
use upend_base::addressing::{Address, Addressable};
use upend_base::constants::{ATTR_ADDED, ATTR_LABEL};
use upend_base::entry::{Entry, EntryValue, InvariantEntry};
use upend_base::hash::{b58_decode, b58_encode, sha256hash};
use upend_base::lang::Query;
use upend_db::hierarchies::{list_roots, resolve_path, UHierPath};
use upend_db::jobs;
use upend_db::stores::{Blob, UpStore};
use upend_db::UpEndDatabase;
use url::Url;
use uuid::Uuid;
@ -671,17 +671,14 @@ pub async fn get_address(
} else if let Some(url) = query.get("url_content") {
let url = Url::parse(url).map_err(ErrorBadRequest)?;
let (bytes, _) = web::block(|| fetch_external(url)).await??;
let hash_result = hash(&bytes);
let hash_result = sha256hash(&bytes);
(Address::Hash(hash_result), false)
} else if let Some(type_str) = query.get("type") {
match type_str.as_str() {
"Hash" => (upend::database::constants::TYPE_HASH_ADDRESS.clone(), true),
"Uuid" => (upend::database::constants::TYPE_UUID_ADDRESS.clone(), true),
"Attribute" => (
upend::database::constants::TYPE_ATTRIBUTE_ADDRESS.clone(),
true,
),
"Url" => (upend::database::constants::TYPE_URL_ADDRESS.clone(), true),
"Hash" => (upend_base::constants::TYPE_HASH_ADDRESS.clone(), true),
"Uuid" => (upend_base::constants::TYPE_UUID_ADDRESS.clone(), true),
"Attribute" => (upend_base::constants::TYPE_ATTRIBUTE_ADDRESS.clone(), true),
"Url" => (upend_base::constants::TYPE_URL_ADDRESS.clone(), true),
_ => return Err(ErrorBadRequest(format!("Unknown type: {type_str}"))),
}
} else {
@ -1051,7 +1048,7 @@ mod tests {
let upend = Arc::new(open_result.db);
let store = Arc::new(Box::new(
upend::database::stores::fs::FsStore::from_path(temp_dir.path()).unwrap(),
upend_db::stores::fs::FsStore::from_path(temp_dir.path()).unwrap(),
) as Box<dyn UpStore + Send + Sync>);
let job_container = jobs::JobContainer::new();

4
cli/src/serve.rs
View File

@ -1,5 +1,5 @@
use std::{path::Path};
use crate::routes;
use std::path::Path;
pub fn get_app<P, S>(
ui_path: Option<P>,
@ -41,7 +41,7 @@ where
.wrap(cors)
.wrap(
actix_web::middleware::DefaultHeaders::new()
.add(("UPEND-VERSION", upend::common::build::PKG_VERSION)),
.add(("UPEND-VERSION", crate::common::build::PKG_VERSION)),
)
.app_data(actix_web::web::PayloadConfig::new(4_294_967_296))
.app_data(actix_web::web::Data::new(state))

61
db/Cargo.toml Normal file
View File

@ -0,0 +1,61 @@
[package]
name = "upend-db"
version = "0.0.1"
homepage = "https://upend.dev/"
repository = "https://git.thm.place/thm/upend"
authors = ["Tomáš Mládek <t@mldk.cz>"]
license = "AGPL-3.0-or-later"
edition = "2018"
[dependencies]
upend-base = { path = "../base", features = ["diesel"] }
log = "0.4"
tracing = "0.1"
tracing-subscriber = { version = "0.3", features = ["env-filter"] }
anyhow = "1.0"
rayon = "1.4.0"
num_cpus = "1.13"
lazy_static = "1.4.0"
once_cell = "1.7.2"
lru = "0.7.0"
diesel = { version = "1.4", features = [
"sqlite",
"r2d2",
"chrono",
"serde_json",
] }
diesel_migrations = "1.4"
libsqlite3-sys = { version = "^0", features = ["bundled"] }
chrono = { version = "0.4", features = ["serde"] }
serde = { version = "1.0", features = ["derive"] }
serde_json = "1.0"
lexpr = "0.2.6"
regex = "1"
multibase = "0.9"
multihash = { version = "*", default-features = false, features = [
"alloc",
"multihash-impl",
"sha2",
"identity",
] }
uuid = { version = "0.8", features = ["v4"] }
url = { version = "2", features = ["serde"] }
filebuffer = "0.4.0"
tempfile = "^3.2.0"
walkdir = "2"
tree_magic_mini = { version = "3.0.2", features = ["with-gpl-data"] }
nonempty = "0.6.0"
shadow-rs = "0.17"
[build-dependencies]
shadow-rs = "0.17"

3
db/build.rs Normal file
View File

@ -0,0 +1,3 @@
fn main() -> shadow_rs::SdResult<()> {
shadow_rs::new()
}

0
migrations/fsvault/00_initial/down.sql → db/migrations/fsvault/00_initial/down.sql
View File

0
migrations/fsvault/00_initial/up.sql → db/migrations/fsvault/00_initial/up.sql
View File

0
migrations/upend/00_initial/down.sql → db/migrations/upend/00_initial/down.sql
View File

0
migrations/upend/00_initial/up.sql → db/migrations/upend/00_initial/up.sql
View File

0
src/common.rs → db/src/common.rs
View File

14
db/src/constants.rs Normal file
View File

@ -0,0 +1,14 @@
use crate::addressing::Address;
use crate::entry::InvariantEntry;
lazy_static! {
pub static ref HIER_ROOT_INVARIANT: InvariantEntry = InvariantEntry {
attribute: String::from(ATTR_KEY),
value: "HIER_ROOT".into(),
};
pub static ref HIER_ROOT_ADDR: Address = HIER_ROOT_INVARIANT.entity().unwrap();
pub static ref TYPE_HASH_ADDRESS: Address = Address::Hash(crate::util::hash::Hash(vec![]));
pub static ref TYPE_UUID_ADDRESS: Address = Address::Uuid(uuid::Uuid::nil());
pub static ref TYPE_ATTRIBUTE_ADDRESS: Address = Address::Attribute("".to_string());
pub static ref TYPE_URL_ADDRESS: Address = Address::Url(url::Url::parse("up:").unwrap());
}

10
src/database/engine.rs → db/src/engine.rs
View File

@ -1,11 +1,9 @@
use std::collections::HashMap;
use std::iter::zip;
use super::entry::EntryValue;
use super::inner::models::Entry;
use super::inner::schema::data;
use super::lang::{PatternQuery, Query, QueryComponent, QueryPart, QueryQualifier};
use crate::database::inner::models;
use crate::inner::models;
use anyhow::Result;
use diesel::expression::grouped::Grouped;
use diesel::expression::operators::{And, Not, Or};
@ -19,6 +17,8 @@ use diesel::{
};
use diesel::{BoxableExpression, QueryDsl};
use diesel::{ExpressionMethods, TextExpressionMethods};
use upend_base::entry::EntryValue;
use upend_base::lang::{PatternQuery, Query, QueryComponent, QueryPart, QueryQualifier};
#[derive(Debug, Clone)]
pub struct QueryExecutionError(String);
@ -35,7 +35,7 @@ pub fn execute(
connection: &PooledConnection<ConnectionManager<SqliteConnection>>,
query: Query,
) -> Result<Vec<Entry>, QueryExecutionError> {
use crate::database::inner::schema::data::dsl::*;
use crate::inner::schema::data::dsl::*;
if let Some(predicates) = to_sqlite_predicates(query.clone())? {
let db_query = data.filter(predicates);
@ -135,7 +135,7 @@ impl EntryWithVars {
if let QueryComponent::Variable(Some(var_name)) = &query.entity {
vars.insert(
var_name.clone(),
crate::util::hash::b58_encode(&entry.entity),
upend_base::hash::b58_encode(&entry.entity),
);
}

86
db/src/entry.rs Normal file
View File

@ -0,0 +1,86 @@
use crate::inner::models;
use anyhow::{anyhow, Result};
use std::convert::TryFrom;
use upend_base::addressing::{Address, Addressable};
use upend_base::entry::{Entry, EntryValue, ImmutableEntry};
impl TryFrom<&models::Entry> for Entry {
type Error = anyhow::Error;
fn try_from(e: &models::Entry) -> Result<Self, Self::Error> {
if let Some(value_str) = &e.value_str {
Ok(Entry {
entity: Address::decode(&e.entity)?,
attribute: e.attribute.clone(),
value: value_str.parse()?,
provenance: e.provenance.clone(),
timestamp: e.timestamp,
})
} else if let Some(value_num) = e.value_num {
Ok(Entry {
entity: Address::decode(&e.entity)?,
attribute: e.attribute.clone(),
value: EntryValue::Number(value_num),
provenance: e.provenance.clone(),
timestamp: e.timestamp,
})
} else {
Ok(Entry {
entity: Address::decode(&e.entity)?,
attribute: e.attribute.clone(),
value: EntryValue::Number(f64::NAN),
provenance: e.provenance.clone(),
timestamp: e.timestamp,
})
}
}
}
impl TryFrom<&Entry> for models::Entry {
type Error = anyhow::Error;
fn try_from(e: &Entry) -> Result<Self, Self::Error> {
if e.attribute.is_empty() {
return Err(anyhow!("Attribute cannot be empty."));
}
let base_entry = models::Entry {
identity: e.address()?.encode()?,
entity_searchable: match &e.entity {
Address::Attribute(attr) => Some(attr.clone()),
Address::Url(url) => Some(url.to_string()),
_ => None,
},
entity: e.entity.encode()?,
attribute: e.attribute.clone(),
value_str: None,
value_num: None,
immutable: false,
provenance: e.provenance.clone(),
timestamp: e.timestamp,
};
match e.value {
EntryValue::Number(n) => Ok(models::Entry {
value_str: None,
value_num: Some(n),
..base_entry
}),
_ => Ok(models::Entry {
value_str: Some(e.value.to_string()?),
value_num: None,
..base_entry
}),
}
}
}
impl TryFrom<&ImmutableEntry> for models::Entry {
type Error = anyhow::Error;
fn try_from(e: &ImmutableEntry) -> Result<Self, Self::Error> {
Ok(models::Entry {
immutable: true,
..models::Entry::try_from(&e.0)?
})
}
}

12
src/database/hierarchies.rs → db/src/hierarchies.rs
View File

@ -6,12 +6,12 @@ use lru::LruCache;
use tracing::trace;
use uuid::Uuid;
use crate::addressing::Address;
use crate::database::constants::ATTR_LABEL;
use crate::database::entry::Entry;
use crate::database::lang::{PatternQuery, Query, QueryComponent, QueryPart};
use upend_base::addressing::Address;
use upend_base::constants::ATTR_LABEL;
use upend_base::constants::{ATTR_IN, HIER_ROOT_ADDR, HIER_ROOT_INVARIANT};
use upend_base::entry::Entry;
use upend_base::lang::{PatternQuery, Query, QueryComponent, QueryPart};
use super::constants::{ATTR_IN, HIER_ROOT_ADDR, HIER_ROOT_INVARIANT};
use super::UpEndConnection;
#[derive(Debug, Clone, Eq, PartialEq, Hash)]
@ -242,7 +242,7 @@ pub fn initialize_hier(connection: &UpEndConnection) -> Result<()> {
mod tests {
use anyhow::Result;
use crate::database::UpEndDatabase;
use crate::UpEndDatabase;
use tempfile::TempDir;
use super::*;

0
src/database/inner/mod.rs → db/src/inner/mod.rs
View File

0
src/database/inner/models.rs → db/src/inner/models.rs
View File

5
src/database/inner/schema.rs → db/src/inner/schema.rs
View File

@ -20,7 +20,4 @@ table! {
}
}
allow_tables_to_appear_in_same_query!(
data,
meta,
);
allow_tables_to_appear_in_same_query!(data, meta,);

0
src/util/jobs.rs → db/src/jobs.rs
View File

50
src/database/mod.rs → db/src/lib.rs
View File

@ -1,25 +1,29 @@
#![macro_use]
#[macro_use]
extern crate diesel;
#[macro_use]
extern crate diesel_migrations;
#[macro_use]
extern crate lazy_static;
#[macro_use]
mod macros;
pub mod stores;
pub mod constants;
pub mod engine;
pub mod entry;
pub mod hierarchies;
pub mod inner;
pub mod lang;
pub mod jobs;
pub mod stores;
mod common;
mod inner;
mod util;
use crate::addressing::{Address, Addressable};
use crate::common::build;
use crate::database::engine::execute;
use crate::database::entry::{Entry, EntryValue, ImmutableEntry};
use crate::database::inner::models;
use crate::database::inner::schema::data;
use crate::database::lang::Query;
use crate::util::hash::Hash;
use crate::engine::execute;
use crate::inner::models;
use crate::inner::schema::data;
use crate::util::LoggerSink;
use anyhow::{anyhow, Result};
use diesel::prelude::*;
@ -34,6 +38,10 @@ use std::path::{Path, PathBuf};
use std::sync::{Arc, Mutex, RwLock};
use std::time::Duration;
use tracing::{debug, error, trace, warn};
use upend_base::addressing::{Address, Addressable};
use upend_base::entry::{Entry, EntryValue, ImmutableEntry};
use upend_base::hash::Digest;
use upend_base::lang::Query;
#[derive(Debug)]
pub struct ConnectionOptions {
@ -193,7 +201,7 @@ impl UpEndConnection {
}
pub fn get_meta<S: AsRef<str>>(&self, key: S) -> Result<String> {
use crate::database::inner::schema::meta::dsl;
use crate::inner::schema::meta::dsl;
let key = key.as_ref();
debug!("Querying META:{key}");
@ -209,8 +217,8 @@ impl UpEndConnection {
.map(|mv| mv.value.clone())
}
pub fn retrieve_entry(&self, hash: &Hash) -> Result<Option<Entry>> {
use crate::database::inner::schema::data::dsl::*;
pub fn retrieve_entry(&self, hash: &Digest) -> Result<Option<Entry>> {
use crate::inner::schema::data::dsl::*;
let _lock = self.lock.read().unwrap();
let conn = self.pool.get()?;
@ -231,7 +239,7 @@ impl UpEndConnection {
}
pub fn retrieve_object(&self, object_address: &Address) -> Result<Vec<Entry>> {
use crate::database::inner::schema::data::dsl::*;
use crate::inner::schema::data::dsl::*;
let _lock = self.lock.read().unwrap();
let conn = self.pool.get()?;
@ -268,7 +276,7 @@ impl UpEndConnection {
}
pub fn remove_object(&self, object_address: Address) -> Result<usize> {
use crate::database::inner::schema::data::dsl::*;
use crate::inner::schema::data::dsl::*;
debug!("Deleting {}!", object_address);
@ -333,7 +341,7 @@ impl UpEndConnection {
// #[deprecated]
pub fn get_all_addresses(&self) -> Result<Vec<Address>> {
use crate::database::inner::schema::data::dsl::*;
use crate::inner::schema::data::dsl::*;
let _lock = self.lock.read().unwrap();
let conn = self.pool.get()?;
@ -351,7 +359,7 @@ impl UpEndConnection {
// #[deprecated]
pub fn get_all_attributes(&self) -> Result<Vec<String>> {
use crate::database::inner::schema::data::dsl::*;
use crate::inner::schema::data::dsl::*;
let _lock = self.lock.read().unwrap();
let conn = self.pool.get()?;
@ -368,7 +376,7 @@ impl UpEndConnection {
#[cfg(test)]
mod test {
use crate::database::constants::ATTR_LABEL;
use upend_base::constants::ATTR_LABEL;
use super::*;
use tempfile::TempDir;

4
src/database/macros.rs → db/src/macros.rs
View File

@ -4,7 +4,7 @@ macro_rules! upend_insert_val {
$db_connection.insert_entry(Entry {
entity: $entity.clone(),
attribute: String::from($attribute),
value: upend::database::entry::EntryValue::String(String::from($value)),
value: upend_base::entry::EntryValue::String(String::from($value)),
provenance: "SYSTEM INIT".to_string(),
timestamp: chrono::Utc::now().naive_utc(),
})
@ -17,7 +17,7 @@ macro_rules! upend_insert_addr {
$db_connection.insert_entry(Entry {
entity: $entity.clone(),
attribute: String::from($attribute),
value: upend::database::entry::EntryValue::Address($addr.clone()),
value: upend_base::entry::EntryValue::Address($addr.clone()),
provenance: "SYSTEM INIT".to_string(),
timestamp: chrono::Utc::now().naive_utc(),
})

10
src/database/stores/fs/db.rs → db/src/stores/fs/db.rs
View File

@ -1,9 +1,9 @@
use std::path::PathBuf;
use crate::util::hash::Hash;
use chrono::NaiveDateTime;
use diesel::Queryable;
use serde::Serialize;
use upend_base::hash::Digest;
table! {
files (id) {
@ -20,7 +20,7 @@ table! {
#[derive(Queryable, Serialize, Clone, Debug)]
pub struct File {
pub id: i32,
pub hash: Hash,
pub hash: Digest,
pub path: String,
pub valid: bool,
pub added: NaiveDateTime,
@ -32,7 +32,7 @@ pub struct File {
#[derive(Serialize, Clone, Debug)]
pub struct OutFile {
pub id: i32,
pub hash: Hash,
pub hash: Digest,
pub path: PathBuf,
pub valid: bool,
pub added: NaiveDateTime,
@ -50,8 +50,8 @@ pub struct NewFile {
pub mtime: Option<NaiveDateTime>,
}
impl From<File> for crate::addressing::Address {
impl From<File> for upend_base::addressing::Address {
fn from(file: File) -> Self {
crate::addressing::Address::Hash(file.hash)
upend_base::addressing::Address::Hash(file.hash)
}
}

47
src/database/stores/fs/mod.rs → db/src/stores/fs/mod.rs
View File

@ -1,19 +1,10 @@
use self::db::files;
use super::{Blob, StoreError, UpStore, UpdatePathOutcome};
use crate::addressing::Address;
use crate::database::constants::{
ATTR_ADDED, ATTR_BY, ATTR_IN, ATTR_LABEL, ATTR_OF, TYPE_HASH_ADDRESS,
};
use crate::database::entry::Entry;
use crate::database::hierarchies::{
resolve_path, resolve_path_cached, ResolveCache, UHierPath, UNode,
};
use crate::database::{
ConnectionOptions, LoggingHandler, UpEndConnection, UpEndDatabase, UPEND_SUBDIR,
};
use crate::util::hash::{b58_encode, Hash, Hashable};
use crate::util::jobs::{JobContainer, JobHandle};
use crate::hierarchies::{resolve_path, resolve_path_cached, ResolveCache, UHierPath, UNode};
use crate::jobs::{JobContainer, JobHandle};
use crate::util::hash_at_path;
use crate::{ConnectionOptions, LoggingHandler, UpEndConnection, UpEndDatabase, UPEND_SUBDIR};
use anyhow::{anyhow, Error, Result};
use chrono::prelude::*;
use diesel::r2d2::{self, ConnectionManager, ManageConnection};
@ -30,6 +21,10 @@ use std::sync::{Arc, Mutex, RwLock};
use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH};
use std::{fs, iter};
use tracing::{debug, error, info, trace, warn};
use upend_base::addressing::Address;
use upend_base::constants::{ATTR_ADDED, ATTR_BY, ATTR_IN, ATTR_LABEL, ATTR_OF, TYPE_HASH_ADDRESS};
use upend_base::entry::Entry;
use upend_base::hash::{b58_encode, Digest};
use walkdir::WalkDir;
mod db;
@ -257,7 +252,7 @@ impl FsStore {
.to_str()
.ok_or(anyhow!("Path not valid unicode!"))?;
let mut file_hash: Option<Hash> = None;
let mut file_hash: Option<Digest> = None;
// Get size & mtime for quick comparison
let metadata = fs::metadata(&path)?;
@ -294,7 +289,7 @@ impl FsStore {
let mut same_hash = false;
if !quick_check || !same_mtime {
file_hash = Some(path.hash()?);
file_hash = Some(hash_at_path(&path)?);
same_hash = file_hash.as_ref().unwrap() == &existing_file.hash;
}
@ -327,7 +322,7 @@ impl FsStore {
// If not, add it!
if file_hash.is_none() {
file_hash = Some(path.hash()?);
file_hash = Some(hash_at_path(&path)?);
}
let mime_type = tree_magic_mini::from_filepath(&path).map(|s| s.to_string());
@ -351,7 +346,7 @@ impl FsStore {
&self,
connection: &UpEndConnection,
path: &Path,
hash: Hash,
hash: Digest,
name_hint: Option<String>,
) -> Result<Address> {
let normalized_path = self.normalize_path(path)?;
@ -386,7 +381,7 @@ impl FsStore {
&self,
connection: &UpEndConnection,
normalized_path: &Path,
hash: Hash,
hash: Digest,
name: Option<String>,
size: i64,
mtime: Option<NaiveDateTime>,
@ -499,7 +494,7 @@ impl FsStore {
debug!(
"Inserting {} ({})...",
&file.path,
Address::Hash(Hash((file.hash).clone()))
Address::Hash(Digest((file.hash).clone()))
);
let _lock = self.lock.write().unwrap();
@ -518,7 +513,7 @@ impl FsStore {
.unwrap())
}
fn retrieve_file(&self, obj_hash: &Hash) -> Result<Vec<db::OutFile>> {
fn retrieve_file(&self, obj_hash: &Digest) -> Result<Vec<db::OutFile>> {
use self::db::files::dsl::*;
let _lock = self.lock.read().unwrap();
@ -602,7 +597,7 @@ impl From<db::File> for Blob {
}
impl UpStore for FsStore {
fn retrieve(&self, hash: &crate::util::hash::Hash) -> Result<Vec<Blob>, super::StoreError> {
fn retrieve(&self, hash: &upend_base::hash::Digest) -> Result<Vec<Blob>, super::StoreError> {
Ok(self
.retrieve_file(hash)
.map_err(|e| StoreError::Unknown(e.to_string()))?
@ -625,11 +620,9 @@ impl UpStore for FsStore {
connection: UpEndConnection,
blob: Blob,
name_hint: Option<String>,
) -> Result<Hash, super::StoreError> {
) -> Result<Digest, super::StoreError> {
let file_path = blob.get_file_path();
let hash = file_path
.hash()
.map_err(|e| StoreError::Unknown(e.to_string()))?;
let hash = hash_at_path(file_path).map_err(|e| StoreError::Unknown(e.to_string()))?;
let existing_files = self.retrieve(&hash)?;
@ -732,8 +725,8 @@ impl UpStore for FsStore {
#[cfg(test)]
mod test {
use crate::database::UpEndDatabase;
use crate::util::jobs::JobContainer;
use crate::jobs::JobContainer;
use crate::UpEndDatabase;
use super::*;
use std::fs::File;

7
src/database/stores/mod.rs → db/src/stores/mod.rs
View File

@ -1,7 +1,8 @@
use std::path::{Path, PathBuf};
use super::{UpEndConnection, UpEndDatabase};
use crate::util::{hash::Hash, jobs::JobContainer};
use crate::jobs::JobContainer;
use upend_base::hash::Digest;
pub mod fs;
@ -52,14 +53,14 @@ pub enum UpdatePathOutcome {
}
pub trait UpStore {
fn retrieve(&self, hash: &Hash) -> Result<Vec<Blob>>;
fn retrieve(&self, hash: &Digest) -> Result<Vec<Blob>>;
fn retrieve_all(&self) -> Result<Vec<Blob>>;
fn store(
&self,
connection: UpEndConnection,
blob: Blob,
name_hint: Option<String>,
) -> Result<Hash>;
) -> Result<Digest>;
fn update(
&self,
database: &UpEndDatabase,

15
src/util/mod.rs → db/src/util.rs
View File

@ -1,7 +1,8 @@
pub mod hash;
pub mod jobs;
use std::path::Path;
use tracing::debug;
use filebuffer::FileBuffer;
use tracing::{debug, trace};
use upend_base::hash::Digest;
#[derive(Default)]
pub struct LoggerSink {
@ -32,3 +33,11 @@ impl std::io::Write for LoggerSink {
Ok(())
}
}
pub fn hash_at_path<P: AsRef<Path>>(path: P) -> anyhow::Result<Digest> {
let path = path.as_ref();
trace!("Hashing {:?}...", path);
let fbuffer = FileBuffer::open(path)?;
trace!("Finished hashing {:?}...", path);
Ok(upend_base::hash::sha256hash(&fbuffer))
}

8
src/config.rs
View File

@ -1,8 +0,0 @@
#[derive(Clone, Debug)]
pub struct UpEndConfig {
pub vault_name: Option<String>,
pub desktop_enabled: bool,
pub trust_executables: bool,
pub secret: String,
pub key: Option<String>,
}

14
src/lib.rs
View File

@ -1,14 +0,0 @@
#[macro_use]
extern crate diesel;
#[macro_use]
extern crate diesel_migrations;
#[macro_use]
extern crate lazy_static;
extern crate self as upend;
pub mod database;
pub mod util;
pub mod addressing;
pub mod common;
pub mod config;