thm
/
upend
1
0
Fork 0
Code Issues 75 Pull Requests Packages Projects Releases 5 Wiki Activity

separate value_string and value_number columns 

to be able to utilize SQL queries better for ranges, comparisons, etc.
feat/vaults
Tomáš Mládek 2022-01-28 18:17:14 +01:00
parent fc3956e770
commit 8c8d58847a
No known key found for this signature in database
GPG Key ID: ED21612889E75EC5
10 changed files with 163 additions and 114 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
migrations/upend/00_initial_structure/up.sql
View File

@ -20,19 +20,19 @@ CREATE TABLE files
); );
CREATE INDEX files_hash ON files (hash); CREATE INDEX files_hash ON files (hash);
CREATE INDEX files_path ON files (path);
CREATE INDEX files_valid ON files (valid); CREATE INDEX files_valid ON files (valid);
CREATE TABLE data CREATE TABLE data
( (
identity BLOB PRIMARY KEY NOT NULL, identity BLOB PRIMARY KEY NOT NULL,
entity BLOB NOT NULL, entity BLOB NOT NULL,
attribute VARCHAR NOT NULL, attribute VARCHAR NOT NULL,
value VARCHAR NOT NULL, value_str VARCHAR,
immutable BOOLEAN NOT NULL, value_num NUMERIC,
UNIQUE (entity, attribute, value) immutable BOOLEAN NOT NULL
); );
CREATE INDEX data_entity ON data (entity); CREATE INDEX data_entity ON data (entity);
CREATE INDEX data_attribute ON data (attribute); CREATE INDEX data_attribute ON data (attribute);
CREATE INDEX data_value ON data (value); CREATE INDEX data_value_str ON data (value_str);
CREATE INDEX data_value_num ON data (value_num);

4
src/database/constants.rs
View File

@ -16,12 +16,12 @@ pub const LABEL_ATTR: &str = "LBL";
lazy_static! { lazy_static! {
pub static ref TYPE_INVARIANT: InvariantEntry = InvariantEntry { pub static ref TYPE_INVARIANT: InvariantEntry = InvariantEntry {
attribute: String::from(TYPE_BASE_ATTR), attribute: String::from(TYPE_BASE_ATTR),
value: EntryValue::Value(serde_json::Value::from(TYPE_TYPE_VAL)), value: EntryValue::String(String::from(TYPE_TYPE_VAL)),
}; };
pub static ref TYPE_ADDR: Address = TYPE_INVARIANT.entity().unwrap(); pub static ref TYPE_ADDR: Address = TYPE_INVARIANT.entity().unwrap();
pub static ref HIER_INVARIANT: InvariantEntry = InvariantEntry { pub static ref HIER_INVARIANT: InvariantEntry = InvariantEntry {
attribute: String::from(TYPE_BASE_ATTR), attribute: String::from(TYPE_BASE_ATTR),
value: EntryValue::Value(serde_json::Value::from(HIER_TYPE_VAL)), value: EntryValue::String(String::from(HIER_TYPE_VAL)),
}; };
pub static ref HIER_ADDR: Address = HIER_INVARIANT.entity().unwrap(); pub static ref HIER_ADDR: Address = HIER_INVARIANT.entity().unwrap();
} }

117
src/database/entry.rs
View File

@ -32,7 +32,8 @@ pub struct InvariantEntry {
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
#[serde(tag = "t", content = "c")] #[serde(tag = "t", content = "c")]
pub enum EntryValue { pub enum EntryValue {
Value(serde_json::Value), String(String),
Number(f64),
Address(Address), Address(Address),
Invalid, Invalid,
} }
@ -41,11 +42,23 @@ impl TryFrom<&models::Entry> for Entry {
type Error = anyhow::Error; type Error = anyhow::Error;
fn try_from(e: &models::Entry) -> Result<Self, Self::Error> { fn try_from(e: &models::Entry) -> Result<Self, Self::Error> {
Ok(Entry { if let Some(value_str) = &e.value_str {
entity: Address::decode(&e.entity)?, Ok(Entry {
attribute: e.attribute.clone(), entity: Address::decode(&e.entity)?,
value: e.value.parse().unwrap(), attribute: e.attribute.clone(),
}) value: value_str.parse()?,
})
} else if let Some(value_num) = e.value_num {
Ok(Entry {
entity: Address::decode(&e.entity)?,
attribute: e.attribute.clone(),
value: EntryValue::Number(value_num),
})
} else {
Err(anyhow!(
"Inconsistent database: Both values of entry are NULL!"
))
}
} }
} }
@ -53,13 +66,24 @@ impl TryFrom<&Entry> for models::Entry {
type Error = anyhow::Error; type Error = anyhow::Error;
fn try_from(e: &Entry) -> Result<Self, Self::Error> { fn try_from(e: &Entry) -> Result<Self, Self::Error> {
Ok(models::Entry { match e.value {
identity: e.address()?.encode()?, EntryValue::Number(n) => Ok(models::Entry {
entity: e.entity.encode()?, identity: e.address()?.encode()?,
attribute: e.attribute.clone(), entity: e.entity.encode()?,
value: e.value.to_string()?, attribute: e.attribute.clone(),
immutable: false, value_str: None,
}) value_num: Some(n),
immutable: false,
}),
_ => Ok(models::Entry {
identity: e.address()?.encode()?,
entity: e.entity.encode()?,
attribute: e.attribute.clone(),
value_str: Some(e.value.to_string()?),
value_num: None,
immutable: false,
}),
}
} }
} }
@ -67,13 +91,24 @@ impl TryFrom<&ImmutableEntry> for models::Entry {
type Error = anyhow::Error; type Error = anyhow::Error;
fn try_from(e: &ImmutableEntry) -> Result<Self, Self::Error> { fn try_from(e: &ImmutableEntry) -> Result<Self, Self::Error> {
Ok(models::Entry { match e.0.value {
identity: e.0.address()?.encode()?, EntryValue::Number(n) => Ok(models::Entry {
entity: e.0.entity.encode()?, identity: e.0.address()?.encode()?,
attribute: e.0.attribute.clone(), entity: e.0.entity.encode()?,
value: e.0.value.to_string()?, attribute: e.0.attribute.clone(),
immutable: true, value_str: None,
}) value_num: Some(n),
immutable: false,
}),
_ => Ok(models::Entry {
identity: e.0.address()?.encode()?,
entity: e.0.entity.encode()?,
attribute: e.0.attribute.clone(),
value_str: Some(e.0.value.to_string()?),
value_num: None,
immutable: false,
}),
}
} }
} }
@ -144,7 +179,8 @@ impl Addressable for InvariantEntry {}
impl EntryValue { impl EntryValue {
pub fn to_string(&self) -> Result<String> { pub fn to_string(&self) -> Result<String> {
let (type_char, content) = match self { let (type_char, content) = match self {
EntryValue::Value(value) => ('J', serde_json::to_string(value)?), EntryValue::String(value) => ('S', value.to_owned()),
EntryValue::Number(n) => ('N', n.to_string()),
EntryValue::Address(address) => ('O', address.to_string()), EntryValue::Address(address) => ('O', address.to_string()),
EntryValue::Invalid => return Err(anyhow!("Cannot serialize invalid Entity value.")), EntryValue::Invalid => return Err(anyhow!("Cannot serialize invalid Entity value.")),
}; };
@ -157,10 +193,8 @@ impl std::fmt::Display for EntryValue {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
let (entry_type, entry_value) = match self { let (entry_type, entry_value) = match self {
EntryValue::Address(address) => ("ADDRESS", address.to_string()), EntryValue::Address(address) => ("ADDRESS", address.to_string()),
EntryValue::Value(value) => ( EntryValue::String(string) => ("STRING", string.to_owned()),
"VALUE", EntryValue::Number(n) => ("NUMBER", n.to_string()),
serde_json::to_string(value).unwrap_or_else(|_| String::from("?!?!?!")),
),
EntryValue::Invalid => ("INVALID", "INVALID".to_string()), EntryValue::Invalid => ("INVALID", "INVALID".to_string()),
}; };
write!(f, "{}: {}", entry_type, entry_value) write!(f, "{}: {}", entry_type, entry_value)
@ -176,9 +210,10 @@ impl std::str::FromStr for EntryValue {
} else { } else {
let (type_char, content) = s.split_at(1); let (type_char, content) = s.split_at(1);
match (type_char, content) { match (type_char, content) {
("J", content) => { ("S", content) => Ok(EntryValue::String(String::from(content))),
if let Ok(value) = serde_json::from_str(content) { ("N", content) => {
Ok(EntryValue::Value(value)) if let Ok(n) = content.parse::<f64>() {
Ok(EntryValue::Number(n))
} else { } else {
Ok(EntryValue::Invalid) Ok(EntryValue::Invalid)
} }
@ -195,3 +230,29 @@ impl std::str::FromStr for EntryValue {
} }
} }
} }
#[cfg(test)]
mod tests {
use super::*;
use anyhow::Result;
#[test]
fn test_value_from_to_string() -> Result<()> {
let entry = EntryValue::String("hello".to_string());
let encoded = entry.to_string()?;
let decoded = encoded.parse::<EntryValue>()?;
assert_eq!(entry, decoded);
let entry = EntryValue::Number(1337.93);
let encoded = entry.to_string()?;
let decoded = encoded.parse::<EntryValue>()?;
assert_eq!(entry, decoded);
let entry = EntryValue::Address(Address::Url("https://upendproject.net".to_string()));
let encoded = entry.to_string()?;
let decoded = encoded.parse::<EntryValue>()?;
assert_eq!(entry, decoded);
Ok(())
}
}

87
src/database/hierarchies.rs
View File

@ -4,7 +4,6 @@ use std::sync::{Arc, Mutex};
use anyhow::{anyhow, Result}; use anyhow::{anyhow, Result};
use log::trace; use log::trace;
use lru::LruCache; use lru::LruCache;
use serde_json::Value;
use uuid::Uuid; use uuid::Uuid;
use crate::addressing::{Address, Addressable}; use crate::addressing::{Address, Addressable};
@ -95,26 +94,24 @@ impl PointerEntries for Vec<Entry> {
} }
pub fn list_roots(connection: &UpEndConnection) -> Result<Vec<Address>> { pub fn list_roots(connection: &UpEndConnection) -> Result<Vec<Address>> {
let all_directories: Vec<Entry> = connection.query( let all_directories: Vec<Entry> =
Query::SingleQuery(QueryPart::Matches(EntryQuery { connection.query(Query::SingleQuery(QueryPart::Matches(EntryQuery {
entity: QueryComponent::Any, entity: QueryComponent::Any,
attribute: QueryComponent::Exact(IS_OF_TYPE_ATTR.to_string()), attribute: QueryComponent::Exact(IS_OF_TYPE_ATTR.to_string()),
value: QueryComponent::Exact(EntryValue::Address(HIER_ADDR.clone())), value: QueryComponent::Exact(EntryValue::Address(HIER_ADDR.clone())),
})), })))?;
)?;
// TODO: this is horrible // TODO: this is horrible
let directories_with_parents: Vec<Address> = connection.query( let directories_with_parents: Vec<Address> = connection
Query::SingleQuery(QueryPart::Matches(EntryQuery { .query(Query::SingleQuery(QueryPart::Matches(EntryQuery {
entity: QueryComponent::Any, entity: QueryComponent::Any,
attribute: QueryComponent::Exact(HIER_HAS_ATTR.to_string()), attribute: QueryComponent::Exact(HIER_HAS_ATTR.to_string()),
value: QueryComponent::Any, value: QueryComponent::Any,
})), })))?
)? .extract_pointers()
.extract_pointers() .into_iter()
.into_iter() .map(|(_, val)| val)
.map(|(_, val)| val) .collect();
.collect();
Ok(all_directories Ok(all_directories
.into_iter() .into_iter()
@ -143,30 +140,26 @@ pub fn fetch_or_create_dir(
_lock = FETCH_CREATE_LOCK.lock().unwrap(); _lock = FETCH_CREATE_LOCK.lock().unwrap();
} }
let matching_directories = connection.query( let matching_directories = connection
Query::SingleQuery(QueryPart::Matches(EntryQuery { .query(Query::SingleQuery(QueryPart::Matches(EntryQuery {
entity: QueryComponent::Any, entity: QueryComponent::Any,
attribute: QueryComponent::Exact(String::from(LABEL_ATTR)), attribute: QueryComponent::Exact(String::from(LABEL_ATTR)),
value: QueryComponent::Exact(EntryValue::Value(Value::String( value: QueryComponent::Exact(EntryValue::String(directory.as_ref().clone())),
directory.as_ref().clone(), })))?
))), .into_iter()
})), .map(|e: Entry| e.entity);
)?
.into_iter()
.map(|e: Entry| e.entity);
let parent_has: Vec<Address> = match parent.clone() { let parent_has: Vec<Address> = match parent.clone() {
Some(parent) => connection.query( Some(parent) => connection
Query::SingleQuery(QueryPart::Matches(EntryQuery { .query(Query::SingleQuery(QueryPart::Matches(EntryQuery {
entity: QueryComponent::Exact(parent), entity: QueryComponent::Exact(parent),
attribute: QueryComponent::Exact(String::from(HIER_HAS_ATTR)), attribute: QueryComponent::Exact(String::from(HIER_HAS_ATTR)),
value: QueryComponent::Any, value: QueryComponent::Any,
})), })))?
)? .extract_pointers()
.extract_pointers() .into_iter()
.into_iter() .map(|(_, val)| val)
.map(|(_, val)| val) .collect(),
.collect(),
None => list_roots(connection)?, None => list_roots(connection)?,
}; };
@ -188,7 +181,7 @@ pub fn fetch_or_create_dir(
let directory_entry = Entry { let directory_entry = Entry {
entity: new_directory_address.clone(), entity: new_directory_address.clone(),
attribute: String::from(LABEL_ATTR), attribute: String::from(LABEL_ATTR),
value: EntryValue::Value(Value::String(directory.as_ref().clone())), value: EntryValue::String(directory.as_ref().clone()),
}; };
connection.insert_entry(directory_entry)?; connection.insert_entry(directory_entry)?;
@ -330,21 +323,11 @@ mod tests {
let open_result = UpEndDatabase::open(&temp_dir, None, true).unwrap(); let open_result = UpEndDatabase::open(&temp_dir, None, true).unwrap();
let connection = open_result.db.connection().unwrap(); let connection = open_result.db.connection().unwrap();
let foo_result = fetch_or_create_dir( let foo_result = fetch_or_create_dir(&connection, None, UNode("foo".to_string()), true);
&connection,
None,
UNode("foo".to_string()),
true,
);
assert!(foo_result.is_ok()); assert!(foo_result.is_ok());
let foo_result = foo_result.unwrap(); let foo_result = foo_result.unwrap();
let bar_result = fetch_or_create_dir( let bar_result = fetch_or_create_dir(&connection, None, UNode("bar".to_string()), true);
&connection,
None,
UNode("bar".to_string()),
true,
);
assert!(bar_result.is_ok()); assert!(bar_result.is_ok());
let bar_result = bar_result.unwrap(); let bar_result = bar_result.unwrap();
@ -360,11 +343,7 @@ mod tests {
let roots = list_roots(&connection); let roots = list_roots(&connection);
assert_eq!(roots.unwrap(), [foo_result, bar_result.clone()]); assert_eq!(roots.unwrap(), [foo_result, bar_result.clone()]);
let resolve_result = resolve_path( let resolve_result = resolve_path(&connection, &"bar/baz".parse().unwrap(), false);
&connection,
&"bar/baz".parse().unwrap(),
false,
);
assert!(resolve_result.is_ok()); assert!(resolve_result.is_ok());
assert_eq!( assert_eq!(
@ -372,18 +351,10 @@ mod tests {
vec![bar_result.clone(), baz_result.clone()] vec![bar_result.clone(), baz_result.clone()]
); );
let resolve_result = resolve_path( let resolve_result = resolve_path(&connection, &"bar/baz/bax".parse().unwrap(), false);
&connection,
&"bar/baz/bax".parse().unwrap(),
false,
);
assert!(resolve_result.is_err()); assert!(resolve_result.is_err());
let resolve_result = resolve_path( let resolve_result = resolve_path(&connection, &"bar/baz/bax".parse().unwrap(), true);
&connection,
&"bar/baz/bax".parse().unwrap(),
true,
);
assert!(resolve_result.is_ok()); assert!(resolve_result.is_ok());
let bax_result = fetch_or_create_dir( let bax_result = fetch_or_create_dir(

3
src/database/inner/models.rs
View File

@ -45,6 +45,7 @@ pub struct Entry {
pub identity: Vec<u8>, pub identity: Vec<u8>,
pub entity: Vec<u8>, pub entity: Vec<u8>,
pub attribute: String, pub attribute: String,
pub value: String, pub value_str: Option<String>,
pub value_num: Option<f64>,
pub immutable: bool, pub immutable: bool,
} }

3
src/database/inner/schema.rs
View File

@ -3,7 +3,8 @@ table! {
identity -> Binary, identity -> Binary,
entity -> Binary, entity -> Binary,
attribute -> Text, attribute -> Text,
value -> Text, value_str -> Nullable<Text>,
value_num -> Nullable<Double>,
immutable -> Bool, immutable -> Bool,
} }
} }

30
src/database/lang.rs
View File

@ -278,16 +278,32 @@ impl Query {
}; };
match &eq.value { match &eq.value {
QueryComponent::Exact(q_value) => { QueryComponent::Exact(q_value) => match q_value {
subqueries.push(Box::new(data::value.eq(q_value.to_string()?))) EntryValue::Number(n) => {
} subqueries.push(Box::new(data::value_num.eq(*n)))
}
_ => subqueries
.push(Box::new(data::value_str.eq(q_value.to_string()?))),
},
QueryComponent::In(q_values) => { QueryComponent::In(q_values) => {
let values: Result<Vec<_>, _> = let first = q_values.first().ok_or(anyhow!(
q_values.iter().map(|v| v.to_string()).collect(); "Malformed expression: Inner value cannot be empty."
subqueries.push(Box::new(data::value.eq_any(values?))) ))?;
match first {
EntryValue::Number(_) => todo!(),
_ => subqueries.push(Box::new(
data::value_str.eq_any(
q_values
.iter()
.map(|v| v.to_string())
.collect::<Result<Vec<String>>>()?,
),
)),
}
} }
QueryComponent::Contains(q_value) => subqueries QueryComponent::Contains(q_value) => subqueries
.push(Box::new(data::value.like(format!("J%{}%", q_value)))), .push(Box::new(data::value_str.like(format!("J%{}%", q_value)))),
QueryComponent::Any => {} QueryComponent::Any => {}
}; };

2
src/database/macros.rs
View File

@ -4,7 +4,7 @@ macro_rules! upend_insert_val {
Entry { Entry {
entity: $entity.clone(), entity: $entity.clone(),
attribute: String::from($attribute), attribute: String::from($attribute),
value: crate::database::entry::EntryValue::Value(serde_json::Value::from($value)), value: crate::database::entry::EntryValue::String(String::from($value)),
}, },
) )
}}; }};

4
src/database/mod.rs
View File

@ -277,7 +277,7 @@ impl UpEndConnection {
let primary = data let primary = data
.filter(entity.eq(object_address.encode()?)) .filter(entity.eq(object_address.encode()?))
.or_filter(value.eq(EntryValue::Address(object_address).to_string()?)) .or_filter(value_str.eq(EntryValue::Address(object_address).to_string()?))
.load::<models::Entry>(&self.conn)?; .load::<models::Entry>(&self.conn)?;
let entries = primary let entries = primary
@ -314,7 +314,7 @@ impl UpEndConnection {
let matches = data let matches = data
.filter(identity.eq(object_address.encode()?)) .filter(identity.eq(object_address.encode()?))
.or_filter(entity.eq(object_address.encode()?)) .or_filter(entity.eq(object_address.encode()?))
.or_filter(value.eq(EntryValue::Address(object_address).to_string()?)); .or_filter(value_str.eq(EntryValue::Address(object_address).to_string()?));
Ok(diesel::delete(matches).execute(&self.conn)?) Ok(diesel::delete(matches).execute(&self.conn)?)
} }

11
src/filesystem.rs
View File

@ -22,7 +22,6 @@ use chrono::prelude::*;
use log::{debug, error, info, warn}; use log::{debug, error, info, warn};
use lru::LruCache; use lru::LruCache;
use rayon::prelude::*; use rayon::prelude::*;
use serde_json::Value;
use walkdir::WalkDir; use walkdir::WalkDir;
const BLOB_TYPE: &str = "BLOB"; const BLOB_TYPE: &str = "BLOB";
@ -34,7 +33,7 @@ const FILE_SIZE_KEY: &str = "FILE_SIZE";
lazy_static! { lazy_static! {
static ref BLOB_TYPE_INVARIANT: InvariantEntry = InvariantEntry { static ref BLOB_TYPE_INVARIANT: InvariantEntry = InvariantEntry {
attribute: String::from(TYPE_BASE_ATTR), attribute: String::from(TYPE_BASE_ATTR),
value: EntryValue::Value(Value::from(BLOB_TYPE)), value: EntryValue::String(String::from(BLOB_TYPE)),
}; };
static ref BLOB_TYPE_ADDR: Address = BLOB_TYPE_INVARIANT.entity().unwrap(); static ref BLOB_TYPE_ADDR: Address = BLOB_TYPE_INVARIANT.entity().unwrap();
} }
@ -391,13 +390,13 @@ fn insert_file_with_metadata(
let size_entry = Entry { let size_entry = Entry {
entity: Address::Hash(hash.clone()), entity: Address::Hash(hash.clone()),
attribute: FILE_SIZE_KEY.to_string(), attribute: FILE_SIZE_KEY.to_string(),
value: EntryValue::Value(Value::from(size)), value: EntryValue::Number(size as f64),
}; };
let mime_entry = mime_type.map(|mime_type| Entry { let mime_entry = mime_type.map(|mime_type| Entry {
entity: Address::Hash(hash.clone()), entity: Address::Hash(hash.clone()),
attribute: FILE_MIME_KEY.to_string(), attribute: FILE_MIME_KEY.to_string(),
value: EntryValue::Value(Value::String(mime_type)), value: EntryValue::String(mime_type),
}); });
// Add the appropriate entries w/r/t virtual filesystem location // Add the appropriate entries w/r/t virtual filesystem location
@ -437,9 +436,9 @@ fn insert_file_with_metadata(
let name_entry = Entry { let name_entry = Entry {
entity: dir_has_entry_addr, entity: dir_has_entry_addr,
attribute: ALIAS_KEY.to_string(), attribute: ALIAS_KEY.to_string(),
value: EntryValue::Value(Value::String( value: EntryValue::String(
filename.as_os_str().to_string_lossy().to_string(), filename.as_os_str().to_string_lossy().to_string(),
)), ),
}; };
connection.insert_entry(name_entry)?; connection.insert_entry(name_entry)?;