thm
/
upend
1
0
Fork 0
Code Issues 73 Pull Requests Packages Projects Releases 5 Wiki Activity

separate value_string and value_number columns 

to be able to utilize SQL queries better for ranges, comparisons, etc.
feat/vaults
Tomáš Mládek 2022-01-28 18:17:14 +01:00
parent fc3956e770
commit 8c8d58847a
No known key found for this signature in database
GPG Key ID: ED21612889E75EC5
10 changed files with 163 additions and 114 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
migrations/upend/00_initial_structure/up.sql
View File

@ -20,7 +20,6 @@ CREATE TABLE files
);
CREATE INDEX files_hash ON files (hash);
CREATE INDEX files_path ON files (path);
CREATE INDEX files_valid ON files (valid);
CREATE TABLE data
@ -28,11 +27,12 @@ CREATE TABLE data
identity BLOB PRIMARY KEY NOT NULL,
entity BLOB NOT NULL,
attribute VARCHAR NOT NULL,
value VARCHAR NOT NULL,
immutable BOOLEAN NOT NULL,
UNIQUE (entity, attribute, value)
value_str VARCHAR,
value_num NUMERIC,
immutable BOOLEAN NOT NULL
);
CREATE INDEX data_entity ON data (entity);
CREATE INDEX data_attribute ON data (attribute);
CREATE INDEX data_value ON data (value);
CREATE INDEX data_value_str ON data (value_str);
CREATE INDEX data_value_num ON data (value_num);

4
src/database/constants.rs
View File

@ -16,12 +16,12 @@ pub const LABEL_ATTR: &str = "LBL";
lazy_static! {
pub static ref TYPE_INVARIANT: InvariantEntry = InvariantEntry {
attribute: String::from(TYPE_BASE_ATTR),
value: EntryValue::Value(serde_json::Value::from(TYPE_TYPE_VAL)),
value: EntryValue::String(String::from(TYPE_TYPE_VAL)),
};
pub static ref TYPE_ADDR: Address = TYPE_INVARIANT.entity().unwrap();
pub static ref HIER_INVARIANT: InvariantEntry = InvariantEntry {
attribute: String::from(TYPE_BASE_ATTR),
value: EntryValue::Value(serde_json::Value::from(HIER_TYPE_VAL)),
value: EntryValue::String(String::from(HIER_TYPE_VAL)),
};
pub static ref HIER_ADDR: Address = HIER_INVARIANT.entity().unwrap();
}

95
src/database/entry.rs
View File

@ -32,7 +32,8 @@ pub struct InvariantEntry {
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
#[serde(tag = "t", content = "c")]
pub enum EntryValue {
Value(serde_json::Value),
String(String),
Number(f64),
Address(Address),
Invalid,
}
@ -41,11 +42,23 @@ impl TryFrom<&models::Entry> for Entry {
type Error = anyhow::Error;
fn try_from(e: &models::Entry) -> Result<Self, Self::Error> {
if let Some(value_str) = &e.value_str {
Ok(Entry {
entity: Address::decode(&e.entity)?,
attribute: e.attribute.clone(),
value: e.value.parse().unwrap(),
value: value_str.parse()?,
})
} else if let Some(value_num) = e.value_num {
Ok(Entry {
entity: Address::decode(&e.entity)?,
attribute: e.attribute.clone(),
value: EntryValue::Number(value_num),
})
} else {
Err(anyhow!(
"Inconsistent database: Both values of entry are NULL!"
))
}
}
}
@ -53,13 +66,24 @@ impl TryFrom<&Entry> for models::Entry {
type Error = anyhow::Error;
fn try_from(e: &Entry) -> Result<Self, Self::Error> {
Ok(models::Entry {
match e.value {
EntryValue::Number(n) => Ok(models::Entry {
identity: e.address()?.encode()?,
entity: e.entity.encode()?,
attribute: e.attribute.clone(),
value: e.value.to_string()?,
value_str: None,
value_num: Some(n),
immutable: false,
})
}),
_ => Ok(models::Entry {
identity: e.address()?.encode()?,
entity: e.entity.encode()?,
attribute: e.attribute.clone(),
value_str: Some(e.value.to_string()?),
value_num: None,
immutable: false,
}),
}
}
}
@ -67,13 +91,24 @@ impl TryFrom<&ImmutableEntry> for models::Entry {
type Error = anyhow::Error;
fn try_from(e: &ImmutableEntry) -> Result<Self, Self::Error> {
Ok(models::Entry {
match e.0.value {
EntryValue::Number(n) => Ok(models::Entry {
identity: e.0.address()?.encode()?,
entity: e.0.entity.encode()?,
attribute: e.0.attribute.clone(),
value: e.0.value.to_string()?,
immutable: true,
})
value_str: None,
value_num: Some(n),
immutable: false,
}),
_ => Ok(models::Entry {
identity: e.0.address()?.encode()?,
entity: e.0.entity.encode()?,
attribute: e.0.attribute.clone(),
value_str: Some(e.0.value.to_string()?),
value_num: None,
immutable: false,
}),
}
}
}
@ -144,7 +179,8 @@ impl Addressable for InvariantEntry {}
impl EntryValue {
pub fn to_string(&self) -> Result<String> {
let (type_char, content) = match self {
EntryValue::Value(value) => ('J', serde_json::to_string(value)?),
EntryValue::String(value) => ('S', value.to_owned()),
EntryValue::Number(n) => ('N', n.to_string()),
EntryValue::Address(address) => ('O', address.to_string()),
EntryValue::Invalid => return Err(anyhow!("Cannot serialize invalid Entity value.")),
};
@ -157,10 +193,8 @@ impl std::fmt::Display for EntryValue {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
let (entry_type, entry_value) = match self {
EntryValue::Address(address) => ("ADDRESS", address.to_string()),
EntryValue::Value(value) => (
"VALUE",
serde_json::to_string(value).unwrap_or_else(|_| String::from("?!?!?!")),
),
EntryValue::String(string) => ("STRING", string.to_owned()),
EntryValue::Number(n) => ("NUMBER", n.to_string()),
EntryValue::Invalid => ("INVALID", "INVALID".to_string()),
};
write!(f, "{}: {}", entry_type, entry_value)
@ -176,9 +210,10 @@ impl std::str::FromStr for EntryValue {
} else {
let (type_char, content) = s.split_at(1);
match (type_char, content) {
("J", content) => {
if let Ok(value) = serde_json::from_str(content) {
Ok(EntryValue::Value(value))
("S", content) => Ok(EntryValue::String(String::from(content))),
("N", content) => {
if let Ok(n) = content.parse::<f64>() {
Ok(EntryValue::Number(n))
} else {
Ok(EntryValue::Invalid)
}
@ -195,3 +230,29 @@ impl std::str::FromStr for EntryValue {
}
}
}
#[cfg(test)]
mod tests {
use super::*;
use anyhow::Result;
#[test]
fn test_value_from_to_string() -> Result<()> {
let entry = EntryValue::String("hello".to_string());
let encoded = entry.to_string()?;
let decoded = encoded.parse::<EntryValue>()?;
assert_eq!(entry, decoded);
let entry = EntryValue::Number(1337.93);
let encoded = entry.to_string()?;
let decoded = encoded.parse::<EntryValue>()?;
assert_eq!(entry, decoded);
let entry = EntryValue::Address(Address::Url("https://upendproject.net".to_string()));
let encoded = entry.to_string()?;
let decoded = encoded.parse::<EntryValue>()?;
assert_eq!(entry, decoded);
Ok(())
}
}

67
src/database/hierarchies.rs
View File

@ -4,7 +4,6 @@ use std::sync::{Arc, Mutex};
use anyhow::{anyhow, Result};
use log::trace;
use lru::LruCache;
use serde_json::Value;
use uuid::Uuid;
use crate::addressing::{Address, Addressable};
@ -95,22 +94,20 @@ impl PointerEntries for Vec<Entry> {
}
pub fn list_roots(connection: &UpEndConnection) -> Result<Vec<Address>> {
let all_directories: Vec<Entry> = connection.query(
Query::SingleQuery(QueryPart::Matches(EntryQuery {
let all_directories: Vec<Entry> =
connection.query(Query::SingleQuery(QueryPart::Matches(EntryQuery {
entity: QueryComponent::Any,
attribute: QueryComponent::Exact(IS_OF_TYPE_ATTR.to_string()),
value: QueryComponent::Exact(EntryValue::Address(HIER_ADDR.clone())),
})),
)?;
})))?;
// TODO: this is horrible
let directories_with_parents: Vec<Address> = connection.query(
Query::SingleQuery(QueryPart::Matches(EntryQuery {
let directories_with_parents: Vec<Address> = connection
.query(Query::SingleQuery(QueryPart::Matches(EntryQuery {
entity: QueryComponent::Any,
attribute: QueryComponent::Exact(HIER_HAS_ATTR.to_string()),
value: QueryComponent::Any,
})),
)?
})))?
.extract_pointers()
.into_iter()
.map(|(_, val)| val)
@ -143,26 +140,22 @@ pub fn fetch_or_create_dir(
_lock = FETCH_CREATE_LOCK.lock().unwrap();
}
let matching_directories = connection.query(
Query::SingleQuery(QueryPart::Matches(EntryQuery {
let matching_directories = connection
.query(Query::SingleQuery(QueryPart::Matches(EntryQuery {
entity: QueryComponent::Any,
attribute: QueryComponent::Exact(String::from(LABEL_ATTR)),
value: QueryComponent::Exact(EntryValue::Value(Value::String(
directory.as_ref().clone(),
))),
})),
)?
value: QueryComponent::Exact(EntryValue::String(directory.as_ref().clone())),
})))?
.into_iter()
.map(|e: Entry| e.entity);
let parent_has: Vec<Address> = match parent.clone() {
Some(parent) => connection.query(
Query::SingleQuery(QueryPart::Matches(EntryQuery {
Some(parent) => connection
.query(Query::SingleQuery(QueryPart::Matches(EntryQuery {
entity: QueryComponent::Exact(parent),
attribute: QueryComponent::Exact(String::from(HIER_HAS_ATTR)),
value: QueryComponent::Any,
})),
)?
})))?
.extract_pointers()
.into_iter()
.map(|(_, val)| val)
@ -188,7 +181,7 @@ pub fn fetch_or_create_dir(
let directory_entry = Entry {
entity: new_directory_address.clone(),
attribute: String::from(LABEL_ATTR),
value: EntryValue::Value(Value::String(directory.as_ref().clone())),
value: EntryValue::String(directory.as_ref().clone()),
};
connection.insert_entry(directory_entry)?;
@ -330,21 +323,11 @@ mod tests {
let open_result = UpEndDatabase::open(&temp_dir, None, true).unwrap();
let connection = open_result.db.connection().unwrap();
let foo_result = fetch_or_create_dir(
&connection,
None,
UNode("foo".to_string()),
true,
);
let foo_result = fetch_or_create_dir(&connection, None, UNode("foo".to_string()), true);
assert!(foo_result.is_ok());
let foo_result = foo_result.unwrap();
let bar_result = fetch_or_create_dir(
&connection,
None,
UNode("bar".to_string()),
true,
);
let bar_result = fetch_or_create_dir(&connection, None, UNode("bar".to_string()), true);
assert!(bar_result.is_ok());
let bar_result = bar_result.unwrap();
@ -360,11 +343,7 @@ mod tests {
let roots = list_roots(&connection);
assert_eq!(roots.unwrap(), [foo_result, bar_result.clone()]);
let resolve_result = resolve_path(
&connection,
&"bar/baz".parse().unwrap(),
false,
);
let resolve_result = resolve_path(&connection, &"bar/baz".parse().unwrap(), false);
assert!(resolve_result.is_ok());
assert_eq!(
@ -372,18 +351,10 @@ mod tests {
vec![bar_result.clone(), baz_result.clone()]
);
let resolve_result = resolve_path(
&connection,
&"bar/baz/bax".parse().unwrap(),
false,
);
let resolve_result = resolve_path(&connection, &"bar/baz/bax".parse().unwrap(), false);
assert!(resolve_result.is_err());
let resolve_result = resolve_path(
&connection,
&"bar/baz/bax".parse().unwrap(),
true,
);
let resolve_result = resolve_path(&connection, &"bar/baz/bax".parse().unwrap(), true);
assert!(resolve_result.is_ok());
let bax_result = fetch_or_create_dir(

3
src/database/inner/models.rs
View File

@ -45,6 +45,7 @@ pub struct Entry {
pub identity: Vec<u8>,
pub entity: Vec<u8>,
pub attribute: String,
pub value: String,
pub value_str: Option<String>,
pub value_num: Option<f64>,
pub immutable: bool,
}

3
src/database/inner/schema.rs
View File

@ -3,7 +3,8 @@ table! {
identity -> Binary,
entity -> Binary,
attribute -> Text,
value -> Text,
value_str -> Nullable<Text>,
value_num -> Nullable<Double>,
immutable -> Bool,
}
}

28
src/database/lang.rs
View File

@ -278,16 +278,32 @@ impl Query {
};
match &eq.value {
QueryComponent::Exact(q_value) => {
subqueries.push(Box::new(data::value.eq(q_value.to_string()?)))
QueryComponent::Exact(q_value) => match q_value {
EntryValue::Number(n) => {
subqueries.push(Box::new(data::value_num.eq(*n)))
}
_ => subqueries
.push(Box::new(data::value_str.eq(q_value.to_string()?))),
},
QueryComponent::In(q_values) => {
let values: Result<Vec<_>, _> =
q_values.iter().map(|v| v.to_string()).collect();
subqueries.push(Box::new(data::value.eq_any(values?)))
let first = q_values.first().ok_or(anyhow!(
"Malformed expression: Inner value cannot be empty."
))?;
match first {
EntryValue::Number(_) => todo!(),
_ => subqueries.push(Box::new(
data::value_str.eq_any(
q_values
.iter()
.map(|v| v.to_string())
.collect::<Result<Vec<String>>>()?,
),
)),
}
}
QueryComponent::Contains(q_value) => subqueries
.push(Box::new(data::value.like(format!("J%{}%", q_value)))),
.push(Box::new(data::value_str.like(format!("J%{}%", q_value)))),
QueryComponent::Any => {}
};

2
src/database/macros.rs
View File

@ -4,7 +4,7 @@ macro_rules! upend_insert_val {
Entry {
entity: $entity.clone(),
attribute: String::from($attribute),
value: crate::database::entry::EntryValue::Value(serde_json::Value::from($value)),
value: crate::database::entry::EntryValue::String(String::from($value)),
},
)
}};

4
src/database/mod.rs
View File

@ -277,7 +277,7 @@ impl UpEndConnection {
let primary = data
.filter(entity.eq(object_address.encode()?))
.or_filter(value.eq(EntryValue::Address(object_address).to_string()?))
.or_filter(value_str.eq(EntryValue::Address(object_address).to_string()?))
.load::<models::Entry>(&self.conn)?;
let entries = primary
@ -314,7 +314,7 @@ impl UpEndConnection {
let matches = data
.filter(identity.eq(object_address.encode()?))
.or_filter(entity.eq(object_address.encode()?))
.or_filter(value.eq(EntryValue::Address(object_address).to_string()?));
.or_filter(value_str.eq(EntryValue::Address(object_address).to_string()?));
Ok(diesel::delete(matches).execute(&self.conn)?)
}

11
src/filesystem.rs
View File

@ -22,7 +22,6 @@ use chrono::prelude::*;
use log::{debug, error, info, warn};
use lru::LruCache;
use rayon::prelude::*;
use serde_json::Value;
use walkdir::WalkDir;
const BLOB_TYPE: &str = "BLOB";
@ -34,7 +33,7 @@ const FILE_SIZE_KEY: &str = "FILE_SIZE";
lazy_static! {
static ref BLOB_TYPE_INVARIANT: InvariantEntry = InvariantEntry {
attribute: String::from(TYPE_BASE_ATTR),
value: EntryValue::Value(Value::from(BLOB_TYPE)),
value: EntryValue::String(String::from(BLOB_TYPE)),
};
static ref BLOB_TYPE_ADDR: Address = BLOB_TYPE_INVARIANT.entity().unwrap();
}
@ -391,13 +390,13 @@ fn insert_file_with_metadata(
let size_entry = Entry {
entity: Address::Hash(hash.clone()),
attribute: FILE_SIZE_KEY.to_string(),
value: EntryValue::Value(Value::from(size)),
value: EntryValue::Number(size as f64),
};
let mime_entry = mime_type.map(|mime_type| Entry {
entity: Address::Hash(hash.clone()),
attribute: FILE_MIME_KEY.to_string(),
value: EntryValue::Value(Value::String(mime_type)),
value: EntryValue::String(mime_type),
});
// Add the appropriate entries w/r/t virtual filesystem location
@ -437,9 +436,9 @@ fn insert_file_with_metadata(
let name_entry = Entry {
entity: dir_has_entry_addr,
attribute: ALIAS_KEY.to_string(),
value: EntryValue::Value(Value::String(
value: EntryValue::String(
filename.as_os_str().to_string_lossy().to_string(),
)),
),
};
connection.insert_entry(name_entry)?;