thm
/
upend
1
0
Fork 0
Code Issues 73 Pull Requests Packages Projects Releases 5 Wiki Activity

separate value_string and value_number columns 

to be able to utilize SQL queries better for ranges, comparisons, etc.
feat/vaults
Tomáš Mládek 2022-01-28 18:17:14 +01:00
parent fc3956e770
commit 8c8d58847a
No known key found for this signature in database
GPG Key ID: ED21612889E75EC5
10 changed files with 163 additions and 114 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
migrations/upend/00_initial_structure/up.sql
View File

@ -20,19 +20,19 @@ CREATE TABLE files
);
CREATE INDEX files_hash ON files (hash);
CREATE INDEX files_path ON files (path);
CREATE INDEX files_valid ON files (valid);
CREATE TABLE data
(
identity BLOB PRIMARY KEY NOT NULL,
entity BLOB NOT NULL,
attribute VARCHAR NOT NULL,
value VARCHAR NOT NULL,
immutable BOOLEAN NOT NULL,
UNIQUE (entity, attribute, value)
identity BLOB PRIMARY KEY NOT NULL,
entity BLOB NOT NULL,
attribute VARCHAR NOT NULL,
value_str VARCHAR,
value_num NUMERIC,
immutable BOOLEAN NOT NULL
);
CREATE INDEX data_entity ON data (entity);
CREATE INDEX data_attribute ON data (attribute);
CREATE INDEX data_value ON data (value);
CREATE INDEX data_value_str ON data (value_str);
CREATE INDEX data_value_num ON data (value_num);

4
src/database/constants.rs
View File

@ -16,12 +16,12 @@ pub const LABEL_ATTR: &str = "LBL";
lazy_static! {
pub static ref TYPE_INVARIANT: InvariantEntry = InvariantEntry {
attribute: String::from(TYPE_BASE_ATTR),
value: EntryValue::Value(serde_json::Value::from(TYPE_TYPE_VAL)),
value: EntryValue::String(String::from(TYPE_TYPE_VAL)),
};
pub static ref TYPE_ADDR: Address = TYPE_INVARIANT.entity().unwrap();
pub static ref HIER_INVARIANT: InvariantEntry = InvariantEntry {
attribute: String::from(TYPE_BASE_ATTR),
value: EntryValue::Value(serde_json::Value::from(HIER_TYPE_VAL)),
value: EntryValue::String(String::from(HIER_TYPE_VAL)),
};
pub static ref HIER_ADDR: Address = HIER_INVARIANT.entity().unwrap();
}

117
src/database/entry.rs
View File

@ -32,7 +32,8 @@ pub struct InvariantEntry {
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
#[serde(tag = "t", content = "c")]
pub enum EntryValue {
Value(serde_json::Value),
String(String),
Number(f64),
Address(Address),
Invalid,
}
@ -41,11 +42,23 @@ impl TryFrom<&models::Entry> for Entry {
type Error = anyhow::Error;
fn try_from(e: &models::Entry) -> Result<Self, Self::Error> {
Ok(Entry {
entity: Address::decode(&e.entity)?,
attribute: e.attribute.clone(),
value: e.value.parse().unwrap(),
})
if let Some(value_str) = &e.value_str {
Ok(Entry {
entity: Address::decode(&e.entity)?,
attribute: e.attribute.clone(),
value: value_str.parse()?,
})
} else if let Some(value_num) = e.value_num {
Ok(Entry {
entity: Address::decode(&e.entity)?,
attribute: e.attribute.clone(),
value: EntryValue::Number(value_num),
})
} else {
Err(anyhow!(
"Inconsistent database: Both values of entry are NULL!"
))
}
}
}
@ -53,13 +66,24 @@ impl TryFrom<&Entry> for models::Entry {
type Error = anyhow::Error;
fn try_from(e: &Entry) -> Result<Self, Self::Error> {
Ok(models::Entry {
identity: e.address()?.encode()?,
entity: e.entity.encode()?,
attribute: e.attribute.clone(),
value: e.value.to_string()?,
immutable: false,
})
match e.value {
EntryValue::Number(n) => Ok(models::Entry {
identity: e.address()?.encode()?,
entity: e.entity.encode()?,
attribute: e.attribute.clone(),
value_str: None,
value_num: Some(n),
immutable: false,
}),
_ => Ok(models::Entry {
identity: e.address()?.encode()?,
entity: e.entity.encode()?,
attribute: e.attribute.clone(),
value_str: Some(e.value.to_string()?),
value_num: None,
immutable: false,
}),
}
}
}
@ -67,13 +91,24 @@ impl TryFrom<&ImmutableEntry> for models::Entry {
type Error = anyhow::Error;
fn try_from(e: &ImmutableEntry) -> Result<Self, Self::Error> {
Ok(models::Entry {
identity: e.0.address()?.encode()?,
entity: e.0.entity.encode()?,
attribute: e.0.attribute.clone(),
value: e.0.value.to_string()?,
immutable: true,
})
match e.0.value {
EntryValue::Number(n) => Ok(models::Entry {
identity: e.0.address()?.encode()?,
entity: e.0.entity.encode()?,
attribute: e.0.attribute.clone(),
value_str: None,
value_num: Some(n),
immutable: false,
}),
_ => Ok(models::Entry {
identity: e.0.address()?.encode()?,
entity: e.0.entity.encode()?,
attribute: e.0.attribute.clone(),
value_str: Some(e.0.value.to_string()?),
value_num: None,
immutable: false,
}),
}
}
}
@ -144,7 +179,8 @@ impl Addressable for InvariantEntry {}
impl EntryValue {
pub fn to_string(&self) -> Result<String> {
let (type_char, content) = match self {
EntryValue::Value(value) => ('J', serde_json::to_string(value)?),
EntryValue::String(value) => ('S', value.to_owned()),
EntryValue::Number(n) => ('N', n.to_string()),
EntryValue::Address(address) => ('O', address.to_string()),
EntryValue::Invalid => return Err(anyhow!("Cannot serialize invalid Entity value.")),
};
@ -157,10 +193,8 @@ impl std::fmt::Display for EntryValue {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
let (entry_type, entry_value) = match self {
EntryValue::Address(address) => ("ADDRESS", address.to_string()),
EntryValue::Value(value) => (
"VALUE",
serde_json::to_string(value).unwrap_or_else(|_| String::from("?!?!?!")),
),
EntryValue::String(string) => ("STRING", string.to_owned()),
EntryValue::Number(n) => ("NUMBER", n.to_string()),
EntryValue::Invalid => ("INVALID", "INVALID".to_string()),
};
write!(f, "{}: {}", entry_type, entry_value)
@ -176,9 +210,10 @@ impl std::str::FromStr for EntryValue {
} else {
let (type_char, content) = s.split_at(1);
match (type_char, content) {
("J", content) => {
if let Ok(value) = serde_json::from_str(content) {
Ok(EntryValue::Value(value))
("S", content) => Ok(EntryValue::String(String::from(content))),
("N", content) => {
if let Ok(n) = content.parse::<f64>() {
Ok(EntryValue::Number(n))
} else {
Ok(EntryValue::Invalid)
}
@ -195,3 +230,29 @@ impl std::str::FromStr for EntryValue {
}
}
}
#[cfg(test)]
mod tests {
use super::*;
use anyhow::Result;
#[test]
fn test_value_from_to_string() -> Result<()> {
let entry = EntryValue::String("hello".to_string());
let encoded = entry.to_string()?;
let decoded = encoded.parse::<EntryValue>()?;
assert_eq!(entry, decoded);
let entry = EntryValue::Number(1337.93);
let encoded = entry.to_string()?;
let decoded = encoded.parse::<EntryValue>()?;
assert_eq!(entry, decoded);
let entry = EntryValue::Address(Address::Url("https://upendproject.net".to_string()));
let encoded = entry.to_string()?;
let decoded = encoded.parse::<EntryValue>()?;
assert_eq!(entry, decoded);
Ok(())
}
}

87
src/database/hierarchies.rs
View File

@ -4,7 +4,6 @@ use std::sync::{Arc, Mutex};
use anyhow::{anyhow, Result};
use log::trace;
use lru::LruCache;
use serde_json::Value;
use uuid::Uuid;
use crate::addressing::{Address, Addressable};
@ -95,26 +94,24 @@ impl PointerEntries for Vec<Entry> {
}
pub fn list_roots(connection: &UpEndConnection) -> Result<Vec<Address>> {
let all_directories: Vec<Entry> = connection.query(
Query::SingleQuery(QueryPart::Matches(EntryQuery {
let all_directories: Vec<Entry> =
connection.query(Query::SingleQuery(QueryPart::Matches(EntryQuery {
entity: QueryComponent::Any,
attribute: QueryComponent::Exact(IS_OF_TYPE_ATTR.to_string()),
value: QueryComponent::Exact(EntryValue::Address(HIER_ADDR.clone())),
})),
)?;
})))?;
// TODO: this is horrible
let directories_with_parents: Vec<Address> = connection.query(
Query::SingleQuery(QueryPart::Matches(EntryQuery {
let directories_with_parents: Vec<Address> = connection
.query(Query::SingleQuery(QueryPart::Matches(EntryQuery {
entity: QueryComponent::Any,
attribute: QueryComponent::Exact(HIER_HAS_ATTR.to_string()),
value: QueryComponent::Any,
})),
)?
.extract_pointers()
.into_iter()
.map(|(_, val)| val)
.collect();
})))?
.extract_pointers()
.into_iter()
.map(|(_, val)| val)
.collect();
Ok(all_directories
.into_iter()
@ -143,30 +140,26 @@ pub fn fetch_or_create_dir(
_lock = FETCH_CREATE_LOCK.lock().unwrap();
}
let matching_directories = connection.query(
Query::SingleQuery(QueryPart::Matches(EntryQuery {
let matching_directories = connection
.query(Query::SingleQuery(QueryPart::Matches(EntryQuery {
entity: QueryComponent::Any,
attribute: QueryComponent::Exact(String::from(LABEL_ATTR)),
value: QueryComponent::Exact(EntryValue::Value(Value::String(
directory.as_ref().clone(),
))),
})),
)?
.into_iter()
.map(|e: Entry| e.entity);
value: QueryComponent::Exact(EntryValue::String(directory.as_ref().clone())),
})))?
.into_iter()
.map(|e: Entry| e.entity);
let parent_has: Vec<Address> = match parent.clone() {
Some(parent) => connection.query(
Query::SingleQuery(QueryPart::Matches(EntryQuery {
Some(parent) => connection
.query(Query::SingleQuery(QueryPart::Matches(EntryQuery {
entity: QueryComponent::Exact(parent),
attribute: QueryComponent::Exact(String::from(HIER_HAS_ATTR)),
value: QueryComponent::Any,
})),
)?
.extract_pointers()
.into_iter()
.map(|(_, val)| val)
.collect(),
})))?
.extract_pointers()
.into_iter()
.map(|(_, val)| val)
.collect(),
None => list_roots(connection)?,
};
@ -188,7 +181,7 @@ pub fn fetch_or_create_dir(
let directory_entry = Entry {
entity: new_directory_address.clone(),
attribute: String::from(LABEL_ATTR),
value: EntryValue::Value(Value::String(directory.as_ref().clone())),
value: EntryValue::String(directory.as_ref().clone()),
};
connection.insert_entry(directory_entry)?;
@ -330,21 +323,11 @@ mod tests {
let open_result = UpEndDatabase::open(&temp_dir, None, true).unwrap();
let connection = open_result.db.connection().unwrap();
let foo_result = fetch_or_create_dir(
&connection,
None,
UNode("foo".to_string()),
true,
);
let foo_result = fetch_or_create_dir(&connection, None, UNode("foo".to_string()), true);
assert!(foo_result.is_ok());
let foo_result = foo_result.unwrap();
let bar_result = fetch_or_create_dir(
&connection,
None,
UNode("bar".to_string()),
true,
);
let bar_result = fetch_or_create_dir(&connection, None, UNode("bar".to_string()), true);
assert!(bar_result.is_ok());
let bar_result = bar_result.unwrap();
@ -360,11 +343,7 @@ mod tests {
let roots = list_roots(&connection);
assert_eq!(roots.unwrap(), [foo_result, bar_result.clone()]);
let resolve_result = resolve_path(
&connection,
&"bar/baz".parse().unwrap(),
false,
);
let resolve_result = resolve_path(&connection, &"bar/baz".parse().unwrap(), false);
assert!(resolve_result.is_ok());
assert_eq!(
@ -372,18 +351,10 @@ mod tests {
vec![bar_result.clone(), baz_result.clone()]
);
let resolve_result = resolve_path(
&connection,
&"bar/baz/bax".parse().unwrap(),
false,
);
let resolve_result = resolve_path(&connection, &"bar/baz/bax".parse().unwrap(), false);
assert!(resolve_result.is_err());
let resolve_result = resolve_path(
&connection,
&"bar/baz/bax".parse().unwrap(),
true,
);
let resolve_result = resolve_path(&connection, &"bar/baz/bax".parse().unwrap(), true);
assert!(resolve_result.is_ok());
let bax_result = fetch_or_create_dir(

3
src/database/inner/models.rs
View File

@ -45,6 +45,7 @@ pub struct Entry {
pub identity: Vec<u8>,
pub entity: Vec<u8>,
pub attribute: String,
pub value: String,
pub value_str: Option<String>,
pub value_num: Option<f64>,
pub immutable: bool,
}

3
src/database/inner/schema.rs
View File

@ -3,7 +3,8 @@ table! {
identity -> Binary,
entity -> Binary,
attribute -> Text,
value -> Text,
value_str -> Nullable<Text>,
value_num -> Nullable<Double>,
immutable -> Bool,
}
}

30
src/database/lang.rs
View File

@ -278,16 +278,32 @@ impl Query {
};
match &eq.value {
QueryComponent::Exact(q_value) => {
subqueries.push(Box::new(data::value.eq(q_value.to_string()?)))
}
QueryComponent::Exact(q_value) => match q_value {
EntryValue::Number(n) => {
subqueries.push(Box::new(data::value_num.eq(*n)))
}
_ => subqueries
.push(Box::new(data::value_str.eq(q_value.to_string()?))),
},
QueryComponent::In(q_values) => {
let values: Result<Vec<_>, _> =
q_values.iter().map(|v| v.to_string()).collect();
subqueries.push(Box::new(data::value.eq_any(values?)))
let first = q_values.first().ok_or(anyhow!(
"Malformed expression: Inner value cannot be empty."
))?;
match first {
EntryValue::Number(_) => todo!(),
_ => subqueries.push(Box::new(
data::value_str.eq_any(
q_values
.iter()
.map(|v| v.to_string())
.collect::<Result<Vec<String>>>()?,
),
)),
}
}
QueryComponent::Contains(q_value) => subqueries
.push(Box::new(data::value.like(format!("J%{}%", q_value)))),
.push(Box::new(data::value_str.like(format!("J%{}%", q_value)))),
QueryComponent::Any => {}
};

2
src/database/macros.rs
View File

@ -4,7 +4,7 @@ macro_rules! upend_insert_val {
Entry {
entity: $entity.clone(),
attribute: String::from($attribute),
value: crate::database::entry::EntryValue::Value(serde_json::Value::from($value)),
value: crate::database::entry::EntryValue::String(String::from($value)),
},
)
}};

4
src/database/mod.rs
View File

@ -277,7 +277,7 @@ impl UpEndConnection {
let primary = data
.filter(entity.eq(object_address.encode()?))
.or_filter(value.eq(EntryValue::Address(object_address).to_string()?))
.or_filter(value_str.eq(EntryValue::Address(object_address).to_string()?))
.load::<models::Entry>(&self.conn)?;
let entries = primary
@ -314,7 +314,7 @@ impl UpEndConnection {
let matches = data
.filter(identity.eq(object_address.encode()?))
.or_filter(entity.eq(object_address.encode()?))
.or_filter(value.eq(EntryValue::Address(object_address).to_string()?));
.or_filter(value_str.eq(EntryValue::Address(object_address).to_string()?));
Ok(diesel::delete(matches).execute(&self.conn)?)
}

11
src/filesystem.rs
View File

@ -22,7 +22,6 @@ use chrono::prelude::*;
use log::{debug, error, info, warn};
use lru::LruCache;
use rayon::prelude::*;
use serde_json::Value;
use walkdir::WalkDir;
const BLOB_TYPE: &str = "BLOB";
@ -34,7 +33,7 @@ const FILE_SIZE_KEY: &str = "FILE_SIZE";
lazy_static! {
static ref BLOB_TYPE_INVARIANT: InvariantEntry = InvariantEntry {
attribute: String::from(TYPE_BASE_ATTR),
value: EntryValue::Value(Value::from(BLOB_TYPE)),
value: EntryValue::String(String::from(BLOB_TYPE)),
};
static ref BLOB_TYPE_ADDR: Address = BLOB_TYPE_INVARIANT.entity().unwrap();
}
@ -391,13 +390,13 @@ fn insert_file_with_metadata(
let size_entry = Entry {
entity: Address::Hash(hash.clone()),
attribute: FILE_SIZE_KEY.to_string(),
value: EntryValue::Value(Value::from(size)),
value: EntryValue::Number(size as f64),
};
let mime_entry = mime_type.map(|mime_type| Entry {
entity: Address::Hash(hash.clone()),
attribute: FILE_MIME_KEY.to_string(),
value: EntryValue::Value(Value::String(mime_type)),
value: EntryValue::String(mime_type),
});
// Add the appropriate entries w/r/t virtual filesystem location
@ -437,9 +436,9 @@ fn insert_file_with_metadata(
let name_entry = Entry {
entity: dir_has_entry_addr,
attribute: ALIAS_KEY.to_string(),
value: EntryValue::Value(Value::String(
value: EntryValue::String(
filename.as_os_str().to_string_lossy().to_string(),
)),
),
};
connection.insert_entry(name_entry)?;