separate value_string and value_number columns
to be able to utilize SQL queries better for ranges, comparisons, etc.
This commit is contained in:
parent
fc3956e770
commit
8c8d58847a
10 changed files with 163 additions and 114 deletions
|
@ -20,7 +20,6 @@ CREATE TABLE files
|
|||
);
|
||||
|
||||
CREATE INDEX files_hash ON files (hash);
|
||||
CREATE INDEX files_path ON files (path);
|
||||
CREATE INDEX files_valid ON files (valid);
|
||||
|
||||
CREATE TABLE data
|
||||
|
@ -28,11 +27,12 @@ CREATE TABLE data
|
|||
identity BLOB PRIMARY KEY NOT NULL,
|
||||
entity BLOB NOT NULL,
|
||||
attribute VARCHAR NOT NULL,
|
||||
value VARCHAR NOT NULL,
|
||||
immutable BOOLEAN NOT NULL,
|
||||
UNIQUE (entity, attribute, value)
|
||||
value_str VARCHAR,
|
||||
value_num NUMERIC,
|
||||
immutable BOOLEAN NOT NULL
|
||||
);
|
||||
|
||||
CREATE INDEX data_entity ON data (entity);
|
||||
CREATE INDEX data_attribute ON data (attribute);
|
||||
CREATE INDEX data_value ON data (value);
|
||||
CREATE INDEX data_value_str ON data (value_str);
|
||||
CREATE INDEX data_value_num ON data (value_num);
|
||||
|
|
|
@ -16,12 +16,12 @@ pub const LABEL_ATTR: &str = "LBL";
|
|||
lazy_static! {
|
||||
pub static ref TYPE_INVARIANT: InvariantEntry = InvariantEntry {
|
||||
attribute: String::from(TYPE_BASE_ATTR),
|
||||
value: EntryValue::Value(serde_json::Value::from(TYPE_TYPE_VAL)),
|
||||
value: EntryValue::String(String::from(TYPE_TYPE_VAL)),
|
||||
};
|
||||
pub static ref TYPE_ADDR: Address = TYPE_INVARIANT.entity().unwrap();
|
||||
pub static ref HIER_INVARIANT: InvariantEntry = InvariantEntry {
|
||||
attribute: String::from(TYPE_BASE_ATTR),
|
||||
value: EntryValue::Value(serde_json::Value::from(HIER_TYPE_VAL)),
|
||||
value: EntryValue::String(String::from(HIER_TYPE_VAL)),
|
||||
};
|
||||
pub static ref HIER_ADDR: Address = HIER_INVARIANT.entity().unwrap();
|
||||
}
|
||||
|
|
|
@ -32,7 +32,8 @@ pub struct InvariantEntry {
|
|||
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
|
||||
#[serde(tag = "t", content = "c")]
|
||||
pub enum EntryValue {
|
||||
Value(serde_json::Value),
|
||||
String(String),
|
||||
Number(f64),
|
||||
Address(Address),
|
||||
Invalid,
|
||||
}
|
||||
|
@ -41,11 +42,23 @@ impl TryFrom<&models::Entry> for Entry {
|
|||
type Error = anyhow::Error;
|
||||
|
||||
fn try_from(e: &models::Entry) -> Result<Self, Self::Error> {
|
||||
if let Some(value_str) = &e.value_str {
|
||||
Ok(Entry {
|
||||
entity: Address::decode(&e.entity)?,
|
||||
attribute: e.attribute.clone(),
|
||||
value: e.value.parse().unwrap(),
|
||||
value: value_str.parse()?,
|
||||
})
|
||||
} else if let Some(value_num) = e.value_num {
|
||||
Ok(Entry {
|
||||
entity: Address::decode(&e.entity)?,
|
||||
attribute: e.attribute.clone(),
|
||||
value: EntryValue::Number(value_num),
|
||||
})
|
||||
} else {
|
||||
Err(anyhow!(
|
||||
"Inconsistent database: Both values of entry are NULL!"
|
||||
))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -53,13 +66,24 @@ impl TryFrom<&Entry> for models::Entry {
|
|||
type Error = anyhow::Error;
|
||||
|
||||
fn try_from(e: &Entry) -> Result<Self, Self::Error> {
|
||||
Ok(models::Entry {
|
||||
match e.value {
|
||||
EntryValue::Number(n) => Ok(models::Entry {
|
||||
identity: e.address()?.encode()?,
|
||||
entity: e.entity.encode()?,
|
||||
attribute: e.attribute.clone(),
|
||||
value: e.value.to_string()?,
|
||||
value_str: None,
|
||||
value_num: Some(n),
|
||||
immutable: false,
|
||||
})
|
||||
}),
|
||||
_ => Ok(models::Entry {
|
||||
identity: e.address()?.encode()?,
|
||||
entity: e.entity.encode()?,
|
||||
attribute: e.attribute.clone(),
|
||||
value_str: Some(e.value.to_string()?),
|
||||
value_num: None,
|
||||
immutable: false,
|
||||
}),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -67,13 +91,24 @@ impl TryFrom<&ImmutableEntry> for models::Entry {
|
|||
type Error = anyhow::Error;
|
||||
|
||||
fn try_from(e: &ImmutableEntry) -> Result<Self, Self::Error> {
|
||||
Ok(models::Entry {
|
||||
match e.0.value {
|
||||
EntryValue::Number(n) => Ok(models::Entry {
|
||||
identity: e.0.address()?.encode()?,
|
||||
entity: e.0.entity.encode()?,
|
||||
attribute: e.0.attribute.clone(),
|
||||
value: e.0.value.to_string()?,
|
||||
immutable: true,
|
||||
})
|
||||
value_str: None,
|
||||
value_num: Some(n),
|
||||
immutable: false,
|
||||
}),
|
||||
_ => Ok(models::Entry {
|
||||
identity: e.0.address()?.encode()?,
|
||||
entity: e.0.entity.encode()?,
|
||||
attribute: e.0.attribute.clone(),
|
||||
value_str: Some(e.0.value.to_string()?),
|
||||
value_num: None,
|
||||
immutable: false,
|
||||
}),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -144,7 +179,8 @@ impl Addressable for InvariantEntry {}
|
|||
impl EntryValue {
|
||||
pub fn to_string(&self) -> Result<String> {
|
||||
let (type_char, content) = match self {
|
||||
EntryValue::Value(value) => ('J', serde_json::to_string(value)?),
|
||||
EntryValue::String(value) => ('S', value.to_owned()),
|
||||
EntryValue::Number(n) => ('N', n.to_string()),
|
||||
EntryValue::Address(address) => ('O', address.to_string()),
|
||||
EntryValue::Invalid => return Err(anyhow!("Cannot serialize invalid Entity value.")),
|
||||
};
|
||||
|
@ -157,10 +193,8 @@ impl std::fmt::Display for EntryValue {
|
|||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
let (entry_type, entry_value) = match self {
|
||||
EntryValue::Address(address) => ("ADDRESS", address.to_string()),
|
||||
EntryValue::Value(value) => (
|
||||
"VALUE",
|
||||
serde_json::to_string(value).unwrap_or_else(|_| String::from("?!?!?!")),
|
||||
),
|
||||
EntryValue::String(string) => ("STRING", string.to_owned()),
|
||||
EntryValue::Number(n) => ("NUMBER", n.to_string()),
|
||||
EntryValue::Invalid => ("INVALID", "INVALID".to_string()),
|
||||
};
|
||||
write!(f, "{}: {}", entry_type, entry_value)
|
||||
|
@ -176,9 +210,10 @@ impl std::str::FromStr for EntryValue {
|
|||
} else {
|
||||
let (type_char, content) = s.split_at(1);
|
||||
match (type_char, content) {
|
||||
("J", content) => {
|
||||
if let Ok(value) = serde_json::from_str(content) {
|
||||
Ok(EntryValue::Value(value))
|
||||
("S", content) => Ok(EntryValue::String(String::from(content))),
|
||||
("N", content) => {
|
||||
if let Ok(n) = content.parse::<f64>() {
|
||||
Ok(EntryValue::Number(n))
|
||||
} else {
|
||||
Ok(EntryValue::Invalid)
|
||||
}
|
||||
|
@ -195,3 +230,29 @@ impl std::str::FromStr for EntryValue {
|
|||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use anyhow::Result;
|
||||
|
||||
#[test]
|
||||
fn test_value_from_to_string() -> Result<()> {
|
||||
let entry = EntryValue::String("hello".to_string());
|
||||
let encoded = entry.to_string()?;
|
||||
let decoded = encoded.parse::<EntryValue>()?;
|
||||
assert_eq!(entry, decoded);
|
||||
|
||||
let entry = EntryValue::Number(1337.93);
|
||||
let encoded = entry.to_string()?;
|
||||
let decoded = encoded.parse::<EntryValue>()?;
|
||||
assert_eq!(entry, decoded);
|
||||
|
||||
let entry = EntryValue::Address(Address::Url("https://upendproject.net".to_string()));
|
||||
let encoded = entry.to_string()?;
|
||||
let decoded = encoded.parse::<EntryValue>()?;
|
||||
assert_eq!(entry, decoded);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
|
|
@ -4,7 +4,6 @@ use std::sync::{Arc, Mutex};
|
|||
use anyhow::{anyhow, Result};
|
||||
use log::trace;
|
||||
use lru::LruCache;
|
||||
use serde_json::Value;
|
||||
use uuid::Uuid;
|
||||
|
||||
use crate::addressing::{Address, Addressable};
|
||||
|
@ -95,22 +94,20 @@ impl PointerEntries for Vec<Entry> {
|
|||
}
|
||||
|
||||
pub fn list_roots(connection: &UpEndConnection) -> Result<Vec<Address>> {
|
||||
let all_directories: Vec<Entry> = connection.query(
|
||||
Query::SingleQuery(QueryPart::Matches(EntryQuery {
|
||||
let all_directories: Vec<Entry> =
|
||||
connection.query(Query::SingleQuery(QueryPart::Matches(EntryQuery {
|
||||
entity: QueryComponent::Any,
|
||||
attribute: QueryComponent::Exact(IS_OF_TYPE_ATTR.to_string()),
|
||||
value: QueryComponent::Exact(EntryValue::Address(HIER_ADDR.clone())),
|
||||
})),
|
||||
)?;
|
||||
})))?;
|
||||
|
||||
// TODO: this is horrible
|
||||
let directories_with_parents: Vec<Address> = connection.query(
|
||||
Query::SingleQuery(QueryPart::Matches(EntryQuery {
|
||||
let directories_with_parents: Vec<Address> = connection
|
||||
.query(Query::SingleQuery(QueryPart::Matches(EntryQuery {
|
||||
entity: QueryComponent::Any,
|
||||
attribute: QueryComponent::Exact(HIER_HAS_ATTR.to_string()),
|
||||
value: QueryComponent::Any,
|
||||
})),
|
||||
)?
|
||||
})))?
|
||||
.extract_pointers()
|
||||
.into_iter()
|
||||
.map(|(_, val)| val)
|
||||
|
@ -143,26 +140,22 @@ pub fn fetch_or_create_dir(
|
|||
_lock = FETCH_CREATE_LOCK.lock().unwrap();
|
||||
}
|
||||
|
||||
let matching_directories = connection.query(
|
||||
Query::SingleQuery(QueryPart::Matches(EntryQuery {
|
||||
let matching_directories = connection
|
||||
.query(Query::SingleQuery(QueryPart::Matches(EntryQuery {
|
||||
entity: QueryComponent::Any,
|
||||
attribute: QueryComponent::Exact(String::from(LABEL_ATTR)),
|
||||
value: QueryComponent::Exact(EntryValue::Value(Value::String(
|
||||
directory.as_ref().clone(),
|
||||
))),
|
||||
})),
|
||||
)?
|
||||
value: QueryComponent::Exact(EntryValue::String(directory.as_ref().clone())),
|
||||
})))?
|
||||
.into_iter()
|
||||
.map(|e: Entry| e.entity);
|
||||
|
||||
let parent_has: Vec<Address> = match parent.clone() {
|
||||
Some(parent) => connection.query(
|
||||
Query::SingleQuery(QueryPart::Matches(EntryQuery {
|
||||
Some(parent) => connection
|
||||
.query(Query::SingleQuery(QueryPart::Matches(EntryQuery {
|
||||
entity: QueryComponent::Exact(parent),
|
||||
attribute: QueryComponent::Exact(String::from(HIER_HAS_ATTR)),
|
||||
value: QueryComponent::Any,
|
||||
})),
|
||||
)?
|
||||
})))?
|
||||
.extract_pointers()
|
||||
.into_iter()
|
||||
.map(|(_, val)| val)
|
||||
|
@ -188,7 +181,7 @@ pub fn fetch_or_create_dir(
|
|||
let directory_entry = Entry {
|
||||
entity: new_directory_address.clone(),
|
||||
attribute: String::from(LABEL_ATTR),
|
||||
value: EntryValue::Value(Value::String(directory.as_ref().clone())),
|
||||
value: EntryValue::String(directory.as_ref().clone()),
|
||||
};
|
||||
connection.insert_entry(directory_entry)?;
|
||||
|
||||
|
@ -330,21 +323,11 @@ mod tests {
|
|||
let open_result = UpEndDatabase::open(&temp_dir, None, true).unwrap();
|
||||
let connection = open_result.db.connection().unwrap();
|
||||
|
||||
let foo_result = fetch_or_create_dir(
|
||||
&connection,
|
||||
None,
|
||||
UNode("foo".to_string()),
|
||||
true,
|
||||
);
|
||||
let foo_result = fetch_or_create_dir(&connection, None, UNode("foo".to_string()), true);
|
||||
assert!(foo_result.is_ok());
|
||||
let foo_result = foo_result.unwrap();
|
||||
|
||||
let bar_result = fetch_or_create_dir(
|
||||
&connection,
|
||||
None,
|
||||
UNode("bar".to_string()),
|
||||
true,
|
||||
);
|
||||
let bar_result = fetch_or_create_dir(&connection, None, UNode("bar".to_string()), true);
|
||||
assert!(bar_result.is_ok());
|
||||
let bar_result = bar_result.unwrap();
|
||||
|
||||
|
@ -360,11 +343,7 @@ mod tests {
|
|||
let roots = list_roots(&connection);
|
||||
assert_eq!(roots.unwrap(), [foo_result, bar_result.clone()]);
|
||||
|
||||
let resolve_result = resolve_path(
|
||||
&connection,
|
||||
&"bar/baz".parse().unwrap(),
|
||||
false,
|
||||
);
|
||||
let resolve_result = resolve_path(&connection, &"bar/baz".parse().unwrap(), false);
|
||||
|
||||
assert!(resolve_result.is_ok());
|
||||
assert_eq!(
|
||||
|
@ -372,18 +351,10 @@ mod tests {
|
|||
vec![bar_result.clone(), baz_result.clone()]
|
||||
);
|
||||
|
||||
let resolve_result = resolve_path(
|
||||
&connection,
|
||||
&"bar/baz/bax".parse().unwrap(),
|
||||
false,
|
||||
);
|
||||
let resolve_result = resolve_path(&connection, &"bar/baz/bax".parse().unwrap(), false);
|
||||
assert!(resolve_result.is_err());
|
||||
|
||||
let resolve_result = resolve_path(
|
||||
&connection,
|
||||
&"bar/baz/bax".parse().unwrap(),
|
||||
true,
|
||||
);
|
||||
let resolve_result = resolve_path(&connection, &"bar/baz/bax".parse().unwrap(), true);
|
||||
assert!(resolve_result.is_ok());
|
||||
|
||||
let bax_result = fetch_or_create_dir(
|
||||
|
|
|
@ -45,6 +45,7 @@ pub struct Entry {
|
|||
pub identity: Vec<u8>,
|
||||
pub entity: Vec<u8>,
|
||||
pub attribute: String,
|
||||
pub value: String,
|
||||
pub value_str: Option<String>,
|
||||
pub value_num: Option<f64>,
|
||||
pub immutable: bool,
|
||||
}
|
||||
|
|
|
@ -3,7 +3,8 @@ table! {
|
|||
identity -> Binary,
|
||||
entity -> Binary,
|
||||
attribute -> Text,
|
||||
value -> Text,
|
||||
value_str -> Nullable<Text>,
|
||||
value_num -> Nullable<Double>,
|
||||
immutable -> Bool,
|
||||
}
|
||||
}
|
||||
|
|
|
@ -278,16 +278,32 @@ impl Query {
|
|||
};
|
||||
|
||||
match &eq.value {
|
||||
QueryComponent::Exact(q_value) => {
|
||||
subqueries.push(Box::new(data::value.eq(q_value.to_string()?)))
|
||||
QueryComponent::Exact(q_value) => match q_value {
|
||||
EntryValue::Number(n) => {
|
||||
subqueries.push(Box::new(data::value_num.eq(*n)))
|
||||
}
|
||||
_ => subqueries
|
||||
.push(Box::new(data::value_str.eq(q_value.to_string()?))),
|
||||
},
|
||||
QueryComponent::In(q_values) => {
|
||||
let values: Result<Vec<_>, _> =
|
||||
q_values.iter().map(|v| v.to_string()).collect();
|
||||
subqueries.push(Box::new(data::value.eq_any(values?)))
|
||||
let first = q_values.first().ok_or(anyhow!(
|
||||
"Malformed expression: Inner value cannot be empty."
|
||||
))?;
|
||||
|
||||
match first {
|
||||
EntryValue::Number(_) => todo!(),
|
||||
_ => subqueries.push(Box::new(
|
||||
data::value_str.eq_any(
|
||||
q_values
|
||||
.iter()
|
||||
.map(|v| v.to_string())
|
||||
.collect::<Result<Vec<String>>>()?,
|
||||
),
|
||||
)),
|
||||
}
|
||||
}
|
||||
QueryComponent::Contains(q_value) => subqueries
|
||||
.push(Box::new(data::value.like(format!("J%{}%", q_value)))),
|
||||
.push(Box::new(data::value_str.like(format!("J%{}%", q_value)))),
|
||||
QueryComponent::Any => {}
|
||||
};
|
||||
|
||||
|
|
|
@ -4,7 +4,7 @@ macro_rules! upend_insert_val {
|
|||
Entry {
|
||||
entity: $entity.clone(),
|
||||
attribute: String::from($attribute),
|
||||
value: crate::database::entry::EntryValue::Value(serde_json::Value::from($value)),
|
||||
value: crate::database::entry::EntryValue::String(String::from($value)),
|
||||
},
|
||||
)
|
||||
}};
|
||||
|
|
|
@ -277,7 +277,7 @@ impl UpEndConnection {
|
|||
|
||||
let primary = data
|
||||
.filter(entity.eq(object_address.encode()?))
|
||||
.or_filter(value.eq(EntryValue::Address(object_address).to_string()?))
|
||||
.or_filter(value_str.eq(EntryValue::Address(object_address).to_string()?))
|
||||
.load::<models::Entry>(&self.conn)?;
|
||||
|
||||
let entries = primary
|
||||
|
@ -314,7 +314,7 @@ impl UpEndConnection {
|
|||
let matches = data
|
||||
.filter(identity.eq(object_address.encode()?))
|
||||
.or_filter(entity.eq(object_address.encode()?))
|
||||
.or_filter(value.eq(EntryValue::Address(object_address).to_string()?));
|
||||
.or_filter(value_str.eq(EntryValue::Address(object_address).to_string()?));
|
||||
|
||||
Ok(diesel::delete(matches).execute(&self.conn)?)
|
||||
}
|
||||
|
|
|
@ -22,7 +22,6 @@ use chrono::prelude::*;
|
|||
use log::{debug, error, info, warn};
|
||||
use lru::LruCache;
|
||||
use rayon::prelude::*;
|
||||
use serde_json::Value;
|
||||
use walkdir::WalkDir;
|
||||
|
||||
const BLOB_TYPE: &str = "BLOB";
|
||||
|
@ -34,7 +33,7 @@ const FILE_SIZE_KEY: &str = "FILE_SIZE";
|
|||
lazy_static! {
|
||||
static ref BLOB_TYPE_INVARIANT: InvariantEntry = InvariantEntry {
|
||||
attribute: String::from(TYPE_BASE_ATTR),
|
||||
value: EntryValue::Value(Value::from(BLOB_TYPE)),
|
||||
value: EntryValue::String(String::from(BLOB_TYPE)),
|
||||
};
|
||||
static ref BLOB_TYPE_ADDR: Address = BLOB_TYPE_INVARIANT.entity().unwrap();
|
||||
}
|
||||
|
@ -391,13 +390,13 @@ fn insert_file_with_metadata(
|
|||
let size_entry = Entry {
|
||||
entity: Address::Hash(hash.clone()),
|
||||
attribute: FILE_SIZE_KEY.to_string(),
|
||||
value: EntryValue::Value(Value::from(size)),
|
||||
value: EntryValue::Number(size as f64),
|
||||
};
|
||||
|
||||
let mime_entry = mime_type.map(|mime_type| Entry {
|
||||
entity: Address::Hash(hash.clone()),
|
||||
attribute: FILE_MIME_KEY.to_string(),
|
||||
value: EntryValue::Value(Value::String(mime_type)),
|
||||
value: EntryValue::String(mime_type),
|
||||
});
|
||||
|
||||
// Add the appropriate entries w/r/t virtual filesystem location
|
||||
|
@ -437,9 +436,9 @@ fn insert_file_with_metadata(
|
|||
let name_entry = Entry {
|
||||
entity: dir_has_entry_addr,
|
||||
attribute: ALIAS_KEY.to_string(),
|
||||
value: EntryValue::Value(Value::String(
|
||||
value: EntryValue::String(
|
||||
filename.as_os_str().to_string_lossy().to_string(),
|
||||
)),
|
||||
),
|
||||
};
|
||||
connection.insert_entry(name_entry)?;
|
||||
|
||||
|
|
Loading…
Reference in a new issue