thm
/
upend
1
0
Fork 0
Code Issues 73 Pull Requests Packages Projects Releases 5 Wiki Activity

refactor: Attributes are their proper type instead of strings
ci/woodpecker/push/woodpecker Pipeline was successful Details 

Also adds checking for non-emptiness and upper-casing
feat/tables
Tomáš Mládek 2024-02-15 19:10:22 +01:00
parent c5c157a856
commit afa5bd088d
16 changed files with 210 additions and 164 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

23
base/src/addressing.rs
View File

@ -1,3 +1,4 @@
use crate::entry::Attribute;
use crate::error::{AddressComponentsDecodeError, UpEndError};
use crate::hash::{
b58_decode, b58_encode, AsMultihash, AsMultihashError, LargeMultihash, UpMultihash, IDENTITY,
@ -18,7 +19,7 @@ use wasm_bindgen::prelude::*;
pub enum Address {
Hash(UpMultihash),
Uuid(Uuid),
Attribute(String),
Attribute(Attribute),
Url(Url),
}
@ -62,7 +63,7 @@ impl Address {
),
Self::Attribute(attribute) => (
UP_ATTRIBUTE,
LargeMultihash::wrap(IDENTITY, attribute.as_bytes())
LargeMultihash::wrap(IDENTITY, attribute.to_string().as_bytes())
.map_err(UpEndError::from_any)?,
),
Self::Url(url) => (
@ -103,7 +104,10 @@ impl Address {
Uuid::from_slice(digest.as_slice()).map_err(UpEndError::from_any)?,
)),
UP_ATTRIBUTE => Ok(Address::Attribute(
String::from_utf8(digest).map_err(UpEndError::from_any)?,
String::from_utf8(digest)
.map_err(UpEndError::from_any)?
.as_str()
.parse()?,
)),
UP_URL => Ok(Address::Url(
Url::parse(&String::from_utf8(digest).map_err(UpEndError::from_any)?)
@ -120,7 +124,7 @@ impl Address {
let (entity_type, entity_content) = match self {
Address::Hash(uphash) => ("Hash", Some(b58_encode(uphash.to_bytes()))),
Address::Uuid(uuid) => ("Uuid", Some(uuid.to_string())),
Address::Attribute(attribute) => ("Attribute", Some(attribute.clone())),
Address::Attribute(attribute) => ("Attribute", Some(attribute.to_string())),
Address::Url(url) => ("Url", Some(url.to_string())),
};
@ -133,11 +137,12 @@ impl Address {
pub fn from_components(components: AddressComponents) -> Result<Self, UpEndError> {
// TODO: make this automatically derive from `Address` definition
let address = match components {
AddressComponents { t, c } if t == "Attribute" => {
Address::Attribute(c.ok_or(UpEndError::AddressComponentsDecodeError(
AddressComponents { t, c } if t == "Attribute" => Address::Attribute(
c.ok_or(UpEndError::AddressComponentsDecodeError(
AddressComponentsDecodeError::MissingValue,
))?)
}
))?
.parse()?,
),
AddressComponents { t, c } if t == "Url" => Address::Url(if let Some(string) = c {
Url::parse(&string).map_err(|e| {
UpEndError::AddressComponentsDecodeError(
@ -276,7 +281,7 @@ mod tests {
#[test]
fn test_attribute_codec() -> Result<(), UpEndError> {
let addr = Address::Attribute(String::from("ATTRIBUTE"));
let addr = Address::Attribute("ATTRIBUTE".parse().unwrap());
let encoded = addr.encode()?;
let decoded = Address::decode(&encoded)?;
assert_eq!(addr, decoded);

5
base/src/constants.rs
View File

@ -1,4 +1,5 @@
use crate::addressing::Address;
use crate::entry::Attribute;
use crate::entry::InvariantEntry;
use crate::hash::{LargeMultihash, UpMultihash};
@ -19,13 +20,13 @@ pub const ATTR_KEY: &str = "KEY";
lazy_static! {
pub static ref HIER_ROOT_INVARIANT: InvariantEntry = InvariantEntry {
attribute: String::from(ATTR_KEY),
attribute: ATTR_KEY.parse().unwrap(),
value: "HIER_ROOT".into(),
};
pub static ref HIER_ROOT_ADDR: Address = HIER_ROOT_INVARIANT.entity().unwrap();
pub static ref TYPE_HASH_ADDRESS: Address =
Address::Hash(UpMultihash::from(LargeMultihash::default()));
pub static ref TYPE_UUID_ADDRESS: Address = Address::Uuid(uuid::Uuid::nil());
pub static ref TYPE_ATTRIBUTE_ADDRESS: Address = Address::Attribute("".to_string());
pub static ref TYPE_ATTRIBUTE_ADDRESS: Address = Address::Attribute(Attribute::null());
pub static ref TYPE_URL_ADDRESS: Address = Address::Url(url::Url::parse("up:").unwrap());
}

59
base/src/entry.rs
View File

@ -1,17 +1,52 @@
use crate::addressing::Address;
use crate::error::UpEndError;
use crate::hash::{b58_decode, sha256hash, AsMultihash, AsMultihashError, UpMultihash};
use crate::lang::Attribute;
use chrono::NaiveDateTime;
use serde::{Deserialize, Serialize};
use std::convert::TryFrom;
use std::io::{Cursor, Write};
use std::str::FromStr;
use url::Url;
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, Hash)]
pub struct Attribute(String);
impl Attribute {
pub fn null() -> Self {
Self("".to_string())
}
}
impl std::fmt::Display for Attribute {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
write!(f, "{}", self.0)
}
}
impl FromStr for Attribute {
type Err = UpEndError;
fn from_str(value: &str) -> Result<Self, Self::Err> {
if value.is_empty() {
Err(UpEndError::EmptyAttribute)
} else {
Ok(Self(value.to_uppercase()))
}
}
}
impl<S> PartialEq<S> for Attribute
where
S: AsRef<str>,
{
fn eq(&self, other: &S) -> bool {
self.0.eq_ignore_ascii_case(other.as_ref())
}
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct Entry {
pub entity: Address,
pub attribute: String,
pub attribute: Attribute,
pub value: EntryValue,
pub provenance: String,
pub timestamp: NaiveDateTime,
@ -22,7 +57,7 @@ pub struct ImmutableEntry(pub Entry);
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct InvariantEntry {
pub attribute: String,
pub attribute: Attribute,
pub value: EntryValue,
}
@ -36,18 +71,6 @@ pub enum EntryValue {
Invalid,
}
impl Default for Entry {
fn default() -> Self {
Self {
entity: Address::Uuid(uuid::Uuid::nil()),
attribute: Default::default(),
value: EntryValue::Null,
provenance: "SYSTEM".into(),
timestamp: NaiveDateTime::from_timestamp_opt(0, 0).unwrap(),
}
}
}
impl TryFrom<&InvariantEntry> for Entry {
type Error = UpEndError;
@ -57,7 +80,7 @@ impl TryFrom<&InvariantEntry> for Entry {
attribute: invariant.attribute.clone(),
value: invariant.value.clone(),
provenance: "INVARIANT".to_string(),
..Default::default()
timestamp: NaiveDateTime::from_timestamp_opt(0, 0).unwrap(),
})
}
}
@ -66,7 +89,7 @@ impl InvariantEntry {
pub fn entity(&self) -> Result<Address, UpEndError> {
let mut entity = Cursor::new(vec![0u8; 0]);
entity
.write_all(self.attribute.as_bytes())
.write_all(self.attribute.0.as_bytes())
.map_err(UpEndError::from_any)?;
entity
.write_all(self.value.to_string()?.as_bytes())
@ -92,7 +115,7 @@ impl AsMultihash for Entry {
.map_err(|e| AsMultihashError(e.to_string()))?
.as_slice(),
)?;
result.write_all(self.attribute.as_bytes())?;
result.write_all(self.attribute.0.as_bytes())?;
result.write_all(
self.value
.to_string()

2
base/src/error.rs
View File

@ -3,6 +3,7 @@ pub enum UpEndError {
HashDecodeError(String),
AddressParseError(String),
AddressComponentsDecodeError(AddressComponentsDecodeError),
EmptyAttribute,
CannotSerializeInvalid,
QueryParseError(String),
Other(String),
@ -35,6 +36,7 @@ impl std::fmt::Display for UpEndError {
String::from("Invalid EntryValues cannot be serialized."),
UpEndError::QueryParseError(err) => format!("Error parsing query: {err}"),
UpEndError::Other(err) => format!("Unknown error: {err}"),
UpEndError::EmptyAttribute => String::from("Attribute cannot be empty."),
}
)
}

16
base/src/lang.rs
View File

@ -1,4 +1,5 @@
use crate::addressing::Address;
use crate::entry::Attribute;
use crate::entry::EntryValue;
use crate::error::UpEndError;
use nonempty::NonEmpty;
@ -6,15 +7,6 @@ use std::borrow::Borrow;
use std::convert::TryFrom;
use std::str::FromStr;
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct Attribute(pub String);
impl From<&str> for Attribute {
fn from(str: &str) -> Self {
Self(str.to_string())
}
}
#[derive(Debug, Clone, PartialEq)]
pub enum QueryComponent<T>
where
@ -79,7 +71,7 @@ impl TryFrom<lexpr::Value> for Attribute {
fn try_from(value: lexpr::Value) -> Result<Self, Self::Error> {
match value {
lexpr::Value::String(str) => Ok(Attribute(str.to_string())),
lexpr::Value::String(str) => str.parse(),
_ => Err(UpEndError::QueryParseError(
"Can only convert to attribute from string.".into(),
)),
@ -331,7 +323,7 @@ mod test {
query,
Query::SingleQuery(QueryPart::Matches(PatternQuery {
entity: QueryComponent::Variable(None),
attribute: QueryComponent::Exact("FOO".into()),
attribute: QueryComponent::Exact("FOO".parse().unwrap()),
value: QueryComponent::Variable(None)
}))
);
@ -372,7 +364,7 @@ mod test {
query,
Query::SingleQuery(QueryPart::Matches(PatternQuery {
entity: QueryComponent::Variable(None),
attribute: QueryComponent::In(vec!("FOO".into(), "BAR".into())),
attribute: QueryComponent::In(vec!("FOO".parse().unwrap(), "BAR".parse().unwrap())),
value: QueryComponent::Variable(None)
}))
);

45
cli/src/extractors/audio.rs
View File

@ -18,14 +18,15 @@ use upend_db::{
lazy_static! {
pub static ref ID3_TYPE_INVARIANT: InvariantEntry = InvariantEntry {
attribute: String::from(ATTR_KEY),
attribute: ATTR_KEY.parse().unwrap(),
value: "TYPE_ID3".into(),
};
pub static ref ID3_TYPE_LABEL: Entry = Entry {
entity: ID3_TYPE_INVARIANT.entity().unwrap(),
attribute: ATTR_LABEL.into(),
attribute: ATTR_LABEL.parse().unwrap(),
value: "ID3".into(),
..Default::default()
provenance: "INVARIANT".to_string(),
timestamp: chrono::Utc::now().naive_utc(),
};
}
@ -59,13 +60,14 @@ impl Extractor for ID3Extractor {
let tags = id3::Tag::read_from_path(file_path)?;
let mut result: Vec<Entry> = tags
.frames()
.flat_map(|frame| match frame.content() {
id3::Content::Text(text) => vec![
let mut result: Vec<Entry> = vec![];
for frame in tags.frames() {
if let id3::Content::Text(text) = frame.content() {
result.extend(vec![
Entry {
entity: address.clone(),
attribute: format!("ID3_{}", frame.id()),
attribute: format!("ID3_{}", frame.id()).parse()?,
value: match frame.id() {
"TYER" | "TBPM" => EntryValue::guess_from(text),
_ => text.clone().into(),
@ -74,16 +76,15 @@ impl Extractor for ID3Extractor {
timestamp: chrono::Utc::now().naive_utc(),
},
Entry {
entity: Address::Attribute(format!("ID3_{}", frame.id())),
attribute: ATTR_LABEL.into(),
entity: (format!("ID3_{}", frame.id())).parse()?,
attribute: ATTR_LABEL.parse().unwrap(),
value: format!("ID3: {}", frame.name()).into(),
provenance: "SYSTEM EXTRACTOR".to_string(),
timestamp: chrono::Utc::now().naive_utc(),
},
],
_ => vec![],
})
.collect();
]);
}
}
let mut has_pictures = false;
for (idx, picture) in tags.pictures().enumerate() {
@ -99,7 +100,7 @@ impl Extractor for ID3Extractor {
)?;
result.push(Entry {
entity: address.clone(),
attribute: "ID3_PICTURE".to_string(),
attribute: "ID3_PICTURE".parse()?,
value: EntryValue::Address(Address::Hash(hash)),
provenance: "SYSTEM EXTRACTOR".to_string(),
timestamp: chrono::Utc::now().naive_utc(),
@ -108,8 +109,8 @@ impl Extractor for ID3Extractor {
}
if has_pictures {
result.push(Entry {
entity: Address::Attribute("ID3_PICTURE".to_string()),
attribute: ATTR_LABEL.into(),
entity: "ID3_PICTURE".parse()?,
attribute: ATTR_LABEL.parse().unwrap(),
value: "ID3 Embedded Image".into(),
provenance: "SYSTEM EXTRACTOR".to_string(),
timestamp: chrono::Utc::now().naive_utc(),
@ -123,9 +124,10 @@ impl Extractor for ID3Extractor {
.filter(|e| e.attribute != ATTR_LABEL)
.map(|e| Entry {
entity: Address::Attribute(e.attribute.clone()),
attribute: ATTR_OF.into(),
attribute: ATTR_OF.parse().unwrap(),
value: EntryValue::Address(ID3_TYPE_INVARIANT.entity().unwrap()),
..Default::default()
provenance: "SYSTEM EXTRACTOR".to_string(),
timestamp: chrono::Utc::now().naive_utc(),
})
.collect::<Vec<Entry>>(),
);
@ -134,9 +136,10 @@ impl Extractor for ID3Extractor {
ID3_TYPE_LABEL.clone(),
Entry {
entity: address.clone(),
attribute: ATTR_IN.into(),
attribute: ATTR_IN.parse().unwrap(),
value: EntryValue::Address(ID3_TYPE_INVARIANT.entity().unwrap()),
..Default::default()
provenance: "SYSTEM EXTRACTOR".to_string(),
timestamp: chrono::Utc::now().naive_utc(),
},
]);
}

80
cli/src/extractors/exif.rs
View File

@ -3,6 +3,7 @@ use std::sync::Arc;
use super::Extractor;
use anyhow::{anyhow, Result};
use lazy_static::lazy_static;
use upend_base::entry::Attribute;
use upend_base::{
addressing::Address,
constants::{ATTR_IN, ATTR_KEY, ATTR_LABEL, ATTR_OF},
@ -21,14 +22,15 @@ pub struct ExifExtractor;
lazy_static! {
pub static ref EXIF_TYPE_INVARIANT: InvariantEntry = InvariantEntry {
attribute: String::from(ATTR_KEY),
attribute: ATTR_KEY.parse().unwrap(),
value: "TYPE_EXIF".into(),
};
pub static ref EXIF_TYPE_LABEL: Entry = Entry {
entity: EXIF_TYPE_INVARIANT.entity().unwrap(),
attribute: ATTR_LABEL.into(),
attribute: ATTR_LABEL.parse().unwrap(),
value: "EXIF".into(),
..Default::default()
provenance: "INVARIANT".to_string(),
timestamp: chrono::Utc::now().naive_utc(),
};
}
@ -63,42 +65,40 @@ impl Extractor for ExifExtractor {
let exifreader = exif::Reader::new();
let exif = exifreader.read_from_container(&mut bufreader)?;
let mut result: Vec<Entry> = exif
let mut result: Vec<Entry> = vec![];
for field in exif
.fields()
.filter(|field| !matches!(field.value, exif::Value::Undefined(..)))
.flat_map(|field| {
if let Some(tag_description) = field.tag.description() {
let attribute = format!("EXIF_{}", field.tag.1);
{
if let Some(tag_description) = field.tag.description() {
let attribute: Attribute = format!("EXIF_{}", field.tag.1).parse()?;
vec![
Entry {
entity: address.clone(),
attribute: attribute.clone(),
value: match field.tag {
exif::Tag::ExifVersion => {
EntryValue::String(format!("{}", field.display_value()))
}
_ => EntryValue::guess_from(format!(
"{}",
field.display_value()
)),
},
provenance: "SYSTEM EXTRACTOR".to_string(),
timestamp: chrono::Utc::now().naive_utc(),
result.extend(vec![
Entry {
entity: address.clone(),
attribute: attribute.clone(),
value: match field.tag {
exif::Tag::ExifVersion => {
EntryValue::String(format!("{}", field.display_value()))
}
_ => {
EntryValue::guess_from(format!("{}", field.display_value()))
}
},
Entry {
entity: Address::Attribute(attribute),
attribute: ATTR_LABEL.into(),
value: format!("EXIF: {}", tag_description).into(),
provenance: "SYSTEM EXTRACTOR".to_string(),
timestamp: chrono::Utc::now().naive_utc(),
},
]
} else {
vec![]
}
})
.collect();
provenance: "SYSTEM EXTRACTOR".to_string(),
timestamp: chrono::Utc::now().naive_utc(),
},
Entry {
entity: Address::Attribute(attribute),
attribute: ATTR_LABEL.parse().unwrap(),
value: format!("EXIF: {}", tag_description).into(),
provenance: "SYSTEM EXTRACTOR".to_string(),
timestamp: chrono::Utc::now().naive_utc(),
},
]);
}
}
if !result.is_empty() {
result.extend(
@ -107,9 +107,10 @@ impl Extractor for ExifExtractor {
.filter(|e| e.attribute != ATTR_LABEL)
.map(|e| Entry {
entity: Address::Attribute(e.attribute.clone()),
attribute: ATTR_OF.into(),
attribute: ATTR_OF.parse().unwrap(),
value: EntryValue::Address(EXIF_TYPE_INVARIANT.entity().unwrap()),
..Default::default()
provenance: "SYSTEM EXTRACTOR".to_string(),
timestamp: chrono::Utc::now().naive_utc(),
})
.collect::<Vec<Entry>>(),
);
@ -120,9 +121,10 @@ impl Extractor for ExifExtractor {
EXIF_TYPE_LABEL.clone(),
Entry {
entity: address.clone(),
attribute: ATTR_IN.into(),
attribute: ATTR_IN.parse().unwrap(),
value: EntryValue::Address(EXIF_TYPE_INVARIANT.entity().unwrap()),
..Default::default()
provenance: "SYSTEM EXTRACTOR".to_string(),
timestamp: chrono::Utc::now().naive_utc(),
},
]);
}

21
cli/src/extractors/media.rs
View File

@ -18,20 +18,22 @@ const DURATION_KEY: &str = "MEDIA_DURATION";
lazy_static! {
pub static ref MEDIA_TYPE_INVARIANT: InvariantEntry = InvariantEntry {
attribute: String::from(ATTR_KEY),
attribute: ATTR_KEY.parse().unwrap(),
value: "TYPE_MEDIA".into(),
};
pub static ref MEDIA_TYPE_LABEL: Entry = Entry {
entity: MEDIA_TYPE_INVARIANT.entity().unwrap(),
attribute: ATTR_LABEL.into(),
attribute: ATTR_LABEL.parse().unwrap(),
value: "Multimedia".into(),
..Default::default()
provenance: "INVARIANT".to_string(),
timestamp: chrono::Utc::now().naive_utc(),
};
pub static ref DURATION_OF_MEDIA: Entry = Entry {
entity: Address::Attribute(DURATION_KEY.to_string()),
attribute: ATTR_OF.into(),
entity: DURATION_KEY.parse().unwrap(),
attribute: ATTR_OF.parse().unwrap(),
value: EntryValue::Address(MEDIA_TYPE_INVARIANT.entity().unwrap()),
..Default::default()
provenance: "INVARIANT".to_string(),
timestamp: chrono::Utc::now().naive_utc(),
};
}
@ -90,7 +92,7 @@ impl Extractor for MediaExtractor {
let result = vec![
Entry {
entity: address.clone(),
attribute: DURATION_KEY.to_string(),
attribute: DURATION_KEY.parse().unwrap(),
value: EntryValue::Number(duration),
provenance: "SYSTEM EXTRACTOR".to_string(),
timestamp: chrono::Utc::now().naive_utc(),
@ -102,9 +104,10 @@ impl Extractor for MediaExtractor {
DURATION_OF_MEDIA.clone(),
Entry {
entity: address.clone(),
attribute: ATTR_IN.into(),
attribute: ATTR_IN.parse().unwrap(),
value: EntryValue::Address(MEDIA_TYPE_INVARIANT.entity().unwrap()),
..Default::default()
provenance: "SYSTEM EXTRACTOR".to_string(),
timestamp: chrono::Utc::now().naive_utc(),
},
];

16
cli/src/extractors/web.rs
View File

@ -40,21 +40,21 @@ impl Extractor for WebExtractor {
let mut entries = vec![
html.title.as_ref().map(|html_title| Entry {
entity: address.clone(),
attribute: "HTML_TITLE".to_string(),
attribute: "HTML_TITLE".parse().unwrap(),
value: html_title.clone().into(),
provenance: "SYSTEM EXTRACTOR".to_string(),
timestamp: chrono::Utc::now().naive_utc(),
}),
html.title.map(|html_title| Entry {
entity: address.clone(),
attribute: ATTR_LABEL.to_string(),
attribute: ATTR_LABEL.parse().unwrap(),
value: html_title.into(),
provenance: "SYSTEM EXTRACTOR".to_string(),
timestamp: chrono::Utc::now().naive_utc(),
}),
html.description.map(|html_desc| Entry {
entity: address.clone(),
attribute: "HTML_DESCRIPTION".to_string(),
attribute: "HTML_DESCRIPTION".parse().unwrap(),
value: html_desc.into(),
provenance: "SYSTEM EXTRACTOR".to_string(),
timestamp: chrono::Utc::now().naive_utc(),
@ -65,7 +65,7 @@ impl Extractor for WebExtractor {
if attribute == "OG_TITLE" {
entries.push(Some(Entry {
entity: address.clone(),
attribute: ATTR_LABEL.to_string(),
attribute: ATTR_LABEL.parse()?,
value: value.clone().into(),
provenance: "SYSTEM EXTRACTOR".to_string(),
timestamp: chrono::Utc::now().naive_utc(),
@ -74,7 +74,7 @@ impl Extractor for WebExtractor {
entries.push(Some(Entry {
entity: address.clone(),
attribute,
attribute: attribute.parse()?,
value: value.into(),
provenance: "SYSTEM EXTRACTOR".to_string(),
timestamp: chrono::Utc::now().naive_utc(),
@ -83,7 +83,7 @@ impl Extractor for WebExtractor {
for image in html.opengraph.images {
entries.push(Some(Entry {
entity: address.clone(),
attribute: "OG_IMAGE".to_string(),
attribute: "OG_IMAGE".parse()?,
value: image.url.into(),
provenance: "SYSTEM EXTRACTOR".to_string(),
timestamp: chrono::Utc::now().naive_utc(),
@ -99,10 +99,10 @@ impl Extractor for WebExtractor {
vec![
Entry {
entity: Address::Attribute(e.attribute.clone()),
attribute: ATTR_OF.to_string(),
attribute: ATTR_OF.parse().unwrap(),
value: EntryValue::Address(TYPE_URL_ADDRESS.clone()),
provenance: "SYSTEM EXTRACTOR".to_string(),
..Default::default()
timestamp: chrono::Utc::now().naive_utc(),
},
e,
]

20
cli/src/routes.rs
View File

@ -47,6 +47,7 @@ use url::Url;
#[cfg(feature = "desktop")]
use is_executable::IsExecutable;
use upend_base::error::UpEndError;
#[derive(Clone)]
pub struct State {
@ -417,7 +418,7 @@ pub async fn put_object(
if let Some(entity) = in_entry.entity {
Ok(Entry {
entity: entity.try_into()?,
attribute: in_entry.attribute,
attribute: in_entry.attribute.parse()?,
value: in_entry.value,
provenance: (match &provenance {
Some(s) => format!("API {}", s),
@ -429,7 +430,7 @@ pub async fn put_object(
})
} else {
Ok(Entry::try_from(&InvariantEntry {
attribute: in_entry.attribute,
attribute: in_entry.attribute.parse()?,
value: in_entry.value,
})?)
}
@ -481,7 +482,7 @@ pub async fn put_object(
if connection.retrieve_object(&address)?.is_empty() {
connection.insert_entry(Entry {
entity: address.clone(),
attribute: ATTR_ADDED.to_string(),
attribute: ATTR_ADDED.parse().unwrap(),
value: EntryValue::Number(
SystemTime::now()
.duration_since(UNIX_EPOCH)
@ -622,7 +623,7 @@ pub async fn put_object_attribute(
let new_attr_entry = Entry {
entity: address,
attribute,
attribute: attribute.parse()?,
value: value.into_inner(),
provenance: (match &query.provenance {
Some(s) => format!("API {}", s),
@ -688,7 +689,14 @@ pub async fn get_address(
web::Query(query): web::Query<HashMap<String, String>>,
) -> Result<HttpResponse, Error> {
let (address, immutable) = if let Some(attribute) = query.get("attribute") {
(Address::Attribute(attribute.into()), true)
(
Address::Attribute(
attribute
.parse()
.map_err(|e: UpEndError| ErrorBadRequest(e.to_string()))?,
),
true,
)
} else if let Some(url) = query.get("url") {
(
Address::Url(Url::parse(url).map_err(ErrorBadRequest)?),
@ -1145,7 +1153,7 @@ mod tests {
assert_eq!(result["entity"]["t"], "Hash");
assert_eq!(result["entity"]["c"], digest_str);
let address = Address::Attribute("TEST".to_string());
let address = Address::Attribute("TEST".parse().unwrap());
let req = actix_web::test::TestRequest::get()
.uri(&format!("/api/obj/{}", address))
.to_request();

19
db/src/engine.rs
View File

@ -18,7 +18,8 @@ use diesel::{BoxableExpression, QueryDsl};
use diesel::{ExpressionMethods, TextExpressionMethods};
use upend_base::addressing::Address;
use upend_base::entry::{EntryPart, EntryValue};
use upend_base::lang::{Attribute, Query, QueryComponent, QueryPart, QueryQualifier};
use upend_base::error::UpEndError;
use upend_base::lang::{Query, QueryComponent, QueryPart, QueryQualifier};
#[derive(Debug, Clone)]
pub struct QueryExecutionError(String);
@ -31,6 +32,12 @@ impl std::fmt::Display for QueryExecutionError {
impl std::error::Error for QueryExecutionError {}
impl From<UpEndError> for QueryExecutionError {
fn from(e: UpEndError) -> Self {
QueryExecutionError(e.to_string())
}
}
pub fn execute(
connection: &PooledConnection<ConnectionManager<SqliteConnection>>,
query: Query,
@ -126,7 +133,7 @@ pub fn execute(
}
EntryPart::Attribute(a) => {
Some(EntryValue::Address(Address::Attribute(
a.0.clone(),
a.clone(),
)))
}
EntryPart::Value(v) => Some(v.clone()),
@ -172,9 +179,9 @@ pub fn execute(
subquery_results
.iter()
.map(|e| {
EntryPart::Attribute(Attribute(e.attribute.clone()))
e.attribute.parse().map(|a| EntryPart::Attribute(a))
})
.collect(),
.collect::<Result<Vec<EntryPart>, _>>()?,
);
}
@ -266,10 +273,10 @@ fn to_sqlite_predicates(query: Query) -> Result<SqlResult, QueryExecutionError>
match &eq.attribute {
QueryComponent::Exact(q_attribute) => {
subqueries.push(Box::new(data::attribute.eq(q_attribute.0.clone())))
subqueries.push(Box::new(data::attribute.eq(q_attribute.to_string())))
}
QueryComponent::In(q_attributes) => subqueries.push(Box::new(
data::attribute.eq_any(q_attributes.iter().map(|a| &a.0).cloned()),
data::attribute.eq_any(q_attributes.iter().map(|a| a.to_string())),
)),
QueryComponent::Contains(q_attribute) => subqueries
.push(Box::new(data::attribute.like(format!("%{}%", q_attribute)))),

19
db/src/entry.rs
View File

@ -1,25 +1,25 @@
use crate::inner::models;
use anyhow::{anyhow, Result};
use std::convert::TryFrom;
use upend_base::addressing::{Address, Addressable};
use upend_base::entry::{Entry, EntryValue, ImmutableEntry};
use upend_base::error::UpEndError;
impl TryFrom<&models::Entry> for Entry {
type Error = anyhow::Error;
type Error = UpEndError;
fn try_from(e: &models::Entry) -> Result<Self, Self::Error> {
if let Some(value_str) = &e.value_str {
Ok(Entry {
entity: Address::decode(&e.entity)?,
attribute: e.attribute.clone(),
value: value_str.parse()?,
attribute: e.attribute.parse()?,
value: value_str.parse().unwrap(),
provenance: e.provenance.clone(),
timestamp: e.timestamp,
})
} else if let Some(value_num) = e.value_num {
Ok(Entry {
entity: Address::decode(&e.entity)?,
attribute: e.attribute.clone(),
attribute: e.attribute.parse()?,
value: EntryValue::Number(value_num),
provenance: e.provenance.clone(),
timestamp: e.timestamp,
@ -27,7 +27,7 @@ impl TryFrom<&models::Entry> for Entry {
} else {
Ok(Entry {
entity: Address::decode(&e.entity)?,
attribute: e.attribute.clone(),
attribute: e.attribute.parse()?,
value: EntryValue::Number(f64::NAN),
provenance: e.provenance.clone(),
timestamp: e.timestamp,
@ -40,18 +40,15 @@ impl TryFrom<&Entry> for models::Entry {
type Error = anyhow::Error;
fn try_from(e: &Entry) -> Result<Self, Self::Error> {
if e.attribute.is_empty() {
return Err(anyhow!("Attribute cannot be empty."));
}
let base_entry = models::Entry {
identity: e.address()?.encode()?,
entity_searchable: match &e.entity {
Address::Attribute(attr) => Some(attr.clone()),
Address::Attribute(attr) => Some(attr.to_string()),
Address::Url(url) => Some(url.to_string()),
_ => None,
},
entity: e.entity.encode()?,
attribute: e.attribute.clone(),
attribute: e.attribute.to_string(),
value_str: None,
value_num: None,
immutable: false,

12
db/src/hierarchies.rs
View File

@ -74,7 +74,7 @@ pub fn list_roots(connection: &UpEndConnection) -> Result<Vec<Address>> {
Ok(connection
.query(Query::SingleQuery(QueryPart::Matches(PatternQuery {
entity: QueryComponent::Variable(None),
attribute: QueryComponent::Exact(ATTR_IN.into()),
attribute: QueryComponent::Exact(ATTR_IN.parse().unwrap()),
value: QueryComponent::Exact((*HIER_ROOT_ADDR).clone().into()),
})))?
.into_iter()
@ -105,7 +105,7 @@ pub fn fetch_or_create_dir(
let matching_directories = connection
.query(Query::SingleQuery(QueryPart::Matches(PatternQuery {
entity: QueryComponent::Variable(None),
attribute: QueryComponent::Exact(ATTR_LABEL.into()),
attribute: QueryComponent::Exact(ATTR_LABEL.parse().unwrap()),
value: QueryComponent::Exact(directory.to_string().into()),
})))?
.into_iter()
@ -115,7 +115,7 @@ pub fn fetch_or_create_dir(
Some(parent) => connection
.query(Query::SingleQuery(QueryPart::Matches(PatternQuery {
entity: QueryComponent::Variable(None),
attribute: QueryComponent::Exact(ATTR_IN.into()),
attribute: QueryComponent::Exact(ATTR_IN.parse().unwrap()),
value: QueryComponent::Exact(parent.into()),
})))?
.into_iter()
@ -135,7 +135,7 @@ pub fn fetch_or_create_dir(
let directory_entry = Entry {
entity: new_directory_address.clone(),
attribute: String::from(ATTR_LABEL),
attribute: ATTR_LABEL.parse().unwrap(),
value: directory.to_string().into(),
provenance: "SYSTEM FS".to_string(),
timestamp: chrono::Utc::now().naive_utc(),
@ -145,7 +145,7 @@ pub fn fetch_or_create_dir(
connection.insert_entry(if let Some(parent) = parent {
Entry {
entity: new_directory_address.clone(),
attribute: String::from(ATTR_IN),
attribute: ATTR_IN.parse().unwrap(),
value: parent.into(),
provenance: "SYSTEM FS".to_string(),
timestamp: chrono::Utc::now().naive_utc(),
@ -153,7 +153,7 @@ pub fn fetch_or_create_dir(
} else {
Entry {
entity: new_directory_address.clone(),
attribute: String::from(ATTR_IN),
attribute: ATTR_IN.parse().unwrap(),
value: HIER_ROOT_ADDR.clone().into(),
provenance: "SYSTEM FS".to_string(),
timestamp: chrono::Utc::now().naive_utc(),

17
db/src/lib.rs
View File

@ -40,7 +40,7 @@ use std::sync::{Arc, Mutex, RwLock};
use std::time::Duration;
use tracing::{debug, error, trace, warn};
use upend_base::addressing::{Address, Addressable};
use upend_base::entry::{Entry, EntryValue, ImmutableEntry};
use upend_base::entry::{Attribute, Entry, EntryValue, ImmutableEntry};
use upend_base::error::UpEndError;
use upend_base::hash::UpMultihash;
use upend_base::lang::Query;
@ -308,7 +308,7 @@ impl UpEndConnection {
let entries = primary
.iter()
.map(Entry::try_from)
.collect::<Result<Vec<Entry>>>()?;
.collect::<Result<Vec<Entry>, UpEndError>>()?;
let secondary = data
.filter(
@ -326,7 +326,7 @@ impl UpEndConnection {
let secondary_entries = secondary
.iter()
.map(Entry::try_from)
.collect::<Result<Vec<Entry>>>()?;
.collect::<Result<Vec<Entry>, UpEndError>>()?;
Ok([entries, secondary_entries].concat())
}
@ -414,7 +414,7 @@ impl UpEndConnection {
}
// #[deprecated]
pub fn get_all_attributes(&self) -> Result<Vec<String>> {
pub fn get_all_attributes(&self) -> Result<Vec<Attribute>> {
use crate::inner::schema::data::dsl::*;
let _lock = self.lock.read().unwrap();
@ -426,7 +426,10 @@ impl UpEndConnection {
.order_by(attribute)
.load::<String>(&conn)?;
Ok(result)
Ok(result
.into_iter()
.map(|a| a.parse())
.collect::<Result<Vec<Attribute>, UpEndError>>()?)
}
pub fn get_stats(&self) -> Result<serde_json::Value> {
@ -478,10 +481,10 @@ impl UpEndConnection {
)
.load(&conn)?;
result
Ok(result
.iter()
.map(Entry::try_from)
.collect::<Result<Vec<Entry>>>()
.collect::<Result<Vec<Entry>, UpEndError>>()?)
}
}

4
db/src/macros.rs
View File

@ -3,7 +3,7 @@ macro_rules! upend_insert_val {
($db_connection:expr, $entity:expr, $attribute:expr, $value:expr) => {{
$db_connection.insert_entry(Entry {
entity: $entity.clone(),
attribute: String::from($attribute),
attribute: $attribute.parse().unwrap(),
value: upend_base::entry::EntryValue::String(String::from($value)),
provenance: "SYSTEM INIT".to_string(),
timestamp: chrono::Utc::now().naive_utc(),
@ -16,7 +16,7 @@ macro_rules! upend_insert_addr {
($db_connection:expr, $entity:expr, $attribute:expr, $addr:expr) => {{
$db_connection.insert_entry(Entry {
entity: $entity.clone(),
attribute: String::from($attribute),
attribute: $attribute.parse().unwrap(),
value: upend_base::entry::EntryValue::Address($addr.clone()),
provenance: "SYSTEM INIT".to_string(),
timestamp: chrono::Utc::now().naive_utc(),

16
db/src/stores/fs/mod.rs
View File

@ -108,13 +108,13 @@ impl FsStore {
trace!("Initializing DB types.");
upend_insert_addr!(
upconnection,
Address::Attribute(FILE_SIZE_KEY.to_string()),
Address::Attribute(FILE_SIZE_KEY.parse().unwrap()),
ATTR_OF,
TYPE_HASH_ADDRESS
)?;
upend_insert_addr!(
upconnection,
Address::Attribute(FILE_MIME_KEY.to_string()),
Address::Attribute(FILE_MIME_KEY.parse().unwrap()),
ATTR_OF,
TYPE_HASH_ADDRESS
)?;
@ -438,7 +438,7 @@ impl FsStore {
// Metadata
let size_entry = Entry {
entity: blob_address.clone(),
attribute: FILE_SIZE_KEY.to_string(),
attribute: FILE_SIZE_KEY.parse().unwrap(),
value: (size as f64).into(),
provenance: "SYSTEM INIT".to_string(),
timestamp: chrono::Utc::now().naive_utc(),
@ -447,7 +447,7 @@ impl FsStore {
let mime_type = tree_magic_mini::from_filepath(path).map(|s| s.to_string());
let mime_entry = mime_type.map(|mime_type| Entry {
entity: blob_address.clone(),
attribute: FILE_MIME_KEY.to_string(),
attribute: FILE_MIME_KEY.parse().unwrap(),
value: mime_type.into(),
provenance: "SYSTEM INIT".to_string(),
timestamp: chrono::Utc::now().naive_utc(),
@ -455,7 +455,7 @@ impl FsStore {
let added_entry = Entry {
entity: blob_address.clone(),
attribute: ATTR_ADDED.to_string(),
attribute: ATTR_ADDED.parse().unwrap(),
value: (SystemTime::now()
.duration_since(UNIX_EPOCH)
.unwrap()
@ -480,7 +480,7 @@ impl FsStore {
let label_entry = Entry {
entity: blob_address.clone(),
attribute: ATTR_LABEL.to_string(),
attribute: ATTR_LABEL.parse().unwrap(),
value: name
.unwrap_or_else(|| filename.as_os_str().to_string_lossy().to_string())
.into(),
@ -498,7 +498,7 @@ impl FsStore {
let dir_has_entry = Entry {
entity: blob_address.clone(),
attribute: ATTR_IN.to_string(),
attribute: ATTR_IN.parse().unwrap(),
value: parent_dir.clone().into(),
provenance: "SYSTEM INIT".to_string(),
timestamp: chrono::Utc::now().naive_utc(),
@ -507,7 +507,7 @@ impl FsStore {
let alias_entry = Entry {
entity: dir_has_entry_addr,
attribute: ATTR_BY.to_string(),
attribute: ATTR_BY.parse().unwrap(),
value: label_entry_addr.into(),
provenance: "SYSTEM INIT".to_string(),
timestamp: chrono::Utc::now().naive_utc(),