From afa5bd088db7d423bde646a01da038413fe3926c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1=C5=A1=20Ml=C3=A1dek?= Date: Thu, 15 Feb 2024 19:10:22 +0100 Subject: [PATCH] refactor: Attributes are their proper type instead of strings Also adds checking for non-emptiness and upper-casing --- base/src/addressing.rs | 23 ++++++----- base/src/constants.rs | 5 ++- base/src/entry.rs | 59 ++++++++++++++++++--------- base/src/error.rs | 2 + base/src/lang.rs | 16 ++------ cli/src/extractors/audio.rs | 45 +++++++++++---------- cli/src/extractors/exif.rs | 80 +++++++++++++++++++------------------ cli/src/extractors/media.rs | 21 +++++----- cli/src/extractors/web.rs | 16 ++++---- cli/src/routes.rs | 20 +++++++--- db/src/engine.rs | 19 ++++++--- db/src/entry.rs | 19 ++++----- db/src/hierarchies.rs | 12 +++--- db/src/lib.rs | 17 ++++---- db/src/macros.rs | 4 +- db/src/stores/fs/mod.rs | 16 ++++---- 16 files changed, 210 insertions(+), 164 deletions(-) diff --git a/base/src/addressing.rs b/base/src/addressing.rs index 4f6f287..e960b00 100644 --- a/base/src/addressing.rs +++ b/base/src/addressing.rs @@ -1,3 +1,4 @@ +use crate::entry::Attribute; use crate::error::{AddressComponentsDecodeError, UpEndError}; use crate::hash::{ b58_decode, b58_encode, AsMultihash, AsMultihashError, LargeMultihash, UpMultihash, IDENTITY, @@ -18,7 +19,7 @@ use wasm_bindgen::prelude::*; pub enum Address { Hash(UpMultihash), Uuid(Uuid), - Attribute(String), + Attribute(Attribute), Url(Url), } @@ -62,7 +63,7 @@ impl Address { ), Self::Attribute(attribute) => ( UP_ATTRIBUTE, - LargeMultihash::wrap(IDENTITY, attribute.as_bytes()) + LargeMultihash::wrap(IDENTITY, attribute.to_string().as_bytes()) .map_err(UpEndError::from_any)?, ), Self::Url(url) => ( @@ -103,7 +104,10 @@ impl Address { Uuid::from_slice(digest.as_slice()).map_err(UpEndError::from_any)?, )), UP_ATTRIBUTE => Ok(Address::Attribute( - String::from_utf8(digest).map_err(UpEndError::from_any)?, + String::from_utf8(digest) + .map_err(UpEndError::from_any)? + .as_str() + .parse()?, )), UP_URL => Ok(Address::Url( Url::parse(&String::from_utf8(digest).map_err(UpEndError::from_any)?) @@ -120,7 +124,7 @@ impl Address { let (entity_type, entity_content) = match self { Address::Hash(uphash) => ("Hash", Some(b58_encode(uphash.to_bytes()))), Address::Uuid(uuid) => ("Uuid", Some(uuid.to_string())), - Address::Attribute(attribute) => ("Attribute", Some(attribute.clone())), + Address::Attribute(attribute) => ("Attribute", Some(attribute.to_string())), Address::Url(url) => ("Url", Some(url.to_string())), }; @@ -133,11 +137,12 @@ impl Address { pub fn from_components(components: AddressComponents) -> Result { // TODO: make this automatically derive from `Address` definition let address = match components { - AddressComponents { t, c } if t == "Attribute" => { - Address::Attribute(c.ok_or(UpEndError::AddressComponentsDecodeError( + AddressComponents { t, c } if t == "Attribute" => Address::Attribute( + c.ok_or(UpEndError::AddressComponentsDecodeError( AddressComponentsDecodeError::MissingValue, - ))?) - } + ))? + .parse()?, + ), AddressComponents { t, c } if t == "Url" => Address::Url(if let Some(string) = c { Url::parse(&string).map_err(|e| { UpEndError::AddressComponentsDecodeError( @@ -276,7 +281,7 @@ mod tests { #[test] fn test_attribute_codec() -> Result<(), UpEndError> { - let addr = Address::Attribute(String::from("ATTRIBUTE")); + let addr = Address::Attribute("ATTRIBUTE".parse().unwrap()); let encoded = addr.encode()?; let decoded = Address::decode(&encoded)?; assert_eq!(addr, decoded); diff --git a/base/src/constants.rs b/base/src/constants.rs index 96ae18d..87715ab 100644 --- a/base/src/constants.rs +++ b/base/src/constants.rs @@ -1,4 +1,5 @@ use crate::addressing::Address; +use crate::entry::Attribute; use crate::entry::InvariantEntry; use crate::hash::{LargeMultihash, UpMultihash}; @@ -19,13 +20,13 @@ pub const ATTR_KEY: &str = "KEY"; lazy_static! { pub static ref HIER_ROOT_INVARIANT: InvariantEntry = InvariantEntry { - attribute: String::from(ATTR_KEY), + attribute: ATTR_KEY.parse().unwrap(), value: "HIER_ROOT".into(), }; pub static ref HIER_ROOT_ADDR: Address = HIER_ROOT_INVARIANT.entity().unwrap(); pub static ref TYPE_HASH_ADDRESS: Address = Address::Hash(UpMultihash::from(LargeMultihash::default())); pub static ref TYPE_UUID_ADDRESS: Address = Address::Uuid(uuid::Uuid::nil()); - pub static ref TYPE_ATTRIBUTE_ADDRESS: Address = Address::Attribute("".to_string()); + pub static ref TYPE_ATTRIBUTE_ADDRESS: Address = Address::Attribute(Attribute::null()); pub static ref TYPE_URL_ADDRESS: Address = Address::Url(url::Url::parse("up:").unwrap()); } diff --git a/base/src/entry.rs b/base/src/entry.rs index 4088e51..381c782 100644 --- a/base/src/entry.rs +++ b/base/src/entry.rs @@ -1,17 +1,52 @@ use crate::addressing::Address; use crate::error::UpEndError; use crate::hash::{b58_decode, sha256hash, AsMultihash, AsMultihashError, UpMultihash}; -use crate::lang::Attribute; use chrono::NaiveDateTime; use serde::{Deserialize, Serialize}; use std::convert::TryFrom; use std::io::{Cursor, Write}; +use std::str::FromStr; use url::Url; +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, Hash)] +pub struct Attribute(String); + +impl Attribute { + pub fn null() -> Self { + Self("".to_string()) + } +} +impl std::fmt::Display for Attribute { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{}", self.0) + } +} + +impl FromStr for Attribute { + type Err = UpEndError; + + fn from_str(value: &str) -> Result { + if value.is_empty() { + Err(UpEndError::EmptyAttribute) + } else { + Ok(Self(value.to_uppercase())) + } + } +} + +impl PartialEq for Attribute +where + S: AsRef, +{ + fn eq(&self, other: &S) -> bool { + self.0.eq_ignore_ascii_case(other.as_ref()) + } +} + #[derive(Debug, Clone, Serialize, Deserialize)] pub struct Entry { pub entity: Address, - pub attribute: String, + pub attribute: Attribute, pub value: EntryValue, pub provenance: String, pub timestamp: NaiveDateTime, @@ -22,7 +57,7 @@ pub struct ImmutableEntry(pub Entry); #[derive(Debug, Clone, Serialize, Deserialize)] pub struct InvariantEntry { - pub attribute: String, + pub attribute: Attribute, pub value: EntryValue, } @@ -36,18 +71,6 @@ pub enum EntryValue { Invalid, } -impl Default for Entry { - fn default() -> Self { - Self { - entity: Address::Uuid(uuid::Uuid::nil()), - attribute: Default::default(), - value: EntryValue::Null, - provenance: "SYSTEM".into(), - timestamp: NaiveDateTime::from_timestamp_opt(0, 0).unwrap(), - } - } -} - impl TryFrom<&InvariantEntry> for Entry { type Error = UpEndError; @@ -57,7 +80,7 @@ impl TryFrom<&InvariantEntry> for Entry { attribute: invariant.attribute.clone(), value: invariant.value.clone(), provenance: "INVARIANT".to_string(), - ..Default::default() + timestamp: NaiveDateTime::from_timestamp_opt(0, 0).unwrap(), }) } } @@ -66,7 +89,7 @@ impl InvariantEntry { pub fn entity(&self) -> Result { let mut entity = Cursor::new(vec![0u8; 0]); entity - .write_all(self.attribute.as_bytes()) + .write_all(self.attribute.0.as_bytes()) .map_err(UpEndError::from_any)?; entity .write_all(self.value.to_string()?.as_bytes()) @@ -92,7 +115,7 @@ impl AsMultihash for Entry { .map_err(|e| AsMultihashError(e.to_string()))? .as_slice(), )?; - result.write_all(self.attribute.as_bytes())?; + result.write_all(self.attribute.0.as_bytes())?; result.write_all( self.value .to_string() diff --git a/base/src/error.rs b/base/src/error.rs index ed0bf62..db0bad0 100644 --- a/base/src/error.rs +++ b/base/src/error.rs @@ -3,6 +3,7 @@ pub enum UpEndError { HashDecodeError(String), AddressParseError(String), AddressComponentsDecodeError(AddressComponentsDecodeError), + EmptyAttribute, CannotSerializeInvalid, QueryParseError(String), Other(String), @@ -35,6 +36,7 @@ impl std::fmt::Display for UpEndError { String::from("Invalid EntryValues cannot be serialized."), UpEndError::QueryParseError(err) => format!("Error parsing query: {err}"), UpEndError::Other(err) => format!("Unknown error: {err}"), + UpEndError::EmptyAttribute => String::from("Attribute cannot be empty."), } ) } diff --git a/base/src/lang.rs b/base/src/lang.rs index 3218df8..3e36e6c 100644 --- a/base/src/lang.rs +++ b/base/src/lang.rs @@ -1,4 +1,5 @@ use crate::addressing::Address; +use crate::entry::Attribute; use crate::entry::EntryValue; use crate::error::UpEndError; use nonempty::NonEmpty; @@ -6,15 +7,6 @@ use std::borrow::Borrow; use std::convert::TryFrom; use std::str::FromStr; -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct Attribute(pub String); - -impl From<&str> for Attribute { - fn from(str: &str) -> Self { - Self(str.to_string()) - } -} - #[derive(Debug, Clone, PartialEq)] pub enum QueryComponent where @@ -79,7 +71,7 @@ impl TryFrom for Attribute { fn try_from(value: lexpr::Value) -> Result { match value { - lexpr::Value::String(str) => Ok(Attribute(str.to_string())), + lexpr::Value::String(str) => str.parse(), _ => Err(UpEndError::QueryParseError( "Can only convert to attribute from string.".into(), )), @@ -331,7 +323,7 @@ mod test { query, Query::SingleQuery(QueryPart::Matches(PatternQuery { entity: QueryComponent::Variable(None), - attribute: QueryComponent::Exact("FOO".into()), + attribute: QueryComponent::Exact("FOO".parse().unwrap()), value: QueryComponent::Variable(None) })) ); @@ -372,7 +364,7 @@ mod test { query, Query::SingleQuery(QueryPart::Matches(PatternQuery { entity: QueryComponent::Variable(None), - attribute: QueryComponent::In(vec!("FOO".into(), "BAR".into())), + attribute: QueryComponent::In(vec!("FOO".parse().unwrap(), "BAR".parse().unwrap())), value: QueryComponent::Variable(None) })) ); diff --git a/cli/src/extractors/audio.rs b/cli/src/extractors/audio.rs index 372c022..bceb434 100644 --- a/cli/src/extractors/audio.rs +++ b/cli/src/extractors/audio.rs @@ -18,14 +18,15 @@ use upend_db::{ lazy_static! { pub static ref ID3_TYPE_INVARIANT: InvariantEntry = InvariantEntry { - attribute: String::from(ATTR_KEY), + attribute: ATTR_KEY.parse().unwrap(), value: "TYPE_ID3".into(), }; pub static ref ID3_TYPE_LABEL: Entry = Entry { entity: ID3_TYPE_INVARIANT.entity().unwrap(), - attribute: ATTR_LABEL.into(), + attribute: ATTR_LABEL.parse().unwrap(), value: "ID3".into(), - ..Default::default() + provenance: "INVARIANT".to_string(), + timestamp: chrono::Utc::now().naive_utc(), }; } @@ -59,13 +60,14 @@ impl Extractor for ID3Extractor { let tags = id3::Tag::read_from_path(file_path)?; - let mut result: Vec = tags - .frames() - .flat_map(|frame| match frame.content() { - id3::Content::Text(text) => vec![ + let mut result: Vec = vec![]; + + for frame in tags.frames() { + if let id3::Content::Text(text) = frame.content() { + result.extend(vec![ Entry { entity: address.clone(), - attribute: format!("ID3_{}", frame.id()), + attribute: format!("ID3_{}", frame.id()).parse()?, value: match frame.id() { "TYER" | "TBPM" => EntryValue::guess_from(text), _ => text.clone().into(), @@ -74,16 +76,15 @@ impl Extractor for ID3Extractor { timestamp: chrono::Utc::now().naive_utc(), }, Entry { - entity: Address::Attribute(format!("ID3_{}", frame.id())), - attribute: ATTR_LABEL.into(), + entity: (format!("ID3_{}", frame.id())).parse()?, + attribute: ATTR_LABEL.parse().unwrap(), value: format!("ID3: {}", frame.name()).into(), provenance: "SYSTEM EXTRACTOR".to_string(), timestamp: chrono::Utc::now().naive_utc(), }, - ], - _ => vec![], - }) - .collect(); + ]); + } + } let mut has_pictures = false; for (idx, picture) in tags.pictures().enumerate() { @@ -99,7 +100,7 @@ impl Extractor for ID3Extractor { )?; result.push(Entry { entity: address.clone(), - attribute: "ID3_PICTURE".to_string(), + attribute: "ID3_PICTURE".parse()?, value: EntryValue::Address(Address::Hash(hash)), provenance: "SYSTEM EXTRACTOR".to_string(), timestamp: chrono::Utc::now().naive_utc(), @@ -108,8 +109,8 @@ impl Extractor for ID3Extractor { } if has_pictures { result.push(Entry { - entity: Address::Attribute("ID3_PICTURE".to_string()), - attribute: ATTR_LABEL.into(), + entity: "ID3_PICTURE".parse()?, + attribute: ATTR_LABEL.parse().unwrap(), value: "ID3 Embedded Image".into(), provenance: "SYSTEM EXTRACTOR".to_string(), timestamp: chrono::Utc::now().naive_utc(), @@ -123,9 +124,10 @@ impl Extractor for ID3Extractor { .filter(|e| e.attribute != ATTR_LABEL) .map(|e| Entry { entity: Address::Attribute(e.attribute.clone()), - attribute: ATTR_OF.into(), + attribute: ATTR_OF.parse().unwrap(), value: EntryValue::Address(ID3_TYPE_INVARIANT.entity().unwrap()), - ..Default::default() + provenance: "SYSTEM EXTRACTOR".to_string(), + timestamp: chrono::Utc::now().naive_utc(), }) .collect::>(), ); @@ -134,9 +136,10 @@ impl Extractor for ID3Extractor { ID3_TYPE_LABEL.clone(), Entry { entity: address.clone(), - attribute: ATTR_IN.into(), + attribute: ATTR_IN.parse().unwrap(), value: EntryValue::Address(ID3_TYPE_INVARIANT.entity().unwrap()), - ..Default::default() + provenance: "SYSTEM EXTRACTOR".to_string(), + timestamp: chrono::Utc::now().naive_utc(), }, ]); } diff --git a/cli/src/extractors/exif.rs b/cli/src/extractors/exif.rs index 919457a..8ddf074 100644 --- a/cli/src/extractors/exif.rs +++ b/cli/src/extractors/exif.rs @@ -3,6 +3,7 @@ use std::sync::Arc; use super::Extractor; use anyhow::{anyhow, Result}; use lazy_static::lazy_static; +use upend_base::entry::Attribute; use upend_base::{ addressing::Address, constants::{ATTR_IN, ATTR_KEY, ATTR_LABEL, ATTR_OF}, @@ -21,14 +22,15 @@ pub struct ExifExtractor; lazy_static! { pub static ref EXIF_TYPE_INVARIANT: InvariantEntry = InvariantEntry { - attribute: String::from(ATTR_KEY), + attribute: ATTR_KEY.parse().unwrap(), value: "TYPE_EXIF".into(), }; pub static ref EXIF_TYPE_LABEL: Entry = Entry { entity: EXIF_TYPE_INVARIANT.entity().unwrap(), - attribute: ATTR_LABEL.into(), + attribute: ATTR_LABEL.parse().unwrap(), value: "EXIF".into(), - ..Default::default() + provenance: "INVARIANT".to_string(), + timestamp: chrono::Utc::now().naive_utc(), }; } @@ -63,42 +65,40 @@ impl Extractor for ExifExtractor { let exifreader = exif::Reader::new(); let exif = exifreader.read_from_container(&mut bufreader)?; - let mut result: Vec = exif + let mut result: Vec = vec![]; + + for field in exif .fields() .filter(|field| !matches!(field.value, exif::Value::Undefined(..))) - .flat_map(|field| { - if let Some(tag_description) = field.tag.description() { - let attribute = format!("EXIF_{}", field.tag.1); + { + if let Some(tag_description) = field.tag.description() { + let attribute: Attribute = format!("EXIF_{}", field.tag.1).parse()?; - vec![ - Entry { - entity: address.clone(), - attribute: attribute.clone(), - value: match field.tag { - exif::Tag::ExifVersion => { - EntryValue::String(format!("{}", field.display_value())) - } - _ => EntryValue::guess_from(format!( - "{}", - field.display_value() - )), - }, - provenance: "SYSTEM EXTRACTOR".to_string(), - timestamp: chrono::Utc::now().naive_utc(), + result.extend(vec![ + Entry { + entity: address.clone(), + attribute: attribute.clone(), + value: match field.tag { + exif::Tag::ExifVersion => { + EntryValue::String(format!("{}", field.display_value())) + } + _ => { + EntryValue::guess_from(format!("{}", field.display_value())) + } }, - Entry { - entity: Address::Attribute(attribute), - attribute: ATTR_LABEL.into(), - value: format!("EXIF: {}", tag_description).into(), - provenance: "SYSTEM EXTRACTOR".to_string(), - timestamp: chrono::Utc::now().naive_utc(), - }, - ] - } else { - vec![] - } - }) - .collect(); + provenance: "SYSTEM EXTRACTOR".to_string(), + timestamp: chrono::Utc::now().naive_utc(), + }, + Entry { + entity: Address::Attribute(attribute), + attribute: ATTR_LABEL.parse().unwrap(), + value: format!("EXIF: {}", tag_description).into(), + provenance: "SYSTEM EXTRACTOR".to_string(), + timestamp: chrono::Utc::now().naive_utc(), + }, + ]); + } + } if !result.is_empty() { result.extend( @@ -107,9 +107,10 @@ impl Extractor for ExifExtractor { .filter(|e| e.attribute != ATTR_LABEL) .map(|e| Entry { entity: Address::Attribute(e.attribute.clone()), - attribute: ATTR_OF.into(), + attribute: ATTR_OF.parse().unwrap(), value: EntryValue::Address(EXIF_TYPE_INVARIANT.entity().unwrap()), - ..Default::default() + provenance: "SYSTEM EXTRACTOR".to_string(), + timestamp: chrono::Utc::now().naive_utc(), }) .collect::>(), ); @@ -120,9 +121,10 @@ impl Extractor for ExifExtractor { EXIF_TYPE_LABEL.clone(), Entry { entity: address.clone(), - attribute: ATTR_IN.into(), + attribute: ATTR_IN.parse().unwrap(), value: EntryValue::Address(EXIF_TYPE_INVARIANT.entity().unwrap()), - ..Default::default() + provenance: "SYSTEM EXTRACTOR".to_string(), + timestamp: chrono::Utc::now().naive_utc(), }, ]); } diff --git a/cli/src/extractors/media.rs b/cli/src/extractors/media.rs index 53f0bd1..3782ed8 100644 --- a/cli/src/extractors/media.rs +++ b/cli/src/extractors/media.rs @@ -18,20 +18,22 @@ const DURATION_KEY: &str = "MEDIA_DURATION"; lazy_static! { pub static ref MEDIA_TYPE_INVARIANT: InvariantEntry = InvariantEntry { - attribute: String::from(ATTR_KEY), + attribute: ATTR_KEY.parse().unwrap(), value: "TYPE_MEDIA".into(), }; pub static ref MEDIA_TYPE_LABEL: Entry = Entry { entity: MEDIA_TYPE_INVARIANT.entity().unwrap(), - attribute: ATTR_LABEL.into(), + attribute: ATTR_LABEL.parse().unwrap(), value: "Multimedia".into(), - ..Default::default() + provenance: "INVARIANT".to_string(), + timestamp: chrono::Utc::now().naive_utc(), }; pub static ref DURATION_OF_MEDIA: Entry = Entry { - entity: Address::Attribute(DURATION_KEY.to_string()), - attribute: ATTR_OF.into(), + entity: DURATION_KEY.parse().unwrap(), + attribute: ATTR_OF.parse().unwrap(), value: EntryValue::Address(MEDIA_TYPE_INVARIANT.entity().unwrap()), - ..Default::default() + provenance: "INVARIANT".to_string(), + timestamp: chrono::Utc::now().naive_utc(), }; } @@ -90,7 +92,7 @@ impl Extractor for MediaExtractor { let result = vec![ Entry { entity: address.clone(), - attribute: DURATION_KEY.to_string(), + attribute: DURATION_KEY.parse().unwrap(), value: EntryValue::Number(duration), provenance: "SYSTEM EXTRACTOR".to_string(), timestamp: chrono::Utc::now().naive_utc(), @@ -102,9 +104,10 @@ impl Extractor for MediaExtractor { DURATION_OF_MEDIA.clone(), Entry { entity: address.clone(), - attribute: ATTR_IN.into(), + attribute: ATTR_IN.parse().unwrap(), value: EntryValue::Address(MEDIA_TYPE_INVARIANT.entity().unwrap()), - ..Default::default() + provenance: "SYSTEM EXTRACTOR".to_string(), + timestamp: chrono::Utc::now().naive_utc(), }, ]; diff --git a/cli/src/extractors/web.rs b/cli/src/extractors/web.rs index 69ac882..8c72ade 100644 --- a/cli/src/extractors/web.rs +++ b/cli/src/extractors/web.rs @@ -40,21 +40,21 @@ impl Extractor for WebExtractor { let mut entries = vec![ html.title.as_ref().map(|html_title| Entry { entity: address.clone(), - attribute: "HTML_TITLE".to_string(), + attribute: "HTML_TITLE".parse().unwrap(), value: html_title.clone().into(), provenance: "SYSTEM EXTRACTOR".to_string(), timestamp: chrono::Utc::now().naive_utc(), }), html.title.map(|html_title| Entry { entity: address.clone(), - attribute: ATTR_LABEL.to_string(), + attribute: ATTR_LABEL.parse().unwrap(), value: html_title.into(), provenance: "SYSTEM EXTRACTOR".to_string(), timestamp: chrono::Utc::now().naive_utc(), }), html.description.map(|html_desc| Entry { entity: address.clone(), - attribute: "HTML_DESCRIPTION".to_string(), + attribute: "HTML_DESCRIPTION".parse().unwrap(), value: html_desc.into(), provenance: "SYSTEM EXTRACTOR".to_string(), timestamp: chrono::Utc::now().naive_utc(), @@ -65,7 +65,7 @@ impl Extractor for WebExtractor { if attribute == "OG_TITLE" { entries.push(Some(Entry { entity: address.clone(), - attribute: ATTR_LABEL.to_string(), + attribute: ATTR_LABEL.parse()?, value: value.clone().into(), provenance: "SYSTEM EXTRACTOR".to_string(), timestamp: chrono::Utc::now().naive_utc(), @@ -74,7 +74,7 @@ impl Extractor for WebExtractor { entries.push(Some(Entry { entity: address.clone(), - attribute, + attribute: attribute.parse()?, value: value.into(), provenance: "SYSTEM EXTRACTOR".to_string(), timestamp: chrono::Utc::now().naive_utc(), @@ -83,7 +83,7 @@ impl Extractor for WebExtractor { for image in html.opengraph.images { entries.push(Some(Entry { entity: address.clone(), - attribute: "OG_IMAGE".to_string(), + attribute: "OG_IMAGE".parse()?, value: image.url.into(), provenance: "SYSTEM EXTRACTOR".to_string(), timestamp: chrono::Utc::now().naive_utc(), @@ -99,10 +99,10 @@ impl Extractor for WebExtractor { vec![ Entry { entity: Address::Attribute(e.attribute.clone()), - attribute: ATTR_OF.to_string(), + attribute: ATTR_OF.parse().unwrap(), value: EntryValue::Address(TYPE_URL_ADDRESS.clone()), provenance: "SYSTEM EXTRACTOR".to_string(), - ..Default::default() + timestamp: chrono::Utc::now().naive_utc(), }, e, ] diff --git a/cli/src/routes.rs b/cli/src/routes.rs index 0ca0cab..abeb2bd 100644 --- a/cli/src/routes.rs +++ b/cli/src/routes.rs @@ -47,6 +47,7 @@ use url::Url; #[cfg(feature = "desktop")] use is_executable::IsExecutable; +use upend_base::error::UpEndError; #[derive(Clone)] pub struct State { @@ -417,7 +418,7 @@ pub async fn put_object( if let Some(entity) = in_entry.entity { Ok(Entry { entity: entity.try_into()?, - attribute: in_entry.attribute, + attribute: in_entry.attribute.parse()?, value: in_entry.value, provenance: (match &provenance { Some(s) => format!("API {}", s), @@ -429,7 +430,7 @@ pub async fn put_object( }) } else { Ok(Entry::try_from(&InvariantEntry { - attribute: in_entry.attribute, + attribute: in_entry.attribute.parse()?, value: in_entry.value, })?) } @@ -481,7 +482,7 @@ pub async fn put_object( if connection.retrieve_object(&address)?.is_empty() { connection.insert_entry(Entry { entity: address.clone(), - attribute: ATTR_ADDED.to_string(), + attribute: ATTR_ADDED.parse().unwrap(), value: EntryValue::Number( SystemTime::now() .duration_since(UNIX_EPOCH) @@ -622,7 +623,7 @@ pub async fn put_object_attribute( let new_attr_entry = Entry { entity: address, - attribute, + attribute: attribute.parse()?, value: value.into_inner(), provenance: (match &query.provenance { Some(s) => format!("API {}", s), @@ -688,7 +689,14 @@ pub async fn get_address( web::Query(query): web::Query>, ) -> Result { let (address, immutable) = if let Some(attribute) = query.get("attribute") { - (Address::Attribute(attribute.into()), true) + ( + Address::Attribute( + attribute + .parse() + .map_err(|e: UpEndError| ErrorBadRequest(e.to_string()))?, + ), + true, + ) } else if let Some(url) = query.get("url") { ( Address::Url(Url::parse(url).map_err(ErrorBadRequest)?), @@ -1145,7 +1153,7 @@ mod tests { assert_eq!(result["entity"]["t"], "Hash"); assert_eq!(result["entity"]["c"], digest_str); - let address = Address::Attribute("TEST".to_string()); + let address = Address::Attribute("TEST".parse().unwrap()); let req = actix_web::test::TestRequest::get() .uri(&format!("/api/obj/{}", address)) .to_request(); diff --git a/db/src/engine.rs b/db/src/engine.rs index fbcfd92..b516f46 100644 --- a/db/src/engine.rs +++ b/db/src/engine.rs @@ -18,7 +18,8 @@ use diesel::{BoxableExpression, QueryDsl}; use diesel::{ExpressionMethods, TextExpressionMethods}; use upend_base::addressing::Address; use upend_base::entry::{EntryPart, EntryValue}; -use upend_base::lang::{Attribute, Query, QueryComponent, QueryPart, QueryQualifier}; +use upend_base::error::UpEndError; +use upend_base::lang::{Query, QueryComponent, QueryPart, QueryQualifier}; #[derive(Debug, Clone)] pub struct QueryExecutionError(String); @@ -31,6 +32,12 @@ impl std::fmt::Display for QueryExecutionError { impl std::error::Error for QueryExecutionError {} +impl From for QueryExecutionError { + fn from(e: UpEndError) -> Self { + QueryExecutionError(e.to_string()) + } +} + pub fn execute( connection: &PooledConnection>, query: Query, @@ -126,7 +133,7 @@ pub fn execute( } EntryPart::Attribute(a) => { Some(EntryValue::Address(Address::Attribute( - a.0.clone(), + a.clone(), ))) } EntryPart::Value(v) => Some(v.clone()), @@ -172,9 +179,9 @@ pub fn execute( subquery_results .iter() .map(|e| { - EntryPart::Attribute(Attribute(e.attribute.clone())) + e.attribute.parse().map(|a| EntryPart::Attribute(a)) }) - .collect(), + .collect::, _>>()?, ); } @@ -266,10 +273,10 @@ fn to_sqlite_predicates(query: Query) -> Result match &eq.attribute { QueryComponent::Exact(q_attribute) => { - subqueries.push(Box::new(data::attribute.eq(q_attribute.0.clone()))) + subqueries.push(Box::new(data::attribute.eq(q_attribute.to_string()))) } QueryComponent::In(q_attributes) => subqueries.push(Box::new( - data::attribute.eq_any(q_attributes.iter().map(|a| &a.0).cloned()), + data::attribute.eq_any(q_attributes.iter().map(|a| a.to_string())), )), QueryComponent::Contains(q_attribute) => subqueries .push(Box::new(data::attribute.like(format!("%{}%", q_attribute)))), diff --git a/db/src/entry.rs b/db/src/entry.rs index 813c993..89d7814 100644 --- a/db/src/entry.rs +++ b/db/src/entry.rs @@ -1,25 +1,25 @@ use crate::inner::models; -use anyhow::{anyhow, Result}; use std::convert::TryFrom; use upend_base::addressing::{Address, Addressable}; use upend_base::entry::{Entry, EntryValue, ImmutableEntry}; +use upend_base::error::UpEndError; impl TryFrom<&models::Entry> for Entry { - type Error = anyhow::Error; + type Error = UpEndError; fn try_from(e: &models::Entry) -> Result { if let Some(value_str) = &e.value_str { Ok(Entry { entity: Address::decode(&e.entity)?, - attribute: e.attribute.clone(), - value: value_str.parse()?, + attribute: e.attribute.parse()?, + value: value_str.parse().unwrap(), provenance: e.provenance.clone(), timestamp: e.timestamp, }) } else if let Some(value_num) = e.value_num { Ok(Entry { entity: Address::decode(&e.entity)?, - attribute: e.attribute.clone(), + attribute: e.attribute.parse()?, value: EntryValue::Number(value_num), provenance: e.provenance.clone(), timestamp: e.timestamp, @@ -27,7 +27,7 @@ impl TryFrom<&models::Entry> for Entry { } else { Ok(Entry { entity: Address::decode(&e.entity)?, - attribute: e.attribute.clone(), + attribute: e.attribute.parse()?, value: EntryValue::Number(f64::NAN), provenance: e.provenance.clone(), timestamp: e.timestamp, @@ -40,18 +40,15 @@ impl TryFrom<&Entry> for models::Entry { type Error = anyhow::Error; fn try_from(e: &Entry) -> Result { - if e.attribute.is_empty() { - return Err(anyhow!("Attribute cannot be empty.")); - } let base_entry = models::Entry { identity: e.address()?.encode()?, entity_searchable: match &e.entity { - Address::Attribute(attr) => Some(attr.clone()), + Address::Attribute(attr) => Some(attr.to_string()), Address::Url(url) => Some(url.to_string()), _ => None, }, entity: e.entity.encode()?, - attribute: e.attribute.clone(), + attribute: e.attribute.to_string(), value_str: None, value_num: None, immutable: false, diff --git a/db/src/hierarchies.rs b/db/src/hierarchies.rs index a2a3020..01714f7 100644 --- a/db/src/hierarchies.rs +++ b/db/src/hierarchies.rs @@ -74,7 +74,7 @@ pub fn list_roots(connection: &UpEndConnection) -> Result> { Ok(connection .query(Query::SingleQuery(QueryPart::Matches(PatternQuery { entity: QueryComponent::Variable(None), - attribute: QueryComponent::Exact(ATTR_IN.into()), + attribute: QueryComponent::Exact(ATTR_IN.parse().unwrap()), value: QueryComponent::Exact((*HIER_ROOT_ADDR).clone().into()), })))? .into_iter() @@ -105,7 +105,7 @@ pub fn fetch_or_create_dir( let matching_directories = connection .query(Query::SingleQuery(QueryPart::Matches(PatternQuery { entity: QueryComponent::Variable(None), - attribute: QueryComponent::Exact(ATTR_LABEL.into()), + attribute: QueryComponent::Exact(ATTR_LABEL.parse().unwrap()), value: QueryComponent::Exact(directory.to_string().into()), })))? .into_iter() @@ -115,7 +115,7 @@ pub fn fetch_or_create_dir( Some(parent) => connection .query(Query::SingleQuery(QueryPart::Matches(PatternQuery { entity: QueryComponent::Variable(None), - attribute: QueryComponent::Exact(ATTR_IN.into()), + attribute: QueryComponent::Exact(ATTR_IN.parse().unwrap()), value: QueryComponent::Exact(parent.into()), })))? .into_iter() @@ -135,7 +135,7 @@ pub fn fetch_or_create_dir( let directory_entry = Entry { entity: new_directory_address.clone(), - attribute: String::from(ATTR_LABEL), + attribute: ATTR_LABEL.parse().unwrap(), value: directory.to_string().into(), provenance: "SYSTEM FS".to_string(), timestamp: chrono::Utc::now().naive_utc(), @@ -145,7 +145,7 @@ pub fn fetch_or_create_dir( connection.insert_entry(if let Some(parent) = parent { Entry { entity: new_directory_address.clone(), - attribute: String::from(ATTR_IN), + attribute: ATTR_IN.parse().unwrap(), value: parent.into(), provenance: "SYSTEM FS".to_string(), timestamp: chrono::Utc::now().naive_utc(), @@ -153,7 +153,7 @@ pub fn fetch_or_create_dir( } else { Entry { entity: new_directory_address.clone(), - attribute: String::from(ATTR_IN), + attribute: ATTR_IN.parse().unwrap(), value: HIER_ROOT_ADDR.clone().into(), provenance: "SYSTEM FS".to_string(), timestamp: chrono::Utc::now().naive_utc(), diff --git a/db/src/lib.rs b/db/src/lib.rs index d0733d8..0edbe66 100644 --- a/db/src/lib.rs +++ b/db/src/lib.rs @@ -40,7 +40,7 @@ use std::sync::{Arc, Mutex, RwLock}; use std::time::Duration; use tracing::{debug, error, trace, warn}; use upend_base::addressing::{Address, Addressable}; -use upend_base::entry::{Entry, EntryValue, ImmutableEntry}; +use upend_base::entry::{Attribute, Entry, EntryValue, ImmutableEntry}; use upend_base::error::UpEndError; use upend_base::hash::UpMultihash; use upend_base::lang::Query; @@ -308,7 +308,7 @@ impl UpEndConnection { let entries = primary .iter() .map(Entry::try_from) - .collect::>>()?; + .collect::, UpEndError>>()?; let secondary = data .filter( @@ -326,7 +326,7 @@ impl UpEndConnection { let secondary_entries = secondary .iter() .map(Entry::try_from) - .collect::>>()?; + .collect::, UpEndError>>()?; Ok([entries, secondary_entries].concat()) } @@ -414,7 +414,7 @@ impl UpEndConnection { } // #[deprecated] - pub fn get_all_attributes(&self) -> Result> { + pub fn get_all_attributes(&self) -> Result> { use crate::inner::schema::data::dsl::*; let _lock = self.lock.read().unwrap(); @@ -426,7 +426,10 @@ impl UpEndConnection { .order_by(attribute) .load::(&conn)?; - Ok(result) + Ok(result + .into_iter() + .map(|a| a.parse()) + .collect::, UpEndError>>()?) } pub fn get_stats(&self) -> Result { @@ -478,10 +481,10 @@ impl UpEndConnection { ) .load(&conn)?; - result + Ok(result .iter() .map(Entry::try_from) - .collect::>>() + .collect::, UpEndError>>()?) } } diff --git a/db/src/macros.rs b/db/src/macros.rs index c2915c5..44dab6e 100644 --- a/db/src/macros.rs +++ b/db/src/macros.rs @@ -3,7 +3,7 @@ macro_rules! upend_insert_val { ($db_connection:expr, $entity:expr, $attribute:expr, $value:expr) => {{ $db_connection.insert_entry(Entry { entity: $entity.clone(), - attribute: String::from($attribute), + attribute: $attribute.parse().unwrap(), value: upend_base::entry::EntryValue::String(String::from($value)), provenance: "SYSTEM INIT".to_string(), timestamp: chrono::Utc::now().naive_utc(), @@ -16,7 +16,7 @@ macro_rules! upend_insert_addr { ($db_connection:expr, $entity:expr, $attribute:expr, $addr:expr) => {{ $db_connection.insert_entry(Entry { entity: $entity.clone(), - attribute: String::from($attribute), + attribute: $attribute.parse().unwrap(), value: upend_base::entry::EntryValue::Address($addr.clone()), provenance: "SYSTEM INIT".to_string(), timestamp: chrono::Utc::now().naive_utc(), diff --git a/db/src/stores/fs/mod.rs b/db/src/stores/fs/mod.rs index e7409ab..84dcf90 100644 --- a/db/src/stores/fs/mod.rs +++ b/db/src/stores/fs/mod.rs @@ -108,13 +108,13 @@ impl FsStore { trace!("Initializing DB types."); upend_insert_addr!( upconnection, - Address::Attribute(FILE_SIZE_KEY.to_string()), + Address::Attribute(FILE_SIZE_KEY.parse().unwrap()), ATTR_OF, TYPE_HASH_ADDRESS )?; upend_insert_addr!( upconnection, - Address::Attribute(FILE_MIME_KEY.to_string()), + Address::Attribute(FILE_MIME_KEY.parse().unwrap()), ATTR_OF, TYPE_HASH_ADDRESS )?; @@ -438,7 +438,7 @@ impl FsStore { // Metadata let size_entry = Entry { entity: blob_address.clone(), - attribute: FILE_SIZE_KEY.to_string(), + attribute: FILE_SIZE_KEY.parse().unwrap(), value: (size as f64).into(), provenance: "SYSTEM INIT".to_string(), timestamp: chrono::Utc::now().naive_utc(), @@ -447,7 +447,7 @@ impl FsStore { let mime_type = tree_magic_mini::from_filepath(path).map(|s| s.to_string()); let mime_entry = mime_type.map(|mime_type| Entry { entity: blob_address.clone(), - attribute: FILE_MIME_KEY.to_string(), + attribute: FILE_MIME_KEY.parse().unwrap(), value: mime_type.into(), provenance: "SYSTEM INIT".to_string(), timestamp: chrono::Utc::now().naive_utc(), @@ -455,7 +455,7 @@ impl FsStore { let added_entry = Entry { entity: blob_address.clone(), - attribute: ATTR_ADDED.to_string(), + attribute: ATTR_ADDED.parse().unwrap(), value: (SystemTime::now() .duration_since(UNIX_EPOCH) .unwrap() @@ -480,7 +480,7 @@ impl FsStore { let label_entry = Entry { entity: blob_address.clone(), - attribute: ATTR_LABEL.to_string(), + attribute: ATTR_LABEL.parse().unwrap(), value: name .unwrap_or_else(|| filename.as_os_str().to_string_lossy().to_string()) .into(), @@ -498,7 +498,7 @@ impl FsStore { let dir_has_entry = Entry { entity: blob_address.clone(), - attribute: ATTR_IN.to_string(), + attribute: ATTR_IN.parse().unwrap(), value: parent_dir.clone().into(), provenance: "SYSTEM INIT".to_string(), timestamp: chrono::Utc::now().naive_utc(), @@ -507,7 +507,7 @@ impl FsStore { let alias_entry = Entry { entity: dir_has_entry_addr, - attribute: ATTR_BY.to_string(), + attribute: ATTR_BY.parse().unwrap(), value: label_entry_addr.into(), provenance: "SYSTEM INIT".to_string(), timestamp: chrono::Utc::now().naive_utc(),