From 3a6db03ac19ed7837678a6fbd564621d0f4b1806 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1=C5=A1=20Ml=C3=A1dek?= Date: Tue, 29 Aug 2023 18:27:55 +0200 Subject: [PATCH] wip --- base/src/constants.rs | 4 +- base/src/entry.rs | 80 +++++++++++++------------------------ base/src/error.rs | 8 ++++ base/src/lang.rs | 38 ++++++++++++++---- cli/src/routes.rs | 2 +- db/src/engine.rs | 12 +----- db/src/entry.rs | 4 +- db/src/hierarchies.rs | 6 +-- db/src/lib.rs | 4 +- db/src/macros.rs | 2 +- db/src/stores/fs/mod.rs | 19 +++++---- tools/upend_text/Cargo.toml | 14 +++++++ tools/upend_text/src/lib.rs | 45 +++++++++++++++++++++ 13 files changed, 150 insertions(+), 88 deletions(-) create mode 100644 tools/upend_text/Cargo.toml create mode 100644 tools/upend_text/src/lib.rs diff --git a/base/src/constants.rs b/base/src/constants.rs index 96ae18d..89de318 100644 --- a/base/src/constants.rs +++ b/base/src/constants.rs @@ -1,4 +1,4 @@ -use crate::addressing::Address; +use crate::addressing::{Address, Addressable}; use crate::entry::InvariantEntry; use crate::hash::{LargeMultihash, UpMultihash}; @@ -20,7 +20,7 @@ pub const ATTR_KEY: &str = "KEY"; lazy_static! { pub static ref HIER_ROOT_INVARIANT: InvariantEntry = InvariantEntry { attribute: String::from(ATTR_KEY), - value: "HIER_ROOT".into(), + value: "HIER_ROOT".address().unwrap().into(), }; pub static ref HIER_ROOT_ADDR: Address = HIER_ROOT_INVARIANT.entity().unwrap(); pub static ref TYPE_HASH_ADDRESS: Address = diff --git a/base/src/entry.rs b/base/src/entry.rs index 931e987..de5e5a4 100644 --- a/base/src/entry.rs +++ b/base/src/entry.rs @@ -1,4 +1,4 @@ -use crate::addressing::Address; +use crate::addressing::{Address, Addressable}; use crate::error::UpEndError; use crate::hash::{b58_decode, sha256hash, AsMultihash, AsMultihashError, UpMultihash}; use chrono::NaiveDateTime; @@ -28,11 +28,9 @@ pub struct InvariantEntry { #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] #[serde(tag = "t", content = "c")] pub enum EntryValue { - String(String), - Number(f64), Address(Address), + Number(f64), Null, - Invalid, } impl Default for Entry { @@ -68,7 +66,7 @@ impl InvariantEntry { .write_all(self.attribute.as_bytes()) .map_err(UpEndError::from_any)?; entity - .write_all(self.value.to_string()?.as_bytes()) + .write_all(self.value.to_string().as_bytes()) .map_err(UpEndError::from_any)?; Ok(Address::Hash( sha256hash(entity.into_inner()).map_err(UpEndError::from_any)?, @@ -92,12 +90,7 @@ impl AsMultihash for Entry { .as_slice(), )?; result.write_all(self.attribute.as_bytes())?; - result.write_all( - self.value - .to_string() - .map_err(|e| AsMultihashError(e.to_string()))? - .as_bytes(), - )?; + result.write_all(self.value.to_string().as_bytes())?; sha256hash(result.get_ref()) } } @@ -111,27 +104,25 @@ impl AsMultihash for InvariantEntry { } impl EntryValue { - pub fn to_string(&self) -> Result { + pub fn to_string(&self) -> String { let (type_char, content) = match self { - EntryValue::String(value) => ('S', value.to_owned()), - EntryValue::Number(n) => ('N', n.to_string()), EntryValue::Address(address) => ('O', address.to_string()), + EntryValue::Number(n) => ('N', n.to_string()), EntryValue::Null => ('X', "".to_string()), - EntryValue::Invalid => return Err(UpEndError::CannotSerializeInvalid), }; - Ok(format!("{}{}", type_char, content)) + format!("{}{}", type_char, content) } - pub fn guess_from>(string: S) -> Self { + pub fn guess_from>(string: S) -> Result { let string = string.as_ref(); match string.parse::() { - Ok(num) => EntryValue::Number(num), + Ok(num) => Ok(EntryValue::Number(num)), Err(_) => { if let Ok(url) = Url::parse(string) { - EntryValue::Address(Address::Url(url)) + Ok(EntryValue::Address(Address::Url(url))) } else { - EntryValue::String(string.to_string()) + Ok(EntryValue::Address(string.address()?)) } } } @@ -139,34 +130,32 @@ impl EntryValue { } impl std::str::FromStr for EntryValue { - type Err = std::convert::Infallible; + type Err = UpEndError; fn from_str(s: &str) -> Result { if s.len() < 2 { match s.chars().next() { - Some('S') => Ok(EntryValue::String("".into())), Some('X') => Ok(EntryValue::Null), - _ => Ok(EntryValue::Invalid), + _ => Err(UpEndError::EntryValueInvalid(s.to_string())), } } else { let (type_char, content) = s.split_at(1); match (type_char, content) { - ("S", content) => Ok(EntryValue::String(String::from(content))), ("N", content) => { if let Ok(n) = content.parse::() { Ok(EntryValue::Number(n)) } else { - Ok(EntryValue::Invalid) + Err(UpEndError::EntryValueInvalid(s.to_string())) } } ("O", content) => { if let Ok(addr) = b58_decode(content).and_then(|v| Address::decode(&v)) { Ok(EntryValue::Address(addr)) } else { - Ok(EntryValue::Invalid) + Err(UpEndError::EntryValueInvalid(s.to_string())) } } - _ => Ok(EntryValue::Invalid), + _ => Err(UpEndError::EntryValueInvalid(s.to_string())), } } } @@ -182,27 +171,13 @@ impl std::fmt::Display for EntryValue { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { let (entry_type, entry_value) = match self { EntryValue::Address(address) => ("ADDRESS", address.to_string()), - EntryValue::String(string) => ("STRING", string.to_owned()), EntryValue::Number(n) => ("NUMBER", n.to_string()), EntryValue::Null => ("NULL", "NULL".to_string()), - EntryValue::Invalid => ("INVALID", "INVALID".to_string()), }; write!(f, "{}: {}", entry_type, entry_value) } } -impl From<&str> for EntryValue { - fn from(str: &str) -> Self { - Self::String(str.to_string()) - } -} - -impl From for EntryValue { - fn from(str: String) -> Self { - Self::String(str) - } -} - impl From for EntryValue { fn from(num: f64) -> Self { Self::Number(num) @@ -221,28 +196,28 @@ mod tests { #[test] fn test_value_from_to_string() -> Result<(), UpEndError> { - let entry = EntryValue::String("hello".to_string()); - let encoded = entry.to_string()?; + let entry = EntryValue::Address("hello".address().unwrap()); + let encoded = entry.to_string(); let decoded = encoded.parse::().unwrap(); assert_eq!(entry, decoded); let entry = EntryValue::Number(1337.93); - let encoded = entry.to_string()?; + let encoded = entry.to_string(); let decoded = encoded.parse::().unwrap(); assert_eq!(entry, decoded); let entry = EntryValue::Address(Address::Url(Url::parse("https://upend.dev").unwrap())); - let encoded = entry.to_string()?; + let encoded = entry.to_string(); let decoded = encoded.parse::().unwrap(); assert_eq!(entry, decoded); - let entry = EntryValue::String("".to_string()); - let encoded = entry.to_string()?; + let entry = EntryValue::Address("".address().unwrap()); + let encoded = entry.to_string(); let decoded = encoded.parse::().unwrap(); assert_eq!(entry, decoded); let entry = EntryValue::Null; - let encoded = entry.to_string()?; + let encoded = entry.to_string(); let decoded = encoded.parse::().unwrap(); assert_eq!(entry, decoded); @@ -251,7 +226,6 @@ mod tests { #[test] fn test_into() { - assert_eq!(EntryValue::String(String::from("UPEND")), "UPEND".into()); assert_eq!(EntryValue::Number(1337.93), 1337.93.into()); let addr = Address::Url(Url::parse("https://upend.dev").unwrap()); assert_eq!(EntryValue::Address(addr.clone()), addr.into()); @@ -260,15 +234,15 @@ mod tests { #[test] fn test_guess_value() { assert_eq!( - EntryValue::guess_from("UPEND"), - EntryValue::String("UPEND".into()) + EntryValue::guess_from("UPEND").unwrap(), + EntryValue::Address("UPEND".address().unwrap()) ); assert_eq!( - EntryValue::guess_from("1337.93"), + EntryValue::guess_from("1337.93").unwrap(), EntryValue::Number(1337.93) ); assert_eq!( - EntryValue::guess_from("https://upend.dev"), + EntryValue::guess_from("https://upend.dev").unwrap(), EntryValue::Address(Address::Url(Url::parse("https://upend.dev").unwrap())) ); } diff --git a/base/src/error.rs b/base/src/error.rs index ed0bf62..e7cbb14 100644 --- a/base/src/error.rs +++ b/base/src/error.rs @@ -2,6 +2,7 @@ pub enum UpEndError { HashDecodeError(String), AddressParseError(String), + EntryValueInvalid(String), AddressComponentsDecodeError(AddressComponentsDecodeError), CannotSerializeInvalid, QueryParseError(String), @@ -23,6 +24,7 @@ impl std::fmt::Display for UpEndError { match self { UpEndError::HashDecodeError(err) => format!("Could not decode hash: {err}"), UpEndError::AddressParseError(err) => format!("Error parsing address: {err}"), + UpEndError::EntryValueInvalid(err) => format!("Invalid Entry value: {err}"), UpEndError::AddressComponentsDecodeError(cde) => match cde { AddressComponentsDecodeError::UnknownType(t) => format!("Unknown type: \"{t}\""), @@ -47,3 +49,9 @@ impl UpEndError { UpEndError::Other(error.to_string()) } } + +impl From for UpEndError { + fn from(error: crate::hash::AsMultihashError) -> Self { + UpEndError::Other(error.to_string()) + } +} diff --git a/base/src/lang.rs b/base/src/lang.rs index 3218df8..f339637 100644 --- a/base/src/lang.rs +++ b/base/src/lang.rs @@ -1,4 +1,4 @@ -use crate::addressing::Address; +use crate::addressing::{Address, Addressable}; use crate::entry::EntryValue; use crate::error::UpEndError; use nonempty::NonEmpty; @@ -64,8 +64,28 @@ impl TryFrom for EntryValue { lexpr::Value::Number(num) => Ok(EntryValue::Number(num.as_f64().ok_or_else(|| { UpEndError::QueryParseError(format!("Error processing number ({num:?}).")) })?)), - lexpr::Value::Char(chr) => Ok(EntryValue::String(chr.to_string())), - lexpr::Value::String(str) => Ok(EntryValue::String(str.to_string())), + lexpr::Value::Char(chr) => Ok(EntryValue::Address( + chr.to_string() + .address() + .map_err(|e| { + UpEndError::QueryParseError(format!( + "Error hashing character `{}`: {:}", + chr, e + )) + })? + .into(), + )), + lexpr::Value::String(str) => Ok(EntryValue::Address( + str.to_string() + .address() + .map_err(|e| { + UpEndError::QueryParseError(format!( + "Error hashing string `{}`: {:}", + str, e + )) + })? + .into(), + )), lexpr::Value::Symbol(_) => Ok(EntryValue::Address(Address::try_from(value.clone())?)), _ => Err(UpEndError::QueryParseError( "Value can only be a string, number or address.".into(), @@ -298,7 +318,7 @@ impl FromStr for Query { #[cfg(test)] mod test { - use crate::error::UpEndError; + use crate::{addressing::Addressable, error::UpEndError}; use super::*; use url::Url; @@ -377,7 +397,10 @@ mod test { })) ); - let values: Vec = vec!["FOO".into(), "BAR".into()]; + let values: Vec = vec![ + "FOO".address().unwrap().into(), + "BAR".address().unwrap().into(), + ]; let query = r#"(matches ? ? (in "FOO" "BAR"))"#.parse::()?; assert_eq!( query, @@ -400,7 +423,8 @@ mod test { ); // Invalid queries - let values: Vec = vec!["FOO".into(), EntryValue::Number(1337.93)]; + let values: Vec = + vec!["FOO".address().unwrap().into(), EntryValue::Number(1337.93)]; let query = r#"(matches ? ? (in "FOO" 1337.93))"#.parse::()?; assert_eq!( @@ -412,7 +436,7 @@ mod test { })) ); - let values = vec![EntryValue::Number(1337.93), "FOO".into()]; + let values = vec![EntryValue::Number(1337.93), "FOO".address().unwrap().into()]; let query = r#"(matches ? ? (in 1337.93 "FOO"))"#.parse::()?; assert_eq!( diff --git a/cli/src/routes.rs b/cli/src/routes.rs index fc705f4..9923a8d 100644 --- a/cli/src/routes.rs +++ b/cli/src/routes.rs @@ -698,7 +698,7 @@ pub async fn get_all_attributes(state: web::Data) -> Result Result match &eq.value { QueryComponent::Exact(q_value) => match q_value { EntryValue::Number(n) => subqueries.push(Box::new(data::value_num.eq(*n))), - _ => subqueries.push(Box::new(data::value_str.eq( - q_value.to_string().map_err(|e| { - QueryExecutionError(format!("failed producing sql: {e}")) - })?, - ))), + _ => subqueries.push(Box::new(data::value_str.eq(q_value.to_string()))), }, QueryComponent::In(q_values) => { let first = q_values.first().ok_or_else(|| { @@ -239,11 +235,7 @@ fn to_sqlite_predicates(query: Query) -> Result string values! (Found {v})" ))) } else { - v.to_string().map_err(|e| { - QueryExecutionError(format!( - "failed producing sql: {e}" - )) - }) + Ok(v.to_string()) } }) .collect::, QueryExecutionError>>()?, diff --git a/db/src/entry.rs b/db/src/entry.rs index 813c993..fc3801f 100644 --- a/db/src/entry.rs +++ b/db/src/entry.rs @@ -48,7 +48,7 @@ impl TryFrom<&Entry> for models::Entry { entity_searchable: match &e.entity { Address::Attribute(attr) => Some(attr.clone()), Address::Url(url) => Some(url.to_string()), - _ => None, + _ => None, // TODO }, entity: e.entity.encode()?, attribute: e.attribute.clone(), @@ -66,7 +66,7 @@ impl TryFrom<&Entry> for models::Entry { ..base_entry }), _ => Ok(models::Entry { - value_str: Some(e.value.to_string()?), + value_str: Some(e.value.to_string()), value_num: None, ..base_entry }), diff --git a/db/src/hierarchies.rs b/db/src/hierarchies.rs index e15e2ef..97db2d2 100644 --- a/db/src/hierarchies.rs +++ b/db/src/hierarchies.rs @@ -6,7 +6,7 @@ use lru::LruCache; use tracing::trace; use uuid::Uuid; -use upend_base::addressing::Address; +use upend_base::addressing::{Address, Addressable}; use upend_base::constants::ATTR_LABEL; use upend_base::constants::{ATTR_IN, HIER_ROOT_ADDR, HIER_ROOT_INVARIANT}; use upend_base::entry::Entry; @@ -112,7 +112,7 @@ pub fn fetch_or_create_dir( .query(Query::SingleQuery(QueryPart::Matches(PatternQuery { entity: QueryComponent::Variable(None), attribute: QueryComponent::Exact(ATTR_LABEL.into()), - value: QueryComponent::Exact(String::from(directory.clone()).into()), + value: QueryComponent::Exact(String::from(directory.clone()).address()?.into()), })))? .into_iter() .map(|e: Entry| e.entity); @@ -142,7 +142,7 @@ pub fn fetch_or_create_dir( let directory_entry = Entry { entity: new_directory_address.clone(), attribute: String::from(ATTR_LABEL), - value: String::from(directory).into(), + value: String::from(directory).address()?.into(), provenance: "SYSTEM FS".to_string(), timestamp: chrono::Utc::now().naive_utc(), }; diff --git a/db/src/lib.rs b/db/src/lib.rs index 47a134d..bf431c7 100644 --- a/db/src/lib.rs +++ b/db/src/lib.rs @@ -247,7 +247,7 @@ impl UpEndConnection { let primary = data .filter(entity.eq(object_address.encode()?)) - .or_filter(value_str.eq(EntryValue::Address(object_address.clone()).to_string()?)) + .or_filter(value_str.eq(EntryValue::Address(object_address.clone()).to_string())) .load::(&conn)?; let entries = primary @@ -287,7 +287,7 @@ impl UpEndConnection { let matches = data .filter(identity.eq(object_address.encode()?)) .or_filter(entity.eq(object_address.encode()?)) - .or_filter(value_str.eq(EntryValue::Address(object_address).to_string()?)); + .or_filter(value_str.eq(EntryValue::Address(object_address).to_string())); Ok(diesel::delete(matches).execute(&conn)?) } diff --git a/db/src/macros.rs b/db/src/macros.rs index c2915c5..8625809 100644 --- a/db/src/macros.rs +++ b/db/src/macros.rs @@ -4,7 +4,7 @@ macro_rules! upend_insert_val { $db_connection.insert_entry(Entry { entity: $entity.clone(), attribute: String::from($attribute), - value: upend_base::entry::EntryValue::String(String::from($value)), + value: upend_base::entry::EntryValue::Address(String::from($value).address().unwrap()), provenance: "SYSTEM INIT".to_string(), timestamp: chrono::Utc::now().naive_utc(), }) diff --git a/db/src/stores/fs/mod.rs b/db/src/stores/fs/mod.rs index 5639c7a..10524b9 100644 --- a/db/src/stores/fs/mod.rs +++ b/db/src/stores/fs/mod.rs @@ -21,7 +21,7 @@ use std::sync::{Arc, Mutex, RwLock}; use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; use std::{fs, iter}; use tracing::{error, info, trace, warn}; -use upend_base::addressing::Address; +use upend_base::addressing::{Address, Addressable}; use upend_base::constants::{ATTR_ADDED, ATTR_BY, ATTR_IN, ATTR_LABEL, ATTR_OF, TYPE_HASH_ADDRESS}; use upend_base::entry::Entry; use upend_base::hash::{b58_encode, UpMultihash}; @@ -410,12 +410,16 @@ impl FsStore { timestamp: chrono::Utc::now().naive_utc(), }; - let mime_entry = mime_type.map(|mime_type| Entry { - entity: blob_address.clone(), - attribute: FILE_MIME_KEY.to_string(), - value: mime_type.into(), - provenance: "SYSTEM INIT".to_string(), - timestamp: chrono::Utc::now().naive_utc(), + let mime_entry = mime_type.and_then(|mime_type| { + mime_type.address().ok().and_then(|mime_type_address| { + Some(Entry { + entity: blob_address.clone(), + attribute: FILE_MIME_KEY.to_string(), + value: mime_type_address.into(), + provenance: "SYSTEM INIT".to_string(), + timestamp: chrono::Utc::now().naive_utc(), + }) + }) }); let added_entry = Entry { @@ -472,6 +476,7 @@ impl FsStore { attribute: ATTR_LABEL.to_string(), value: name .unwrap_or_else(|| filename.as_os_str().to_string_lossy().to_string()) + .address()? .into(), provenance: "SYSTEM INIT".to_string(), timestamp: chrono::Utc::now().naive_utc(), diff --git a/tools/upend_text/Cargo.toml b/tools/upend_text/Cargo.toml new file mode 100644 index 0000000..5fd018a --- /dev/null +++ b/tools/upend_text/Cargo.toml @@ -0,0 +1,14 @@ +[package] +name = "upend_text" +version = "0.1.0" +edition = "2021" + +[lib] +path = "src/lib.rs" +crate-type = ["cdylib", "rlib"] + +[dependencies] +pulldown-cmark = "0.9.3" +serde = "1.0.188" +wasm-bindgen = "0.2.87" +wee_alloc = "0.4.5" diff --git a/tools/upend_text/src/lib.rs b/tools/upend_text/src/lib.rs new file mode 100644 index 0000000..a98d7d3 --- /dev/null +++ b/tools/upend_text/src/lib.rs @@ -0,0 +1,45 @@ +use pulldown_cmark::{Options, Parser}; + +pub struct Annotation { + pub id: String, + pub body: String, + pub target: String, +} + +pub struct MarkdownResult { + pub text: String, + pub annotations: Vec, +} + +#[derive(Debug)] +pub struct UpEndTextError(pub String); + +impl std::fmt::Display for UpEndTextError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{:#?}", self.0) + } +} + +pub fn markdown_to_annotations(markdown: &str) -> Result { + let mut options = Options::empty(); + options.insert(Options::ENABLE_STRIKETHROUGH); + let parser = Parser::new_ext(markdown, options); + + for ev in parser { + println!("{:?}", ev); + } + + todo!() +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn it_works() { + let markdown_input = "Hello world, this is a ~~complicated~~ *very simple* example."; + let result = markdown_to_annotations(markdown_input); + result.unwrap(); + } +}