wip, test: add tests for only-entity queries

base/src/entry.rs

@@ -7,7 +7,7 @@ use std::convert::TryFrom;
 use std::io::{Cursor, Write};
 use url::Url;
 
-#[derive(Debug, Clone, Serialize, Deserialize)]
+#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
 pub struct Entry {
     pub entity: Address,
     pub attribute: String,

base/src/lang.rs

@@ -1,6 +1,7 @@
 use crate::addressing::Address;
 use crate::entry::EntryValue;
 use crate::error::UpEndError;
+use chrono::NaiveDateTime;
 use nonempty::NonEmpty;
 use std::borrow::Borrow;
 use std::convert::TryFrom;
@@ -15,8 +16,8 @@ impl From<&str> for Attribute {
     }
 }
 
-#[derive(Debug, Clone, PartialEq)]
+#[derive(Debug, Clone, PartialEq, Default)]
-pub enum QueryComponent<T>
+pub enum PatternQueryComponent<T>
 where
     T: TryFrom<lexpr::Value>,
 {
@@ -24,13 +25,23 @@ where
     In(Vec<T>),
     Contains(String),
     Variable(Option<String>),
+    #[default]
+    Discard,
 }
 
-#[derive(Debug, Clone, PartialEq)]
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub struct Provenance(pub String);
+
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub struct Timestamp(pub NaiveDateTime);
+
+#[derive(Debug, Clone, PartialEq, Default)]
 pub struct PatternQuery {
-    pub entity: QueryComponent<Address>,
+    pub entity: PatternQueryComponent<Address>,
-    pub attribute: QueryComponent<Attribute>,
+    pub attribute: PatternQueryComponent<Attribute>,
-    pub value: QueryComponent<EntryValue>,
+    pub value: PatternQueryComponent<EntryValue>,
+    pub provenance: Option<PatternQueryComponent<Provenance>>,
+    pub timestamp: Option<PatternQueryComponent<Timestamp>>,
 }
 
 impl TryFrom<lexpr::Value> for Address {
@@ -87,6 +98,42 @@ impl TryFrom<lexpr::Value> for Attribute {
     }
 }
 
+impl TryFrom<lexpr::Value> for Provenance {
+    type Error = UpEndError;
+
+    fn try_from(value: lexpr::Value) -> Result<Self, Self::Error> {
+        match value {
+            lexpr::Value::String(str) => Ok(Provenance(str.to_string())),
+            _ => Err(UpEndError::QueryParseError(
+                "Can only convert to provenance from string.".into(),
+            )),
+        }
+    }
+}
+
+impl TryFrom<lexpr::Value> for Timestamp {
+    type Error = UpEndError;
+
+    fn try_from(value: lexpr::Value) -> Result<Self, Self::Error> {
+        match value {
+            lexpr::Value::Number(num) => {
+                if let Some(num) = num.as_i64() {
+                    Ok(Timestamp(NaiveDateTime::from_timestamp_opt(num, 0).ok_or(
+                        UpEndError::QueryParseError("Couldn't parse timestamp.".into()),
+                    )?))
+                } else {
+                    Err(UpEndError::QueryParseError(
+                        "Couldn't parse number as i64.".into(),
+                    ))
+                }
+            }
+            _ => Err(UpEndError::QueryParseError(
+                "Can only convert to attribute from string.".into(),
+            )),
+        }
+    }
+}
+
 #[derive(Debug, Clone, PartialEq)]
 pub enum QueryPart {
     Matches(PatternQuery),
@@ -120,7 +167,7 @@ impl TryFrom<&lexpr::Value> for Query {
     fn try_from(expression: &lexpr::Value) -> Result<Self, Self::Error> {
         fn parse_component<T: TryFrom<lexpr::Value>>(
             value: &lexpr::Value,
-        ) -> Result<QueryComponent<T>, UpEndError>
+        ) -> Result<PatternQueryComponent<T>, UpEndError>
         where
             UpEndError: From<<T as TryFrom<lexpr::Value>>::Error>,
         {
@@ -137,7 +184,7 @@ impl TryFrom<&lexpr::Value> for Query {
                                         .map(|value| T::try_from(value.clone()))
                                         .collect();
 
-                                    Ok(QueryComponent::In(values?))
+                                    Ok(PatternQueryComponent::In(values?))
                                 } else {
                                     Err(UpEndError::QueryParseError(
                                         "Malformed expression: Inner value cannot be empty.".into(),
@@ -150,7 +197,7 @@ impl TryFrom<&lexpr::Value> for Query {
                                     2 => {
                                         let value = cons_vec.remove(1);
                                         if let lexpr::Value::String(str) = value {
-                                            Ok(QueryComponent::Contains(str.into_string()))
+                                            Ok(PatternQueryComponent::Contains(str.into_string()))
                                         } else {
                                             Err(UpEndError::QueryParseError("Malformed expression: 'Contains' argument must be a string.".into()))
                                         }
@@ -174,13 +221,16 @@ impl TryFrom<&lexpr::Value> for Query {
                 }
                 lexpr::Value::Symbol(symbol) if symbol.starts_with('?') => {
                     let var_name = symbol.strip_prefix('?').unwrap();
-                    Ok(QueryComponent::Variable(if var_name.is_empty() {
+                    Ok(PatternQueryComponent::Variable(if var_name.is_empty() {
                         None
                     } else {
                         Some(var_name.into())
                     }))
                 }
-                _ => Ok(QueryComponent::Exact(T::try_from(value.clone())?)),
+                lexpr::Value::Symbol(symbol) if symbol.starts_with('_') => {
+                    Ok(PatternQueryComponent::Discard)
+                }
+                _ => Ok(PatternQueryComponent::Exact(T::try_from(value.clone())?)),
             }
         }
 
@@ -189,19 +239,34 @@ impl TryFrom<&lexpr::Value> for Query {
                 match symbol.borrow() {
                     "matches" => {
                         let (cons_vec, _) = value.clone().into_vec();
-                        if let [_, entity, attribute, value] = &cons_vec[..] {
+                        if let [_, entity, attribute, value, rest @ ..] = &cons_vec[..] {
                             let entity = parse_component::<Address>(entity)?;
                             let attribute = parse_component::<Attribute>(attribute)?;
                             let value = parse_component::<EntryValue>(value)?;
+                            let (provenance, timestamp) = match rest {
+                                [] => (None, None),
+                                [provenance] => {
+                                    (Some(parse_component::<Provenance>(provenance)?), None)
+                                }
+                                [provenance, timestamp] => (
+                                    Some(parse_component::<Provenance>(provenance)?),
+                                    Some(parse_component::<Timestamp>(timestamp)?),
+                                ),
+                                _ => Err(UpEndError::QueryParseError(
+                                    "Malformed expression: Too many arguments for `matches`."
+                                        .into(),
+                                ))?,
+                            };
                             Ok(Query::SingleQuery(QueryPart::Matches(PatternQuery {
                                 entity,
                                 attribute,
                                 value,
+                                provenance,
+                                timestamp,
                             })))
                         } else {
                             Err(UpEndError::QueryParseError(
-                                "Malformed expression: Wrong number of arguments to 'matches'."
+                                "Malformed expression: Not enough arguments for `matches`.".into(),
-                                    .into(),
                             ))
                         }
                     }
@@ -309,9 +374,10 @@ mod test {
         assert_eq!(
             query,
             Query::SingleQuery(QueryPart::Matches(PatternQuery {
-                entity: QueryComponent::Variable(None),
+                entity: PatternQueryComponent::Variable(None),
-                attribute: QueryComponent::Variable(None),
+                attribute: PatternQueryComponent::Variable(None),
-                value: QueryComponent::Variable(None)
+                value: PatternQueryComponent::Variable(None),
+                ..Default::default()
             }))
         );
 
@@ -320,9 +386,10 @@ mod test {
         assert_eq!(
             query,
             Query::SingleQuery(QueryPart::Matches(PatternQuery {
-                entity: QueryComponent::Exact(address),
+                entity: PatternQueryComponent::Exact(address),
-                attribute: QueryComponent::Variable(None),
+                attribute: PatternQueryComponent::Variable(None),
-                value: QueryComponent::Variable(None)
+                value: PatternQueryComponent::Variable(None),
+                ..Default::default()
             }))
         );
 
@@ -330,9 +397,10 @@ mod test {
         assert_eq!(
             query,
             Query::SingleQuery(QueryPart::Matches(PatternQuery {
-                entity: QueryComponent::Variable(None),
+                entity: PatternQueryComponent::Variable(None),
-                attribute: QueryComponent::Exact("FOO".into()),
+                attribute: PatternQueryComponent::Exact("FOO".into()),
-                value: QueryComponent::Variable(None)
+                value: PatternQueryComponent::Variable(None),
+                ..Default::default()
             }))
         );
 
@@ -341,9 +409,10 @@ mod test {
         assert_eq!(
             query,
             Query::SingleQuery(QueryPart::Matches(PatternQuery {
-                entity: QueryComponent::Variable(None),
+                entity: PatternQueryComponent::Variable(None),
-                attribute: QueryComponent::Variable(None),
+                attribute: PatternQueryComponent::Variable(None),
-                value: QueryComponent::Exact(value)
+                value: PatternQueryComponent::Exact(value),
+                ..Default::default()
             }))
         );
 
@@ -356,9 +425,10 @@ mod test {
         assert_eq!(
             query,
             Query::SingleQuery(QueryPart::Matches(PatternQuery {
-                entity: QueryComponent::Variable(Some("a".into())),
+                entity: PatternQueryComponent::Variable(Some("a".into())),
-                attribute: QueryComponent::Variable(Some("b".into())),
+                attribute: PatternQueryComponent::Variable(Some("b".into())),
-                value: QueryComponent::Variable(None)
+                value: PatternQueryComponent::Variable(None),
+                ..Default::default()
             }))
         );
 
@@ -371,9 +441,10 @@ mod test {
         assert_eq!(
             query,
             Query::SingleQuery(QueryPart::Matches(PatternQuery {
-                entity: QueryComponent::Variable(None),
+                entity: PatternQueryComponent::Variable(None),
-                attribute: QueryComponent::In(vec!("FOO".into(), "BAR".into())),
+                attribute: PatternQueryComponent::In(vec!("FOO".into(), "BAR".into())),
-                value: QueryComponent::Variable(None)
+                value: PatternQueryComponent::Variable(None),
+                ..Default::default()
             }))
         );
 
@@ -382,9 +453,10 @@ mod test {
         assert_eq!(
             query,
             Query::SingleQuery(QueryPart::Matches(PatternQuery {
-                entity: QueryComponent::Variable(None),
+                entity: PatternQueryComponent::Variable(None),
-                attribute: QueryComponent::Variable(None),
+                attribute: PatternQueryComponent::Variable(None),
-                value: QueryComponent::In(values)
+                value: PatternQueryComponent::In(values),
+                ..Default::default()
             }))
         );
 
@@ -393,9 +465,10 @@ mod test {
         assert_eq!(
             query,
             Query::SingleQuery(QueryPart::Matches(PatternQuery {
-                entity: QueryComponent::Variable(None),
+                entity: PatternQueryComponent::Variable(None),
-                attribute: QueryComponent::Variable(None),
+                attribute: PatternQueryComponent::Variable(None),
-                value: QueryComponent::In(values)
+                value: PatternQueryComponent::In(values),
+                ..Default::default()
             }))
         );
 
@@ -406,9 +479,10 @@ mod test {
         assert_eq!(
             query,
             Query::SingleQuery(QueryPart::Matches(PatternQuery {
-                entity: QueryComponent::Variable(None),
+                entity: PatternQueryComponent::Variable(None),
-                attribute: QueryComponent::Variable(None),
+                attribute: PatternQueryComponent::Variable(None),
-                value: QueryComponent::In(values)
+                value: PatternQueryComponent::In(values),
+                ..Default::default()
             }))
         );
 
@@ -418,9 +492,10 @@ mod test {
         assert_eq!(
             query,
             Query::SingleQuery(QueryPart::Matches(PatternQuery {
-                entity: QueryComponent::Variable(None),
+                entity: PatternQueryComponent::Variable(None),
-                attribute: QueryComponent::Variable(None),
+                attribute: PatternQueryComponent::Variable(None),
-                value: QueryComponent::In(values)
+                value: PatternQueryComponent::In(values),
+                ..Default::default()
             }))
         );
 
@@ -433,9 +508,10 @@ mod test {
         assert_eq!(
             query,
             Query::SingleQuery(QueryPart::Matches(PatternQuery {
-                entity: QueryComponent::Contains("foo".to_string()),
+                entity: PatternQueryComponent::Contains("foo".to_string()),
-                attribute: QueryComponent::Variable(None),
+                attribute: PatternQueryComponent::Variable(None),
-                value: QueryComponent::Variable(None)
+                value: PatternQueryComponent::Variable(None),
+                ..Default::default()
             }))
         );
 
@@ -443,9 +519,10 @@ mod test {
         assert_eq!(
             query,
             Query::SingleQuery(QueryPart::Matches(PatternQuery {
-                entity: QueryComponent::Variable(None),
+                entity: PatternQueryComponent::Variable(None),
-                attribute: QueryComponent::Contains("foo".to_string()),
+                attribute: PatternQueryComponent::Contains("foo".to_string()),
-                value: QueryComponent::Variable(None),
+                value: PatternQueryComponent::Variable(None),
+                ..Default::default()
             }))
         );
 
@@ -453,9 +530,10 @@ mod test {
         assert_eq!(
             query,
             Query::SingleQuery(QueryPart::Matches(PatternQuery {
-                entity: QueryComponent::Variable(None),
+                entity: PatternQueryComponent::Variable(None),
-                attribute: QueryComponent::Variable(None),
+                attribute: PatternQueryComponent::Variable(None),
-                value: QueryComponent::Contains("foo".to_string())
+                value: PatternQueryComponent::Contains("foo".to_string()),
+                ..Default::default()
             }))
         );
 

cli/src/extractors/audio.rs

@@ -134,7 +134,7 @@ impl Extractor for ID3Extractor {
         }
 
         let is_extracted = !connection
-            .query(format!("(matches @{} (contains \"ID3\") ?)", address).parse()?)?
+            .query::<Vec<Entry>>(format!("(matches @{} (contains \"ID3\") ?)", address).parse()?)?
             .is_empty();
 
         if is_extracted {

cli/src/extractors/exif.rs

@@ -153,7 +153,7 @@ impl Extractor for ExifExtractor {
         }
 
         let is_extracted = !connection
-            .query(format!("(matches @{} (contains \"EXIF\") ?)", address).parse()?)?
+            .query::<Vec<Entry>>(format!("(matches @{} (contains \"EXIF\") ?)", address).parse()?)?
             .is_empty();
 
         if is_extracted {

cli/src/extractors/media.rs

@@ -141,7 +141,9 @@ impl Extractor for MediaExtractor {
         }
 
         let is_extracted = !connection
-            .query(format!("(matches @{} (contains \"{}\") ?)", address, DURATION_KEY).parse()?)?
+            .query::<Vec<Entry>>(
+                format!("(matches @{} (contains \"{}\") ?)", address, DURATION_KEY).parse()?,
+            )?
             .is_empty();
 
         if is_extracted {

cli/src/extractors/web.rs

@@ -99,7 +99,7 @@ impl Extractor for WebExtractor {
 
     fn is_needed(&self, address: &Address, connection: &UpEndConnection) -> Result<bool> {
         Ok(connection
-            .query(
+            .query::<Vec<Entry>>(
                 format!(r#"(matches @{address} (in "HTML_TITLE" "HTML_DESCRIPTION") ?)"#)
                     .parse()?,
             )?

cli/src/routes.rs

@@ -39,6 +39,7 @@ use upend_base::lang::Query;
 use upend_db::hierarchies::{list_roots, resolve_path, UHierPath};
 use upend_db::jobs;
 use upend_db::stores::{Blob, UpStore};
+use upend_db::QueryResult;
 use upend_db::UpEndDatabase;
 use url::Url;
 
@@ -263,25 +264,41 @@ pub async fn get_query(state: web::Data<State>, query: String) -> Result<HttpRes
     let connection = state.upend.connection().map_err(ErrorInternalServerError)?;
 
     let in_query: Query = query.parse().map_err(ErrorBadRequest)?;
-    let entries = web::block(move || connection.query(in_query))
+    let query_result = web::block(move || connection.query::<QueryResult>(in_query))
         .await
         .map_err(ErrorInternalServerError)?
         .map_err(ErrorInternalServerError)?;
-    let mut result: HashMap<String, Entry> = HashMap::new();
-    for entry in entries {
-        result.insert(
-            b58_encode(
-                entry
-                    .address()
-                    .map_err(ErrorInternalServerError)?
-                    .encode()
-                    .map_err(ErrorInternalServerError)?,
-            ),
-            entry,
-        );
-    }
 
-    Ok(HttpResponse::Ok().json(&result))
+    let result = match query_result {
+        QueryResult::Entries(entries) => {
+            let mut result: HashMap<String, Entry> = HashMap::new();
+            for entry in entries {
+                result.insert(
+                    b58_encode(
+                        entry
+                            .address()
+                            .map_err(ErrorInternalServerError)?
+                            .encode()
+                            .map_err(ErrorInternalServerError)?,
+                    ),
+                    entry,
+                );
+            }
+
+            json!({ "entries": result })
+        }
+        QueryResult::Entities(entities) => {
+            json!({ "entities": entities })
+        }
+        QueryResult::Attributes(attributes) => {
+            json!({ "attributes": attributes })
+        }
+        QueryResult::Values(values) => {
+            json!({ "values": values })
+        }
+    };
+
+    Ok(HttpResponse::Ok().json(result))
 }
 
 trait EntriesAsHash {
@@ -566,8 +583,8 @@ pub async fn put_object_attribute(
 
     let new_address = web::block(move || {
         connection.transaction::<_, anyhow::Error, _>(|| {
-            let existing_attr_entries =
+            let existing_attr_entries = connection
-                connection.query(format!(r#"(matches @{address} "{attribute}" ?)"#).parse()?)?;
+                .query::<Vec<Entry>>(format!(r#"(matches @{address} "{attribute}" ?)"#).parse()?)?;
 
             for eae in existing_attr_entries {
                 let _ = connection.remove_object(eae.address()?)?;
@@ -679,42 +696,6 @@ pub async fn get_address(
     Ok(response.json(format!("{}", address)))
 }
 
-#[get("/api/all/attributes")]
-pub async fn get_all_attributes(state: web::Data<State>) -> Result<HttpResponse, Error> {
-    let connection = state.upend.connection().map_err(ErrorInternalServerError)?;
-    let attributes = web::block(move || connection.get_all_attributes())
-        .await?
-        .map_err(ErrorInternalServerError)?;
-
-    let connection = state.upend.connection().map_err(ErrorInternalServerError)?;
-    let result: serde_json::Value = attributes
-        .into_iter()
-        .map(|attribute| {
-            json!({
-                "name": attribute,
-                "labels": connection
-                    .retrieve_object(&Address::Attribute(attribute))
-                    .unwrap_or_else(|_| vec![])
-                    .into_iter()
-                    .filter_map(|e| {
-                        if e.attribute == ATTR_LABEL {
-                            if let EntryValue::String(label) = e.value {
-                                Some(label)
-                            } else {
-                                None
-                            }
-                        } else {
-                            None
-                        }
-                    })
-                    .collect::<Vec<String>>(),
-            })
-        })
-        .collect();
-
-    Ok(HttpResponse::Ok().json(result))
-}
-
 #[routes]
 #[get("/api/hier/{path:.*}")]
 #[put("/api/hier/{path:.*}")]

cli/src/serve.rs

@@ -56,7 +56,6 @@ where
         .service(routes::put_object_attribute)
         .service(routes::delete_object)
         .service(routes::get_address)
-        .service(routes::get_all_attributes)
         .service(routes::api_refresh)
         .service(routes::list_hier)
         .service(routes::list_hier_roots)

db/src/engine.rs

@@ -18,7 +18,14 @@ use diesel::{
 use diesel::{BoxableExpression, QueryDsl};
 use diesel::{ExpressionMethods, TextExpressionMethods};
 use upend_base::entry::EntryValue;
-use upend_base::lang::{PatternQuery, Query, QueryComponent, QueryPart, QueryQualifier};
+use upend_base::lang::{PatternQuery, PatternQueryComponent, Query, QueryPart, QueryQualifier};
+
+pub enum InnerQueryResult {
+    Entries(Vec<Entry>),
+    Entities(Vec<Vec<u8>>),
+    Attributes(Vec<String>),
+    Values(Vec<EntryValue>),
+}
 
 #[derive(Debug, Clone)]
 pub struct QueryExecutionError(String);
@@ -33,15 +40,52 @@ impl std::error::Error for QueryExecutionError {}
 
 pub fn execute(
     connection: &PooledConnection<ConnectionManager<SqliteConnection>>,
-    query: Query,
+    query: &Query,
-) -> Result<Vec<Entry>, QueryExecutionError> {
+) -> Result<InnerQueryResult, QueryExecutionError> {
     use crate::inner::schema::data::dsl::*;
 
     if let Some(predicates) = to_sqlite_predicates(query.clone())? {
         let db_query = data.filter(predicates);
-        db_query
+        let entries = db_query
             .load::<models::Entry>(connection)
-            .map_err(|e| QueryExecutionError(e.to_string()))
+            .map_err(|e| QueryExecutionError(e.to_string()))?;
+
+        match query {
+            Query::SingleQuery(query) => match query {
+                QueryPart::Matches(pattern)
+                    if !matches!(pattern.entity, PatternQueryComponent::Discard)
+                        && matches!(pattern.attribute, PatternQueryComponent::Discard)
+                        && matches!(pattern.value, PatternQueryComponent::Discard) =>
+                {
+                    let mut result: Vec<Vec<u8>> = entries.into_iter().map(|e| e.entity).collect();
+                    result.sort_unstable();
+                    result.dedup();
+                    Ok(InnerQueryResult::Entities(result))
+                }
+                QueryPart::Matches(pattern)
+                    if matches!(pattern.entity, PatternQueryComponent::Discard)
+                        && !matches!(pattern.attribute, PatternQueryComponent::Discard)
+                        && matches!(pattern.value, PatternQueryComponent::Discard) =>
+                {
+                    let mut attributes: Vec<String> =
+                        entries.into_iter().map(|e| e.attribute).collect();
+                    attributes.sort_unstable();
+                    attributes.dedup();
+                    Ok(InnerQueryResult::Attributes(attributes))
+                }
+                QueryPart::Matches(pattern)
+                    if matches!(pattern.entity, PatternQueryComponent::Discard)
+                        && matches!(pattern.attribute, PatternQueryComponent::Discard)
+                        && !matches!(pattern.value, PatternQueryComponent::Discard) =>
+                {
+                    Err(QueryExecutionError(
+                        "Only-value entries not yet implemented.".to_string(),
+                    ))
+                }
+                _ => Ok(InnerQueryResult::Entries(entries)),
+            },
+            _ => Ok(InnerQueryResult::Entries(entries)),
+        }
     } else {
         match query {
             Query::SingleQuery(_) => Err(QueryExecutionError(
@@ -57,27 +101,44 @@ pub fn execute(
                     let subquery_results = mq
                         .queries
                         .iter()
-                        .map(|q| execute(connection, *q.clone()))
+                        .map(|q| execute(connection, q))
+                        .map(|q| {
+                            if let Ok(InnerQueryResult::Entries(entries)) = q {
+                                Ok(entries)
+                            } else {
+                                Err(QueryExecutionError(
+                                    "Multiquery must be composed of pattern queries.".to_string(),
+                                ))
+                            }
+                        })
                         .collect::<Result<Vec<Vec<Entry>>, QueryExecutionError>>()?;
+
                     match mq.qualifier {
                         QueryQualifier::Not => unreachable!(),
-                        QueryQualifier::And => Ok(subquery_results
+                        QueryQualifier::And => Ok(InnerQueryResult::Entries(
-                            .into_iter()
+                            subquery_results
-                            .reduce(|acc, cur| {
+                                .into_iter()
-                                acc.into_iter()
+                                .reduce(|acc, cur| {
-                                    .filter(|e| {
+                                    acc.into_iter()
-                                        cur.iter().map(|e| &e.identity).any(|x| x == &e.identity)
+                                        .filter(|e| {
-                                    })
+                                            cur.iter()
-                                    .collect()
+                                                .map(|e| &e.identity)
-                            })
+                                                .any(|x| x == &e.identity)
-                            .unwrap()), // TODO
+                                        })
-                        QueryQualifier::Or => Ok(subquery_results.into_iter().flatten().collect()),
+                                        .collect()
+                                })
+                                .unwrap(),
+                        )), // TODO
+                        QueryQualifier::Or => Ok(InnerQueryResult::Entries(
+                            subquery_results.into_iter().flatten().collect(),
+                        )),
                         QueryQualifier::Join => {
                             let pattern_queries = mq
                                 .queries
+                                .clone()
                                 .into_iter()
                                 .map(|q| match *q {
-                                    Query::SingleQuery(QueryPart::Matches(pq)) => Some(pq),
+                                    Query::SingleQuery(QueryPart::Matches(pq)) => Some(pq.clone()),
                                     _ => None,
                                 })
                                 .collect::<Option<Vec<_>>>();
@@ -109,7 +170,9 @@ pub fn execute(
                                     })
                                     .unwrap(); // TODO
 
-                                Ok(joined.into_iter().map(|ev| ev.entry).collect())
+                                Ok(InnerQueryResult::Entries(
+                                    joined.into_iter().map(|ev| ev.entry).collect(),
+                                ))
                             } else {
                                 Err(QueryExecutionError(
                                     "Cannot join on non-atomic queries.".into(),
@@ -132,18 +195,18 @@ impl EntryWithVars {
     pub fn new(query: &PatternQuery, entry: Entry) -> Self {
         let mut vars = HashMap::new();
 
-        if let QueryComponent::Variable(Some(var_name)) = &query.entity {
+        if let PatternQueryComponent::Variable(Some(var_name)) = &query.entity {
             vars.insert(
                 var_name.clone(),
                 upend_base::hash::b58_encode(&entry.entity),
             );
         }
 
-        if let QueryComponent::Variable(Some(var_name)) = &query.attribute {
+        if let PatternQueryComponent::Variable(Some(var_name)) = &query.attribute {
             vars.insert(var_name.clone(), entry.attribute.clone());
         }
 
-        if let QueryComponent::Variable(Some(var_name)) = &query.value {
+        if let PatternQueryComponent::Variable(Some(var_name)) = &query.value {
             if let Some(value_str) = &entry.value_str {
                 vars.insert(var_name.clone(), value_str.clone());
             }
@@ -164,38 +227,38 @@ fn to_sqlite_predicates(query: Query) -> Result<SqlResult, QueryExecutionError>
                 let mut subqueries: Vec<Box<SqlPredicate>> = vec![];
 
                 match &eq.entity {
-                    QueryComponent::Exact(q_entity) => {
+                    PatternQueryComponent::Exact(q_entity) => {
                         subqueries.push(Box::new(data::entity.eq(q_entity.encode().map_err(
                             |e| QueryExecutionError(format!("failed producing sql: {e}")),
                         )?)))
                     }
-                    QueryComponent::In(q_entities) => {
+                    PatternQueryComponent::In(q_entities) => {
                         let entities: Result<Vec<_>, _> =
                             q_entities.iter().map(|t| t.encode()).collect();
                         subqueries.push(Box::new(data::entity.eq_any(entities.map_err(|e| {
                             QueryExecutionError(format!("failed producing sql: {e}"))
                         })?)))
                     }
-                    QueryComponent::Contains(q_entity) => subqueries.push(Box::new(
+                    PatternQueryComponent::Contains(q_entity) => subqueries.push(Box::new(
                         data::entity_searchable.like(format!("%{}%", q_entity)),
                     )),
-                    QueryComponent::Variable(_) => {}
+                    PatternQueryComponent::Variable(_) | PatternQueryComponent::Discard => {}
                 };
 
                 match &eq.attribute {
-                    QueryComponent::Exact(q_attribute) => {
+                    PatternQueryComponent::Exact(q_attribute) => {
                         subqueries.push(Box::new(data::attribute.eq(q_attribute.0.clone())))
                     }
-                    QueryComponent::In(q_attributes) => subqueries.push(Box::new(
+                    PatternQueryComponent::In(q_attributes) => subqueries.push(Box::new(
                         data::attribute.eq_any(q_attributes.iter().map(|a| &a.0).cloned()),
                     )),
-                    QueryComponent::Contains(q_attribute) => subqueries
+                    PatternQueryComponent::Contains(q_attribute) => subqueries
                         .push(Box::new(data::attribute.like(format!("%{}%", q_attribute)))),
-                    QueryComponent::Variable(_) => {}
+                    PatternQueryComponent::Variable(_) | PatternQueryComponent::Discard => {}
                 };
 
                 match &eq.value {
-                    QueryComponent::Exact(q_value) => match q_value {
+                    PatternQueryComponent::Exact(q_value) => match q_value {
                         EntryValue::Number(n) => subqueries.push(Box::new(data::value_num.eq(*n))),
                         _ => subqueries.push(Box::new(data::value_str.eq(
                             q_value.to_string().map_err(|e| {
@@ -203,7 +266,7 @@ fn to_sqlite_predicates(query: Query) -> Result<SqlResult, QueryExecutionError>
                             })?,
                         ))),
                     },
-                    QueryComponent::In(q_values) => {
+                    PatternQueryComponent::In(q_values) => {
                         let first = q_values.first().ok_or_else(|| {
                             QueryExecutionError(
                                 "Malformed expression: Inner value cannot be empty.".into(),
@@ -251,12 +314,42 @@ fn to_sqlite_predicates(query: Query) -> Result<SqlResult, QueryExecutionError>
                             )),
                         }
                     }
-                    QueryComponent::Contains(q_value) => {
+                    PatternQueryComponent::Contains(q_value) => {
                         subqueries.push(Box::new(data::value_str.like(format!("S%{}%", q_value))))
                     }
-                    QueryComponent::Variable(_) => {}
+                    PatternQueryComponent::Variable(_) | PatternQueryComponent::Discard => {}
                 };
 
+                if let Some(provenance) = &eq.provenance {
+                    match provenance {
+                        PatternQueryComponent::Exact(q_provenance) => {
+                            subqueries.push(Box::new(data::provenance.eq(q_provenance.0.clone())))
+                        }
+                        PatternQueryComponent::In(q_provenances) => subqueries.push(Box::new(
+                            data::provenance.eq_any(q_provenances.iter().map(|a| &a.0).cloned()),
+                        )),
+                        PatternQueryComponent::Contains(q_provenance) => subqueries.push(Box::new(
+                            data::provenance.like(format!("%{}%", q_provenance)),
+                        )),
+                        PatternQueryComponent::Variable(_) | PatternQueryComponent::Discard => {}
+                    }
+                }
+
+                if let Some(timestamp) = &eq.timestamp {
+                    match timestamp {
+                        PatternQueryComponent::Exact(q_timestamp) => {
+                            subqueries.push(Box::new(data::timestamp.eq(q_timestamp.0.clone())))
+                        }
+                        PatternQueryComponent::In(q_timestamps) => subqueries.push(Box::new(
+                            data::timestamp.eq_any(q_timestamps.iter().map(|a| &a.0).cloned()),
+                        )),
+                        PatternQueryComponent::Contains(_) => Err(QueryExecutionError(
+                            "Cannot like-compare timestamps.".into(),
+                        ))?,
+                        PatternQueryComponent::Variable(_) | PatternQueryComponent::Discard => {}
+                    }
+                }
+
                 match subqueries.len() {
                     0 => Ok(Some(Box::new(true.into_sql::<Bool>()))),
                     1 => Ok(Some(subqueries.remove(0))),

db/src/hierarchies.rs

@@ -10,7 +10,7 @@ use upend_base::addressing::Address;
 use upend_base::constants::ATTR_LABEL;
 use upend_base::constants::{ATTR_IN, HIER_ROOT_ADDR, HIER_ROOT_INVARIANT};
 use upend_base::entry::Entry;
-use upend_base::lang::{PatternQuery, Query, QueryComponent, QueryPart};
+use upend_base::lang::{PatternQuery, PatternQueryComponent, Query, QueryPart};
 
 use super::UpEndConnection;
 
@@ -77,15 +77,12 @@ impl std::fmt::Display for UHierPath {
 }
 
 pub fn list_roots(connection: &UpEndConnection) -> Result<Vec<Address>> {
-    Ok(connection
+    connection.query::<Vec<Address>>(Query::SingleQuery(QueryPart::Matches(PatternQuery {
-        .query(Query::SingleQuery(QueryPart::Matches(PatternQuery {
+        entity: PatternQueryComponent::Variable(None),
-            entity: QueryComponent::Variable(None),
+        attribute: PatternQueryComponent::Exact(ATTR_IN.into()),
-            attribute: QueryComponent::Exact(ATTR_IN.into()),
+        value: PatternQueryComponent::Exact((*HIER_ROOT_ADDR).clone().into()),
-            value: QueryComponent::Exact((*HIER_ROOT_ADDR).clone().into()),
+        ..Default::default()
-        })))?
+    })))
-        .into_iter()
-        .map(|e| e.entity)
-        .collect())
 }
 
 lazy_static! {
@@ -108,29 +105,28 @@ pub fn fetch_or_create_dir(
         _lock = FETCH_CREATE_LOCK.lock().unwrap();
     }
 
-    let matching_directories = connection
+    let matching_directories =
-        .query(Query::SingleQuery(QueryPart::Matches(PatternQuery {
+        connection.query::<Vec<Address>>(Query::SingleQuery(QueryPart::Matches(PatternQuery {
-            entity: QueryComponent::Variable(None),
+            entity: PatternQueryComponent::Variable(None),
-            attribute: QueryComponent::Exact(ATTR_LABEL.into()),
+            attribute: PatternQueryComponent::Exact(ATTR_LABEL.into()),
-            value: QueryComponent::Exact(String::from(directory.clone()).into()),
+            value: PatternQueryComponent::Exact(String::from(directory.clone()).into()),
-        })))?
+            ..Default::default()
-        .into_iter()
+        })))?;
-        .map(|e: Entry| e.entity);
 
     let parent_has: Vec<Address> = match parent.clone() {
-        Some(parent) => connection
+        Some(parent) => connection.query::<Vec<Address>>(Query::SingleQuery(
-            .query(Query::SingleQuery(QueryPart::Matches(PatternQuery {
+            QueryPart::Matches(PatternQuery {
-                entity: QueryComponent::Variable(None),
+                entity: PatternQueryComponent::Variable(None),
-                attribute: QueryComponent::Exact(ATTR_IN.into()),
+                attribute: PatternQueryComponent::Exact(ATTR_IN.into()),
-                value: QueryComponent::Exact(parent.into()),
+                value: PatternQueryComponent::Exact(parent.into()),
-            })))?
+                ..Default::default()
-            .into_iter()
+            }),
-            .map(|e| e.entity)
+        ))?,
-            .collect(),
         None => list_roots(connection)?,
     };
 
     let valid_directories: Vec<Address> = matching_directories
+        .into_iter()
         .filter(|a| parent_has.contains(a))
         .collect();
 

db/src/lib.rs

@@ -30,9 +30,10 @@ use diesel::prelude::*;
 use diesel::r2d2::{self, ConnectionManager};
 use diesel::result::{DatabaseErrorKind, Error};
 use diesel::sqlite::SqliteConnection;
+use engine::InnerQueryResult;
 use hierarchies::initialize_hier;
 use shadow_rs::is_release;
-use std::convert::TryFrom;
+use std::convert::{TryFrom, TryInto};
 use std::fs;
 use std::path::{Path, PathBuf};
 use std::sync::{Arc, Mutex, RwLock};
@@ -52,7 +53,7 @@ pub struct ConnectionOptions {
 }
 
 impl ConnectionOptions {
-    pub fn apply(&self, connection: &SqliteConnection) -> QueryResult<()> {
+    pub fn apply(&self, connection: &SqliteConnection) -> diesel::prelude::QueryResult<()> {
         let _lock = self.mutex.lock().unwrap();
 
         if let Some(duration) = self.busy_timeout {
@@ -292,20 +293,20 @@ impl UpEndConnection {
         Ok(diesel::delete(matches).execute(&conn)?)
     }
 
-    pub fn query(&self, query: Query) -> Result<Vec<Entry>> {
+    pub fn query<T>(&self, query: Query) -> Result<T>
+    where
+        T: TryFrom<InnerQueryResult>,
+        <T as TryFrom<InnerQueryResult>>::Error: std::fmt::Display,
+    {
         trace!("Querying: {:?}", query);
 
         let _lock = self.lock.read().unwrap();
         let conn = self.pool.get()?;
 
-        let entries = execute(&conn, query)?;
+        let result = execute(&conn, &query)?;
-        let entries = entries
+        result
-            .iter()
+            .try_into()
-            .map(Entry::try_from)
+            .map_err(|err| anyhow!("Could not convert to requested result type: {err}"))
-            .filter_map(Result::ok)
-            .collect();
-
-        Ok(entries)
     }
 
     pub fn insert_entry(&self, entry: Entry) -> Result<Address> {
@@ -358,22 +359,6 @@ impl UpEndConnection {
         Ok(result)
     }
 
-    #[deprecated]
-    pub fn get_all_attributes(&self) -> Result<Vec<String>> {
-        use crate::inner::schema::data::dsl::*;
-
-        let _lock = self.lock.read().unwrap();
-        let conn = self.pool.get()?;
-
-        let result = data
-            .select(attribute)
-            .distinct()
-            .order_by(attribute)
-            .load::<String>(&conn)?;
-
-        Ok(result)
-    }
-
     pub fn get_stats(&self) -> Result<serde_json::Value> {
         use crate::inner::schema::data::dsl::*;
         let _lock = self.lock.read().unwrap();
@@ -430,6 +415,74 @@ impl UpEndConnection {
     }
 }
 
+pub enum QueryResult {
+    Entries(Vec<Entry>),
+    Entities(Vec<Address>),
+    Attributes(Vec<String>),
+    Values(Vec<EntryValue>),
+}
+
+impl TryFrom<InnerQueryResult> for QueryResult {
+    type Error = anyhow::Error;
+
+    fn try_from(inner_result: InnerQueryResult) -> std::result::Result<Self, Self::Error> {
+        let result = match inner_result {
+            InnerQueryResult::Entries(entries) => QueryResult::Entries(
+                entries
+                    .iter()
+                    .map(Entry::try_from)
+                    .collect::<Result<Vec<Entry>, _>>()?,
+            ),
+            InnerQueryResult::Entities(entities) => QueryResult::Entities(
+                entities
+                    .iter()
+                    .map(|buf| Address::decode(buf))
+                    .collect::<Result<Vec<Address>, _>>()?,
+            ),
+            InnerQueryResult::Attributes(attributes) => QueryResult::Attributes(attributes),
+            InnerQueryResult::Values(values) => QueryResult::Values(values),
+        };
+
+        Ok(result)
+    }
+}
+
+impl TryFrom<InnerQueryResult> for Vec<Address> {
+    type Error = anyhow::Error;
+
+    fn try_from(inner_result: InnerQueryResult) -> std::result::Result<Self, Self::Error> {
+        let result = match inner_result {
+            InnerQueryResult::Entries(entries) => entries
+                .into_iter()
+                .map(|e| Address::decode(&e.entity))
+                .collect::<Result<Vec<Address>, _>>()?,
+            InnerQueryResult::Entities(entities) => entities
+                .into_iter()
+                .map(|buf| Address::decode(&buf))
+                .collect::<Result<Vec<Address>, _>>()?,
+            _ => Err(anyhow!("Insufficient information in query."))?,
+        };
+
+        Ok(result)
+    }
+}
+
+impl TryFrom<InnerQueryResult> for Vec<Entry> {
+    type Error = anyhow::Error;
+
+    fn try_from(inner_result: InnerQueryResult) -> std::result::Result<Self, Self::Error> {
+        let result = match inner_result {
+            InnerQueryResult::Entries(entries) => entries
+                .iter()
+                .map(Entry::try_from)
+                .collect::<Result<Vec<Entry>, _>>()?,
+            _ => Err(anyhow!("Insufficient information in query."))?,
+        };
+
+        Ok(result)
+    }
+}
+
 #[cfg(test)]
 mod test {
     use upend_base::constants::ATTR_LABEL;
@@ -464,16 +517,47 @@ mod test {
 
         let connection = db.connection().unwrap();
 
+        for address in connection.get_all_addresses().unwrap() {
+            connection.remove_object(address).unwrap();
+        }
+
+        // Test elementary inserts and queries
         let random_entity = Address::Uuid(uuid::Uuid::new_v4());
         upend_insert_val!(connection, random_entity, ATTR_LABEL, "FOOBAR").unwrap();
         upend_insert_val!(connection, random_entity, "FLAVOUR", "STRANGE").unwrap();
 
+        let query = format!(r#"(matches ? ? ?)"#).parse().unwrap();
+        let result = connection.query::<Vec<Entry>>(query).unwrap();
+        assert_eq!(result.len(), 2);
+
         let query = format!(r#"(matches @{random_entity} ? ?)"#)
             .parse()
             .unwrap();
-        let result = connection.query(query).unwrap();
+        let result = connection.query::<Vec<Entry>>(query).unwrap();
         assert_eq!(result.len(), 2);
 
+        // Test elementary provenance queries
+        let query = format!(r#"(matches ? ? ? "SYSTEM INIT")"#).parse().unwrap();
+        let result = connection.query::<Vec<Entry>>(query).unwrap();
+        assert_eq!(result.len(), 2);
+
+        let query = format!(r#"(matches ? ? ? "SOMETHING ELSE")"#)
+            .parse()
+            .unwrap();
+        let result = connection.query::<Vec<Entry>>(query).unwrap();
+        assert_eq!(result.len(), 0);
+
+        // Test only-queries
+        let query = format!(r#"(matches ? ? ?)"#).parse().unwrap();
+        let result = connection.query::<Vec<Address>>(query).unwrap();
+        assert_eq!(result.len(), 2);
+
+        let query = format!(r#"(matches ? _ _)"#).parse().unwrap();
+        let result = connection.query::<Vec<Address>>(query).unwrap();
+        println!("{:?}", result);
+        assert_eq!(result.len(), 1);
+
+        // Test IN queries for entities
         let other_entity = Address::Uuid(uuid::Uuid::new_v4());
         upend_insert_val!(connection, random_entity, ATTR_LABEL, "BAZQUX").unwrap();
         upend_insert_val!(connection, random_entity, "CHARGE", "POSITIVE").unwrap();
@@ -481,38 +565,44 @@ mod test {
         let query = format!(r#"(matches (in @{random_entity} @{other_entity}) ? ?)"#)
             .parse()
             .unwrap();
-        let result = connection.query(query).unwrap();
+        let result = connection.query::<Vec<Entry>>(query).unwrap();
         assert_eq!(result.len(), 4);
 
+        // Test IN queries for attributes
         let query = r#"(matches ? (in "FLAVOUR" "CHARGE") ?)"#.parse().unwrap();
-        let result = connection.query(query).unwrap();
+        let result = connection.query::<Vec<Entry>>(query).unwrap();
         assert_eq!(result.len(), 2);
 
+        // Test IN queries for values
         let query = format!(r#"(matches ? "{ATTR_LABEL}" (in "FOOBAR" "BAZQUX"))"#)
             .parse()
             .unwrap();
-        let result = connection.query(query).unwrap();
+        let result = connection.query::<Vec<Entry>>(query).unwrap();
         assert_eq!(result.len(), 2);
 
+        // Test CONTAINS queries for values
         let query = format!(r#"(matches ? "{ATTR_LABEL}" (contains "OOBA"))"#)
             .parse()
             .unwrap();
-        let result = connection.query(query).unwrap();
+        let result = connection.query::<Vec<Entry>>(query).unwrap();
         assert_eq!(result.len(), 1);
 
+        // Test multiple IN queries
         let query = r#"(or (matches ? ? (contains "OOBA")) (matches ? (contains "HARGE") ?) )"#
             .parse()
             .unwrap();
-        let result = connection.query(query).unwrap();
+        let result = connection.query::<Vec<Entry>>(query).unwrap();
         assert_eq!(result.len(), 2);
 
+        // Test multiple queries
         let query =
             format!(r#"(and (matches ? ? (contains "OOBA")) (matches ? "{ATTR_LABEL}" ?) )"#)
                 .parse()
                 .unwrap();
-        let result = connection.query(query).unwrap();
+        let result = connection.query::<Vec<Entry>>(query).unwrap();
         assert_eq!(result.len(), 1);
 
+        // Test composed multi-query
         let query = format!(
             r#"(and
                     (or
@@ -524,9 +614,10 @@ mod test {
         )
         .parse()
         .unwrap();
-        let result = connection.query(query).unwrap();
+        let result = connection.query::<Vec<Entry>>(query).unwrap();
         assert_eq!(result.len(), 1);
 
+        // Test join query
         let query = format!(
             r#"(join 
                     (matches ?a "FLAVOUR" ?)
@@ -535,7 +626,7 @@ mod test {
         )
         .parse()
         .unwrap();
-        let result = connection.query(query).unwrap();
+        let result = connection.query::<Vec<Entry>>(query).unwrap();
         assert_eq!(result.len(), 1);
         assert_eq!(result[0].value, "STRANGE".into());
     }