diff --git a/migrations/upend/00_initial/up.sql b/migrations/upend/00_initial/up.sql index b8f60e6..d7da4b3 100644 --- a/migrations/upend/00_initial/up.sql +++ b/migrations/upend/00_initial/up.sql @@ -17,7 +17,9 @@ CREATE TABLE data attribute VARCHAR NOT NULL, value_str VARCHAR, value_num NUMERIC, - immutable BOOLEAN NOT NULL + immutable BOOLEAN NOT NULL, + provenance VARCHAR NOT NULL, + timestamp DATETIME NOT NULL ); CREATE INDEX data_entity ON data (entity); diff --git a/src/database/entry.rs b/src/database/entry.rs index f1942cd..1f1823d 100644 --- a/src/database/entry.rs +++ b/src/database/entry.rs @@ -2,6 +2,7 @@ use crate::addressing::{Address, Addressable}; use crate::database::inner::models; use crate::util::hash::{b58_decode, hash, Hash, Hashable}; use anyhow::{anyhow, Result}; +use chrono::NaiveDateTime; use regex::Regex; use serde::{Deserialize, Serialize}; use std::convert::TryFrom; @@ -12,6 +13,8 @@ pub struct Entry { pub entity: Address, pub attribute: String, pub value: EntryValue, + pub provenance: String, + pub timestamp: NaiveDateTime, } #[derive(Debug, Clone)] @@ -42,18 +45,24 @@ impl TryFrom<&models::Entry> for Entry { entity: Address::decode(&e.entity)?, attribute: e.attribute.clone(), value: value_str.parse()?, + provenance: e.provenance.clone(), + timestamp: e.timestamp.clone(), }) } else if let Some(value_num) = e.value_num { Ok(Entry { entity: Address::decode(&e.entity)?, attribute: e.attribute.clone(), value: EntryValue::Number(value_num), + provenance: e.provenance.clone(), + timestamp: e.timestamp.clone(), }) } else { Ok(Entry { entity: Address::decode(&e.entity)?, attribute: e.attribute.clone(), value: EntryValue::Number(f64::NAN), + provenance: e.provenance.clone(), + timestamp: e.timestamp.clone(), }) } } @@ -78,6 +87,8 @@ impl TryFrom<&Entry> for models::Entry { value_str: None, value_num: None, immutable: false, + provenance: e.provenance.clone(), + timestamp: e.timestamp.clone(), }; match e.value { @@ -114,6 +125,8 @@ impl TryFrom<&InvariantEntry> for Entry { entity: invariant.entity()?, attribute: invariant.attribute.clone(), value: invariant.value.clone(), + provenance: "INVARIANT".to_string(), + timestamp: chrono::Utc::now().naive_utc(), // ? }) } } diff --git a/src/database/hierarchies.rs b/src/database/hierarchies.rs index f00afea..a1e6596 100644 --- a/src/database/hierarchies.rs +++ b/src/database/hierarchies.rs @@ -175,6 +175,8 @@ pub fn fetch_or_create_dir( entity: new_directory_address.clone(), attribute: String::from(IS_OF_TYPE_ATTR), value: HIER_ADDR.clone().into(), + provenance: "SYSTEM FS".to_string(), + timestamp: chrono::Utc::now().naive_utc(), }; connection.insert_entry(type_entry)?; @@ -182,6 +184,8 @@ pub fn fetch_or_create_dir( entity: new_directory_address.clone(), attribute: String::from(LABEL_ATTR), value: directory.as_ref().clone().into(), + provenance: "SYSTEM FS".to_string(), + timestamp: chrono::Utc::now().naive_utc(), }; connection.insert_entry(directory_entry)?; @@ -190,6 +194,8 @@ pub fn fetch_or_create_dir( entity: parent, attribute: String::from(HIER_HAS_ATTR), value: new_directory_address.clone().into(), + provenance: "SYSTEM FS".to_string(), + timestamp: chrono::Utc::now().naive_utc(), }; connection.insert_entry(has_entry)?; } diff --git a/src/database/inner/models.rs b/src/database/inner/models.rs index 9de42c7..2542692 100644 --- a/src/database/inner/models.rs +++ b/src/database/inner/models.rs @@ -1,4 +1,5 @@ use super::schema::{data, meta}; +use chrono::NaiveDateTime; use serde::Serialize; #[derive(Queryable, Insertable, Serialize, Debug)] @@ -11,6 +12,8 @@ pub struct Entry { pub value_str: Option, pub value_num: Option, pub immutable: bool, + pub provenance: String, + pub timestamp: NaiveDateTime, } #[derive(Queryable, Insertable, Serialize, Clone, Debug)] diff --git a/src/database/inner/schema.rs b/src/database/inner/schema.rs index ba50579..c7172ba 100644 --- a/src/database/inner/schema.rs +++ b/src/database/inner/schema.rs @@ -7,8 +7,11 @@ table! { value_str -> Nullable, value_num -> Nullable, immutable -> Bool, + provenance -> Text, + timestamp -> Timestamp, } } + table! { meta (id) { id -> Integer, @@ -17,4 +20,7 @@ table! { } } -allow_tables_to_appear_in_same_query!(data, meta,); +allow_tables_to_appear_in_same_query!( + data, + meta, +); diff --git a/src/database/macros.rs b/src/database/macros.rs index 9e03783..a52c75b 100644 --- a/src/database/macros.rs +++ b/src/database/macros.rs @@ -4,6 +4,8 @@ macro_rules! upend_insert_val { entity: $entity.clone(), attribute: String::from($attribute), value: crate::database::entry::EntryValue::String(String::from($value)), + provenance: "SYSTEM INIT".to_string(), + timestamp: chrono::Utc::now().naive_utc(), }) }}; } @@ -14,6 +16,8 @@ macro_rules! upend_insert_addr { entity: $entity.clone(), attribute: String::from($attribute), value: crate::database::entry::EntryValue::Address($addr.clone()), + provenance: "SYSTEM INIT".to_string(), + timestamp: chrono::Utc::now().naive_utc(), }) }}; } diff --git a/src/database/stores/fs/mod.rs b/src/database/stores/fs/mod.rs index a7ba4ee..8de4f2a 100644 --- a/src/database/stores/fs/mod.rs +++ b/src/database/stores/fs/mod.rs @@ -405,18 +405,24 @@ impl FsStore { entity: blob_address.clone(), attribute: String::from(IS_OF_TYPE_ATTR), value: BLOB_TYPE_ADDR.clone().into(), + provenance: "SYSTEM INIT".to_string(), + timestamp: chrono::Utc::now().naive_utc(), }; let size_entry = Entry { entity: blob_address.clone(), attribute: FILE_SIZE_KEY.to_string(), value: (size as f64).into(), + provenance: "SYSTEM INIT".to_string(), + timestamp: chrono::Utc::now().naive_utc(), }; let mime_entry = mime_type.map(|mime_type| Entry { entity: blob_address.clone(), attribute: FILE_MIME_KEY.to_string(), value: mime_type.into(), + provenance: "SYSTEM INIT".to_string(), + timestamp: chrono::Utc::now().naive_utc(), }); let added_entry = Entry { @@ -427,6 +433,8 @@ impl FsStore { .unwrap() .as_secs() as f64) .into(), + provenance: "SYSTEM INIT".to_string(), + timestamp: chrono::Utc::now().naive_utc(), }; // Add the appropriate entries w/r/t virtual filesystem location @@ -462,6 +470,8 @@ impl FsStore { entity: parent_dir.clone(), attribute: HIER_HAS_ATTR.to_string(), value: blob_address.clone().into(), + provenance: "SYSTEM INIT".to_string(), + timestamp: chrono::Utc::now().naive_utc(), }; let dir_has_entry_addr = connection.insert_entry(dir_has_entry)?; @@ -469,6 +479,8 @@ impl FsStore { entity: blob_address.clone(), attribute: LABEL_ATTR.to_string(), value: filename.as_os_str().to_string_lossy().to_string().into(), + provenance: "SYSTEM INIT".to_string(), + timestamp: chrono::Utc::now().naive_utc(), }; let label_entry_addr = connection.insert_entry(label_entry)?; @@ -476,6 +488,8 @@ impl FsStore { entity: dir_has_entry_addr, attribute: ALIAS_KEY.to_string(), value: label_entry_addr.into(), + provenance: "SYSTEM INIT".to_string(), + timestamp: chrono::Utc::now().naive_utc(), }; connection.insert_entry(alias_entry)?; diff --git a/src/extractors/audio.rs b/src/extractors/audio.rs index 5c35ae8..3a7d987 100644 --- a/src/extractors/audio.rs +++ b/src/extractors/audio.rs @@ -54,11 +54,15 @@ impl Extractor for ID3Extractor { "TYER" | "TBPM" => EntryValue::guess_from(text), _ => text.clone().into(), }, + provenance: "SYSTEM EXTRACTOR".to_string(), + timestamp: chrono::Utc::now().naive_utc(), }, Entry { entity: Address::Attribute(format!("ID3_{}", frame.id())), attribute: constants::LABEL_ATTR.into(), value: format!("ID3: {}", frame.name()).into(), + provenance: "SYSTEM EXTRACTOR".to_string(), + timestamp: chrono::Utc::now().naive_utc(), }, ], _ => vec![], diff --git a/src/extractors/media.rs b/src/extractors/media.rs index 763c7f9..d5b0366 100644 --- a/src/extractors/media.rs +++ b/src/extractors/media.rs @@ -66,6 +66,8 @@ impl Extractor for MediaExtractor { entity: address.clone(), attribute: DURATION_KEY.to_string(), value: EntryValue::Number(duration), + provenance: "SYSTEM EXTRACTOR".to_string(), + timestamp: chrono::Utc::now().naive_utc(), }]; let _ = job_handle.update_state(JobState::Done); diff --git a/src/extractors/photo.rs b/src/extractors/photo.rs index 6ee1a57..b9fbd24 100644 --- a/src/extractors/photo.rs +++ b/src/extractors/photo.rs @@ -69,11 +69,15 @@ impl Extractor for ExifExtractor { field.display_value() )), }, + provenance: "SYSTEM EXTRACTOR".to_string(), + timestamp: chrono::Utc::now().naive_utc(), }, Entry { entity: Address::Attribute(attribute), attribute: constants::LABEL_ATTR.into(), value: format!("EXIF: {}", tag_description).into(), + provenance: "SYSTEM EXTRACTOR".to_string(), + timestamp: chrono::Utc::now().naive_utc(), }, ] } else { diff --git a/src/extractors/web.rs b/src/extractors/web.rs index 388a840..dfadf35 100644 --- a/src/extractors/web.rs +++ b/src/extractors/web.rs @@ -36,11 +36,15 @@ impl Extractor for WebExtractor { entity: address.clone(), attribute: "HTML_TITLE".to_string(), value: html_title.into(), + provenance: "SYSTEM EXTRACTOR".to_string(), + timestamp: chrono::Utc::now().naive_utc(), }), webpage.html.description.map(|html_desc| Entry { entity: address.clone(), attribute: "HTML_DESCRIPTION".to_string(), value: html_desc.into(), + provenance: "SYSTEM EXTRACTOR".to_string(), + timestamp: chrono::Utc::now().naive_utc(), }), ]; for (key, value) in webpage.html.opengraph.properties { @@ -48,6 +52,8 @@ impl Extractor for WebExtractor { entity: address.clone(), attribute: format!("OG_{}", key.to_uppercase()), value: value.into(), + provenance: "SYSTEM EXTRACTOR".to_string(), + timestamp: chrono::Utc::now().naive_utc(), })) } for image in webpage.html.opengraph.images { @@ -55,6 +61,8 @@ impl Extractor for WebExtractor { entity: address.clone(), attribute: "OG_IMAGE".to_string(), value: image.url.into(), + provenance: "SYSTEM EXTRACTOR".to_string(), + timestamp: chrono::Utc::now().naive_utc(), })) } diff --git a/src/routes.rs b/src/routes.rs index 1fae5e9..16c88da 100644 --- a/src/routes.rs +++ b/src/routes.rs @@ -391,6 +391,8 @@ pub async fn put_object( entity, attribute: in_entry.attribute, value: in_entry.value, + provenance: "USER API".to_string(), + timestamp: chrono::Utc::now().naive_utc(), }; Ok(web::block::<_, _, anyhow::Error>(move || { @@ -441,6 +443,8 @@ pub async fn put_object( entity: address.clone(), attribute: LABEL_ATTR.to_string(), value: url.clone().into(), + provenance: "USER API".to_string(), + timestamp: chrono::Utc::now().naive_utc(), }), }; @@ -469,6 +473,8 @@ pub async fn put_object( .as_secs() as f64, ), + provenance: "USER API".to_string(), + timestamp: chrono::Utc::now().naive_utc(), })?; } @@ -563,6 +569,8 @@ pub async fn put_object_attribute( entity: address, attribute, value: value.into_inner(), + provenance: "USER API".to_string(), + timestamp: chrono::Utc::now().naive_utc(), }; connection.insert_entry(new_attr_entry)