feat!: add provenance & timestamp to Entry

migrations/upend/00_initial/up.sql

@@ -17,7 +17,9 @@ CREATE TABLE data
     attribute         VARCHAR          NOT NULL,
     value_str         VARCHAR,
     value_num         NUMERIC,
-    immutable         BOOLEAN          NOT NULL
+    immutable         BOOLEAN          NOT NULL,
+    provenance        VARCHAR          NOT NULL,
+    timestamp         DATETIME         NOT NULL
 );
 
 CREATE INDEX data_entity ON data (entity);

src/database/entry.rs

@@ -2,6 +2,7 @@ use crate::addressing::{Address, Addressable};
 use crate::database::inner::models;
 use crate::util::hash::{b58_decode, hash, Hash, Hashable};
 use anyhow::{anyhow, Result};
+use chrono::NaiveDateTime;
 use regex::Regex;
 use serde::{Deserialize, Serialize};
 use std::convert::TryFrom;
@@ -12,6 +13,8 @@ pub struct Entry {
     pub entity: Address,
     pub attribute: String,
     pub value: EntryValue,
+    pub provenance: String,
+    pub timestamp: NaiveDateTime,
 }
 
 #[derive(Debug, Clone)]
@@ -42,18 +45,24 @@ impl TryFrom<&models::Entry> for Entry {
                 entity: Address::decode(&e.entity)?,
                 attribute: e.attribute.clone(),
                 value: value_str.parse()?,
+                provenance: e.provenance.clone(),
+                timestamp: e.timestamp.clone(),
             })
         } else if let Some(value_num) = e.value_num {
             Ok(Entry {
                 entity: Address::decode(&e.entity)?,
                 attribute: e.attribute.clone(),
                 value: EntryValue::Number(value_num),
+                provenance: e.provenance.clone(),
+                timestamp: e.timestamp.clone(),
             })
         } else {
             Ok(Entry {
                 entity: Address::decode(&e.entity)?,
                 attribute: e.attribute.clone(),
                 value: EntryValue::Number(f64::NAN),
+                provenance: e.provenance.clone(),
+                timestamp: e.timestamp.clone(),
             })
         }
     }
@@ -78,6 +87,8 @@ impl TryFrom<&Entry> for models::Entry {
             value_str: None,
             value_num: None,
             immutable: false,
+            provenance: e.provenance.clone(),
+            timestamp: e.timestamp.clone(),
         };
 
         match e.value {
@@ -114,6 +125,8 @@ impl TryFrom<&InvariantEntry> for Entry {
             entity: invariant.entity()?,
             attribute: invariant.attribute.clone(),
             value: invariant.value.clone(),
+            provenance: "INVARIANT".to_string(),
+            timestamp: chrono::Utc::now().naive_utc(), // ?
         })
     }
 }

src/database/hierarchies.rs

@@ -175,6 +175,8 @@ pub fn fetch_or_create_dir(
                     entity: new_directory_address.clone(),
                     attribute: String::from(IS_OF_TYPE_ATTR),
                     value: HIER_ADDR.clone().into(),
+                    provenance: "SYSTEM FS".to_string(),
+                    timestamp: chrono::Utc::now().naive_utc(),
                 };
                 connection.insert_entry(type_entry)?;
 
@@ -182,6 +184,8 @@ pub fn fetch_or_create_dir(
                     entity: new_directory_address.clone(),
                     attribute: String::from(LABEL_ATTR),
                     value: directory.as_ref().clone().into(),
+                    provenance: "SYSTEM FS".to_string(),
+                    timestamp: chrono::Utc::now().naive_utc(),
                 };
                 connection.insert_entry(directory_entry)?;
 
@@ -190,6 +194,8 @@ pub fn fetch_or_create_dir(
                         entity: parent,
                         attribute: String::from(HIER_HAS_ATTR),
                         value: new_directory_address.clone().into(),
+                        provenance: "SYSTEM FS".to_string(),
+                        timestamp: chrono::Utc::now().naive_utc(),
                     };
                     connection.insert_entry(has_entry)?;
                 }

src/database/inner/models.rs

@@ -1,4 +1,5 @@
 use super::schema::{data, meta};
+use chrono::NaiveDateTime;
 use serde::Serialize;
 
 #[derive(Queryable, Insertable, Serialize, Debug)]
@@ -11,6 +12,8 @@ pub struct Entry {
     pub value_str: Option<String>,
     pub value_num: Option<f64>,
     pub immutable: bool,
+    pub provenance: String,
+    pub timestamp: NaiveDateTime,
 }
 
 #[derive(Queryable, Insertable, Serialize, Clone, Debug)]

src/database/inner/schema.rs

@@ -7,8 +7,11 @@ table! {
         value_str -> Nullable<Text>,
         value_num -> Nullable<Double>,
         immutable -> Bool,
+        provenance -> Text,
+        timestamp -> Timestamp,
     }
 }
+
 table! {
     meta (id) {
         id -> Integer,
@@ -17,4 +20,7 @@ table! {
     }
 }
 
-allow_tables_to_appear_in_same_query!(data, meta,);
+allow_tables_to_appear_in_same_query!(
+    data,
+    meta,
+);

src/database/macros.rs

@@ -4,6 +4,8 @@ macro_rules! upend_insert_val {
             entity: $entity.clone(),
             attribute: String::from($attribute),
             value: crate::database::entry::EntryValue::String(String::from($value)),
+            provenance: "SYSTEM INIT".to_string(),
+            timestamp: chrono::Utc::now().naive_utc(),
         })
     }};
 }
@@ -14,6 +16,8 @@ macro_rules! upend_insert_addr {
             entity: $entity.clone(),
             attribute: String::from($attribute),
             value: crate::database::entry::EntryValue::Address($addr.clone()),
+            provenance: "SYSTEM INIT".to_string(),
+            timestamp: chrono::Utc::now().naive_utc(),
         })
     }};
 }

src/database/stores/fs/mod.rs

@@ -405,18 +405,24 @@ impl FsStore {
             entity: blob_address.clone(),
             attribute: String::from(IS_OF_TYPE_ATTR),
             value: BLOB_TYPE_ADDR.clone().into(),
+            provenance: "SYSTEM INIT".to_string(),
+            timestamp: chrono::Utc::now().naive_utc(),
         };
 
         let size_entry = Entry {
             entity: blob_address.clone(),
             attribute: FILE_SIZE_KEY.to_string(),
             value: (size as f64).into(),
+            provenance: "SYSTEM INIT".to_string(),
+            timestamp: chrono::Utc::now().naive_utc(),
         };
 
         let mime_entry = mime_type.map(|mime_type| Entry {
             entity: blob_address.clone(),
             attribute: FILE_MIME_KEY.to_string(),
             value: mime_type.into(),
+            provenance: "SYSTEM INIT".to_string(),
+            timestamp: chrono::Utc::now().naive_utc(),
         });
 
         let added_entry = Entry {
@@ -427,6 +433,8 @@ impl FsStore {
                 .unwrap()
                 .as_secs() as f64)
                 .into(),
+            provenance: "SYSTEM INIT".to_string(),
+            timestamp: chrono::Utc::now().naive_utc(),
         };
 
         // Add the appropriate entries w/r/t virtual filesystem location
@@ -462,6 +470,8 @@ impl FsStore {
             entity: parent_dir.clone(),
             attribute: HIER_HAS_ATTR.to_string(),
             value: blob_address.clone().into(),
+            provenance: "SYSTEM INIT".to_string(),
+            timestamp: chrono::Utc::now().naive_utc(),
         };
         let dir_has_entry_addr = connection.insert_entry(dir_has_entry)?;
 
@@ -469,6 +479,8 @@ impl FsStore {
             entity: blob_address.clone(),
             attribute: LABEL_ATTR.to_string(),
             value: filename.as_os_str().to_string_lossy().to_string().into(),
+            provenance: "SYSTEM INIT".to_string(),
+            timestamp: chrono::Utc::now().naive_utc(),
         };
         let label_entry_addr = connection.insert_entry(label_entry)?;
 
@@ -476,6 +488,8 @@ impl FsStore {
             entity: dir_has_entry_addr,
             attribute: ALIAS_KEY.to_string(),
             value: label_entry_addr.into(),
+            provenance: "SYSTEM INIT".to_string(),
+            timestamp: chrono::Utc::now().naive_utc(),
         };
         connection.insert_entry(alias_entry)?;
 

src/extractors/audio.rs

@@ -54,11 +54,15 @@ impl Extractor for ID3Extractor {
                                     "TYER" | "TBPM" => EntryValue::guess_from(text),
                                     _ => text.clone().into(),
                                 },
+                                provenance: "SYSTEM EXTRACTOR".to_string(),
+                                timestamp: chrono::Utc::now().naive_utc(),
                             },
                             Entry {
                                 entity: Address::Attribute(format!("ID3_{}", frame.id())),
                                 attribute: constants::LABEL_ATTR.into(),
                                 value: format!("ID3: {}", frame.name()).into(),
+                                provenance: "SYSTEM EXTRACTOR".to_string(),
+                                timestamp: chrono::Utc::now().naive_utc(),
                             },
                         ],
                         _ => vec![],

src/extractors/media.rs

@@ -66,6 +66,8 @@ impl Extractor for MediaExtractor {
                     entity: address.clone(),
                     attribute: DURATION_KEY.to_string(),
                     value: EntryValue::Number(duration),
+                    provenance: "SYSTEM EXTRACTOR".to_string(),
+                    timestamp: chrono::Utc::now().naive_utc(),
                 }];
 
                 let _ = job_handle.update_state(JobState::Done);

src/extractors/photo.rs

@@ -69,11 +69,15 @@ impl Extractor for ExifExtractor {
                                             field.display_value()
                                         )),
                                     },
+                                    provenance: "SYSTEM EXTRACTOR".to_string(),
+                                    timestamp: chrono::Utc::now().naive_utc(),
                                 },
                                 Entry {
                                     entity: Address::Attribute(attribute),
                                     attribute: constants::LABEL_ATTR.into(),
                                     value: format!("EXIF: {}", tag_description).into(),
+                                    provenance: "SYSTEM EXTRACTOR".to_string(),
+                                    timestamp: chrono::Utc::now().naive_utc(),
                                 },
                             ]
                         } else {

src/extractors/web.rs

@@ -36,11 +36,15 @@ impl Extractor for WebExtractor {
                         entity: address.clone(),
                         attribute: "HTML_TITLE".to_string(),
                         value: html_title.into(),
+                        provenance: "SYSTEM EXTRACTOR".to_string(),
+                        timestamp: chrono::Utc::now().naive_utc(),
                     }),
                     webpage.html.description.map(|html_desc| Entry {
                         entity: address.clone(),
                         attribute: "HTML_DESCRIPTION".to_string(),
                         value: html_desc.into(),
+                        provenance: "SYSTEM EXTRACTOR".to_string(),
+                        timestamp: chrono::Utc::now().naive_utc(),
                     }),
                 ];
                 for (key, value) in webpage.html.opengraph.properties {
@@ -48,6 +52,8 @@ impl Extractor for WebExtractor {
                         entity: address.clone(),
                         attribute: format!("OG_{}", key.to_uppercase()),
                         value: value.into(),
+                        provenance: "SYSTEM EXTRACTOR".to_string(),
+                        timestamp: chrono::Utc::now().naive_utc(),
                     }))
                 }
                 for image in webpage.html.opengraph.images {
@@ -55,6 +61,8 @@ impl Extractor for WebExtractor {
                         entity: address.clone(),
                         attribute: "OG_IMAGE".to_string(),
                         value: image.url.into(),
+                        provenance: "SYSTEM EXTRACTOR".to_string(),
+                        timestamp: chrono::Utc::now().naive_utc(),
                     }))
                 }
 

src/routes.rs

@@ -391,6 +391,8 @@ pub async fn put_object(
                             entity,
                             attribute: in_entry.attribute,
                             value: in_entry.value,
+                            provenance: "USER API".to_string(),
+                            timestamp: chrono::Utc::now().naive_utc(),
                         };
 
                         Ok(web::block::<_, _, anyhow::Error>(move || {
@@ -441,6 +443,8 @@ pub async fn put_object(
                             entity: address.clone(),
                             attribute: LABEL_ATTR.to_string(),
                             value: url.clone().into(),
+                            provenance: "USER API".to_string(),
+                            timestamp: chrono::Utc::now().naive_utc(),
                         }),
                     };
 
@@ -469,6 +473,8 @@ pub async fn put_object(
                                             .as_secs()
                                             as f64,
                                     ),
+                                    provenance: "USER API".to_string(),
+                                    timestamp: chrono::Utc::now().naive_utc(),
                                 })?;
                             }
 
@@ -563,6 +569,8 @@ pub async fn put_object_attribute(
                 entity: address,
                 attribute,
                 value: value.into_inner(),
+                provenance: "USER API".to_string(),
+                timestamp: chrono::Utc::now().naive_utc(),
             };
 
             connection.insert_entry(new_attr_entry)