thm
/
upend
1
0
Fork 0
Code Issues 72 Pull Requests Packages Projects Releases 5 Wiki Activity

feat!: add provenance & timestamp to Entry

feat/type-attributes
Tomáš Mládek 2023-04-02 19:55:51 +02:00
parent 1722336a4a
commit 36e788b9d3
12 changed files with 76 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
migrations/upend/00_initial/up.sql
View File

@ -17,7 +17,9 @@ CREATE TABLE data
attribute VARCHAR NOT NULL,
value_str VARCHAR,
value_num NUMERIC,
immutable BOOLEAN NOT NULL
immutable BOOLEAN NOT NULL,
provenance VARCHAR NOT NULL,
timestamp DATETIME NOT NULL
);
CREATE INDEX data_entity ON data (entity);

13
src/database/entry.rs
View File

@ -2,6 +2,7 @@ use crate::addressing::{Address, Addressable};
use crate::database::inner::models;
use crate::util::hash::{b58_decode, hash, Hash, Hashable};
use anyhow::{anyhow, Result};
use chrono::NaiveDateTime;
use regex::Regex;
use serde::{Deserialize, Serialize};
use std::convert::TryFrom;
@ -12,6 +13,8 @@ pub struct Entry {
pub entity: Address,
pub attribute: String,
pub value: EntryValue,
pub provenance: String,
pub timestamp: NaiveDateTime,
}
#[derive(Debug, Clone)]
@ -42,18 +45,24 @@ impl TryFrom<&models::Entry> for Entry {
entity: Address::decode(&e.entity)?,
attribute: e.attribute.clone(),
value: value_str.parse()?,
provenance: e.provenance.clone(),
timestamp: e.timestamp.clone(),
})
} else if let Some(value_num) = e.value_num {
Ok(Entry {
entity: Address::decode(&e.entity)?,
attribute: e.attribute.clone(),
value: EntryValue::Number(value_num),
provenance: e.provenance.clone(),
timestamp: e.timestamp.clone(),
})
} else {
Ok(Entry {
entity: Address::decode(&e.entity)?,
attribute: e.attribute.clone(),
value: EntryValue::Number(f64::NAN),
provenance: e.provenance.clone(),
timestamp: e.timestamp.clone(),
})
}
}
@ -78,6 +87,8 @@ impl TryFrom<&Entry> for models::Entry {
value_str: None,
value_num: None,
immutable: false,
provenance: e.provenance.clone(),
timestamp: e.timestamp.clone(),
};
match e.value {
@ -114,6 +125,8 @@ impl TryFrom<&InvariantEntry> for Entry {
entity: invariant.entity()?,
attribute: invariant.attribute.clone(),
value: invariant.value.clone(),
provenance: "INVARIANT".to_string(),
timestamp: chrono::Utc::now().naive_utc(), // ?
})
}
}

6
src/database/hierarchies.rs
View File

@ -175,6 +175,8 @@ pub fn fetch_or_create_dir(
entity: new_directory_address.clone(),
attribute: String::from(IS_OF_TYPE_ATTR),
value: HIER_ADDR.clone().into(),
provenance: "SYSTEM FS".to_string(),
timestamp: chrono::Utc::now().naive_utc(),
};
connection.insert_entry(type_entry)?;
@ -182,6 +184,8 @@ pub fn fetch_or_create_dir(
entity: new_directory_address.clone(),
attribute: String::from(LABEL_ATTR),
value: directory.as_ref().clone().into(),
provenance: "SYSTEM FS".to_string(),
timestamp: chrono::Utc::now().naive_utc(),
};
connection.insert_entry(directory_entry)?;
@ -190,6 +194,8 @@ pub fn fetch_or_create_dir(
entity: parent,
attribute: String::from(HIER_HAS_ATTR),
value: new_directory_address.clone().into(),
provenance: "SYSTEM FS".to_string(),
timestamp: chrono::Utc::now().naive_utc(),
};
connection.insert_entry(has_entry)?;
}

3
src/database/inner/models.rs
View File

@ -1,4 +1,5 @@
use super::schema::{data, meta};
use chrono::NaiveDateTime;
use serde::Serialize;
#[derive(Queryable, Insertable, Serialize, Debug)]
@ -11,6 +12,8 @@ pub struct Entry {
pub value_str: Option<String>,
pub value_num: Option<f64>,
pub immutable: bool,
pub provenance: String,
pub timestamp: NaiveDateTime,
}
#[derive(Queryable, Insertable, Serialize, Clone, Debug)]

8
src/database/inner/schema.rs
View File

@ -7,8 +7,11 @@ table! {
value_str -> Nullable<Text>,
value_num -> Nullable<Double>,
immutable -> Bool,
provenance -> Text,
timestamp -> Timestamp,
}
}
table! {
meta (id) {
id -> Integer,
@ -17,4 +20,7 @@ table! {
}
}
allow_tables_to_appear_in_same_query!(data, meta,);
allow_tables_to_appear_in_same_query!(
data,
meta,
);

4
src/database/macros.rs
View File

@ -4,6 +4,8 @@ macro_rules! upend_insert_val {
entity: $entity.clone(),
attribute: String::from($attribute),
value: crate::database::entry::EntryValue::String(String::from($value)),
provenance: "SYSTEM INIT".to_string(),
timestamp: chrono::Utc::now().naive_utc(),
})
}};
}
@ -14,6 +16,8 @@ macro_rules! upend_insert_addr {
entity: $entity.clone(),
attribute: String::from($attribute),
value: crate::database::entry::EntryValue::Address($addr.clone()),
provenance: "SYSTEM INIT".to_string(),
timestamp: chrono::Utc::now().naive_utc(),
})
}};
}

14
src/database/stores/fs/mod.rs
View File

@ -405,18 +405,24 @@ impl FsStore {
entity: blob_address.clone(),
attribute: String::from(IS_OF_TYPE_ATTR),
value: BLOB_TYPE_ADDR.clone().into(),
provenance: "SYSTEM INIT".to_string(),
timestamp: chrono::Utc::now().naive_utc(),
};
let size_entry = Entry {
entity: blob_address.clone(),
attribute: FILE_SIZE_KEY.to_string(),
value: (size as f64).into(),
provenance: "SYSTEM INIT".to_string(),
timestamp: chrono::Utc::now().naive_utc(),
};
let mime_entry = mime_type.map(|mime_type| Entry {
entity: blob_address.clone(),
attribute: FILE_MIME_KEY.to_string(),
value: mime_type.into(),
provenance: "SYSTEM INIT".to_string(),
timestamp: chrono::Utc::now().naive_utc(),
});
let added_entry = Entry {
@ -427,6 +433,8 @@ impl FsStore {
.unwrap()
.as_secs() as f64)
.into(),
provenance: "SYSTEM INIT".to_string(),
timestamp: chrono::Utc::now().naive_utc(),
};
// Add the appropriate entries w/r/t virtual filesystem location
@ -462,6 +470,8 @@ impl FsStore {
entity: parent_dir.clone(),
attribute: HIER_HAS_ATTR.to_string(),
value: blob_address.clone().into(),
provenance: "SYSTEM INIT".to_string(),
timestamp: chrono::Utc::now().naive_utc(),
};
let dir_has_entry_addr = connection.insert_entry(dir_has_entry)?;
@ -469,6 +479,8 @@ impl FsStore {
entity: blob_address.clone(),
attribute: LABEL_ATTR.to_string(),
value: filename.as_os_str().to_string_lossy().to_string().into(),
provenance: "SYSTEM INIT".to_string(),
timestamp: chrono::Utc::now().naive_utc(),
};
let label_entry_addr = connection.insert_entry(label_entry)?;
@ -476,6 +488,8 @@ impl FsStore {
entity: dir_has_entry_addr,
attribute: ALIAS_KEY.to_string(),
value: label_entry_addr.into(),
provenance: "SYSTEM INIT".to_string(),
timestamp: chrono::Utc::now().naive_utc(),
};
connection.insert_entry(alias_entry)?;

4
src/extractors/audio.rs
View File

@ -54,11 +54,15 @@ impl Extractor for ID3Extractor {
"TYER" | "TBPM" => EntryValue::guess_from(text),
_ => text.clone().into(),
},
provenance: "SYSTEM EXTRACTOR".to_string(),
timestamp: chrono::Utc::now().naive_utc(),
},
Entry {
entity: Address::Attribute(format!("ID3_{}", frame.id())),
attribute: constants::LABEL_ATTR.into(),
value: format!("ID3: {}", frame.name()).into(),
provenance: "SYSTEM EXTRACTOR".to_string(),
timestamp: chrono::Utc::now().naive_utc(),
},
],
_ => vec![],

2
src/extractors/media.rs
View File

@ -66,6 +66,8 @@ impl Extractor for MediaExtractor {
entity: address.clone(),
attribute: DURATION_KEY.to_string(),
value: EntryValue::Number(duration),
provenance: "SYSTEM EXTRACTOR".to_string(),
timestamp: chrono::Utc::now().naive_utc(),
}];
let _ = job_handle.update_state(JobState::Done);

4
src/extractors/photo.rs
View File

@ -69,11 +69,15 @@ impl Extractor for ExifExtractor {
field.display_value()
)),
},
provenance: "SYSTEM EXTRACTOR".to_string(),
timestamp: chrono::Utc::now().naive_utc(),
},
Entry {
entity: Address::Attribute(attribute),
attribute: constants::LABEL_ATTR.into(),
value: format!("EXIF: {}", tag_description).into(),
provenance: "SYSTEM EXTRACTOR".to_string(),
timestamp: chrono::Utc::now().naive_utc(),
},
]
} else {

8
src/extractors/web.rs
View File

@ -36,11 +36,15 @@ impl Extractor for WebExtractor {
entity: address.clone(),
attribute: "HTML_TITLE".to_string(),
value: html_title.into(),
provenance: "SYSTEM EXTRACTOR".to_string(),
timestamp: chrono::Utc::now().naive_utc(),
}),
webpage.html.description.map(|html_desc| Entry {
entity: address.clone(),
attribute: "HTML_DESCRIPTION".to_string(),
value: html_desc.into(),
provenance: "SYSTEM EXTRACTOR".to_string(),
timestamp: chrono::Utc::now().naive_utc(),
}),
];
for (key, value) in webpage.html.opengraph.properties {
@ -48,6 +52,8 @@ impl Extractor for WebExtractor {
entity: address.clone(),
attribute: format!("OG_{}", key.to_uppercase()),
value: value.into(),
provenance: "SYSTEM EXTRACTOR".to_string(),
timestamp: chrono::Utc::now().naive_utc(),
}))
}
for image in webpage.html.opengraph.images {
@ -55,6 +61,8 @@ impl Extractor for WebExtractor {
entity: address.clone(),
attribute: "OG_IMAGE".to_string(),
value: image.url.into(),
provenance: "SYSTEM EXTRACTOR".to_string(),
timestamp: chrono::Utc::now().naive_utc(),
}))
}

8
src/routes.rs
View File

@ -391,6 +391,8 @@ pub async fn put_object(
entity,
attribute: in_entry.attribute,
value: in_entry.value,
provenance: "USER API".to_string(),
timestamp: chrono::Utc::now().naive_utc(),
};
Ok(web::block::<_, _, anyhow::Error>(move || {
@ -441,6 +443,8 @@ pub async fn put_object(
entity: address.clone(),
attribute: LABEL_ATTR.to_string(),
value: url.clone().into(),
provenance: "USER API".to_string(),
timestamp: chrono::Utc::now().naive_utc(),
}),
};
@ -469,6 +473,8 @@ pub async fn put_object(
.as_secs()
as f64,
),
provenance: "USER API".to_string(),
timestamp: chrono::Utc::now().naive_utc(),
})?;
}
@ -563,6 +569,8 @@ pub async fn put_object_attribute(
entity: address,
attribute,
value: value.into_inner(),
provenance: "USER API".to_string(),
timestamp: chrono::Utc::now().naive_utc(),
};
connection.insert_entry(new_attr_entry)