thm
/
upend
1
0
Fork 0
Code Issues 73 Pull Requests Packages Projects Releases 5 Wiki Activity

feat: extractors append types

feat/type-attributes
Tomáš Mládek 2023-07-09 19:24:46 +02:00
parent 49ecc7dc5a
commit f4c03fde37
5 changed files with 142 additions and 16 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
base/src/entry.rs
View File

@ -35,6 +35,18 @@ pub enum EntryValue {
Invalid,
}
impl Default for Entry {
fn default() -> Self {
Self {
entity: Address::Uuid(uuid::Uuid::nil()),
attribute: Default::default(),
value: EntryValue::Null,
provenance: "SYSTEM".into(),
timestamp: NaiveDateTime::from_timestamp_opt(0, 0).unwrap(),
}
}
}
impl TryFrom<&InvariantEntry> for Entry {
type Error = UpEndError;
@ -44,7 +56,7 @@ impl TryFrom<&InvariantEntry> for Entry {
attribute: invariant.attribute.clone(),
value: invariant.value.clone(),
provenance: "INVARIANT".to_string(),
timestamp: NaiveDateTime::from_timestamp_opt(0, 0).unwrap(),
..Default::default()
})
}
}

45
cli/src/extractors/audio.rs
View File

@ -2,10 +2,11 @@ use std::sync::Arc;
use super::Extractor;
use anyhow::{anyhow, Result};
use lazy_static::lazy_static;
use upend_base::{
addressing::Address,
constants::ATTR_LABEL,
entry::{Entry, EntryValue},
constants::{ATTR_IN, ATTR_KEY, ATTR_LABEL, ATTR_OF},
entry::{Entry, EntryValue, InvariantEntry},
};
use upend_db::{
jobs::{JobContainer, JobState},
@ -13,6 +14,19 @@ use upend_db::{
UpEndConnection,
};
lazy_static! {
pub static ref ID3_TYPE_INVARIANT: InvariantEntry = InvariantEntry {
attribute: String::from(ATTR_KEY),
value: "TYPE_ID3".into(),
};
pub static ref ID3_TYPE_LABEL: Entry = Entry {
entity: ID3_TYPE_INVARIANT.entity().unwrap(),
attribute: ATTR_LABEL.into(),
value: "ID3".into(),
..Default::default()
};
}
pub struct ID3Extractor;
impl Extractor for ID3Extractor {
@ -43,7 +57,7 @@ impl Extractor for ID3Extractor {
let tags = id3::Tag::read_from_path(file_path)?;
let result: Vec<Entry> = tags
let mut result: Vec<Entry> = tags
.frames()
.flat_map(|frame| match frame.content() {
id3::Content::Text(text) => vec![
@ -69,6 +83,31 @@ impl Extractor for ID3Extractor {
})
.collect();
if !result.is_empty() {
result.extend(
result
.iter()
.filter(|e| e.attribute != ATTR_LABEL)
.map(|e| Entry {
entity: Address::Attribute(e.attribute.clone()),
attribute: ATTR_OF.into(),
value: EntryValue::Address(ID3_TYPE_INVARIANT.entity().unwrap()),
..Default::default()
})
.collect::<Vec<Entry>>(),
);
result.extend(vec![
(&ID3_TYPE_INVARIANT as &InvariantEntry).try_into().unwrap(),
ID3_TYPE_LABEL.clone(),
Entry {
entity: address.clone(),
attribute: ATTR_IN.into(),
value: EntryValue::Address(ID3_TYPE_INVARIANT.entity().unwrap()),
..Default::default()
},
]);
}
let _ = job_handle.update_state(JobState::Done);
Ok(result)

51
cli/src/extractors/media.rs
View File

@ -2,10 +2,11 @@ use std::{process::Command, sync::Arc};
use super::Extractor;
use anyhow::{anyhow, Result};
use lazy_static::lazy_static;
use upend_base::{
addressing::Address,
constants::ATTR_LABEL,
entry::{Entry, EntryValue},
constants::{ATTR_IN, ATTR_KEY, ATTR_LABEL, ATTR_OF},
entry::{Entry, EntryValue, InvariantEntry},
};
use upend_db::{
jobs::{JobContainer, JobState},
@ -15,6 +16,25 @@ use upend_db::{
const DURATION_KEY: &str = "MEDIA_DURATION";
lazy_static! {
pub static ref MEDIA_TYPE_INVARIANT: InvariantEntry = InvariantEntry {
attribute: String::from(ATTR_KEY),
value: "TYPE_MEDIA".into(),
};
pub static ref MEDIA_TYPE_LABEL: Entry = Entry {
entity: MEDIA_TYPE_INVARIANT.entity().unwrap(),
attribute: ATTR_LABEL.into(),
value: "Multimedia".into(),
..Default::default()
};
pub static ref DURATION_OF_MEDIA: Entry = Entry {
entity: Address::Attribute(DURATION_KEY.to_string()),
attribute: ATTR_OF.into(),
value: EntryValue::Address(MEDIA_TYPE_INVARIANT.entity().unwrap()),
..Default::default()
};
}
pub struct MediaExtractor;
impl Extractor for MediaExtractor {
@ -62,13 +82,26 @@ impl Extractor for MediaExtractor {
.trim()
.parse::<f64>()?;
let result = vec![Entry {
entity: address.clone(),
attribute: DURATION_KEY.to_string(),
value: EntryValue::Number(duration),
provenance: "SYSTEM EXTRACTOR".to_string(),
timestamp: chrono::Utc::now().naive_utc(),
}];
let result = vec![
Entry {
entity: address.clone(),
attribute: DURATION_KEY.to_string(),
value: EntryValue::Number(duration),
provenance: "SYSTEM EXTRACTOR".to_string(),
timestamp: chrono::Utc::now().naive_utc(),
},
(&MEDIA_TYPE_INVARIANT as &InvariantEntry)
.try_into()
.unwrap(),
MEDIA_TYPE_LABEL.clone(),
DURATION_OF_MEDIA.clone(),
Entry {
entity: address.clone(),
attribute: ATTR_IN.into(),
value: EntryValue::Address(MEDIA_TYPE_INVARIANT.entity().unwrap()),
..Default::default()
},
];
let _ = job_handle.update_state(JobState::Done);
Ok(result)

1
cli/src/extractors/mod.rs
View File

@ -42,6 +42,7 @@ pub trait Extractor {
) -> Result<usize> {
if self.is_needed(address, connection)? {
let entries = self.get(address, connection, store, job_container)?;
trace!("For \"{address}\", got: {entries:?}");
connection.transaction(|| {
let len = entries.len();

47
cli/src/extractors/photo.rs
View File

@ -2,10 +2,11 @@ use std::sync::Arc;
use super::Extractor;
use anyhow::{anyhow, Result};
use lazy_static::lazy_static;
use upend_base::{
addressing::Address,
constants::ATTR_LABEL,
entry::{Entry, EntryValue},
constants::{ATTR_IN, ATTR_KEY, ATTR_LABEL, ATTR_OF},
entry::{Entry, EntryValue, InvariantEntry},
};
use upend_db::{
jobs::{JobContainer, JobState},
@ -18,6 +19,19 @@ pub struct ExifExtractor;
// TODO: EXIF metadata is oftentimes a constant/enum value. What's the proper
// model for enum-like values in UpEnd?
lazy_static! {
pub static ref PHOTO_TYPE_INVARIANT: InvariantEntry = InvariantEntry {
attribute: String::from(ATTR_KEY),
value: "TYPE_PHOTO".into(),
};
pub static ref PHOTO_TYPE_LABEL: Entry = Entry {
entity: PHOTO_TYPE_INVARIANT.entity().unwrap(),
attribute: ATTR_LABEL.into(),
value: "Photo".into(),
..Default::default()
};
}
impl Extractor for ExifExtractor {
fn get(
&self,
@ -49,7 +63,7 @@ impl Extractor for ExifExtractor {
let exifreader = exif::Reader::new();
let exif = exifreader.read_from_container(&mut bufreader)?;
let result: Vec<Entry> = exif
let mut result: Vec<Entry> = exif
.fields()
.filter(|field| !matches!(field.value, exif::Value::Undefined(..)))
.flat_map(|field| {
@ -86,6 +100,33 @@ impl Extractor for ExifExtractor {
})
.collect();
if !result.is_empty() {
result.extend(
result
.iter()
.filter(|e| e.attribute != ATTR_LABEL)
.map(|e| Entry {
entity: Address::Attribute(e.attribute.clone()),
attribute: ATTR_OF.into(),
value: EntryValue::Address(PHOTO_TYPE_INVARIANT.entity().unwrap()),
..Default::default()
})
.collect::<Vec<Entry>>(),
);
result.extend(vec![
(&PHOTO_TYPE_INVARIANT as &InvariantEntry)
.try_into()
.unwrap(),
PHOTO_TYPE_LABEL.clone(),
Entry {
entity: address.clone(),
attribute: ATTR_IN.into(),
value: EntryValue::Address(PHOTO_TYPE_INVARIANT.entity().unwrap()),
..Default::default()
},
]);
}
let _ = job_handle.update_state(JobState::Done);
Ok(result)