use crate::{ addressing::Address, database::{entry::Entry, stores::UpStore, UpEndConnection, UpEndDatabase}, util::jobs::JobContainer, }; use anyhow::Result; use log::{info, trace}; use rayon::iter::{IntoParallelRefIterator, ParallelIterator}; use std::{ borrow::Borrow, sync::{Arc, Mutex, RwLock}, }; #[cfg(feature = "extractors-web")] pub mod web; #[cfg(feature = "extractors-audio")] pub mod audio; #[cfg(feature = "extractors-photo")] pub mod photo; pub trait Extractor { fn get( &self, address: &Address, connection: &UpEndConnection, store: Arc>, job_container: JobContainer, ) -> Result>; fn is_needed(&self, _address: &Address, _connection: &UpEndConnection) -> Result { Ok(true) } fn insert_info( &self, address: &Address, connection: &UpEndConnection, store: Arc>, job_container: JobContainer, ) -> Result { if self.is_needed(address, connection)? { let entries = self.get(address, connection, store, job_container)?; connection.transaction(|| { let len = entries.len(); for entry in entries { connection.insert_entry(entry)?; } Ok(len) }) } else { Ok(0) } } } pub fn extract_all>( db: D, store: Arc>, mut job_container: JobContainer, ) -> Result { info!("Extracting metadata for all addresses."); let db = db.borrow(); let job_handle = job_container.add_job("EXTRACT_ALL", "Extracting additional metadata...")?; let all_addresses = db.connection()?.get_all_addresses()?; let total = all_addresses.len() as f32; let count = RwLock::new(0_usize); let shared_job_handle = Arc::new(Mutex::new(job_handle)); let result = all_addresses .par_iter() .map(|address| { let connection = db.connection()?; let extract_result = extract(address, &connection, store.clone(), job_container.clone()); let mut cnt = count.write().unwrap(); *cnt += 1; shared_job_handle .lock() .unwrap() .update_progress(*cnt as f32 / total * 100.0)?; extract_result }) .flatten() .sum(); info!( "Done extracting metadata; processed {} addresses, added {} entries.", all_addresses.len(), result ); Ok(result) } pub fn extract( address: &Address, connection: &UpEndConnection, store: Arc>, job_container: JobContainer, ) -> Result { let mut entry_count = 0; trace!("Extracting metadata for {address:?}"); #[cfg(feature = "extractors-web")] { entry_count += web::WebExtractor.insert_info(address, connection, store.clone(), job_container.clone())?; } #[cfg(feature = "extractors-audio")] { entry_count += audio::ID3Extractor.insert_info(address, connection, store.clone(), job_container.clone())?; } #[cfg(feature = "extractors-photo")] { entry_count += photo::ExifExtractor.insert_info(address, connection, store.clone(), job_container)?; } trace!("Extracting metadata for {address:?} - got {entry_count} entries."); Ok(entry_count) }