use crate::{ addressing::Address, database::{entry::Entry, UpEndConnection, UpEndDatabase}, util::jobs::{Job, JobContainer}, }; use anyhow::Result; use log::{info, trace}; use rayon::iter::{IntoParallelRefIterator, ParallelIterator}; use std::{ borrow::Borrow, sync::{Arc, RwLock}, }; #[cfg(feature = "extractors-web")] pub mod web; #[cfg(feature = "extractors-audio")] pub mod audio; pub trait Extractor { fn get( &self, address: &Address, connection: &UpEndConnection, job_container: Arc>, ) -> Result>; fn is_needed(&self, _address: &Address, _connection: &UpEndConnection) -> Result { Ok(true) } fn insert_info( &self, address: &Address, connection: &UpEndConnection, job_container: Arc>, ) -> Result { if self.is_needed(address, connection)? { let entries = self.get(address, connection, job_container)?; connection.transaction(|| { let len = entries.len(); for entry in entries { connection.insert_entry(entry)?; } Ok(len) }) } else { Ok(0) } } } pub fn extract_all>( db: D, job_container: Arc>, ) -> Result { info!("Extracting metadata for all addresses."); let db = db.borrow(); let job_id_result = job_container .write() .unwrap() .add_job(Job::new("EXTRACT_ALL", "Extracting additional metadata...")); match job_id_result { Ok(job_id) => { let all_addresses = db.connection()?.get_all_addresses()?; let total = all_addresses.len() as f32; let count = RwLock::new(0_usize); let result = all_addresses .par_iter() .map(|address| { let connection = db.connection()?; let extract_result = extract(address, &connection, job_container.clone()); let mut cnt = count.write().unwrap(); *cnt += 1; job_container .write() .unwrap() .update_progress(&job_id, *cnt as f32 / total * 100.0)?; extract_result }) .flatten() .sum(); info!( "Done extracting metadata; processed {} addresses, added {} entries.", all_addresses.len(), result ); Ok(result) } Err(err) => Err(err.into()), } } pub fn extract( address: &Address, connection: &UpEndConnection, job_container: Arc>, ) -> Result { let mut entry_count = 0; trace!("Extracting metadata for {address:?}"); #[cfg(feature = "extractors-web")] { entry_count += web::WebExtractor.insert_info(address, connection, job_container.clone())?; } #[cfg(feature = "extractors-audio")] { entry_count += audio::ID3Extractor.insert_info(address, connection, job_container)?; } trace!("Extracting metadata for {address:?} - got {entry_count} entries."); Ok(entry_count) }