use crate::extractors::{self}; use crate::previews::PreviewStore; use crate::util::exec::block_background; use actix_files::NamedFile; use actix_multipart::Multipart; use actix_web::error::{ ErrorBadRequest, ErrorInternalServerError, ErrorNotFound, ErrorUnauthorized, }; use actix_web::{delete, error, get, post, put, web, Either, Error, HttpResponse}; use actix_web::{http, Responder}; use actix_web::{ http::header::{CacheControl, CacheDirective, ContentDisposition, DispositionType}, HttpRequest, }; use anyhow::{anyhow, Result}; use futures::channel::oneshot; use futures_util::TryStreamExt; use serde::{Deserialize, Serialize}; use serde_json::json; use std::collections::HashMap; use std::convert::{TryFrom, TryInto}; use std::io::Write; use std::sync::Arc; use std::time::{SystemTime, UNIX_EPOCH}; use tempfile::NamedTempFile; use tracing::{debug, info, trace}; use upend::addressing::{Address, Addressable}; use upend::common::build; use upend::config::UpEndConfig; use upend::database::constants::{ADDED_ATTR, LABEL_ATTR}; use upend::database::entry::{Entry, EntryValue, InvariantEntry}; use upend::database::hierarchies::{list_roots, resolve_path, UHierPath}; use upend::database::lang::Query; use upend::database::stores::{Blob, UpStore}; use upend::database::UpEndDatabase; use upend::util::hash::{b58_decode, b58_encode}; use upend::util::jobs; use url::Url; use uuid::Uuid; #[cfg(feature = "desktop")] use is_executable::IsExecutable; #[derive(Clone)] pub struct State { pub upend: Arc, pub store: Arc>, pub config: UpEndConfig, pub job_container: jobs::JobContainer, pub preview_store: Option>, pub preview_pool: Option>, } #[derive(Debug, Serialize, Deserialize)] struct JwtClaims { exp: usize, } #[derive(Deserialize)] pub struct LoginRequest { key: String, } #[post("/api/auth/login")] pub async fn login( state: web::Data, payload: web::Json, ) -> Result { if state.config.key.is_none() || Some(&payload.key) == state.config.key.as_ref() { let claims = JwtClaims { exp: (SystemTime::now() .duration_since(UNIX_EPOCH) .map_err(ErrorInternalServerError)? .as_secs() + 7 * 24 * 60 * 60) as usize, }; let token = jsonwebtoken::encode( &jsonwebtoken::Header::default(), &claims, &jsonwebtoken::EncodingKey::from_secret(state.config.secret.as_ref()), ) .map_err(ErrorInternalServerError)?; Ok(HttpResponse::Ok().json(json!({ "token": token }))) } else { Err(ErrorUnauthorized("Incorrect token.")) } } fn check_auth(req: &HttpRequest, state: &State) -> Result<(), actix_web::Error> { if let Some(key) = &state.config.key { if let Some(auth_header) = req.headers().get("Authorization") { let auth_header = auth_header.to_str().map_err(|err| { ErrorBadRequest(format!("Invalid value in Authorization header: {err:?}")) })?; let token = jsonwebtoken::decode::( auth_header, &jsonwebtoken::DecodingKey::from_secret(key.as_ref()), &jsonwebtoken::Validation::default(), ); token .map(|_| ()) .map_err(|err| ErrorUnauthorized(format!("Invalid token: {err:?}"))) } else { Err(ErrorUnauthorized("Authorization required.")) } } else { Ok(()) } } #[derive(Deserialize)] pub struct RawRequest { native: Option, inline: Option, } #[get("/api/raw/{hash}")] pub async fn get_raw( state: web::Data, web::Query(query): web::Query, hash: web::Path, ) -> Result { let address = Address::decode(&b58_decode(hash.into_inner()).map_err(ErrorInternalServerError)?) .map_err(ErrorInternalServerError)?; if let Address::Hash(hash) = address { let hash = Arc::new(hash); let _hash = hash.clone(); let _store = state.store.clone(); let blobs = web::block(move || _store.retrieve(_hash.as_ref())) .await .map_err(ErrorInternalServerError)?; if let Some(blob) = blobs.get(0) { let file_path = blob.get_file_path(); if query.native.is_none() { return Ok(Either::A( NamedFile::open(file_path)? .set_content_disposition(ContentDisposition { disposition: if query.inline.is_some() { DispositionType::Inline } else { DispositionType::Attachment }, parameters: vec![], }) .with_header( http::header::CACHE_CONTROL, CacheControl(vec![ CacheDirective::MaxAge(2678400), CacheDirective::Extension("immutable".into(), None), ]), ), )); } else if state.config.desktop_enabled { #[cfg(feature = "desktop")] { info!("Opening {:?}...", file_path); let mut response = HttpResponse::NoContent(); let path = if !file_path.is_executable() || state.config.trust_executables { file_path } else { response .header( http::header::WARNING, "199 - Opening parent directory due to file being executable.", ) .header( http::header::ACCESS_CONTROL_EXPOSE_HEADERS, http::header::WARNING.to_string(), ); file_path.parent().ok_or_else(|| { ErrorInternalServerError("No parent to open as fallback.") })? }; opener::open(path).map_err(error::ErrorServiceUnavailable)?; return Ok(Either::B(response.finish())); } #[cfg(not(feature = "desktop"))] unreachable!() } else { return Err(error::ErrorNotImplemented("Desktop features not enabled.")); } } let connection = state.upend.connection().map_err(ErrorInternalServerError)?; let _hash = hash.clone(); let entry = web::block(move || connection.retrieve_entry(_hash.as_ref())) .await .map_err(ErrorInternalServerError)?; if let Some(entry) = entry { return Ok(Either::B(HttpResponse::Ok().json(entry))); } Err(error::ErrorNotFound("NOT FOUND")) } else { Err(ErrorBadRequest( "Address does not refer to a rawable object.", )) } } #[get("/api/thumb/{hash}")] pub async fn get_thumbnail( state: web::Data, hash: web::Path, web::Query(query): web::Query>, ) -> Result, Error> { #[cfg(feature = "previews")] if let Some(preview_store) = &state.preview_store { let hash = hash.into_inner(); let address = Address::decode(&b58_decode(&hash).map_err(ErrorInternalServerError)?) .map_err(ErrorInternalServerError)?; if let Address::Hash(address_hash) = address { let preview_store = preview_store.clone(); let (tx, rx) = oneshot::channel(); let _job_container = state.job_container.clone(); state.preview_pool.as_ref().unwrap().spawn(move || { let result = preview_store.get(address_hash, query, _job_container); tx.send(result).unwrap(); }); let preview_result = rx.await.unwrap().map_err(ErrorInternalServerError)?; if let Some(preview_path) = preview_result { let mut file = NamedFile::open(&preview_path)?.disable_content_disposition(); if let Some(mime_type) = tree_magic_mini::from_filepath(&preview_path) { if let Ok(mime) = mime_type.parse() { file = file.set_content_type(mime); } } return Ok(Either::A(file)); } else { return Ok(Either::B( HttpResponse::SeeOther() .header(http::header::LOCATION, format!("../../api/raw/{hash}")) .finish(), )); } } else { return Err(ErrorBadRequest( "Address does not refer to a previewable object.", )); } } Err(error::ErrorNotImplemented("Previews not enabled.")) } #[post("/api/query")] pub async fn get_query(state: web::Data, query: String) -> Result { let connection = state.upend.connection().map_err(ErrorInternalServerError)?; let in_query: Query = query.parse().map_err(ErrorBadRequest)?; let entries = web::block(move || connection.query(in_query)) .await .map_err(ErrorInternalServerError)?; let mut result: HashMap = HashMap::new(); for entry in entries { result.insert( b58_encode( entry .address() .map_err(ErrorInternalServerError)? .encode() .map_err(ErrorInternalServerError)?, ), entry, ); } Ok(HttpResponse::Ok().json(&result)) } trait EntriesAsHash { fn as_hash(&self) -> Result>; } impl EntriesAsHash for Vec { fn as_hash(&self) -> Result> { let mut result: HashMap = HashMap::new(); for entry in self { result.insert(b58_encode(entry.address()?.encode()?), entry); } Ok(result) } } #[get("/api/obj/{address_str}")] pub async fn get_object( state: web::Data, address: web::Path
, ) -> Result { let connection = state.upend.connection().map_err(ErrorInternalServerError)?; let address = address.into_inner(); let _address = address.clone(); let result: Vec = web::block(move || connection.retrieve_object(&_address)) .await .map_err(ErrorInternalServerError)?; debug!("{:?}", result); // TODO: make this automatically derive from `Address` definition let (entity_type, entity_content) = match address { Address::Hash(_) => ("Hash", None), Address::Uuid(_) => ("Uuid", None), Address::Attribute(attribute) => ("Attribute", Some(attribute)), Address::Url(url) => ("Url", Some(url.to_string())), }; Ok(HttpResponse::Ok().json(json!({ "entity": { "t": entity_type, "c": entity_content }, "entries": result.as_hash().map_err(ErrorInternalServerError)? }))) } #[derive(Debug, Clone, Deserialize)] #[serde(untagged)] pub enum InAddress { Address(String), Components { t: String, c: Option }, } impl TryInto
for InAddress { type Error = anyhow::Error; fn try_into(self) -> Result { Ok(match self { InAddress::Address(address) => address.parse()?, InAddress::Components { t, c } => { // I absolutely cannot handle serde magic right now // TODO: make this automatically derive from `Address` definition match t.as_str() { "Attribute" => Address::Attribute(c.ok_or(anyhow!("Missing attribute."))?), "Url" => Address::Url(if let Some(string) = c { Url::parse(&string)? } else { Err(anyhow!("Missing URL."))? }), "Uuid" => match c { Some(c) => c.parse()?, None => Address::Uuid(Uuid::new_v4()), }, _ => c.ok_or(anyhow!("Missing address."))?.parse()?, } } }) } } #[derive(Debug, Clone, Deserialize)] pub struct InEntry { pub entity: Option, pub attribute: String, pub value: EntryValue, } #[derive(Debug, Clone, Deserialize)] #[serde(untagged, deny_unknown_fields)] pub enum PutInput { Entry(InEntry), EntryList(Vec), Address { entity: InAddress }, } #[derive(Deserialize)] pub struct UpdateQuery { provenance: Option, } #[put("/api/obj")] pub async fn put_object( req: HttpRequest, state: web::Data, payload: web::Json, web::Query(query): web::Query, ) -> Result { check_auth(&req, &state)?; let (entry_address, entity_address) = { let connection = state.upend.connection().map_err(ErrorInternalServerError)?; let in_entry = payload.into_inner(); debug!("PUTting {in_entry:?}"); let provenance = query.provenance.clone(); let process_inentry = move |in_entry: InEntry| -> Result { if let Some(entity) = in_entry.entity { Ok(Entry { entity: entity.try_into()?, attribute: in_entry.attribute, value: in_entry.value, provenance: match &provenance { Some(s) => format!("API {}", s), None => "API".to_string(), }, timestamp: chrono::Utc::now().naive_utc(), }) } else { Entry::try_from(&InvariantEntry { attribute: in_entry.attribute, value: in_entry.value, }) } }; match in_entry { PutInput::Entry(in_entry) => { let entry = process_inentry(in_entry).map_err(ErrorBadRequest)?; web::block::<_, _, anyhow::Error>(move || { Ok(( Some(connection.insert_entry(entry.clone())?), Some(entry.entity), )) }) .await .map_err(ErrorInternalServerError)? } PutInput::EntryList(entries) => { web::block(move || { connection.transaction::<_, anyhow::Error, _>(|| { for entry in entries { connection.insert_entry(process_inentry(entry)?)?; } Ok(()) }) }) .await .map_err(ErrorBadRequest)?; (None, None) } PutInput::Address { entity: in_address } => { let address = in_address.try_into().map_err(ErrorBadRequest)?; let label_entry = match &address { Address::Hash(_) | Address::Uuid(_) => None, Address::Attribute(_) => None, Address::Url(url) => Some(Entry { entity: address.clone(), attribute: LABEL_ATTR.to_string(), value: url.clone().into(), provenance: match &query.provenance { Some(s) => format!("API {}", s), None => "API".to_string(), }, timestamp: chrono::Utc::now().naive_utc(), }), }; let _address = address.clone(); let _job_container = state.job_container.clone(); let _store = state.store.clone(); block_background::<_, _, anyhow::Error>(move || { let entry_count = extractors::extract(&_address, &connection, _store, _job_container); debug!("Added {entry_count} extracted entries for {_address:?}"); Ok(()) }); let connection = state.upend.connection().map_err(ErrorInternalServerError)?; web::block(move || { connection.transaction::<_, anyhow::Error, _>(|| { if connection.retrieve_object(&address)?.is_empty() { connection.insert_entry(Entry { entity: address.clone(), attribute: ADDED_ATTR.to_string(), value: EntryValue::Number( SystemTime::now() .duration_since(UNIX_EPOCH) .unwrap() .as_secs() as f64, ), provenance: match &query.provenance { Some(s) => format!("API {}", s), None => "API".to_string(), }, timestamp: chrono::Utc::now().naive_utc(), })?; } if let Some(label_entry) = label_entry { connection.insert_entry(label_entry)?; } Ok((None, Some(address))) }) }) .await .map_err(ErrorInternalServerError)? } } }; Ok(HttpResponse::Ok().json([entry_address, entity_address])) } #[put("/api/blob")] pub async fn put_blob( req: HttpRequest, state: web::Data, mut payload: Multipart, ) -> Result { check_auth(&req, &state)?; if let Some(mut field) = payload.try_next().await? { let content_disposition = field .content_disposition() .ok_or_else(|| HttpResponse::BadRequest().finish())?; let filename = content_disposition.get_filename().map(String::from); let mut file = NamedTempFile::new()?; while let Some(chunk) = field.try_next().await? { file = web::block(move || file.write_all(&chunk).map(|_| file)).await?; } let connection = state.upend.connection().map_err(ErrorInternalServerError)?; let _store = state.store.clone(); let _filename = filename.clone(); let hash = web::block(move || { _store.store(connection, Blob::from_filepath(file.path()), _filename) }) .await .map_err(ErrorInternalServerError)?; let address = Address::Hash(hash); if let Some(ref filename) = filename { let connection = state.upend.connection().map_err(ErrorInternalServerError)?; let _address = address.clone(); let _filename = filename.clone(); let _ = web::block(move || upend_insert_val!(&connection, _address, LABEL_ATTR, _filename)) .await; } let _address = address.clone(); let _job_container = state.job_container.clone(); let _store = state.store.clone(); let connection = state.upend.connection().map_err(ErrorInternalServerError)?; block_background::<_, _, anyhow::Error>(move || { let entry_count = extractors::extract(&_address, &connection, _store, _job_container); debug!("Added {entry_count} extracted entries for {_address:?}"); Ok(()) }); Ok(HttpResponse::Ok().json(address)) } else { Err(ErrorBadRequest("Multipart contains no fields.")) } } #[put("/api/obj/{address}/{attribute}")] pub async fn put_object_attribute( req: HttpRequest, state: web::Data, web::Path((address, attribute)): web::Path<(Address, String)>, value: web::Json, web::Query(query): web::Query, ) -> Result { check_auth(&req, &state)?; let connection = state.upend.connection().map_err(ErrorInternalServerError)?; let new_address = web::block(move || { connection.transaction::<_, anyhow::Error, _>(|| { let existing_attr_entries = connection.query(format!(r#"(matches @{address} "{attribute}" ?)"#).parse()?)?; for eae in existing_attr_entries { let _ = connection.remove_object(eae.address()?)?; } let new_attr_entry = Entry { entity: address, attribute, value: value.into_inner(), provenance: match &query.provenance { Some(s) => format!("API {}", s), None => "API".to_string(), }, timestamp: chrono::Utc::now().naive_utc(), }; connection.insert_entry(new_attr_entry) }) }) .await .map_err(ErrorInternalServerError)?; Ok(HttpResponse::Ok().json(new_address)) } #[delete("/api/obj/{address_str}")] pub async fn delete_object( req: HttpRequest, state: web::Data, address: web::Path
, ) -> Result { check_auth(&req, &state)?; let connection = state.upend.connection().map_err(ErrorInternalServerError)?; let _ = web::block(move || connection.remove_object(address.into_inner())) .await .map_err(ErrorInternalServerError)?; Ok(HttpResponse::Ok().finish()) } // #[post("api/obj/{address_str}")] // pub async fn update_attribute( // state: web::Data, // address_str: web::Path, // mut payload: web::Payload, // ) -> Result { // let body = load_body(&mut payload) // .await // .map_err(error::ErrorBadRequest)?; // let entry_value = serde_json::from_slice::(&body).map_err(ErrorBadRequest)?; // let connection = state.upend.connection().map_err(ErrorInternalServerError)?; // connection // .transaction::<_, anyhow::Error, _>(|| { // let address = Address::decode(&decode(address_str.into_inner())?)?; // let _ = connection.remove_object(address)?; // Ok(()) // }) // .map_err(ErrorInternalServerError)?; // Ok(HttpResponse::Ok().finish()) // } #[derive(Deserialize)] pub struct GetAddressRequest { attribute: Option, // url: Option, } #[get("/api/address")] pub async fn get_address( web::Query(query): web::Query, ) -> Result { let address = match query { GetAddressRequest { attribute: Some(attribute), } => Address::Attribute(attribute), _ => Err(ErrorBadRequest("Specify one of: `attribute`"))?, }; Ok(HttpResponse::Ok() .set_header( http::header::CACHE_CONTROL, CacheControl(vec![ CacheDirective::MaxAge(2678400), CacheDirective::Extension("immutable".into(), None), ]), ) .json(format!("{}", address))) } #[get("/api/all/attributes")] pub async fn get_all_attributes(state: web::Data) -> Result { let connection = state.upend.connection().map_err(ErrorInternalServerError)?; let attributes = web::block(move || connection.get_all_attributes()) .await .map_err(ErrorInternalServerError)?; let connection = state.upend.connection().map_err(ErrorInternalServerError)?; let result: serde_json::Value = attributes .into_iter() .map(|attribute| { json!({ "name": attribute, "labels": connection .retrieve_object(&Address::Attribute(attribute)) .unwrap_or_else(|_| vec![]) .into_iter() .filter_map(|e| { if e.attribute == LABEL_ATTR { if let EntryValue::String(label) = e.value { Some(label) } else { None } } else { None } }) .collect::>(), }) }) .collect(); Ok(HttpResponse::Ok().json(result)) } #[get("/api/hier/{path:.*}")] pub async fn list_hier( state: web::Data, path: web::Path, ) -> Result { let connection = state.upend.connection().map_err(ErrorInternalServerError)?; if path.is_empty() { Ok(HttpResponse::MovedPermanently() .header(http::header::LOCATION, "../../api/hier_roots") .finish()) } else { let upath: UHierPath = path.into_inner().parse().map_err(ErrorBadRequest)?; trace!(r#"Listing path "{}""#, upath); // todo: 500 if actual error occurs let path = web::block(move || resolve_path(&connection, &upath, false)) .await .map_err(ErrorNotFound)?; match path.last() { Some(addr) => Ok(HttpResponse::Found() .header(http::header::LOCATION, format!("../../api/obj/{}", addr)) .finish()), None => Ok(HttpResponse::NotFound().finish()), } } } #[get("/api/hier_roots")] pub async fn list_hier_roots(state: web::Data) -> Result { let connection = state.upend.connection().map_err(ErrorInternalServerError)?; let result = web::block(move || { list_roots(&connection)? .into_iter() .map(|root| connection.retrieve_object(&root)) .collect::>>>() }) .await .map_err(ErrorInternalServerError)? .concat(); Ok(HttpResponse::Ok().json(result.as_hash().map_err(ErrorInternalServerError)?)) } // #[derive(Deserialize)] // pub struct RescanRequest { // full: Option, // } #[post("/api/refresh")] pub async fn api_refresh( req: HttpRequest, state: web::Data, // web::Query(query): web::Query, ) -> Result { check_auth(&req, &state)?; block_background::<_, _, anyhow::Error>(move || { let _ = state .store .update(&state.upend, state.job_container.clone(), false); let _ = crate::extractors::extract_all( state.upend.clone(), state.store.clone(), state.job_container.clone(), ); Ok(()) }); Ok(HttpResponse::Ok().finish()) } #[get("/api/store")] pub async fn store_info(state: web::Data) -> Result { Ok(HttpResponse::Ok().json(json!({ "main": state.store.stats().map_err(ErrorInternalServerError)? }))) } #[derive(Deserialize)] pub struct JobsRequest { full: Option, } #[get("/api/jobs")] pub async fn get_jobs( state: web::Data, web::Query(query): web::Query, ) -> Result { let jobs = state .job_container .get_jobs() .map_err(ErrorInternalServerError)?; Ok(HttpResponse::Ok().json(if query.full.is_some() { jobs } else { jobs.into_iter() .filter(|(_, j)| matches!(j.state, jobs::JobState::InProgress)) .collect() })) } #[get("/api/info")] pub async fn get_info(state: web::Data) -> Result { Ok(HttpResponse::Ok().json(json!({ "name": state.config.vault_name, // "location": &*state.store.path, "version": build::PKG_VERSION, "desktop": state.config.desktop_enabled }))) } #[cfg(test)] mod tests { use super::*; use anyhow::Result; #[test] fn test_in_address() -> Result<()> { let address = Address::Url(Url::parse("https://upend.dev").unwrap()); let in_address = InAddress::Address(address.to_string()); assert_eq!(address, in_address.try_into()?); let in_address = InAddress::Components { t: "Url".into(), c: Some("https://upend.dev".into()), }; assert_eq!(address, in_address.try_into()?); Ok(()) } }